diff options
Diffstat (limited to 'spec/lib')
347 files changed, 9834 insertions, 4354 deletions
diff --git a/spec/lib/api/ci/helpers/runner_helpers_spec.rb b/spec/lib/api/ci/helpers/runner_helpers_spec.rb index b254c419cbc..d32f7e4f0be 100644 --- a/spec/lib/api/ci/helpers/runner_helpers_spec.rb +++ b/spec/lib/api/ci/helpers/runner_helpers_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe API::Ci::Helpers::Runner do +RSpec.describe API::Ci::Helpers::Runner, feature_category: :runner do let(:ip_address) { '1.2.3.4' } let(:runner_class) do Class.new do diff --git a/spec/lib/api/entities/package_spec.rb b/spec/lib/api/entities/package_spec.rb index d63ea7833ac..9288f6fe8eb 100644 --- a/spec/lib/api/entities/package_spec.rb +++ b/spec/lib/api/entities/package_spec.rb @@ -32,4 +32,12 @@ RSpec.describe API::Entities::Package do expect(subject[:_links][:web_path]).to match('/infrastructure_registry/') end end + + context 'when package has no default status' do + let(:package) { create(:package, :error) } + + it 'does not expose web_path in _links' do + expect(subject[:_links]).not_to have_key(:web_path) + end + end end diff --git a/spec/lib/api/entities/plan_limit_spec.rb b/spec/lib/api/entities/plan_limit_spec.rb index a88ea3f4cad..baaaeb0b600 100644 --- a/spec/lib/api/entities/plan_limit_spec.rb +++ b/spec/lib/api/entities/plan_limit_spec.rb @@ -25,7 +25,8 @@ RSpec.describe API::Entities::PlanLimit do :nuget_max_file_size, :pypi_max_file_size, :terraform_module_max_file_size, - :storage_size_limit + :storage_size_limit, + :pipeline_hierarchy_size ) end diff --git a/spec/lib/api/entities/ssh_key_spec.rb b/spec/lib/api/entities/ssh_key_spec.rb index 768ad416fbe..b4310035a66 100644 --- a/spec/lib/api/entities/ssh_key_spec.rb +++ b/spec/lib/api/entities/ssh_key_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe API::Entities::SSHKey do +RSpec.describe API::Entities::SSHKey, feature_category: :authentication_and_authorization do describe '#as_json' do subject { entity.as_json } @@ -15,7 +15,8 @@ RSpec.describe API::Entities::SSHKey do title: key.title, created_at: key.created_at, expires_at: key.expires_at, - key: key.publishable_key + key: key.publishable_key, + usage_type: 'auth_and_signing' ) end end diff --git a/spec/lib/api/every_api_endpoint_spec.rb b/spec/lib/api/every_api_endpoint_spec.rb index 5fe14823a29..c45ff9eb628 100644 --- a/spec/lib/api/every_api_endpoint_spec.rb +++ b/spec/lib/api/every_api_endpoint_spec.rb @@ -32,10 +32,21 @@ RSpec.describe 'Every API endpoint' do next unless used_category next if used_category == :not_owned - [path, used_category] unless feature_categories.include?(used_category) + [klass, path, used_category] unless feature_categories.include?(used_category) end.compact - expect(routes_unknown_category).to be_empty, "#{routes_unknown_category.first(10)} had an unknown category" + message = -> do + list = routes_unknown_category.map do |klass, path, category| + "- #{klass} (#{path}): #{category}" + end + + <<~MESSAGE + Unknown categories found for: + #{list.join("\n")} + MESSAGE + end + + expect(routes_unknown_category).to be_empty, message end # This is required for API::Base.path_for_app to work, as it picks diff --git a/spec/lib/api/helpers/packages_helpers_spec.rb b/spec/lib/api/helpers/packages_helpers_spec.rb index b9c887b3e16..a3b21059334 100644 --- a/spec/lib/api/helpers/packages_helpers_spec.rb +++ b/spec/lib/api/helpers/packages_helpers_spec.rb @@ -238,4 +238,26 @@ RSpec.describe API::Helpers::PackagesHelpers do end end end + + describe '#track_package_event' do + before do + allow(helper).to receive(:current_user).and_return(user) + end + + it_behaves_like 'Snowplow event tracking with RedisHLL context' do + let(:action) { 'push_package' } + let(:scope) { :terraform_module } + let(:category) { described_class.name } + let(:namespace) { project.namespace } + let(:user) { project.creator } + let(:feature_flag_name) { nil } + let(:label) { 'redis_hll_counters.user_packages.user_packages_total_unique_counts_monthly' } + let(:property) { 'i_package_terraform_module_user' } + + subject(:package_action) do + args = { category: category, namespace: namespace, user: user, project: project } + helper.track_package_event(action, scope, **args) + end + end + end end diff --git a/spec/lib/api/helpers/rate_limiter_spec.rb b/spec/lib/api/helpers/rate_limiter_spec.rb index 2fed1cf3604..3640c7e30e7 100644 --- a/spec/lib/api/helpers/rate_limiter_spec.rb +++ b/spec/lib/api/helpers/rate_limiter_spec.rb @@ -19,8 +19,7 @@ RSpec.describe API::Helpers::RateLimiter do @current_user = current_user end - def render_api_error!(**args) - end + def render_api_error!(**args); end end end diff --git a/spec/lib/api/support/git_access_actor_spec.rb b/spec/lib/api/support/git_access_actor_spec.rb index e1c800d25a7..b3e8787583c 100644 --- a/spec/lib/api/support/git_access_actor_spec.rb +++ b/spec/lib/api/support/git_access_actor_spec.rb @@ -9,7 +9,8 @@ RSpec.describe API::Support::GitAccessActor do subject { described_class.new(user: user, key: key) } describe '.from_params' do - let(:key) { create(:key) } + let_it_be(:user) { create(:user) } + let_it_be(:key) { create(:key, user: user) } context 'with params that are valid' do it 'returns an instance of API::Support::GitAccessActor' do @@ -31,6 +32,42 @@ RSpec.describe API::Support::GitAccessActor do expect(described_class.from_params(identifier: "key-#{key.id}").user).to eq(key.user) end end + + context 'when passing a signing key' do + let_it_be(:key) { create(:key, usage_type: :signing, user: user) } + + it 'does not identify the user' do + actor = described_class.from_params({ identifier: "key-#{key.id}" }) + + expect(actor).to be_instance_of(described_class) + expect(actor.user).to be_nil + end + + it 'does not identify the key' do + actor = described_class.from_params({ key_id: key.id }) + + expect(actor).to be_instance_of(described_class) + expect(actor.key).to be_nil + end + end + + context 'when passing an auth-only key' do + let_it_be(:key) { create(:key, usage_type: :auth, user: user) } + + it 'identifies the user' do + actor = described_class.from_params({ identifier: "key-#{key.id}" }) + + expect(actor).to be_instance_of(described_class) + expect(actor.user).to eq(key.user) + end + + it 'identifies the key' do + actor = described_class.from_params({ key_id: key.id }) + + expect(actor).to be_instance_of(described_class) + expect(actor.key).to eq(key) + end + end end describe 'attributes' do diff --git a/spec/lib/atlassian/jira_connect/client_spec.rb b/spec/lib/atlassian/jira_connect/client_spec.rb index 0ae0f02c46e..a8ee28d3714 100644 --- a/spec/lib/atlassian/jira_connect/client_spec.rb +++ b/spec/lib/atlassian/jira_connect/client_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Atlassian::JiraConnect::Client do +RSpec.describe Atlassian::JiraConnect::Client, feature_category: :integrations do include StubRequests subject(:client) { described_class.new('https://gitlab-test.atlassian.net', 'sample_secret') } @@ -119,7 +119,7 @@ RSpec.describe Atlassian::JiraConnect::Client do let(:errors) { [{ 'message' => 'X' }, { 'message' => 'Y' }] } let(:processed) { subject.send(:handle_response, response, 'foo') { |x| [:data, x] } } - context 'the response is 200 OK' do + context 'when the response is 200 OK' do let(:response) { double(code: 200, parsed_response: :foo) } it 'yields to the block' do @@ -127,7 +127,7 @@ RSpec.describe Atlassian::JiraConnect::Client do end end - context 'the response is 202 accepted' do + context 'when the response is 202 accepted' do let(:response) { double(code: 202, parsed_response: :foo) } it 'yields to the block' do @@ -135,15 +135,15 @@ RSpec.describe Atlassian::JiraConnect::Client do end end - context 'the response is 400 bad request' do + context 'when the response is 400 bad request' do let(:response) { double(code: 400, parsed_response: errors) } it 'extracts the errors messages' do - expect(processed).to eq('errorMessages' => %w(X Y), 'responseCode' => 400) + expect(processed).to eq('errorMessages' => %w[X Y], 'responseCode' => 400) end end - context 'the response is 401 forbidden' do + context 'when the response is 401 forbidden' do let(:response) { double(code: 401, parsed_response: nil) } it 'reports that our JWT is wrong' do @@ -151,7 +151,7 @@ RSpec.describe Atlassian::JiraConnect::Client do end end - context 'the response is 403' do + context 'when the response is 403' do let(:response) { double(code: 403, parsed_response: nil) } it 'reports that the App is misconfigured' do @@ -159,7 +159,7 @@ RSpec.describe Atlassian::JiraConnect::Client do end end - context 'the response is 413' do + context 'when the response is 413' do let(:response) { double(code: 413, parsed_response: errors) } it 'extracts the errors messages' do @@ -167,7 +167,7 @@ RSpec.describe Atlassian::JiraConnect::Client do end end - context 'the response is 429' do + context 'when the response is 429' do let(:response) { double(code: 429, parsed_response: nil) } it 'reports that we exceeded the rate limit' do @@ -175,7 +175,7 @@ RSpec.describe Atlassian::JiraConnect::Client do end end - context 'the response is 503' do + context 'when the response is 503' do let(:response) { double(code: 503, parsed_response: nil) } it 'reports that the service is unavailable' do @@ -183,7 +183,7 @@ RSpec.describe Atlassian::JiraConnect::Client do end end - context 'the response is anything else' do + context 'when the response is anything else' do let(:response) { double(code: 1000, parsed_response: :something) } it 'reports that this was unanticipated' do @@ -192,6 +192,26 @@ RSpec.describe Atlassian::JiraConnect::Client do end end + describe '#request_body_schema' do + let(:response) { instance_double(HTTParty::Response, success?: true, code: 200, request: request) } + + context 'with valid JSON request body' do + let(:request) { instance_double(HTTParty::Request, raw_body: '{ "foo": 1, "bar": 2 }') } + + it 'returns the request body' do + expect(subject.send(:request_body_schema, response)).to eq({ "foo" => nil, "bar" => nil }) + end + end + + context 'with invalid JSON request body' do + let(:request) { instance_double(HTTParty::Request, raw_body: 'invalid json') } + + it 'reports the invalid json' do + expect(subject.send(:request_body_schema, response)).to eq('Request body includes invalid JSON') + end + end + end + describe '#store_deploy_info' do let_it_be(:environment) { create(:environment, name: 'DEV', project: project) } let_it_be(:deployments) do @@ -222,7 +242,7 @@ RSpec.describe Atlassian::JiraConnect::Client do before do path = '/rest/deployments/0.1/bulk' - stub_full_request('https://gitlab-test.atlassian.net' + path, method: :post) + stub_full_request("https://gitlab-test.atlassian.net#{path}", method: :post) .with(body: body, headers: expected_headers(path)) .to_return(body: response_body, headers: { 'Content-Type': 'application/json' }) end @@ -232,7 +252,9 @@ RSpec.describe Atlassian::JiraConnect::Client do end it 'only sends information about relevant MRs' do - expect(subject).to receive(:post).with('/rest/deployments/0.1/bulk', { deployments: have_attributes(size: 6) }).and_call_original + expect(subject).to receive(:post).with( + '/rest/deployments/0.1/bulk', { deployments: have_attributes(size: 6) } + ).and_call_original subject.send(:store_deploy_info, project: project, deployments: deployments) end @@ -243,7 +265,7 @@ RSpec.describe Atlassian::JiraConnect::Client do subject.send(:store_deploy_info, project: project, deployments: deployments.take(1)) end - context 'there are errors' do + context 'when there are errors' do let(:rejections) do [{ errors: [{ message: 'X' }, { message: 'Y' }] }, { errors: [{ message: 'Z' }] }] end @@ -251,7 +273,9 @@ RSpec.describe Atlassian::JiraConnect::Client do it 'reports the errors' do response = subject.send(:store_deploy_info, project: project, deployments: deployments) - expect(response['errorMessages']).to eq(%w(X Y Z)) + expect(response['errorMessages']).to eq(%w[X Y Z]) + expect(response['responseCode']).to eq(200) + expect(response['requestBody']).to be_a(Hash) end end end @@ -282,7 +306,7 @@ RSpec.describe Atlassian::JiraConnect::Client do feature_flags.first.update!(description: 'RELEVANT-123') feature_flags.second.update!(description: 'RELEVANT-123') path = '/rest/featureflags/0.1/bulk' - stub_full_request('https://gitlab-test.atlassian.net' + path, method: :post) + stub_full_request("https://gitlab-test.atlassian.net#{path}", method: :post) .with(body: body, headers: expected_headers(path)) .to_return(body: response_body, headers: { 'Content-Type': 'application/json' }) end @@ -292,9 +316,9 @@ RSpec.describe Atlassian::JiraConnect::Client do end it 'only sends information about relevant MRs' do - expect(subject).to receive(:post).with('/rest/featureflags/0.1/bulk', { - flags: have_attributes(size: 2), properties: Hash - }).and_call_original + expect(subject).to receive(:post).with( + '/rest/featureflags/0.1/bulk', { flags: have_attributes(size: 2), properties: Hash } + ).and_call_original subject.send(:store_ff_info, project: project, feature_flags: feature_flags) end @@ -305,7 +329,7 @@ RSpec.describe Atlassian::JiraConnect::Client do subject.send(:store_ff_info, project: project, feature_flags: [feature_flags.last]) end - context 'there are errors' do + context 'when there are errors' do let(:failures) do { a: [{ message: 'X' }, { message: 'Y' }], @@ -343,7 +367,7 @@ RSpec.describe Atlassian::JiraConnect::Client do before do path = '/rest/builds/0.1/bulk' - stub_full_request('https://gitlab-test.atlassian.net' + path, method: :post) + stub_full_request("https://gitlab-test.atlassian.net#{path}", method: :post) .with(body: body, headers: expected_headers(path)) .to_return(body: response_body, headers: { 'Content-Type': 'application/json' }) end @@ -366,7 +390,7 @@ RSpec.describe Atlassian::JiraConnect::Client do subject.send(:store_build_info, project: project, pipelines: pipelines.take(1)) end - context 'there are errors' do + context 'when there are errors' do let(:failures) do [{ errors: [{ message: 'X' }, { message: 'Y' }] }, { errors: [{ message: 'Z' }] }] end @@ -374,7 +398,9 @@ RSpec.describe Atlassian::JiraConnect::Client do it 'reports the errors' do response = subject.send(:store_build_info, project: project, pipelines: pipelines) - expect(response['errorMessages']).to eq(%w(X Y Z)) + expect(response['errorMessages']).to eq(%w[X Y Z]) + expect(response['responseCode']).to eq(200) + expect(response['requestBody']).to be_a(Hash) end end @@ -385,19 +411,21 @@ RSpec.describe Atlassian::JiraConnect::Client do subject.send(:store_build_info, project: project, pipelines: pipelines) end - pipelines << create(:ci_pipeline, head_pipeline_of: create(:merge_request, :jira_branch)) + pipelines << create(:ci_pipeline, project: project, head_pipeline_of: create(:merge_request, :jira_branch, source_project: project)) - expect { subject.send(:store_build_info, project: project, pipelines: pipelines) }.not_to exceed_query_limit(baseline) + expect do + subject.send(:store_build_info, project: project, pipelines: pipelines) + end.not_to exceed_query_limit(baseline) end end describe '#store_dev_info' do - let_it_be(:merge_requests) { create_list(:merge_request, 2, :unique_branches) } + let_it_be(:merge_requests) { create_list(:merge_request, 2, :unique_branches, source_project: project) } before do path = '/rest/devinfo/0.10/bulk' - stub_full_request('https://gitlab-test.atlassian.net' + path, method: :post) + stub_full_request("https://gitlab-test.atlassian.net#{path}", method: :post) .with(headers: expected_headers(path)) end @@ -406,11 +434,16 @@ RSpec.describe Atlassian::JiraConnect::Client do end it 'avoids N+1 database queries' do - control_count = ActiveRecord::QueryRecorder.new { subject.send(:store_dev_info, project: project, merge_requests: merge_requests) }.count + control_count = ActiveRecord::QueryRecorder.new do + subject.send(:store_dev_info, project: project, merge_requests: merge_requests) + end.count - merge_requests << create(:merge_request, :unique_branches) + merge_requests << create(:merge_request, :unique_branches, source_project: project) - expect { subject.send(:store_dev_info, project: project, merge_requests: merge_requests) }.not_to exceed_query_limit(control_count) + expect do + subject.send(:store_dev_info, project: project, + merge_requests: merge_requests) + end.not_to exceed_query_limit(control_count) end end diff --git a/spec/lib/atlassian/jira_connect/jwt/asymmetric_spec.rb b/spec/lib/atlassian/jira_connect/jwt/asymmetric_spec.rb index 86d672067a3..89c85489aea 100644 --- a/spec/lib/atlassian/jira_connect/jwt/asymmetric_spec.rb +++ b/spec/lib/atlassian/jira_connect/jwt/asymmetric_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Atlassian::JiraConnect::Jwt::Asymmetric do +RSpec.describe Atlassian::JiraConnect::Jwt::Asymmetric, feature_category: :integrations do describe '#valid?' do let_it_be(:private_key) { OpenSSL::PKey::RSA.generate 3072 } @@ -17,6 +17,7 @@ RSpec.describe Atlassian::JiraConnect::Jwt::Asymmetric do let(:jwt) { JWT.encode(jwt_claims, private_key, 'RS256', jwt_headers) } let(:public_key) { private_key.public_key } let(:stub_asymmetric_jwt_cdn) { 'https://connect-install-keys.atlassian.com' } + let(:jira_connect_proxy_url_setting) { nil } let(:install_keys_url) { "#{stub_asymmetric_jwt_cdn}/#{public_key_id}" } let(:qsh) do Atlassian::Jwt.create_query_string_hash('https://gitlab.test/events/installed', 'POST', 'https://gitlab.test') @@ -25,6 +26,8 @@ RSpec.describe Atlassian::JiraConnect::Jwt::Asymmetric do before do stub_request(:get, install_keys_url) .to_return(body: public_key.to_s, status: 200) + + stub_application_setting(jira_connect_proxy_url: jira_connect_proxy_url_setting) end it 'returns true when verified with public key from CDN' do @@ -89,10 +92,7 @@ RSpec.describe Atlassian::JiraConnect::Jwt::Asymmetric do context 'with jira_connect_proxy_url setting' do let(:stub_asymmetric_jwt_cdn) { 'https://example.com/-/jira_connect/public_keys' } - - before do - stub_application_setting(jira_connect_proxy_url: 'https://example.com') - end + let(:jira_connect_proxy_url_setting) { 'https://example.com' } it 'requests the settings CDN' do expect(JWT).to receive(:decode).twice.and_call_original @@ -101,22 +101,6 @@ RSpec.describe Atlassian::JiraConnect::Jwt::Asymmetric do expect(WebMock).to have_requested(:get, "https://example.com/-/jira_connect/public_keys/#{public_key_id}") end - - context 'when jira_connect_oauth_self_managed disabled' do - let(:stub_asymmetric_jwt_cdn) { 'https://connect-install-keys.atlassian.com' } - - before do - stub_feature_flags(jira_connect_oauth_self_managed: false) - end - - it 'requests the default CDN' do - expect(JWT).to receive(:decode).twice.and_call_original - - expect(asymmetric_jwt).to be_valid - - expect(WebMock).to have_requested(:get, install_keys_url) - end - end end end diff --git a/spec/lib/atlassian/jira_connect/jwt/symmetric_spec.rb b/spec/lib/atlassian/jira_connect/jwt/symmetric_spec.rb index 61adff7e221..109868f5d95 100644 --- a/spec/lib/atlassian/jira_connect/jwt/symmetric_spec.rb +++ b/spec/lib/atlassian/jira_connect/jwt/symmetric_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Atlassian::JiraConnect::Jwt::Symmetric do +RSpec.describe Atlassian::JiraConnect::Jwt::Symmetric, feature_category: :integrations do let(:shared_secret) { 'secret' } describe '#iss_claim' do diff --git a/spec/lib/atlassian/jira_connect/serializers/author_entity_spec.rb b/spec/lib/atlassian/jira_connect/serializers/author_entity_spec.rb index f31cf929244..e1158bb5988 100644 --- a/spec/lib/atlassian/jira_connect/serializers/author_entity_spec.rb +++ b/spec/lib/atlassian/jira_connect/serializers/author_entity_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Atlassian::JiraConnect::Serializers::AuthorEntity do +RSpec.describe Atlassian::JiraConnect::Serializers::AuthorEntity, feature_category: :integrations do subject { described_class.represent(user).as_json } context 'when object is a User model' do diff --git a/spec/lib/atlassian/jira_connect/serializers/base_entity_spec.rb b/spec/lib/atlassian/jira_connect/serializers/base_entity_spec.rb index d7672c0baf1..f34bec1413b 100644 --- a/spec/lib/atlassian/jira_connect/serializers/base_entity_spec.rb +++ b/spec/lib/atlassian/jira_connect/serializers/base_entity_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Atlassian::JiraConnect::Serializers::BaseEntity do +RSpec.describe Atlassian::JiraConnect::Serializers::BaseEntity, feature_category: :integrations do let(:update_sequence_id) { nil } subject do diff --git a/spec/lib/atlassian/jira_connect/serializers/branch_entity_spec.rb b/spec/lib/atlassian/jira_connect/serializers/branch_entity_spec.rb index e69e2aae94c..86e48a4a0fd 100644 --- a/spec/lib/atlassian/jira_connect/serializers/branch_entity_spec.rb +++ b/spec/lib/atlassian/jira_connect/serializers/branch_entity_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Atlassian::JiraConnect::Serializers::BranchEntity do +RSpec.describe Atlassian::JiraConnect::Serializers::BranchEntity, feature_category: :integrations do let(:project) { create(:project, :repository) } let(:branch) { project.repository.find_branch('improve/awesome') } diff --git a/spec/lib/atlassian/jira_connect/serializers/build_entity_spec.rb b/spec/lib/atlassian/jira_connect/serializers/build_entity_spec.rb index a29f32d35b8..48787f2a0d2 100644 --- a/spec/lib/atlassian/jira_connect/serializers/build_entity_spec.rb +++ b/spec/lib/atlassian/jira_connect/serializers/build_entity_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Atlassian::JiraConnect::Serializers::BuildEntity do +RSpec.describe Atlassian::JiraConnect::Serializers::BuildEntity, feature_category: :integrations do let_it_be(:user) { create_default(:user) } let_it_be(:project) { create_default(:project) } diff --git a/spec/lib/atlassian/jira_connect/serializers/deployment_entity_spec.rb b/spec/lib/atlassian/jira_connect/serializers/deployment_entity_spec.rb index 40b9e83719b..f6fca39fa68 100644 --- a/spec/lib/atlassian/jira_connect/serializers/deployment_entity_spec.rb +++ b/spec/lib/atlassian/jira_connect/serializers/deployment_entity_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Atlassian::JiraConnect::Serializers::DeploymentEntity do +RSpec.describe Atlassian::JiraConnect::Serializers::DeploymentEntity, feature_category: :integrations do let_it_be(:user) { create_default(:user) } let_it_be(:project) { create_default(:project, :repository) } let_it_be(:environment) { create(:environment, name: 'prod', project: project) } diff --git a/spec/lib/atlassian/jira_connect/serializers/feature_flag_entity_spec.rb b/spec/lib/atlassian/jira_connect/serializers/feature_flag_entity_spec.rb index 2d12cd1ed0a..3f84404f38d 100644 --- a/spec/lib/atlassian/jira_connect/serializers/feature_flag_entity_spec.rb +++ b/spec/lib/atlassian/jira_connect/serializers/feature_flag_entity_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Atlassian::JiraConnect::Serializers::FeatureFlagEntity do +RSpec.describe Atlassian::JiraConnect::Serializers::FeatureFlagEntity, feature_category: :integrations do let_it_be(:user) { create_default(:user) } let_it_be(:project) { create_default(:project) } diff --git a/spec/lib/atlassian/jira_connect/serializers/pull_request_entity_spec.rb b/spec/lib/atlassian/jira_connect/serializers/pull_request_entity_spec.rb index 6399fc9053b..5ebb5ffed3b 100644 --- a/spec/lib/atlassian/jira_connect/serializers/pull_request_entity_spec.rb +++ b/spec/lib/atlassian/jira_connect/serializers/pull_request_entity_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Atlassian::JiraConnect::Serializers::PullRequestEntity do +RSpec.describe Atlassian::JiraConnect::Serializers::PullRequestEntity, feature_category: :integrations do let_it_be(:project) { create_default(:project, :repository) } let_it_be(:merge_requests) { create_list(:merge_request, 2, :unique_branches) } let_it_be(:notes) { create_list(:note, 2, system: false, noteable: merge_requests.first) } diff --git a/spec/lib/atlassian/jira_connect/serializers/repository_entity_spec.rb b/spec/lib/atlassian/jira_connect/serializers/repository_entity_spec.rb index 9100398ecc5..2a4fba0f00e 100644 --- a/spec/lib/atlassian/jira_connect/serializers/repository_entity_spec.rb +++ b/spec/lib/atlassian/jira_connect/serializers/repository_entity_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Atlassian::JiraConnect::Serializers::RepositoryEntity do +RSpec.describe Atlassian::JiraConnect::Serializers::RepositoryEntity, feature_category: :integrations do let(:update_sequence_id) { nil } subject do diff --git a/spec/lib/atlassian/jira_connect_spec.rb b/spec/lib/atlassian/jira_connect_spec.rb index d9c34e938b4..14bf13b8fe6 100644 --- a/spec/lib/atlassian/jira_connect_spec.rb +++ b/spec/lib/atlassian/jira_connect_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Atlassian::JiraConnect do +RSpec.describe Atlassian::JiraConnect, feature_category: :integrations do describe '.app_name' do subject { described_class.app_name } @@ -25,5 +25,13 @@ RSpec.describe Atlassian::JiraConnect do expect(app_key).to eq('gitlab-jira-connect-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx') end end + + context 'with jira_connect_proxy_url setting' do + before do + stub_application_setting(jira_connect_proxy_url: 'https://example.com') + end + + it { is_expected.to eq('gitlab-jira-connect-example.com') } + end end end diff --git a/spec/lib/backup/gitaly_backup_spec.rb b/spec/lib/backup/gitaly_backup_spec.rb index 6b0747735ed..7cc8ce2cbae 100644 --- a/spec/lib/backup/gitaly_backup_spec.rb +++ b/spec/lib/backup/gitaly_backup_spec.rb @@ -61,7 +61,7 @@ RSpec.describe Backup::GitalyBackup do it 'erases any existing repository backups' do existing_file = File.join(destination, 'some_existing_file') - IO.write(existing_file, "Some existing file.\n") + File.write(existing_file, "Some existing file.\n") subject.start(:create, destination, backup_id: backup_id) subject.finish! diff --git a/spec/lib/backup/manager_spec.rb b/spec/lib/backup/manager_spec.rb index f85b005f4d1..992dbec73c2 100644 --- a/spec/lib/backup/manager_spec.rb +++ b/spec/lib/backup/manager_spec.rb @@ -166,7 +166,7 @@ RSpec.describe Backup::Manager do describe '#create' do let(:incremental_env) { 'false' } let(:expected_backup_contents) { %w{backup_information.yml task1.tar.gz task2.tar.gz} } - let(:backup_time) { Time.utc(2019, 1, 1) } + let(:backup_time) { Time.zone.parse('2019-1-1') } let(:backup_id) { "1546300800_2019_01_01_#{Gitlab::VERSION}" } let(:full_backup_id) { backup_id } let(:pack_tar_file) { "#{backup_id}_gitlab_backup.tar" } @@ -284,7 +284,7 @@ RSpec.describe Backup::Manager do allow(Dir).to receive(:chdir).and_yield allow(Dir).to receive(:glob).and_return(files) allow(FileUtils).to receive(:rm) - allow(Time).to receive(:now).and_return(Time.utc(2016)) + allow(Time).to receive(:now).and_return(Time.zone.parse('2016-1-1')) end context 'when keep_time is zero' do diff --git a/spec/lib/banzai/filter/attributes_filter_spec.rb b/spec/lib/banzai/filter/attributes_filter_spec.rb new file mode 100644 index 00000000000..cef5e24cdaa --- /dev/null +++ b/spec/lib/banzai/filter/attributes_filter_spec.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Banzai::Filter::AttributesFilter, feature_category: :team_planning do + using RSpec::Parameterized::TableSyntax + include FilterSpecHelper + + def image + %(<img src="example.jpg">) + end + + describe 'attribute syntax' do + context 'when attribute syntax is valid' do + where(:text, :result) do + "#{image}{width=100}" | '<img src="example.jpg" width="100">' + "#{image}{ width=100 }" | '<img src="example.jpg" width="100">' + "#{image}{width=\"100\"}" | '<img src="example.jpg" width="100">' + "#{image}{width=100 width=200}" | '<img src="example.jpg" width="200">' + + "#{image}{.test_class width=100 style=\"width:400\"}" | '<img src="example.jpg" width="100">' + "<img src=\"example.jpg\" class=\"lazy\" />{width=100}" | '<img src="example.jpg" class="lazy" width="100">' + end + + with_them do + it 'adds them to the img' do + expect(filter(text).to_html).to eq result + end + end + end + + context 'when attribute syntax is invalid' do + where(:text, :result) do + "#{image} {width=100}" | '<img src="example.jpg"> {width=100}' + "#{image}{width=100\nheight=100}" | "<img src=\"example.jpg\">{width=100\nheight=100}" + "{width=100 height=100}\n#{image}" | "{width=100 height=100}\n<img src=\"example.jpg\">" + '<h1>header</h1>{width=100}' | '<h1>header</h1>{width=100}' + end + + with_them do + it 'does not recognize as attributes' do + expect(filter(text).to_html).to eq result + end + end + end + end + + describe 'height and width' do + context 'when size attributes are valid' do + where(:text, :result) do + "#{image}{width=100 height=200px}" | '<img src="example.jpg" width="100" height="200px">' + "#{image}{width=100}" | '<img src="example.jpg" width="100">' + "#{image}{width=100px}" | '<img src="example.jpg" width="100px">' + "#{image}{height=100%}" | '<img src="example.jpg" height="100%">' + "#{image}{width=\"100%\"}" | '<img src="example.jpg" width="100%">' + end + + with_them do + it 'adds them to the img' do + expect(filter(text).to_html).to eq result + end + end + end + + context 'when size attributes are invalid' do + where(:text, :result) do + "#{image}{width=100cs}" | '<img src="example.jpg">' + "#{image}{width=auto height=200}" | '<img src="example.jpg" height="200">' + end + + with_them do + it 'ignores them' do + expect(filter(text).to_html).to eq result + end + end + end + end +end diff --git a/spec/lib/banzai/filter/commit_trailers_filter_spec.rb b/spec/lib/banzai/filter/commit_trailers_filter_spec.rb index c22517621c1..3ebe0798972 100644 --- a/spec/lib/banzai/filter/commit_trailers_filter_spec.rb +++ b/spec/lib/banzai/filter/commit_trailers_filter_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' require 'ffaker' -RSpec.describe Banzai::Filter::CommitTrailersFilter do +RSpec.describe Banzai::Filter::CommitTrailersFilter, feature_category: :source_code_management do include FilterSpecHelper include CommitTrailersSpecHelper diff --git a/spec/lib/banzai/filter/inline_grafana_metrics_filter_spec.rb b/spec/lib/banzai/filter/inline_grafana_metrics_filter_spec.rb index d29af311ee5..db0c10a802b 100644 --- a/spec/lib/banzai/filter/inline_grafana_metrics_filter_spec.rb +++ b/spec/lib/banzai/filter/inline_grafana_metrics_filter_spec.rb @@ -29,12 +29,12 @@ RSpec.describe Banzai::Filter::InlineGrafanaMetricsFilter do ) end - it_behaves_like 'a metrics embed filter' - around do |example| travel_to(Time.utc(2019, 3, 17, 13, 10)) { example.run } end + it_behaves_like 'a metrics embed filter' + context 'when grafana is not configured' do before do allow(project).to receive(:grafana_integration).and_return(nil) diff --git a/spec/lib/banzai/filter/inline_observability_filter_spec.rb b/spec/lib/banzai/filter/inline_observability_filter_spec.rb new file mode 100644 index 00000000000..341ada6d2b5 --- /dev/null +++ b/spec/lib/banzai/filter/inline_observability_filter_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Banzai::Filter::InlineObservabilityFilter do + include FilterSpecHelper + + let(:input) { %(<a href="#{url}">example</a>) } + let(:doc) { filter(input) } + + context 'when the document has an external link' do + let(:url) { 'https://foo.com' } + + it 'leaves regular non-observability links unchanged' do + expect(doc.to_s).to eq(input) + end + end + + context 'when the document contains an embeddable observability link' do + let(:url) { 'https://observe.gitlab.com/12345' } + + it 'leaves the original link unchanged' do + expect(doc.at_css('a').to_s).to eq(input) + end + + it 'appends a observability charts placeholder' do + node = doc.at_css('.js-render-observability') + + expect(node).to be_present + expect(node.attribute('data-frame-url').to_s).to eq(url) + end + end +end diff --git a/spec/lib/banzai/filter/references/reference_filter_spec.rb b/spec/lib/banzai/filter/references/reference_filter_spec.rb index b14b9374364..6d7396ef216 100644 --- a/spec/lib/banzai/filter/references/reference_filter_spec.rb +++ b/spec/lib/banzai/filter/references/reference_filter_spec.rb @@ -148,7 +148,7 @@ RSpec.describe Banzai::Filter::References::ReferenceFilter do include_context 'document nodes' let(:node) { Nokogiri::HTML.fragment('text @reference') } - let(:ref_pattern) { %r{(?<!\w)@(?<user>[a-zA-Z0-9_\-\.]*)}x } + let(:ref_pattern) { %r{(?<!\w)@(?<user>[a-zA-Z0-9_\-.]*)}x } context 'when node has no reference pattern' do let(:node) { Nokogiri::HTML.fragment('random text') } diff --git a/spec/lib/banzai/filter/references/user_reference_filter_spec.rb b/spec/lib/banzai/filter/references/user_reference_filter_spec.rb index b153efd9655..d61b71c711d 100644 --- a/spec/lib/banzai/filter/references/user_reference_filter_spec.rb +++ b/spec/lib/banzai/filter/references/user_reference_filter_spec.rb @@ -42,12 +42,12 @@ RSpec.describe Banzai::Filter::References::UserReferenceFilter do context 'mentioning @all' do let(:reference) { User.reference_prefix + 'all' } - it_behaves_like 'a reference containing an element node' - before do project.add_developer(project.creator) end + it_behaves_like 'a reference containing an element node' + it 'supports a special @all mention' do project.add_developer(user) doc = reference_filter("Hey #{reference}", author: user) diff --git a/spec/lib/banzai/filter/repository_link_filter_spec.rb b/spec/lib/banzai/filter/repository_link_filter_spec.rb index 4aeb6e2a722..0df680dc0c8 100644 --- a/spec/lib/banzai/filter/repository_link_filter_spec.rb +++ b/spec/lib/banzai/filter/repository_link_filter_spec.rb @@ -69,7 +69,7 @@ RSpec.describe Banzai::Filter::RepositoryLinkFilter do expect { filter(raw_doc) }.to change { Gitlab::GitalyClient.get_request_count }.by(2) end - shared_examples :preserve_unchanged do + shared_examples 'preserve unchanged' do it 'does not modify any relative URL in anchor' do doc = filter(link('README.md')) expect(doc.at_css('a')['href']).to eq 'README.md' @@ -96,25 +96,25 @@ RSpec.describe Banzai::Filter::RepositoryLinkFilter do context 'with a wiki' do let(:wiki) { double('ProjectWiki') } - include_examples :preserve_unchanged + include_examples 'preserve unchanged' end context 'without a repository' do let(:project) { create(:project) } - include_examples :preserve_unchanged + include_examples 'preserve unchanged' end context 'with an empty repository' do let(:project) { create(:project_empty_repo) } - include_examples :preserve_unchanged + include_examples 'preserve unchanged' end context 'without project repository access' do let(:project) { create(:project, :repository, repository_access_level: ProjectFeature::PRIVATE) } - include_examples :preserve_unchanged + include_examples 'preserve unchanged' end it 'does not raise an exception on invalid URIs' do @@ -147,7 +147,7 @@ RSpec.describe Banzai::Filter::RepositoryLinkFilter do .to eq "/#{project_path}/-/blob/#{ref}/non/existent.file" end - shared_examples :valid_repository do + shared_examples 'valid repository' do it 'handles Gitaly unavailable exceptions gracefully' do allow_next_instance_of(Gitlab::GitalyClient::BlobService) do |blob_service| allow(blob_service).to receive(:get_blob_types).and_raise(GRPC::Unavailable) @@ -364,13 +364,13 @@ RSpec.describe Banzai::Filter::RepositoryLinkFilter do end context 'with a valid commit' do - include_examples :valid_repository + include_examples 'valid repository' end context 'with a valid ref' do # force filter to use ref instead of commit let(:commit) { nil } - include_examples :valid_repository + include_examples 'valid repository' end end diff --git a/spec/lib/banzai/filter/syntax_highlight_filter_spec.rb b/spec/lib/banzai/filter/syntax_highlight_filter_spec.rb index a409c15533b..b4be26ef8d2 100644 --- a/spec/lib/banzai/filter/syntax_highlight_filter_spec.rb +++ b/spec/lib/banzai/filter/syntax_highlight_filter_spec.rb @@ -12,7 +12,7 @@ RSpec.describe Banzai::Filter::SyntaxHighlightFilter do result = filter(%{<pre lang="#{lang}"><code><script>alert(1)</script></code></pre>}) # `(1)` symbols are wrapped by lexer tags. - expect(result.to_html).not_to match(%r{<script>alert.*<\/script>}) + expect(result.to_html).not_to match(%r{<script>alert.*</script>}) # `<>` stands for lexer tags like <span ...>, not <s above. expect(result.to_html).to match(%r{alert(<.*>)?\((<.*>)?1(<.*>)?\)}) @@ -192,4 +192,8 @@ RSpec.describe Banzai::Filter::SyntaxHighlightFilter do include_examples "XSS prevention", "ruby" end + + it_behaves_like "filter timeout" do + let(:text) { '<pre lang="ruby"><code>def fun end</code></pre>' } + end end diff --git a/spec/lib/banzai/filter/timeout_html_pipeline_filter_spec.rb b/spec/lib/banzai/filter/timeout_html_pipeline_filter_spec.rb new file mode 100644 index 00000000000..95d2e54459d --- /dev/null +++ b/spec/lib/banzai/filter/timeout_html_pipeline_filter_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Banzai::Filter::TimeoutHtmlPipelineFilter do + include FilterSpecHelper + + it_behaves_like 'filter timeout' do + let(:text) { '<p>some text</p>' } + end + + it 'raises NotImplementedError' do + expect { filter('test') }.to raise_error NotImplementedError + end +end diff --git a/spec/lib/banzai/pipeline/incident_management/timeline_event_pipeline_spec.rb b/spec/lib/banzai/pipeline/incident_management/timeline_event_pipeline_spec.rb index 4bccae04fda..8d15dbc8f2f 100644 --- a/spec/lib/banzai/pipeline/incident_management/timeline_event_pipeline_spec.rb +++ b/spec/lib/banzai/pipeline/incident_management/timeline_event_pipeline_spec.rb @@ -78,7 +78,7 @@ RSpec.describe Banzai::Pipeline::IncidentManagement::TimelineEventPipeline do it 'replaces existing label to a link' do # rubocop:disable Layout/LineLength is_expected.to match( - %r(<p>.+<a href=\"[\w/]+-/issues\?label_name=#{label.name}\".+style=\"background-color: #\d{6}\".*>#{label.name}</span></a></span> ~unknown</p>) + %r(<p>.+<a href="[\w/]+-/issues\?label_name=#{label.name}".+style="background-color: #\d{6}".*>#{label.name}</span></a></span> ~unknown</p>) ) # rubocop:enable Layout/LineLength end diff --git a/spec/lib/banzai/reference_parser/base_parser_spec.rb b/spec/lib/banzai/reference_parser/base_parser_spec.rb index 9e77137795a..61751b69842 100644 --- a/spec/lib/banzai/reference_parser/base_parser_spec.rb +++ b/spec/lib/banzai/reference_parser/base_parser_spec.rb @@ -18,6 +18,29 @@ RSpec.describe Banzai::ReferenceParser::BaseParser do parser_class.new(context) end + describe '.reference_class' do + context 'when the method is not defined' do + it 'build the reference class' do + dummy = Class.new(described_class) + dummy.reference_type = :issue + + expect(dummy.reference_class).to eq(Issue) + end + end + + context 'when the method is redefined' do + it 'uses specified reference class' do + dummy = Class.new(described_class) do + def self.reference_class + AlertManagement::Alert + end + end + + expect(dummy.reference_class).to eq(AlertManagement::Alert) + end + end + end + describe '.reference_type=' do it 'sets the reference type' do dummy = Class.new(described_class) diff --git a/spec/lib/bitbucket_server/connection_spec.rb b/spec/lib/bitbucket_server/connection_spec.rb index ae73955e1d1..8341ca10f43 100644 --- a/spec/lib/bitbucket_server/connection_spec.rb +++ b/spec/lib/bitbucket_server/connection_spec.rb @@ -56,6 +56,12 @@ RSpec.describe BitbucketServer::Connection do expect { subject.post(url, payload) }.to raise_error(described_class::ConnectionError) end + + it 'throws an exception if the URI is invalid' do + stub_request(:post, url).with(headers: { 'Accept' => 'application/json' }).to_raise(URI::InvalidURIError) + + expect { subject.post(url, payload) }.to raise_error(described_class::ConnectionError) + end end describe '#delete' do diff --git a/spec/lib/bulk_imports/clients/http_spec.rb b/spec/lib/bulk_imports/clients/http_spec.rb index 6962a943755..4fb08fc0478 100644 --- a/spec/lib/bulk_imports/clients/http_spec.rb +++ b/spec/lib/bulk_imports/clients/http_spec.rb @@ -9,13 +9,23 @@ RSpec.describe BulkImports::Clients::HTTP do let(:token) { 'token' } let(:resource) { 'resource' } let(:version) { "#{BulkImport::MIN_MAJOR_VERSION}.0.0" } + let(:enterprise) { false } let(:response_double) { double(code: 200, success?: true, parsed_response: {}) } - let(:version_response) { double(code: 200, success?: true, parsed_response: { 'version' => version }) } + let(:metadata_response) do + double( + code: 200, + success?: true, + parsed_response: { + 'version' => version, + 'enterprise' => enterprise + } + ) + end before do allow(Gitlab::HTTP).to receive(:get) .with('http://gitlab.example/api/v4/version', anything) - .and_return(version_response) + .and_return(metadata_response) end subject { described_class.new(url: url, token: token) } @@ -213,13 +223,27 @@ RSpec.describe BulkImports::Clients::HTTP do expect(Gitlab::HTTP).to receive(:get) .with('http://gitlab.example/api/v4/metadata', anything) - .and_return(version_response) + .and_return(metadata_response) expect(subject.instance_version).to eq(Gitlab::VersionInfo.parse(version)) end end end + describe '#instance_enterprise' do + it 'returns source instance enterprise information' do + expect(subject.instance_enterprise).to eq(false) + end + + context 'when enterprise information is missing' do + let(:enterprise) { nil } + + it 'defaults to true' do + expect(subject.instance_enterprise).to eq(true) + end + end + end + describe '#compatible_for_project_migration?' do context 'when instance version is lower the the expected minimum' do it 'returns false' do @@ -254,7 +278,7 @@ RSpec.describe BulkImports::Clients::HTTP do before do allow(Gitlab::HTTP).to receive(:get) .with('http://website.example/gitlab/api/v4/version', anything) - .and_return(version_response) + .and_return(metadata_response) end it 'performs network request to a relative gitlab url' do diff --git a/spec/lib/bulk_imports/common/pipelines/uploads_pipeline_spec.rb b/spec/lib/bulk_imports/common/pipelines/uploads_pipeline_spec.rb index 7a93365d098..35ca67c8a4c 100644 --- a/spec/lib/bulk_imports/common/pipelines/uploads_pipeline_spec.rb +++ b/spec/lib/bulk_imports/common/pipelines/uploads_pipeline_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe BulkImports::Common::Pipelines::UploadsPipeline do +RSpec.describe BulkImports::Common::Pipelines::UploadsPipeline, feature_category: :import do let_it_be(:project) { create(:project) } let_it_be(:group) { create(:group) } @@ -103,7 +103,9 @@ RSpec.describe BulkImports::Common::Pipelines::UploadsPipeline do context 'when dynamic path is nil' do it 'returns' do - expect { pipeline.load(context, File.join(tmpdir, 'test')) }.not_to change { portable.uploads.count } + path = File.join(tmpdir, 'test') + FileUtils.touch(path) + expect { pipeline.load(context, path) }.not_to change { portable.uploads.count } end end @@ -122,6 +124,20 @@ RSpec.describe BulkImports::Common::Pipelines::UploadsPipeline do expect { pipeline.load(context, symlink) }.not_to change { portable.uploads.count } end end + + context 'when path traverses' do + it 'does not upload the file' do + path_traversal = "#{uploads_dir_path}/avatar/../../../../etc/passwd" + expect { pipeline.load(context, path_traversal) }.to not_change { portable.uploads.count }.and raise_error(Gitlab::Utils::PathTraversalAttackError) + end + end + + context 'when path is outside the tmpdir' do + it 'does not upload the file' do + path = "/etc/passwd" + expect { pipeline.load(context, path) }.to not_change { portable.uploads.count }.and raise_error(StandardError, /not allowed/) + end + end end describe '#after_run' do diff --git a/spec/lib/bulk_imports/projects/pipelines/issues_pipeline_spec.rb b/spec/lib/bulk_imports/projects/pipelines/issues_pipeline_spec.rb index 97fcddefd42..19e3a1fecc3 100644 --- a/spec/lib/bulk_imports/projects/pipelines/issues_pipeline_spec.rb +++ b/spec/lib/bulk_imports/projects/pipelines/issues_pipeline_spec.rb @@ -89,6 +89,7 @@ RSpec.describe BulkImports::Projects::Pipelines::IssuesPipeline do expect(award_emoji.user).to eq(user) end end + context 'issue state' do let(:issue_attributes) { { 'state' => 'closed' } } diff --git a/spec/lib/extracts_ref_spec.rb b/spec/lib/extracts_ref_spec.rb index 3e9a7499fdd..ca8af9413f3 100644 --- a/spec/lib/extracts_ref_spec.rb +++ b/spec/lib/extracts_ref_spec.rb @@ -44,6 +44,19 @@ RSpec.describe ExtractsRef do expect { assign_ref_vars }.not_to raise_error end end + + context 'when a ref_type parameter is provided' do + let(:params) { ActionController::Parameters.new(path: path, ref: ref, ref_type: 'tags') } + + context 'and the use_ref_type_parameter feature flag is enabled' do + it 'sets a fully_qualified_ref variable' do + fully_qualified_ref = "refs/tags/#{ref}" + expect(container.repository).to receive(:commit).with(fully_qualified_ref) + assign_ref_vars + expect(@fully_qualified_ref).to eq(fully_qualified_ref) + end + end + end end it_behaves_like 'extracts refs' diff --git a/spec/lib/feature/definition_spec.rb b/spec/lib/feature/definition_spec.rb index 3d11ad4c0d8..595725d357c 100644 --- a/spec/lib/feature/definition_spec.rb +++ b/spec/lib/feature/definition_spec.rb @@ -25,6 +25,9 @@ RSpec.describe Feature::Definition do using RSpec::Parameterized::TableSyntax where(:param, :value, :result) do + :name | 'colon:separated' | /Feature flag 'colon:separated' is invalid/ + :name | 'space separated' | /Feature flag 'space separated' is invalid/ + :name | 'ALL_CAPS' | /Feature flag 'ALL_CAPS' is invalid/ :name | nil | /Feature flag is missing name/ :path | nil | /Feature flag 'feature_flag' is missing path/ :type | nil | /Feature flag 'feature_flag' is missing type/ @@ -111,6 +114,11 @@ RSpec.describe Feature::Definition do subject { described_class.send(:load_all!) } + after do + FileUtils.rm_rf(store1) + FileUtils.rm_rf(store2) + end + it "when there's no feature flags a list of definitions is empty" do is_expected.to be_empty end @@ -136,11 +144,6 @@ RSpec.describe Feature::Definition do .to raise_error(/Feature flag is missing name/) end - after do - FileUtils.rm_rf(store1) - FileUtils.rm_rf(store2) - end - def write_feature_flag(store, path, content) path = File.join(store, path) dir = File.dirname(path) diff --git a/spec/lib/feature_spec.rb b/spec/lib/feature_spec.rb index ad324406450..c087931d36a 100644 --- a/spec/lib/feature_spec.rb +++ b/spec/lib/feature_spec.rb @@ -162,6 +162,13 @@ RSpec.describe Feature, stub_feature_flags: false do stub_feature_flag_definition(:enabled_feature_flag, default_enabled: true) end + context 'when using redis cache', :use_clean_rails_redis_caching do + it 'does not make recursive feature-flag calls' do + expect(described_class).to receive(:enabled?).once.and_call_original + described_class.enabled?(:disabled_feature_flag) + end + end + context 'when self-recursive' do before do allow(Feature).to receive(:with_feature).and_wrap_original do |original, name, &block| @@ -318,6 +325,31 @@ RSpec.describe Feature, stub_feature_flags: false do end end + context 'with a group member' do + let(:key) { :awesome_feature } + let(:guinea_pigs) { create_list(:user, 3) } + + before do + described_class.reset + stub_feature_flag_definition(key) + Flipper.unregister_groups + Flipper.register(:guinea_pigs) do |actor| + guinea_pigs.include?(actor.thing) + end + described_class.enable(key, described_class.group(:guinea_pigs)) + end + + it 'is true for all group members' do + expect(described_class.enabled?(key, guinea_pigs[0])).to be_truthy + expect(described_class.enabled?(key, guinea_pigs[1])).to be_truthy + expect(described_class.enabled?(key, guinea_pigs[2])).to be_truthy + end + + it 'is false for any other actor' do + expect(described_class.enabled?(key, create(:user))).to be_falsey + end + end + context 'with an individual actor' do let(:actor) { stub_feature_flag_gate('CustomActor:5') } let(:another_actor) { stub_feature_flag_gate('CustomActor:10') } @@ -495,7 +527,7 @@ RSpec.describe Feature, stub_feature_flags: false do let(:expected_extra) {} it 'logs the event' do - expect(Feature.logger).to receive(:info).with(key: key, action: expected_action, **expected_extra) + expect(Feature.logger).to receive(:info).at_least(:once).with(key: key, action: expected_action, **expected_extra) subject end @@ -518,10 +550,10 @@ RSpec.describe Feature, stub_feature_flags: false do end context 'when thing is an actor' do - let(:thing) { create(:project) } + let(:thing) { create(:user) } it_behaves_like 'logging' do - let(:expected_action) { :enable } + let(:expected_action) { eq(:enable) | eq(:remove_opt_out) } let(:expected_extra) { { "extra.thing" => thing.flipper_id.to_s } } end end @@ -544,12 +576,160 @@ RSpec.describe Feature, stub_feature_flags: false do end context 'when thing is an actor' do - let(:thing) { create(:project) } + let(:thing) { create(:user) } + let(:flag_opts) { {} } it_behaves_like 'logging' do let(:expected_action) { :disable } let(:expected_extra) { { "extra.thing" => thing.flipper_id.to_s } } end + + before do + stub_feature_flag_definition(key, flag_opts) + end + + context 'when the feature flag was enabled for this actor' do + before do + described_class.enable(key, thing) + end + + it 'marks this thing as disabled' do + expect { subject }.to change { thing_enabled? }.from(true).to(false) + end + + it 'does not change the global value' do + expect { subject }.not_to change { described_class.enabled?(key) }.from(false) + end + + it 'is possible to re-enable the feature' do + subject + + expect { described_class.enable(key, thing) } + .to change { thing_enabled? }.from(false).to(true) + end + end + + context 'when the feature flag is enabled globally' do + before do + described_class.enable(key) + end + + it 'does not mark this thing as disabled' do + expect { subject }.not_to change { thing_enabled? }.from(true) + end + + it 'does not change the global value' do + expect { subject }.not_to change { described_class.enabled?(key) }.from(true) + end + end + end + end + + describe 'opt_out' do + subject { described_class.opt_out(key, thing) } + + let(:key) { :awesome_feature } + + before do + stub_feature_flag_definition(key) + described_class.enable(key) + end + + context 'when thing is an actor' do + let_it_be(:thing) { create(:project) } + + it 'marks this thing as disabled' do + expect { subject }.to change { thing_enabled? }.from(true).to(false) + end + + it 'does not change the global value' do + expect { subject }.not_to change { described_class.enabled?(key) }.from(true) + end + + it_behaves_like 'logging' do + let(:expected_action) { eq(:opt_out) } + let(:expected_extra) { { "extra.thing" => thing.flipper_id.to_s } } + end + + it 'stores the opt-out information as a gate' do + subject + + flag = described_class.get(key) + + expect(flag.actors_value).to include(include(thing.flipper_id)) + expect(flag.actors_value).not_to include(thing.flipper_id) + end + end + + context 'when thing is a group' do + let(:thing) { Feature.group(:guinea_pigs) } + let(:guinea_pigs) { create_list(:user, 3) } + + before do + Feature.reset + Flipper.unregister_groups + Flipper.register(:guinea_pigs) do |actor| + guinea_pigs.include?(actor.thing) + end + end + + it 'has no effect' do + expect { subject }.not_to change { described_class.enabled?(key, guinea_pigs.first) }.from(true) + end + end + end + + describe 'remove_opt_out' do + subject { described_class.remove_opt_out(key, thing) } + + let(:key) { :awesome_feature } + + before do + stub_feature_flag_definition(key) + described_class.enable(key) + described_class.opt_out(key, thing) + end + + context 'when thing is an actor' do + let_it_be(:thing) { create(:project) } + + it 're-enables this thing' do + expect { subject }.to change { thing_enabled? }.from(false).to(true) + end + + it 'does not change the global value' do + expect { subject }.not_to change { described_class.enabled?(key) }.from(true) + end + + it_behaves_like 'logging' do + let(:expected_action) { eq(:remove_opt_out) } + let(:expected_extra) { { "extra.thing" => thing.flipper_id.to_s } } + end + + it 'removes the opt-out information' do + subject + + flag = described_class.get(key) + + expect(flag.actors_value).to be_empty + end + end + + context 'when thing is a group' do + let(:thing) { Feature.group(:guinea_pigs) } + let(:guinea_pigs) { create_list(:user, 3) } + + before do + Feature.reset + Flipper.unregister_groups + Flipper.register(:guinea_pigs) do |actor| + guinea_pigs.include?(actor.thing) + end + end + + it 'has no effect' do + expect { subject }.not_to change { described_class.enabled?(key, guinea_pigs.first) }.from(true) + end end end @@ -563,6 +743,16 @@ RSpec.describe Feature, stub_feature_flags: false do let(:expected_action) { :enable_percentage_of_time } let(:expected_extra) { { "extra.percentage" => percentage.to_s } } end + + context 'when the flag is on' do + before do + described_class.enable(key) + end + + it 'fails with InvalidOperation' do + expect { subject }.to raise_error(described_class::InvalidOperation) + end + end end describe '.disable_percentage_of_time' do @@ -586,6 +776,16 @@ RSpec.describe Feature, stub_feature_flags: false do let(:expected_action) { :enable_percentage_of_actors } let(:expected_extra) { { "extra.percentage" => percentage.to_s } } end + + context 'when the flag is on' do + before do + described_class.enable(key) + end + + it 'fails with InvalidOperation' do + expect { subject }.to raise_error(described_class::InvalidOperation) + end + end end describe '.disable_percentage_of_actors' do @@ -603,6 +803,7 @@ RSpec.describe Feature, stub_feature_flags: false do subject { described_class.remove(key) } let(:key) { :awesome_feature } + let(:actor) { create(:user) } before do described_class.enable(key) @@ -617,13 +818,10 @@ RSpec.describe Feature, stub_feature_flags: false do it 'returns nil' do expect(described_class.remove(:non_persisted_feature_flag)).to be_nil end - end - context 'for a persisted feature' do - it 'returns true' do - described_class.enable(:persisted_feature_flag) - - expect(described_class.remove(:persisted_feature_flag)).to be_truthy + it 'returns true, and cleans up' do + expect(subject).to be_truthy + expect(described_class.persisted_names).not_to include(key) end end end @@ -712,6 +910,10 @@ RSpec.describe Feature, stub_feature_flags: false do end end + before do + stub_feature_flag_definition(:enabled_feature_flag) + end + it 'gives the correct value when enabling for an additional actor' do described_class.enable(:enabled_feature_flag, actor) initial_gate_values = active_record_adapter.get(described_class.get(:enabled_feature_flag)) @@ -834,4 +1036,8 @@ RSpec.describe Feature, stub_feature_flags: false do end end end + + def thing_enabled? + described_class.enabled?(key, thing) + end end diff --git a/spec/lib/generators/model/mocks/migration_file.txt b/spec/lib/generators/model/mocks/migration_file.txt index e92c2d2b534..091e086ba65 100644 --- a/spec/lib/generators/model/mocks/migration_file.txt +++ b/spec/lib/generators/model/mocks/migration_file.txt @@ -3,7 +3,7 @@ # See https://docs.gitlab.com/ee/development/migration_style_guide.html # for more information on how to write migrations for GitLab. -class CreateModelGeneratorTestFoos < Gitlab::Database::Migration[2.0] +class CreateModelGeneratorTestFoos < Gitlab::Database::Migration[2.1] # When using the methods "add_concurrent_index" or "remove_concurrent_index" # you must disable the use of transactions # as these methods can not run in an existing transaction. diff --git a/spec/lib/gitlab/analytics/cycle_analytics/base_query_builder_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/base_query_builder_spec.rb index 24f8fb40445..271022e7c55 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/base_query_builder_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/base_query_builder_spec.rb @@ -22,10 +22,7 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder do project.add_maintainer(user) mr1.metrics.update!(merged_at: 1.month.ago) mr2.metrics.update!(merged_at: Time.now) - end - - around do |example| - Timecop.freeze { example.run } + freeze_time end describe 'date range parameters' do diff --git a/spec/lib/gitlab/analytics/cycle_analytics/median_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/median_spec.rb index 258f4a0d019..4db5d64164e 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/median_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/median_spec.rb @@ -18,10 +18,6 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::Median do subject { described_class.new(stage: stage, query: query).seconds } - around do |example| - Timecop.freeze { example.run } - end - it 'retruns nil when no results' do expect(subject).to eq(nil) end @@ -30,11 +26,11 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::Median do merge_request1 = create(:merge_request, source_branch: '1', target_project: project, source_project: project) merge_request2 = create(:merge_request, source_branch: '2', target_project: project, source_project: project) - travel_to(5.minutes.from_now) do + travel(5.minutes) do merge_request1.metrics.update!(merged_at: Time.zone.now) end - travel_to(10.minutes.from_now) do + travel(10.minutes) do merge_request2.metrics.update!(merged_at: Time.zone.now) end diff --git a/spec/lib/gitlab/analytics/cycle_analytics/records_fetcher_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/records_fetcher_spec.rb index 34d5158a5ab..7f70a4cfc4e 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/records_fetcher_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/records_fetcher_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::Analytics::CycleAnalytics::RecordsFetcher do - around do |example| - Timecop.freeze { example.run } + before_all do + freeze_time end let(:params) { { from: 1.year.ago, current_user: user } } diff --git a/spec/lib/gitlab/analytics/cycle_analytics/stage_events_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/stage_events_spec.rb new file mode 100644 index 00000000000..f5a8035b9b4 --- /dev/null +++ b/spec/lib/gitlab/analytics/cycle_analytics/stage_events_spec.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Analytics::CycleAnalytics::StageEvents, feature_category: :value_stream_management do + describe '#selectable_events' do + subject(:selectable_events) { described_class.selectable_events } + + it 'excludes internal events' do + expect(selectable_events).to include(Gitlab::Analytics::CycleAnalytics::StageEvents::MergeRequestCreated) + expect(selectable_events).to exclude(Gitlab::Analytics::CycleAnalytics::StageEvents::IssueStageEnd) + end + end +end diff --git a/spec/lib/gitlab/api_authentication/sent_through_builder_spec.rb b/spec/lib/gitlab/api_authentication/sent_through_builder_spec.rb index 845e317f3aa..aa088a5d6d5 100644 --- a/spec/lib/gitlab/api_authentication/sent_through_builder_spec.rb +++ b/spec/lib/gitlab/api_authentication/sent_through_builder_spec.rb @@ -8,10 +8,10 @@ RSpec.describe Gitlab::APIAuthentication::SentThroughBuilder do let(:locators) { Array.new(3) { double } } it 'adds a strategy for each of locators x resolvers' do - strategies = locators.to_h { |l| [l, []] } + strategies = locators.index_with { [] } described_class.new(strategies, resolvers).sent_through(*locators) - expect(strategies).to eq(locators.to_h { |l| [l, resolvers] }) + expect(strategies).to eq(locators.index_with { resolvers }) end end end diff --git a/spec/lib/gitlab/application_context_spec.rb b/spec/lib/gitlab/application_context_spec.rb index 58d462aa27f..20c1536b9e6 100644 --- a/spec/lib/gitlab/application_context_spec.rb +++ b/spec/lib/gitlab/application_context_spec.rb @@ -141,7 +141,8 @@ RSpec.describe Gitlab::ApplicationContext do describe 'setting the client' do let_it_be(:remote_ip) { '127.0.0.1' } let_it_be(:runner) { create(:ci_runner) } - let_it_be(:options) { { remote_ip: remote_ip, runner: runner, user: user } } + let_it_be(:job) { create(:ci_build, :pending, :queued, user: user, project: project) } + let_it_be(:options) { { remote_ip: remote_ip, runner: runner, user: user, job: job } } using RSpec::Parameterized::TableSyntax @@ -150,6 +151,7 @@ RSpec.describe Gitlab::ApplicationContext do [:remote_ip, :runner] | :runner [:remote_ip, :runner, :user] | :runner [:remote_ip, :user] | :user + [:job] | :user end with_them do diff --git a/spec/lib/gitlab/asciidoc_spec.rb b/spec/lib/gitlab/asciidoc_spec.rb index d2eb9209f42..3d9076c6fa7 100644 --- a/spec/lib/gitlab/asciidoc_spec.rb +++ b/spec/lib/gitlab/asciidoc_spec.rb @@ -697,7 +697,7 @@ module Gitlab end end - shared_examples :invalid_include do + shared_examples 'invalid include' do let(:include_path) { 'dk.png' } before do @@ -716,7 +716,7 @@ module Gitlab context 'with path to a binary file' do let(:blob) { fake_blob(path: 'dk.png', binary: true) } - include_examples :invalid_include + include_examples 'invalid include' end context 'with path to file in external storage' do @@ -727,7 +727,7 @@ module Gitlab project.update_attribute(:lfs_enabled, true) end - include_examples :invalid_include + include_examples 'invalid include' end context 'with path to a textual file' do @@ -737,7 +737,7 @@ module Gitlab create_file(file_path, "Content from #{include_path}") end - shared_examples :valid_include do + shared_examples 'valid include' do [ ['/doc/sample.adoc', 'doc/sample.adoc', 'absolute path'], ['sample.adoc', 'doc/api/sample.adoc', 'relative path'], @@ -760,24 +760,24 @@ module Gitlab context 'when requested path is a file in the repo' do let(:requested_path) { 'doc/api/README.adoc' } - include_examples :valid_include + include_examples 'valid include' context 'without a commit (only ref)' do let(:commit) { nil } - include_examples :valid_include + include_examples 'valid include' end end context 'when requested path is a directory in the repo' do let(:requested_path) { 'doc/api/' } - include_examples :valid_include + include_examples 'valid include' context 'without a commit (only ref)' do let(:commit) { nil } - include_examples :valid_include + include_examples 'valid include' end end end diff --git a/spec/lib/gitlab/audit/auditor_spec.rb b/spec/lib/gitlab/audit/auditor_spec.rb index f743515e616..4b16333d913 100644 --- a/spec/lib/gitlab/audit/auditor_spec.rb +++ b/spec/lib/gitlab/audit/auditor_spec.rb @@ -68,6 +68,7 @@ RSpec.describe Gitlab::Audit::Auditor do expect(logger).to have_received(:info).with( hash_including( + 'id' => AuditEvent.last.id, 'author_id' => author.id, 'author_name' => author.name, 'entity_id' => group.id, @@ -112,6 +113,7 @@ RSpec.describe Gitlab::Audit::Auditor do expect(logger).to have_received(:info).with( hash_including( + 'id' => AuditEvent.last.id, 'author_id' => author.id, 'author_name' => author.name, 'entity_id' => group.id, @@ -244,7 +246,9 @@ RSpec.describe Gitlab::Audit::Auditor do let(:audit!) { auditor.audit(context) } before do - allow(AuditEvent).to receive(:bulk_insert!).and_raise(ActiveRecord::RecordInvalid) + expect_next_instance_of(AuditEvent) do |instance| + allow(instance).to receive(:save!).and_raise(ActiveRecord::RecordInvalid) + end allow(Gitlab::ErrorTracking).to receive(:track_exception) end @@ -261,5 +265,27 @@ RSpec.describe Gitlab::Audit::Auditor do expect { auditor.audit(context) }.not_to raise_exception end end + + context 'when audit event is not saved in database due to some database infra issue' do + let(:audit!) { auditor.audit(context) } + + before do + allow_any_instance_of(auditor) do |auditor_instance| + allow(auditor_instance).to receive(:log_to_database).and_return(nil) + end + end + + it 'calls log_to_file_and_stream with in memory events' do + audit! + + expect_any_instance_of(auditor) do |auditor_instance| + expect(auditor_instance).to receive(:log_to_file_and_stream).with(include(kind_of(AuditEvent))) + end + end + + it 'does not throw exception' do + expect { auditor.audit(context) }.not_to raise_exception + end + end end end diff --git a/spec/lib/gitlab/audit/type/definition_spec.rb b/spec/lib/gitlab/audit/type/definition_spec.rb index 9f4282a4ec0..d1d6b0d7a78 100644 --- a/spec/lib/gitlab/audit/type/definition_spec.rb +++ b/spec/lib/gitlab/audit/type/definition_spec.rb @@ -8,7 +8,7 @@ RSpec.describe Gitlab::Audit::Type::Definition do description: 'Group deploy token is deleted', introduced_by_issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/1', introduced_by_mr: 'https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1', - group: 'govern::compliance', + feature_category: 'continuous_delivery', milestone: '15.4', saved_to_database: true, streamed: true } @@ -18,6 +18,12 @@ RSpec.describe Gitlab::Audit::Type::Definition do let(:definition) { described_class.new(path, attributes) } let(:yaml_content) { attributes.deep_stringify_keys.to_yaml } + around do |example| + described_class.clear_memoization(:definitions) + example.run + described_class.clear_memoization(:definitions) + end + describe '#key' do subject { definition.key } @@ -36,7 +42,7 @@ RSpec.describe Gitlab::Audit::Type::Definition do :description | nil | %r{property '/description' is not of type: string} :introduced_by_issue | nil | %r{property '/introduced_by_issue' is not of type: string} :introduced_by_mr | nil | %r{property '/introduced_by_mr' is not of type: string} - :group | nil | %r{property '/group' is not of type: string} + :feature_category | nil | %r{property '/feature_category' is not of type: string} :milestone | nil | %r{property '/milestone' is not of type: string} end # rubocop:enable Layout/LineLength @@ -103,7 +109,7 @@ RSpec.describe Gitlab::Audit::Type::Definition do expect(audit_event_type_definition.name).to eq "group_deploy_token_destroyed" expect(audit_event_type_definition.description).to eq "Group deploy token is deleted" - expect(audit_event_type_definition.group).to eq "govern::compliance" + expect(audit_event_type_definition.feature_category).to eq "continuous_delivery" expect(audit_event_type_definition.milestone).to eq "15.4" expect(audit_event_type_definition.saved_to_database).to be true expect(audit_event_type_definition.streamed).to be true @@ -111,6 +117,65 @@ RSpec.describe Gitlab::Audit::Type::Definition do end end + describe '.event_names' do + before do + allow(described_class).to receive(:definitions) do + { definition.key => definition } + end + end + + it 'returns names of event types as string array' do + expect(described_class.event_names).to match_array([definition.attributes[:name]]) + end + end + + describe '.defined?' do + before do + allow(described_class).to receive(:definitions) do + { definition.key => definition } + end + end + + it 'returns true if definition for the event name exists' do + expect(described_class.defined?('group_deploy_token_destroyed')).to be_truthy + end + + it 'returns false if definition for the event name exists' do + expect(described_class.defined?('random_event_name')).to be_falsey + end + end + + describe '.stream_only?' do + let(:stream_only_event_attributes) do + { name: 'policy_project_updated', + description: 'This event is triggered whenever the security policy project is updated for a project', + introduced_by_issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/2', + introduced_by_mr: 'https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2', + feature_category: 'security_policy_management', + milestone: '15.6', + saved_to_database: false, + streamed: true } + end + + let(:stream_only_event_path) { File.join('types', 'policy_project_updated.yml') } + let(:stream_only_event_definition) { described_class.new(stream_only_event_path, stream_only_event_attributes) } + + before do + allow(described_class).to receive(:definitions) do + { definition.key => definition, + stream_only_event_definition.key => stream_only_event_definition } + end + end + + it 'returns true for a stream only event' do + expect(described_class.stream_only?('group_deploy_token_destroyed')).to be_falsey + end + + it 'returns false for an event that is saved to database' do + expect(described_class.stream_only?('policy_project_updated')).to be_truthy + end + end + describe '.load_from_file' do it 'properly loads a definition from file' do expect_file_read(path, content: yaml_content) @@ -186,6 +251,12 @@ RSpec.describe Gitlab::Audit::Type::Definition do end end + describe 'validate that all the YAML definitions matches the audit event type schema' do + it 'successfully loads all the YAML definitions' do + expect { described_class.definitions }.not_to raise_error + end + end + describe '.definitions' do let(:store1) { Dir.mktmpdir('path1') } diff --git a/spec/lib/gitlab/auth/auth_finders_spec.rb b/spec/lib/gitlab/auth/auth_finders_spec.rb index 9283c31a207..484b4702497 100644 --- a/spec/lib/gitlab/auth/auth_finders_spec.rb +++ b/spec/lib/gitlab/auth/auth_finders_spec.rb @@ -69,7 +69,7 @@ RSpec.describe Gitlab::Auth::AuthFinders do expect(subject).to eq(user) expect(@current_authenticated_job).to eq job expect(subject).to be_from_ci_job_token - expect(subject.ci_job_token_scope.source_project).to eq(job.project) + expect(subject.ci_job_token_scope.current_project).to eq(job.project) end end @@ -100,7 +100,7 @@ RSpec.describe Gitlab::Auth::AuthFinders do expect(subject).to eq(user) expect(@current_authenticated_job).to eq job expect(subject).to be_from_ci_job_token - expect(subject.ci_job_token_scope.source_project).to eq(job.project) + expect(subject.ci_job_token_scope.current_project).to eq(job.project) end else it 'returns nil' do diff --git a/spec/lib/gitlab/auth/current_user_mode_spec.rb b/spec/lib/gitlab/auth/current_user_mode_spec.rb index a21f0931b78..0a68a4a0ae2 100644 --- a/spec/lib/gitlab/auth/current_user_mode_spec.rb +++ b/spec/lib/gitlab/auth/current_user_mode_spec.rb @@ -194,10 +194,41 @@ RSpec.describe Gitlab::Auth::CurrentUserMode, :request_store do it 'creates a timestamp in the session' do subject.request_admin_mode! + subject.enable_admin_mode!(password: user.password) expect(session).to include(expected_session_entry(be_within(1.second).of(Time.now))) end + + it 'returns true after successful enable' do + subject.request_admin_mode! + + expect(subject.enable_admin_mode!(password: user.password)).to eq(true) + end + + it 'returns false after unsuccessful enable' do + subject.request_admin_mode! + + expect(subject.enable_admin_mode!(password: 'wrong password')).to eq(false) + end + + context 'when user is not an admin' do + let(:user) { build_stubbed(:user) } + + it 'returns false' do + subject.request_admin_mode! + + expect(subject.enable_admin_mode!(password: user.password)).to eq(false) + end + end + + context 'when admin mode is not requested' do + it 'raises error' do + expect do + subject.enable_admin_mode!(password: user.password) + end.to raise_error(Gitlab::Auth::CurrentUserMode::NotRequestedError) + end + end end describe '#disable_admin_mode!' do diff --git a/spec/lib/gitlab/auth/ldap/config_spec.rb b/spec/lib/gitlab/auth/ldap/config_spec.rb index 3be983857bc..160fd78b2b9 100644 --- a/spec/lib/gitlab/auth/ldap/config_spec.rb +++ b/spec/lib/gitlab/auth/ldap/config_spec.rb @@ -99,7 +99,7 @@ AtlErSqafbECNDSwS5BX8yDpu5yRBJ4xegO/rNlmb8ICRYkuJapD1xXicFOsmfUK expect { described_class.new }.to raise_error ArgumentError end - it 'works' do + it 'returns an instance of Gitlab::Auth::Ldap::Config' do expect(config).to be_a described_class end @@ -122,7 +122,8 @@ AtlErSqafbECNDSwS5BX8yDpu5yRBJ4xegO/rNlmb8ICRYkuJapD1xXicFOsmfUK host: 'ldap.example.com', port: 386, hosts: nil, - encryption: nil + encryption: nil, + instrumentation_service: ActiveSupport::Notifications ) end diff --git a/spec/lib/gitlab/auth/ldap/user_spec.rb b/spec/lib/gitlab/auth/ldap/user_spec.rb index b471a89b491..5771b1cd609 100644 --- a/spec/lib/gitlab/auth/ldap/user_spec.rb +++ b/spec/lib/gitlab/auth/ldap/user_spec.rb @@ -133,7 +133,7 @@ RSpec.describe Gitlab::Auth::Ldap::User do context 'when user confirmation email is enabled' do before do - stub_application_setting send_user_confirmation_email: true + stub_application_setting_enum('email_confirmation_setting', 'hard') end it 'creates and confirms the user anyway' do diff --git a/spec/lib/gitlab/auth/o_auth/user_spec.rb b/spec/lib/gitlab/auth/o_auth/user_spec.rb index 95a518afcf1..bb81621ec92 100644 --- a/spec/lib/gitlab/auth/o_auth/user_spec.rb +++ b/spec/lib/gitlab/auth/o_auth/user_spec.rb @@ -108,7 +108,7 @@ RSpec.describe Gitlab::Auth::OAuth::User do context 'when user confirmation email is enabled' do before do - stub_application_setting send_user_confirmation_email: true + stub_application_setting_enum('email_confirmation_setting', 'hard') end it 'creates and confirms the user anyway' do diff --git a/spec/lib/gitlab/auth/saml/user_spec.rb b/spec/lib/gitlab/auth/saml/user_spec.rb index 796512bc52b..a8a5d8ae5df 100644 --- a/spec/lib/gitlab/auth/saml/user_spec.rb +++ b/spec/lib/gitlab/auth/saml/user_spec.rb @@ -46,6 +46,10 @@ RSpec.describe Gitlab::Auth::Saml::User do end context 'external groups' do + before do + stub_saml_group_config(%w(Interns)) + end + context 'are defined' do it 'marks the user as external' do stub_saml_group_config(%w(Freelancers)) @@ -55,10 +59,6 @@ RSpec.describe Gitlab::Auth::Saml::User do end end - before do - stub_saml_group_config(%w(Interns)) - end - context 'are defined but the user does not belong there' do it 'does not mark the user as external' do saml_user.save # rubocop:disable Rails/SaveBang @@ -317,7 +317,7 @@ RSpec.describe Gitlab::Auth::Saml::User do context 'when user confirmation email is enabled' do before do - stub_application_setting send_user_confirmation_email: true + stub_application_setting_enum('email_confirmation_setting', 'hard') end it 'creates and confirms the user anyway' do diff --git a/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb b/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb index b239de841b6..84f6411eae6 100644 --- a/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb +++ b/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb @@ -22,14 +22,14 @@ RSpec.describe Gitlab::Auth::UniqueIpsLimiter, :clean_gitlab_redis_shared_state end it 'resets count after specified time window' do - Timecop.freeze do + freeze_time do expect(described_class.update_and_return_ips_count(user.id, 'ip2')).to eq(1) expect(described_class.update_and_return_ips_count(user.id, 'ip3')).to eq(2) + end - travel_to(Time.now.utc + described_class.config.unique_ips_limit_time_window) do - expect(described_class.update_and_return_ips_count(user.id, 'ip4')).to eq(1) - expect(described_class.update_and_return_ips_count(user.id, 'ip5')).to eq(2) - end + travel_to(Time.now.utc + described_class.config.unique_ips_limit_time_window) do + expect(described_class.update_and_return_ips_count(user.id, 'ip4')).to eq(1) + expect(described_class.update_and_return_ips_count(user.id, 'ip5')).to eq(2) end end end diff --git a/spec/lib/gitlab/background_migration/backfill_environment_tiers_spec.rb b/spec/lib/gitlab/background_migration/backfill_environment_tiers_spec.rb new file mode 100644 index 00000000000..788ed40b61e --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_environment_tiers_spec.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillEnvironmentTiers, + :migration, schema: 20221205151917, feature_category: :continuous_delivery do + let!(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } + let!(:project) { table(:projects).create!(namespace_id: namespace.id, project_namespace_id: namespace.id) } + + let(:migration) do + described_class.new(start_id: 1, end_id: 1000, + batch_table: :environments, batch_column: :id, + sub_batch_size: 10, pause_ms: 0, + connection: ApplicationRecord.connection) + end + + describe '#perform' do + let!(:production) { table(:environments).create!(name: 'production', slug: 'production', project_id: project.id) } + let!(:staging) { table(:environments).create!(name: 'staging', slug: 'staging', project_id: project.id) } + let!(:testing) { table(:environments).create!(name: 'testing', slug: 'testing', project_id: project.id) } + + let!(:development) do + table(:environments).create!(name: 'development', slug: 'development', project_id: project.id) + end + + let!(:other) { table(:environments).create!(name: 'other', slug: 'other', project_id: project.id) } + + it 'backfill tiers for all environments in range' do + expect(production.tier).to be_nil + expect(staging.tier).to be_nil + expect(testing.tier).to be_nil + expect(development.tier).to be_nil + expect(other.tier).to be_nil + + migration.perform + + expect(production.reload.tier).to eq(described_class::PRODUCTION_TIER) + expect(staging.reload.tier).to eq(described_class::STAGING_TIER) + expect(testing.reload.tier).to eq(described_class::TESTING_TIER) + expect(development.reload.tier).to eq(described_class::DEVELOPMENT_TIER) + expect(other.reload.tier).to eq(described_class::OTHER_TIER) + end + end + + # Equivalent to spec/models/environment_spec.rb#guess_tier + describe 'same behavior with guess tier' do + using RSpec::Parameterized::TableSyntax + + let(:environment) { table(:environments).create!(name: name, slug: name, project_id: project.id) } + + where(:name, :tier) do + 'review/feature' | described_class::DEVELOPMENT_TIER + 'review/product' | described_class::DEVELOPMENT_TIER + 'DEV' | described_class::DEVELOPMENT_TIER + 'development' | described_class::DEVELOPMENT_TIER + 'trunk' | described_class::DEVELOPMENT_TIER + 'dev' | described_class::DEVELOPMENT_TIER + 'review/app' | described_class::DEVELOPMENT_TIER + 'PRODUCTION' | described_class::PRODUCTION_TIER + 'prod' | described_class::PRODUCTION_TIER + 'prod-east-2' | described_class::PRODUCTION_TIER + 'us-prod-east' | described_class::PRODUCTION_TIER + 'fe-production' | described_class::PRODUCTION_TIER + 'test' | described_class::TESTING_TIER + 'TEST' | described_class::TESTING_TIER + 'testing' | described_class::TESTING_TIER + 'testing-prd' | described_class::TESTING_TIER + 'acceptance-testing' | described_class::TESTING_TIER + 'production-test' | described_class::TESTING_TIER + 'test-production' | described_class::TESTING_TIER + 'QC' | described_class::TESTING_TIER + 'qa-env-2' | described_class::TESTING_TIER + 'gstg' | described_class::STAGING_TIER + 'staging' | described_class::STAGING_TIER + 'stage' | described_class::STAGING_TIER + 'Model' | described_class::STAGING_TIER + 'MODL' | described_class::STAGING_TIER + 'Pre-production' | described_class::STAGING_TIER + 'pre' | described_class::STAGING_TIER + 'Demo' | described_class::STAGING_TIER + 'staging' | described_class::STAGING_TIER + 'pre-prod' | described_class::STAGING_TIER + 'blue-kit-stage' | described_class::STAGING_TIER + 'nonprod' | described_class::STAGING_TIER + 'nonlive' | described_class::STAGING_TIER + 'non-prod' | described_class::STAGING_TIER + 'non-live' | described_class::STAGING_TIER + 'gprd' | described_class::PRODUCTION_TIER + 'gprd-cny' | described_class::PRODUCTION_TIER + 'production' | described_class::PRODUCTION_TIER + 'Production' | described_class::PRODUCTION_TIER + 'PRODUCTION' | described_class::PRODUCTION_TIER + 'Production/eu' | described_class::PRODUCTION_TIER + 'production/eu' | described_class::PRODUCTION_TIER + 'PRODUCTION/EU' | described_class::PRODUCTION_TIER + 'productioneu' | described_class::PRODUCTION_TIER + 'store-produce' | described_class::PRODUCTION_TIER + 'unproductive' | described_class::PRODUCTION_TIER + 'production/www.gitlab.com' | described_class::PRODUCTION_TIER + 'prod' | described_class::PRODUCTION_TIER + 'PROD' | described_class::PRODUCTION_TIER + 'Live' | described_class::PRODUCTION_TIER + 'canary' | described_class::OTHER_TIER + 'other' | described_class::OTHER_TIER + 'EXP' | described_class::OTHER_TIER + 'something-else' | described_class::OTHER_TIER + end + + with_them do + it 'backfill tiers for all environments in range' do + expect(environment.tier).to be_nil + + migration.perform + + expect(environment.reload.tier).to eq(tier) + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb b/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb index 8947262ae9e..479afb56210 100644 --- a/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb @@ -23,6 +23,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillImportedIssueSearchData, let!(:issue) do table(:issues).create!( project_id: project.id, + namespace_id: project.project_namespace_id, title: 'Patterson', description: FFaker::HipsterIpsum.paragraph ) @@ -71,6 +72,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillImportedIssueSearchData, let!(:issue2) do table(:issues).create!( project_id: project.id, + namespace_id: project.project_namespace_id, title: 'Chatterton', description: FFaker::HipsterIpsum.paragraph ) diff --git a/spec/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2_spec.rb b/spec/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2_spec.rb index 65f5f8368df..8db45ac0f57 100644 --- a/spec/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2_spec.rb @@ -3,11 +3,11 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::BackfillJiraTrackerDeploymentType2, :migration, schema: 20210301200959 do - let_it_be(:jira_integration_temp) { described_class::JiraServiceTemp } - let_it_be(:jira_tracker_data_temp) { described_class::JiraTrackerDataTemp } - let_it_be(:atlassian_host) { 'https://api.atlassian.net' } - let_it_be(:mixedcase_host) { 'https://api.AtlassiaN.nEt' } - let_it_be(:server_host) { 'https://my.server.net' } + let!(:jira_integration_temp) { described_class::JiraServiceTemp } + let!(:jira_tracker_data_temp) { described_class::JiraTrackerDataTemp } + let!(:atlassian_host) { 'https://api.atlassian.net' } + let!(:mixedcase_host) { 'https://api.AtlassiaN.nEt' } + let!(:server_host) { 'https://my.server.net' } let(:jira_integration) { jira_integration_temp.create!(type: 'JiraService', active: true, category: 'issue_tracker') } diff --git a/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb index 77d6cc43114..01daf16d10c 100644 --- a/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb @@ -3,9 +3,9 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::BackfillProjectNamespaceDetails, :migration do - let_it_be(:namespace_details) { table(:namespace_details) } - let_it_be(:namespaces) { table(:namespaces) } - let_it_be(:projects) { table(:projects) } + let!(:namespace_details) { table(:namespace_details) } + let!(:namespaces) { table(:namespaces) } + let!(:projects) { table(:projects) } subject(:perform_migration) do described_class.new(start_id: projects.minimum(:id), diff --git a/spec/lib/gitlab/background_migration/backfill_project_namespace_on_issues_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_namespace_on_issues_spec.rb index 3ca7d28f09d..5fa92759cf9 100644 --- a/spec/lib/gitlab/background_migration/backfill_project_namespace_on_issues_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_project_namespace_on_issues_spec.rb @@ -1,12 +1,14 @@ # frozen_string_literal: true require 'spec_helper' -# todo: this will need to specify schema version once we introduce the not null constraint on issues#namespace_id -# https://gitlab.com/gitlab-org/gitlab/-/issues/367835 -RSpec.describe Gitlab::BackgroundMigration::BackfillProjectNamespaceOnIssues do + +RSpec.describe Gitlab::BackgroundMigration::BackfillProjectNamespaceOnIssues, + :migration, schema: 20221118103352, feature_category: :team_planning do let(:namespaces) { table(:namespaces) } let(:projects) { table(:projects) } let(:issues) { table(:issues) } + let(:issue_base_type_enum_value) { 0 } + let(:issue_type) { table(:work_item_types).find_by!(namespace_id: nil, base_type: issue_base_type_enum_value) } let(:namespace1) { namespaces.create!(name: 'batchtest1', type: 'Group', path: 'space1') } let(:namespace2) { namespaces.create!(name: 'batchtest2', type: 'Group', parent_id: namespace1.id, path: 'space2') } @@ -18,12 +20,12 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillProjectNamespaceOnIssues do let(:proj1) { projects.create!(name: 'proj1', path: 'proj1', namespace_id: namespace1.id, project_namespace_id: proj_namespace1.id) } let(:proj2) { projects.create!(name: 'proj2', path: 'proj2', namespace_id: namespace2.id, project_namespace_id: proj_namespace2.id) } - let!(:proj1_issue_with_namespace) { issues.create!(title: 'issue1', project_id: proj1.id, namespace_id: proj_namespace1.id) } - let!(:proj1_issue_without_namespace1) { issues.create!(title: 'issue2', project_id: proj1.id) } - let!(:proj1_issue_without_namespace2) { issues.create!(title: 'issue3', project_id: proj1.id) } - let!(:proj2_issue_with_namespace) { issues.create!(title: 'issue4', project_id: proj2.id, namespace_id: proj_namespace2.id) } - let!(:proj2_issue_without_namespace1) { issues.create!(title: 'issue5', project_id: proj2.id) } - let!(:proj2_issue_without_namespace2) { issues.create!(title: 'issue6', project_id: proj2.id) } + let!(:proj1_issue_with_namespace) { issues.create!(title: 'issue1', project_id: proj1.id, namespace_id: proj_namespace1.id, work_item_type_id: issue_type.id) } + let!(:proj1_issue_without_namespace1) { issues.create!(title: 'issue2', project_id: proj1.id, work_item_type_id: issue_type.id) } + let!(:proj1_issue_without_namespace2) { issues.create!(title: 'issue3', project_id: proj1.id, work_item_type_id: issue_type.id) } + let!(:proj2_issue_with_namespace) { issues.create!(title: 'issue4', project_id: proj2.id, namespace_id: proj_namespace2.id, work_item_type_id: issue_type.id) } + let!(:proj2_issue_without_namespace1) { issues.create!(title: 'issue5', project_id: proj2.id, work_item_type_id: issue_type.id) } + let!(:proj2_issue_without_namespace2) { issues.create!(title: 'issue6', project_id: proj2.id, work_item_type_id: issue_type.id) } # rubocop:enable Layout/LineLength let(:migration) do diff --git a/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb b/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb index 6ef474ad7f9..5f93424faf6 100644 --- a/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb @@ -45,7 +45,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, :mi it 'sets work_item_type_id only for the given type' do expect(all_issues).to all(have_attributes(work_item_type_id: nil)) - expect { migrate }.to make_queries_matching(/UPDATE \"issues\" SET "work_item_type_id"/, 2) + expect { migrate }.to make_queries_matching(/UPDATE "issues" SET "work_item_type_id"/, 2) all_issues.each(&:reload) expect([issue1, issue2, issue3]).to all(have_attributes(work_item_type_id: issue_type.id)) diff --git a/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb b/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb index 95be14cefb1..7280ca0b58e 100644 --- a/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb +++ b/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb @@ -158,6 +158,28 @@ RSpec.describe Gitlab::BackgroundMigration::BatchedMigrationJob do end end + describe '.feature_category' do + context 'when jobs does not have feature_category attribute set' do + let(:job_class) { Class.new(described_class) } + + it 'returns :database as default' do + expect(job_class.feature_category).to eq(:database) + end + end + + context 'when jobs have feature_category attribute set' do + let(:job_class) do + Class.new(described_class) do + feature_category :delivery + end + end + + it 'returns the provided value' do + expect(job_class.feature_category).to eq(:delivery) + end + end + end + describe 'descendants', :eager_load do it 'have the same method signature for #perform' do expected_arity = described_class.instance_method(:perform).arity diff --git a/spec/lib/gitlab/background_migration/batching_strategies/loose_index_scan_batching_strategy_spec.rb b/spec/lib/gitlab/background_migration/batching_strategies/loose_index_scan_batching_strategy_spec.rb index 1a00fd7c8b3..72958700ca2 100644 --- a/spec/lib/gitlab/background_migration/batching_strategies/loose_index_scan_batching_strategy_spec.rb +++ b/spec/lib/gitlab/background_migration/batching_strategies/loose_index_scan_batching_strategy_spec.rb @@ -7,6 +7,8 @@ RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::LooseIndexScanBa let(:namespaces) { table(:namespaces) } let(:projects) { table(:projects) } let(:issues) { table(:issues) } + let(:issue_base_type_enum_value) { 0 } + let(:issue_type) { table(:work_item_types).find_by!(namespace_id: nil, base_type: issue_base_type_enum_value) } let!(:namespace1) { namespaces.create!(name: 'ns1', path: 'ns1') } let!(:namespace2) { namespaces.create!(name: 'ns2', path: 'ns2') } @@ -19,13 +21,15 @@ RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::LooseIndexScanBa let!(:project4) { projects.create!(name: 'p4', namespace_id: namespace4.id, project_namespace_id: namespace4.id) } let!(:project5) { projects.create!(name: 'p5', namespace_id: namespace5.id, project_namespace_id: namespace5.id) } - let!(:issue1) { issues.create!(title: 'title', description: 'description', project_id: project2.id) } - let!(:issue2) { issues.create!(title: 'title', description: 'description', project_id: project1.id) } - let!(:issue3) { issues.create!(title: 'title', description: 'description', project_id: project2.id) } - let!(:issue4) { issues.create!(title: 'title', description: 'description', project_id: project3.id) } - let!(:issue5) { issues.create!(title: 'title', description: 'description', project_id: project2.id) } - let!(:issue6) { issues.create!(title: 'title', description: 'description', project_id: project4.id) } - let!(:issue7) { issues.create!(title: 'title', description: 'description', project_id: project5.id) } + # rubocop:disable Layout/LineLength + let!(:issue1) { issues.create!(title: 'title', description: 'description', project_id: project2.id, namespace_id: project2.project_namespace_id, work_item_type_id: issue_type.id) } + let!(:issue2) { issues.create!(title: 'title', description: 'description', project_id: project1.id, namespace_id: project1.project_namespace_id, work_item_type_id: issue_type.id) } + let!(:issue3) { issues.create!(title: 'title', description: 'description', project_id: project2.id, namespace_id: project2.project_namespace_id, work_item_type_id: issue_type.id) } + let!(:issue4) { issues.create!(title: 'title', description: 'description', project_id: project3.id, namespace_id: project3.project_namespace_id, work_item_type_id: issue_type.id) } + let!(:issue5) { issues.create!(title: 'title', description: 'description', project_id: project2.id, namespace_id: project2.project_namespace_id, work_item_type_id: issue_type.id) } + let!(:issue6) { issues.create!(title: 'title', description: 'description', project_id: project4.id, namespace_id: project4.project_namespace_id, work_item_type_id: issue_type.id) } + let!(:issue7) { issues.create!(title: 'title', description: 'description', project_id: project5.id, namespace_id: project5.project_namespace_id, work_item_type_id: issue_type.id) } + # rubocop:enable Layout/LineLength it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchingStrategies::BaseStrategy } diff --git a/spec/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities_spec.rb b/spec/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities_spec.rb index afa955a6056..c03962c8d21 100644 --- a/spec/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities_spec.rb +++ b/spec/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities_spec.rb @@ -5,9 +5,9 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedOperationalVulnerabilities, :migration do include MigrationHelpers::VulnerabilitiesHelper - let_it_be(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - let_it_be(:users) { table(:users) } - let_it_be(:user) do + let!(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } + let!(:users) { table(:users) } + let!(:user) do users.create!( name: "Example User", email: "user@example.com", @@ -17,7 +17,7 @@ RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedOperationalVulnerabili ) end - let_it_be(:project) do + let!(:project) do table(:projects).create!( id: 123, namespace_id: namespace.id, @@ -25,9 +25,9 @@ RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedOperationalVulnerabili ) end - let_it_be(:scanners) { table(:vulnerability_scanners) } - let_it_be(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } - let_it_be(:different_scanner) do + let!(:scanners) { table(:vulnerability_scanners) } + let!(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } + let!(:different_scanner) do scanners.create!( project_id: project.id, external_id: 'test 2', @@ -35,22 +35,22 @@ RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedOperationalVulnerabili ) end - let_it_be(:vulnerabilities) { table(:vulnerabilities) } - let_it_be(:vulnerability_with_finding) do + let!(:vulnerabilities) { table(:vulnerabilities) } + let!(:vulnerability_with_finding) do create_vulnerability!( project_id: project.id, author_id: user.id ) end - let_it_be(:vulnerability_without_finding) do + let!(:vulnerability_without_finding) do create_vulnerability!( project_id: project.id, author_id: user.id ) end - let_it_be(:cis_vulnerability_without_finding) do + let!(:cis_vulnerability_without_finding) do create_vulnerability!( project_id: project.id, author_id: user.id, @@ -58,7 +58,7 @@ RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedOperationalVulnerabili ) end - let_it_be(:custom_vulnerability_without_finding) do + let!(:custom_vulnerability_without_finding) do create_vulnerability!( project_id: project.id, author_id: user.id, @@ -66,8 +66,8 @@ RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedOperationalVulnerabili ) end - let_it_be(:vulnerability_identifiers) { table(:vulnerability_identifiers) } - let_it_be(:primary_identifier) do + let!(:vulnerability_identifiers) { table(:vulnerability_identifiers) } + let!(:primary_identifier) do vulnerability_identifiers.create!( project_id: project.id, external_type: 'uuid-v5', @@ -76,8 +76,8 @@ RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedOperationalVulnerabili name: 'Identifier for UUIDv5') end - let_it_be(:vulnerabilities_findings) { table(:vulnerability_occurrences) } - let_it_be(:finding) do + let!(:vulnerabilities_findings) { table(:vulnerability_occurrences) } + let!(:finding) do create_finding!( vulnerability_id: vulnerability_with_finding.id, project_id: project.id, diff --git a/spec/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules_spec.rb b/spec/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules_spec.rb new file mode 100644 index 00000000000..c5b46d3f57c --- /dev/null +++ b/spec/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules_spec.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DeleteOrphansApprovalMergeRequestRules do + describe '#perform' do + let(:batch_table) { :approval_merge_request_rules } + let(:batch_column) { :id } + let(:sub_batch_size) { 1 } + let(:pause_ms) { 0 } + let(:connection) { ApplicationRecord.connection } + + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:approval_merge_request_rules) { table(:approval_merge_request_rules) } + let(:security_orchestration_policy_configurations) { table(:security_orchestration_policy_configurations) } + let(:namespace) { namespaces.create!(name: 'name', path: 'path') } + let(:project) do + projects + .create!(name: "project", path: "project", namespace_id: namespace.id, project_namespace_id: namespace.id) + end + + let(:namespace_2) { namespaces.create!(name: 'name_2', path: 'path_2') } + let(:security_project) do + projects + .create!(name: "security_project", path: "security_project", namespace_id: namespace_2.id, + project_namespace_id: namespace_2.id) + end + + let!(:security_orchestration_policy_configuration) do + security_orchestration_policy_configurations + .create!(project_id: project.id, security_policy_management_project_id: security_project.id) + end + + let(:merge_request) do + table(:merge_requests).create!(target_project_id: project.id, target_branch: 'main', source_branch: 'feature') + end + + let!(:approval_rule) do + approval_merge_request_rules.create!( + name: 'rule', + merge_request_id: merge_request.id, + report_type: 4, + security_orchestration_policy_configuration_id: security_orchestration_policy_configuration.id) + end + + let!(:approval_rule_other_report_type) do + approval_merge_request_rules.create!( + name: 'rule 2', + merge_request_id: merge_request.id, + report_type: 1, + security_orchestration_policy_configuration_id: security_orchestration_policy_configuration.id) + end + + let!(:approval_rule_last) do + approval_merge_request_rules.create!(name: 'rule 3', merge_request_id: merge_request.id, report_type: 4) + end + + subject do + described_class.new( + start_id: approval_rule.id, + end_id: approval_rule_last.id, + batch_table: batch_table, + batch_column: batch_column, + sub_batch_size: sub_batch_size, + pause_ms: pause_ms, + connection: connection + ).perform + end + + it 'delete only approval rules without association with the security project and report_type equals to 4' do + expect { subject }.to change { approval_merge_request_rules.count }.from(3).to(2) + end + end +end diff --git a/spec/lib/gitlab/background_migration/delete_orphans_approval_project_rules_spec.rb b/spec/lib/gitlab/background_migration/delete_orphans_approval_project_rules_spec.rb new file mode 100644 index 00000000000..16253255764 --- /dev/null +++ b/spec/lib/gitlab/background_migration/delete_orphans_approval_project_rules_spec.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DeleteOrphansApprovalProjectRules do + describe '#perform' do + let(:batch_table) { :approval_project_rules } + let(:batch_column) { :id } + let(:sub_batch_size) { 1 } + let(:pause_ms) { 0 } + let(:connection) { ApplicationRecord.connection } + + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:approval_project_rules) { table(:approval_project_rules) } + let(:security_orchestration_policy_configurations) { table(:security_orchestration_policy_configurations) } + let(:namespace) { namespaces.create!(name: 'name', path: 'path') } + let(:project) do + projects + .create!(name: "project", path: "project", namespace_id: namespace.id, project_namespace_id: namespace.id) + end + + let(:namespace_2) { namespaces.create!(name: 'name_2', path: 'path_2') } + let(:security_project) do + projects + .create!(name: "security_project", path: "security_project", namespace_id: namespace_2.id, + project_namespace_id: namespace_2.id) + end + + let!(:security_orchestration_policy_configuration) do + security_orchestration_policy_configurations + .create!(project_id: project.id, security_policy_management_project_id: security_project.id) + end + + let!(:project_rule) do + approval_project_rules.create!( + name: 'rule', + project_id: project.id, + report_type: 4, + security_orchestration_policy_configuration_id: security_orchestration_policy_configuration.id) + end + + let!(:project_rule_other_report_type) do + approval_project_rules.create!( + name: 'rule 2', + project_id: project.id, + report_type: 1, + security_orchestration_policy_configuration_id: security_orchestration_policy_configuration.id) + end + + let!(:project_rule_last) do + approval_project_rules.create!(name: 'rule 3', project_id: project.id, report_type: 4) + end + + subject do + described_class.new( + start_id: project_rule.id, + end_id: project_rule_last.id, + batch_table: batch_table, + batch_column: batch_column, + sub_batch_size: sub_batch_size, + pause_ms: pause_ms, + connection: connection + ).perform + end + + it 'delete only approval rules without association with the security project and report_type equals to 4' do + expect { subject }.to change { approval_project_rules.count }.from(3).to(2) + end + end +end diff --git a/spec/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images_spec.rb b/spec/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images_spec.rb index 8a63673bf38..e7b0471810d 100644 --- a/spec/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images_spec.rb +++ b/spec/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images_spec.rb @@ -3,10 +3,10 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::DisableExpirationPoliciesLinkedToNoContainerImages, :migration, schema: 20220326161803 do # rubocop:disable Layout/LineLength - let_it_be(:projects) { table(:projects) } - let_it_be(:container_expiration_policies) { table(:container_expiration_policies) } - let_it_be(:container_repositories) { table(:container_repositories) } - let_it_be(:namespaces) { table(:namespaces) } + let!(:projects) { table(:projects) } + let!(:container_expiration_policies) { table(:container_expiration_policies) } + let!(:container_repositories) { table(:container_repositories) } + let!(:namespaces) { table(:namespaces) } let!(:namespace) { namespaces.create!(name: 'test', path: 'test') } diff --git a/spec/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects_spec.rb b/spec/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects_spec.rb index d20eaef3650..d60874c3159 100644 --- a/spec/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects_spec.rb +++ b/spec/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects_spec.rb @@ -50,7 +50,7 @@ RSpec.describe Gitlab::BackgroundMigration::DisableLegacyOpenSourceLicenseForNoI ) project_statistics_table.create!(project_id: project.id, namespace_id: namespace.id, repository_size: repo_size) - issues_table.create!(project_id: project.id) if with_issue + issues_table.create!(project_id: project.id, namespace_id: project.project_namespace_id) if with_issue project_settings_table.create!(project_id: project.id, legacy_open_source_license_available: true) project diff --git a/spec/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb_spec.rb b/spec/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb_spec.rb new file mode 100644 index 00000000000..b92f1a74551 --- /dev/null +++ b/spec/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DisableLegacyOpenSourceLicenseForProjectsLessThanFiveMb, + :migration, + schema: 20221018095434, + feature_category: :projects do + let(:namespaces_table) { table(:namespaces) } + let(:projects_table) { table(:projects) } + let(:project_settings_table) { table(:project_settings) } + let(:project_statistics_table) { table(:project_statistics) } + + subject(:perform_migration) do + described_class.new(start_id: project_settings_table.minimum(:project_id), + end_id: project_settings_table.maximum(:project_id), + batch_table: :project_settings, + batch_column: :project_id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection) + .perform + end + + it 'sets `legacy_open_source_license_available` to false only for projects less than 5 MB', :aggregate_failures do + project_setting_2_mb = create_legacy_license_project_setting(repo_size: 2) + project_setting_4_mb = create_legacy_license_project_setting(repo_size: 4) + project_setting_5_mb = create_legacy_license_project_setting(repo_size: 5) + project_setting_6_mb = create_legacy_license_project_setting(repo_size: 6) + + record = ActiveRecord::QueryRecorder.new do + expect { perform_migration } + .to change { migrated_attribute(project_setting_2_mb) }.from(true).to(false) + .and change { migrated_attribute(project_setting_4_mb) }.from(true).to(false) + .and not_change { migrated_attribute(project_setting_5_mb) }.from(true) + .and not_change { migrated_attribute(project_setting_6_mb) }.from(true) + end + + expect(record.count).to eq(15) + end + + private + + # @param repo_size: Repo size in MB + def create_legacy_license_project_setting(repo_size:) + path = "path-for-repo-size-#{repo_size}" + namespace = namespaces_table.create!(name: "namespace-#{path}", path: "namespace-#{path}") + project_namespace = + namespaces_table.create!(name: "-project-namespace-#{path}", path: "project-namespace-#{path}", type: 'Project') + project = projects_table + .create!(name: path, path: path, namespace_id: namespace.id, project_namespace_id: project_namespace.id) + + size_in_bytes = 1.megabyte * repo_size + project_statistics_table.create!(project_id: project.id, namespace_id: namespace.id, repository_size: size_in_bytes) + project_settings_table.create!(project_id: project.id, legacy_open_source_license_available: true) + end + + def migrated_attribute(project_setting) + project_settings_table.find(project_setting.project_id).legacy_open_source_license_available + end +end diff --git a/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb b/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb index 5b6722a3384..ba04f2d20a7 100644 --- a/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb +++ b/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb @@ -3,33 +3,33 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::DropInvalidVulnerabilities, schema: 20210301200959 do - let_it_be(:background_migration_jobs) { table(:background_migration_jobs) } - let_it_be(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - let_it_be(:users) { table(:users) } - let_it_be(:user) { create_user! } - let_it_be(:project) { table(:projects).create!(id: 123, namespace_id: namespace.id) } - - let_it_be(:scanners) { table(:vulnerability_scanners) } - let_it_be(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } - let_it_be(:different_scanner) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') } - - let_it_be(:vulnerabilities) { table(:vulnerabilities) } - let_it_be(:vulnerability_with_finding) do + let!(:background_migration_jobs) { table(:background_migration_jobs) } + let!(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } + let!(:users) { table(:users) } + let!(:user) { create_user! } + let!(:project) { table(:projects).create!(id: 123, namespace_id: namespace.id) } + + let!(:scanners) { table(:vulnerability_scanners) } + let!(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } + let!(:different_scanner) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') } + + let!(:vulnerabilities) { table(:vulnerabilities) } + let!(:vulnerability_with_finding) do create_vulnerability!( project_id: project.id, author_id: user.id ) end - let_it_be(:vulnerability_without_finding) do + let!(:vulnerability_without_finding) do create_vulnerability!( project_id: project.id, author_id: user.id ) end - let_it_be(:vulnerability_identifiers) { table(:vulnerability_identifiers) } - let_it_be(:primary_identifier) do + let!(:vulnerability_identifiers) { table(:vulnerability_identifiers) } + let!(:primary_identifier) do vulnerability_identifiers.create!( project_id: project.id, external_type: 'uuid-v5', @@ -38,8 +38,8 @@ RSpec.describe Gitlab::BackgroundMigration::DropInvalidVulnerabilities, schema: name: 'Identifier for UUIDv5') end - let_it_be(:vulnerabilities_findings) { table(:vulnerability_occurrences) } - let_it_be(:finding) do + let!(:vulnerabilities_findings) { table(:vulnerability_occurrences) } + let!(:finding) do create_finding!( vulnerability_id: vulnerability_with_finding.id, project_id: project.id, @@ -94,7 +94,7 @@ RSpec.describe Gitlab::BackgroundMigration::DropInvalidVulnerabilities, schema: vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, name: "test", severity: 7, confidence: 7, report_type: 0, project_fingerprint: '123qweasdzxc', location_fingerprint: 'test', - metadata_version: 'test', raw_metadata: 'test', uuid: 'test') + metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid) vulnerabilities_findings.create!( vulnerability_id: vulnerability_id, project_id: project_id, diff --git a/spec/lib/gitlab/background_migration/populate_container_repository_migration_plan_spec.rb b/spec/lib/gitlab/background_migration/populate_container_repository_migration_plan_spec.rb index 0463f5a0c0d..477167c9074 100644 --- a/spec/lib/gitlab/background_migration/populate_container_repository_migration_plan_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_container_repository_migration_plan_spec.rb @@ -3,12 +3,12 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::PopulateContainerRepositoryMigrationPlan, schema: 20220316202640 do - let_it_be(:container_repositories) { table(:container_repositories) } - let_it_be(:projects) { table(:projects) } - let_it_be(:namespaces) { table(:namespaces) } - let_it_be(:gitlab_subscriptions) { table(:gitlab_subscriptions) } - let_it_be(:plans) { table(:plans) } - let_it_be(:namespace_statistics) { table(:namespace_statistics) } + let!(:container_repositories) { table(:container_repositories) } + let!(:projects) { table(:projects) } + let!(:namespaces) { table(:namespaces) } + let!(:gitlab_subscriptions) { table(:gitlab_subscriptions) } + let!(:plans) { table(:plans) } + let!(:namespace_statistics) { table(:namespace_statistics) } let!(:namepace1) { namespaces.create!(id: 1, type: 'Group', name: 'group1', path: 'group1', traversal_ids: [1]) } let!(:namepace2) { namespaces.create!(id: 2, type: 'Group', name: 'group2', path: 'group2', traversal_ids: [2]) } diff --git a/spec/lib/gitlab/background_migration/populate_namespace_statistics_spec.rb b/spec/lib/gitlab/background_migration/populate_namespace_statistics_spec.rb index 98b2bc437f3..4a7d52ee784 100644 --- a/spec/lib/gitlab/background_migration/populate_namespace_statistics_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_namespace_statistics_spec.rb @@ -3,10 +3,10 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::PopulateNamespaceStatistics do - let_it_be(:namespaces) { table(:namespaces) } - let_it_be(:namespace_statistics) { table(:namespace_statistics) } - let_it_be(:dependency_proxy_manifests) { table(:dependency_proxy_manifests) } - let_it_be(:dependency_proxy_blobs) { table(:dependency_proxy_blobs) } + let!(:namespaces) { table(:namespaces) } + let!(:namespace_statistics) { table(:namespace_statistics) } + let!(:dependency_proxy_manifests) { table(:dependency_proxy_manifests) } + let!(:dependency_proxy_blobs) { table(:dependency_proxy_blobs) } let!(:group1) { namespaces.create!(id: 10, type: 'Group', name: 'group1', path: 'group1') } let!(:group2) { namespaces.create!(id: 20, type: 'Group', name: 'group2', path: 'group2') } diff --git a/spec/lib/gitlab/background_migration/populate_vulnerability_reads_spec.rb b/spec/lib/gitlab/background_migration/populate_vulnerability_reads_spec.rb index fc06012ed20..c0470f26d9e 100644 --- a/spec/lib/gitlab/background_migration/populate_vulnerability_reads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_vulnerability_reads_spec.rb @@ -68,7 +68,7 @@ RSpec.describe Gitlab::BackgroundMigration::PopulateVulnerabilityReads, :migrati # rubocop:disable Metrics/ParameterLists def create_finding!( - vulnerability_id: nil, project_id:, scanner_id:, primary_identifier_id:, + project_id:, scanner_id:, primary_identifier_id:, vulnerability_id: nil, name: "test", severity: 7, confidence: 7, report_type: 0, project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" }, location_fingerprint: 'test', metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid) diff --git a/spec/lib/gitlab/background_migration/prune_stale_project_export_jobs_spec.rb b/spec/lib/gitlab/background_migration/prune_stale_project_export_jobs_spec.rb new file mode 100644 index 00000000000..5150d0ea4b0 --- /dev/null +++ b/spec/lib/gitlab/background_migration/prune_stale_project_export_jobs_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::PruneStaleProjectExportJobs, feature_category: :importers do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:project_export_jobs) { table(:project_export_jobs) } + let(:project_relation_exports) { table(:project_relation_exports) } + let(:uploads) { table(:project_relation_export_uploads) } + + subject(:perform_migration) do + described_class.new(start_id: 1, + end_id: 300, + batch_table: :project_export_jobs, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection) + .perform + end + + it 'removes export jobs and associated relations older than 7 days' do + namespaces.create!(id: 1000, name: "Sally", path: 'sally') + projects.create!(id: 1, namespace_id: 1000, project_namespace_id: 1000) + + project = Project.find 1 + + project_export_jobs.create!(id: 10, project_id: project.id, jid: SecureRandom.hex(10), updated_at: 37.months.ago) + project_export_jobs.create!(id: 20, project_id: project.id, jid: SecureRandom.hex(10), updated_at: 12.months.ago) + project_export_jobs.create!(id: 30, project_id: project.id, jid: SecureRandom.hex(10), updated_at: 8.days.ago) + project_export_jobs.create!(id: 40, project_id: project.id, jid: SecureRandom.hex(10), updated_at: 1.day.ago) + project_export_jobs.create!(id: 50, project_id: project.id, jid: SecureRandom.hex(10), updated_at: 2.days.ago) + project_export_jobs.create!(id: 60, project_id: project.id, jid: SecureRandom.hex(10), updated_at: 6.days.ago) + + project_relation_exports.create!(id: 100, project_export_job_id: 10, relation: 'Project') + project_relation_exports.create!(id: 200, project_export_job_id: 20, relation: 'Project') + project_relation_exports.create!(id: 300, project_export_job_id: 30, relation: 'Project') + project_relation_exports.create!(id: 400, project_export_job_id: 40, relation: 'Project') + project_relation_exports.create!(id: 500, project_export_job_id: 50, relation: 'Project') + project_relation_exports.create!(id: 600, project_export_job_id: 60, relation: 'Project') + + uploads.create!(project_relation_export_id: 100, export_file: "#{SecureRandom.alphanumeric(5)}_export.tar.gz") + uploads.create!(project_relation_export_id: 200, export_file: "#{SecureRandom.alphanumeric(5)}_export.tar.gz") + uploads.create!(project_relation_export_id: 300, export_file: "#{SecureRandom.alphanumeric(5)}_export.tar.gz") + uploads.create!(project_relation_export_id: 400, export_file: "#{SecureRandom.alphanumeric(5)}_export.tar.gz") + uploads.create!(project_relation_export_id: 500, export_file: "#{SecureRandom.alphanumeric(5)}_export.tar.gz") + uploads.create!(project_relation_export_id: 600, export_file: "#{SecureRandom.alphanumeric(5)}_export.tar.gz") + + expect(project_export_jobs.all.size).to eq(6) + expect(project_relation_exports.all.size).to eq(6) + expect(uploads.all.size).to eq(6) + + expect { perform_migration } + .to change { project_export_jobs.count }.by(-3) + .and change { project_relation_exports.count }.by(-3) + .and change { uploads.count }.by(-3) + + expect(project_export_jobs.all.size).to eq(3) + expect(project_relation_exports.all.size).to eq(3) + expect(uploads.all.size).to eq(3) + end +end diff --git a/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb b/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb index 29cc4f34f6d..2271bbfb2f3 100644 --- a/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb +++ b/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb @@ -488,11 +488,10 @@ RSpec.describe Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrence # rubocop:disable Metrics/ParameterLists def create_finding!( - id: nil, - vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, + vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, id: nil, name: "test", severity: 7, confidence: 7, report_type: 0, project_fingerprint: '123qweasdzxc', location_fingerprint: 'test', - metadata_version: 'test', raw_metadata: 'test', uuid: 'test') + metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid) vulnerability_findings.create!({ id: id, vulnerability_id: vulnerability_id, diff --git a/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb index 10597e65910..5fede892463 100644 --- a/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb +++ b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb @@ -20,8 +20,8 @@ RSpec.describe Gitlab::BackgroundMigration::RemoveBackfilledJobArtifactsExpireAt ) end - let_it_be(:namespace) { table(:namespaces).create!(id: 1, name: 'user', path: 'user') } - let_it_be(:project) do + let!(:namespace) { table(:namespaces).create!(id: 1, name: 'user', path: 'user') } + let!(:project) do table(:projects).create!( id: 1, name: 'gitlab1', diff --git a/spec/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings_spec.rb b/spec/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings_spec.rb index 8003159f59e..ed08ae22245 100644 --- a/spec/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings_spec.rb +++ b/spec/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings_spec.rb @@ -134,11 +134,10 @@ RSpec.describe Gitlab::BackgroundMigration::RemoveDuplicateVulnerabilitiesFindin # rubocop:disable Metrics/ParameterLists def create_finding!( - id: nil, - vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, + vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, id: nil, name: "test", severity: 7, confidence: 7, report_type: 0, project_fingerprint: '123qweasdzxc', location_fingerprint: 'test', - metadata_version: 'test', raw_metadata: 'test', uuid: 'test') + metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid) params = { vulnerability_id: vulnerability_id, project_id: project_id, diff --git a/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb b/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb index 33ad74fbee8..1844347f4a9 100644 --- a/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb +++ b/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb @@ -89,7 +89,6 @@ RSpec.describe Gitlab::BackgroundMigration::RemoveOccurrencePipelinesAndDuplicat let!(:unrelated_finding) do create_finding!( id: 9999999, - uuid: "unreleated_finding", vulnerability_id: nil, report_type: 1, location_fingerprint: 'random_location_fingerprint', @@ -133,11 +132,10 @@ RSpec.describe Gitlab::BackgroundMigration::RemoveOccurrencePipelinesAndDuplicat # rubocop:disable Metrics/ParameterLists def create_finding!( - id: nil, - vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, + vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, id: nil, name: "test", severity: 7, confidence: 7, report_type: 0, project_fingerprint: '123qweasdzxc', location_fingerprint: 'test', - metadata_version: 'test', raw_metadata: 'test', uuid: 'test') + metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid) params = { vulnerability_id: vulnerability_id, project_id: project_id, diff --git a/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb b/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb index ccf96e036ae..918df8f4442 100644 --- a/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb +++ b/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb @@ -33,7 +33,7 @@ RSpec.describe Gitlab::BackgroundMigration::RemoveVulnerabilityFindingLinks, :mi location_fingerprint: "location_fingerprint_#{id}", metadata_version: 'metadata_version', raw_metadata: 'raw_metadata', - uuid: "uuid_#{id}" + uuid: SecureRandom.uuid ) end end diff --git a/spec/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item_spec.rb b/spec/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item_spec.rb index 45932defaf9..580465df4d9 100644 --- a/spec/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item_spec.rb +++ b/spec/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item_spec.rb @@ -5,10 +5,22 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::RenameTaskSystemNoteToChecklistItem do let(:notes) { table(:notes) } let(:projects) { table(:projects) } - let(:namespace) { table(:namespaces).create!(name: 'batchtest1', type: 'Group', path: 'space1') } - let(:project) { table(:projects).create!(name: 'proj1', path: 'proj1', namespace_id: namespace.id) } - let(:issue) { table(:issues).create!(title: 'Test issue') } + let(:issue_base_type_enum_value) { 0 } + let(:issue_type) { table(:work_item_types).find_by!(namespace_id: nil, base_type: issue_base_type_enum_value) } + + let(:project) do + table(:projects).create!( + name: 'proj1', path: 'proj1', namespace_id: namespace.id, project_namespace_id: namespace.id + ) + end + + let(:issue) do + table(:issues).create!( + title: 'Test issue', project_id: project.id, + namespace_id: project.project_namespace_id, work_item_type_id: issue_type.id + ) + end let!(:note1) do notes.create!( diff --git a/spec/lib/gitlab/background_migration/reset_status_on_container_repositories_spec.rb b/spec/lib/gitlab/background_migration/reset_status_on_container_repositories_spec.rb new file mode 100644 index 00000000000..d50b04857d6 --- /dev/null +++ b/spec/lib/gitlab/background_migration/reset_status_on_container_repositories_spec.rb @@ -0,0 +1,261 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories, feature_category: :container_registry do + let(:projects_table) { table(:projects) } + let(:namespaces_table) { table(:namespaces) } + let(:container_repositories_table) { table(:container_repositories) } + let(:routes_table) { table(:routes) } + + let!(:root_group) do + namespaces_table.create!(name: 'root_group', path: 'root_group', type: 'Group') do |new_group| + new_group.update!(traversal_ids: [new_group.id]) + end + end + + let!(:group1) do + namespaces_table.create!(name: 'group1', path: 'group1', parent_id: root_group.id, type: 'Group') do |new_group| + new_group.update!(traversal_ids: [root_group.id, new_group.id]) + end + end + + let!(:subgroup1) do + namespaces_table.create!(name: 'subgroup1', path: 'subgroup1', parent_id: group1.id, type: 'Group') do |new_group| + new_group.update!(traversal_ids: [root_group.id, group1.id, new_group.id]) + end + end + + let!(:group2) do + namespaces_table.create!(name: 'group2', path: 'group2', parent_id: root_group.id, type: 'Group') do |new_group| + new_group.update!(traversal_ids: [root_group.id, new_group.id]) + end + end + + let!(:group1_project_namespace) do + namespaces_table.create!(name: 'group1_project', path: 'group1_project', type: 'Project', parent_id: group1.id) + end + + let!(:subgroup1_project_namespace) do + namespaces_table.create!( + name: 'subgroup1_project', + path: 'subgroup1_project', + type: 'Project', + parent_id: subgroup1.id + ) + end + + let!(:group2_project_namespace) do + namespaces_table.create!( + name: 'group2_project', + path: 'group2_project', + type: 'Project', + parent_id: group2.id + ) + end + + let!(:group1_project) do + projects_table.create!( + name: 'group1_project', + path: 'group1_project', + namespace_id: group1.id, + project_namespace_id: group1_project_namespace.id + ) + end + + let!(:subgroup1_project) do + projects_table.create!( + name: 'subgroup1_project', + path: 'subgroup1_project', + namespace_id: subgroup1.id, + project_namespace_id: subgroup1_project_namespace.id + ) + end + + let!(:group2_project) do + projects_table.create!( + name: 'group2_project', + path: 'group2_project', + namespace_id: group2.id, + project_namespace_id: group2_project_namespace.id + ) + end + + let!(:route2) do + routes_table.create!( + source_id: group2_project.id, + source_type: 'Project', + path: 'root_group/group2/group2_project', + namespace_id: group2_project_namespace.id + ) + end + + let!(:delete_scheduled_container_repository1) do + container_repositories_table.create!(project_id: group1_project.id, status: 0, name: 'container_repository1') + end + + let!(:delete_scheduled_container_repository2) do + container_repositories_table.create!(project_id: subgroup1_project.id, status: 0, name: 'container_repository2') + end + + let!(:delete_scheduled_container_repository3) do + container_repositories_table.create!(project_id: group2_project.id, status: 0, name: 'container_repository3') + end + + let!(:delete_ongoing_container_repository4) do + container_repositories_table.create!(project_id: group2_project.id, status: 2, name: 'container_repository4') + end + + let(:migration) do + described_class.new( + start_id: container_repositories_table.minimum(:id), + end_id: container_repositories_table.maximum(:id), + batch_table: :container_repositories, + batch_column: :id, + sub_batch_size: 50, + pause_ms: 0, + connection: ApplicationRecord.connection + ) + end + + describe '#filter_batch' do + it 'scopes the relation to delete scheduled container repositories' do + expected = container_repositories_table.where(status: 0).pluck(:id) + actual = migration.filter_batch(container_repositories_table).pluck(:id) + + expect(actual).to match_array(expected) + end + end + + describe '#perform' do + let(:registry_api_url) { 'http://example.com' } + + subject(:perform) { migration.perform } + + before do + stub_container_registry_config( + enabled: true, + api_url: registry_api_url, + key: 'spec/fixtures/x509_certificate_pk.key' + ) + stub_tags_list(path: 'root_group/group1/group1_project/container_repository1') + stub_tags_list(path: 'root_group/group1/subgroup1/subgroup1_project/container_repository2', tags: []) + stub_tags_list(path: 'root_group/group2/group2_project/container_repository3') + end + + shared_examples 'resetting status of all container repositories scheduled for deletion' do + it 'resets all statuses' do + expect_logging_on( + path: 'root_group/group1/group1_project/container_repository1', + id: delete_scheduled_container_repository1.id, + has_tags: true + ) + expect_logging_on( + path: 'root_group/group1/subgroup1/subgroup1_project/container_repository2', + id: delete_scheduled_container_repository2.id, + has_tags: true + ) + expect_logging_on( + path: 'root_group/group2/group2_project/container_repository3', + id: delete_scheduled_container_repository3.id, + has_tags: true + ) + + expect { perform } + .to change { delete_scheduled_container_repository1.reload.status }.from(0).to(nil) + .and change { delete_scheduled_container_repository3.reload.status }.from(0).to(nil) + .and change { delete_scheduled_container_repository2.reload.status }.from(0).to(nil) + end + end + + it 'resets status of container repositories with tags' do + expect_pull_access_token_on(path: 'root_group/group1/group1_project/container_repository1') + expect_pull_access_token_on(path: 'root_group/group1/subgroup1/subgroup1_project/container_repository2') + expect_pull_access_token_on(path: 'root_group/group2/group2_project/container_repository3') + + expect_logging_on( + path: 'root_group/group1/group1_project/container_repository1', + id: delete_scheduled_container_repository1.id, + has_tags: true + ) + expect_logging_on( + path: 'root_group/group1/subgroup1/subgroup1_project/container_repository2', + id: delete_scheduled_container_repository2.id, + has_tags: false + ) + expect_logging_on( + path: 'root_group/group2/group2_project/container_repository3', + id: delete_scheduled_container_repository3.id, + has_tags: true + ) + + expect { perform } + .to change { delete_scheduled_container_repository1.reload.status }.from(0).to(nil) + .and change { delete_scheduled_container_repository3.reload.status }.from(0).to(nil) + .and not_change { delete_scheduled_container_repository2.reload.status } + end + + context 'with the registry disabled' do + before do + allow(::Gitlab.config.registry).to receive(:enabled).and_return(false) + end + + it_behaves_like 'resetting status of all container repositories scheduled for deletion' + end + + context 'with the registry api url not defined' do + before do + allow(::Gitlab.config.registry).to receive(:api_url).and_return('') + end + + it_behaves_like 'resetting status of all container repositories scheduled for deletion' + end + + context 'with a faraday error' do + before do + client_double = instance_double('::ContainerRegistry::Client') + allow(::ContainerRegistry::Client).to receive(:new).and_return(client_double) + allow(client_double).to receive(:repository_tags).and_raise(Faraday::TimeoutError) + + expect_pull_access_token_on(path: 'root_group/group1/group1_project/container_repository1') + expect_pull_access_token_on(path: 'root_group/group1/subgroup1/subgroup1_project/container_repository2') + expect_pull_access_token_on(path: 'root_group/group2/group2_project/container_repository3') + end + + it_behaves_like 'resetting status of all container repositories scheduled for deletion' + end + + def stub_tags_list(path:, tags: %w[tag1]) + url = "#{registry_api_url}/v2/#{path}/tags/list?n=1" + + stub_request(:get, url) + .with( + headers: { + 'Accept' => ContainerRegistry::Client::ACCEPTED_TYPES.join(', '), + 'Authorization' => /bearer .+/, + 'User-Agent' => "GitLab/#{Gitlab::VERSION}" + } + ) + .to_return( + status: 200, + body: Gitlab::Json.dump(tags: tags), + headers: { 'Content-Type' => 'application/json' } + ) + end + + def expect_pull_access_token_on(path:) + expect(Auth::ContainerRegistryAuthenticationService) + .to receive(:pull_access_token).with(path).and_call_original + end + + def expect_logging_on(path:, id:, has_tags:) + expect(::Gitlab::BackgroundMigration::Logger) + .to receive(:info).with( + migrator: described_class::MIGRATOR, + has_tags: has_tags, + container_repository_id: id, + container_repository_path: path + ) + end + end +end diff --git a/spec/lib/gitlab/background_migration/sanitize_confidential_todos_spec.rb b/spec/lib/gitlab/background_migration/sanitize_confidential_todos_spec.rb index 2c5c47e39c9..c58f2060001 100644 --- a/spec/lib/gitlab/background_migration/sanitize_confidential_todos_spec.rb +++ b/spec/lib/gitlab/background_migration/sanitize_confidential_todos_spec.rb @@ -27,8 +27,15 @@ RSpec.describe Gitlab::BackgroundMigration::SanitizeConfidentialTodos, :migratio project_namespace_id: project_namespace2.id) end - let(:issue1) { issues.create!(project_id: project1.id, issue_type: 1, title: 'issue1', author_id: user.id) } - let(:issue2) { issues.create!(project_id: project2.id, issue_type: 1, title: 'issue2') } + let(:issue1) do + issues.create!( + project_id: project1.id, namespace_id: project_namespace1.id, issue_type: 1, title: 'issue1', author_id: user.id + ) + end + + let(:issue2) do + issues.create!(project_id: project2.id, namespace_id: project_namespace2.id, issue_type: 1, title: 'issue2') + end let(:public_note) { notes.create!(note: 'text', project_id: project1.id) } diff --git a/spec/lib/gitlab/background_migration/update_timelogs_null_spent_at_spec.rb b/spec/lib/gitlab/background_migration/update_timelogs_null_spent_at_spec.rb index 982e3319063..908f11aabc3 100644 --- a/spec/lib/gitlab/background_migration/update_timelogs_null_spent_at_spec.rb +++ b/spec/lib/gitlab/background_migration/update_timelogs_null_spent_at_spec.rb @@ -3,19 +3,19 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::UpdateTimelogsNullSpentAt, schema: 20211215090620 do - let_it_be(:previous_time) { 10.days.ago } - let_it_be(:namespace) { table(:namespaces).create!(name: 'namespace', path: 'namespace') } - let_it_be(:project) { table(:projects).create!(namespace_id: namespace.id) } - let_it_be(:issue) { table(:issues).create!(project_id: project.id) } - let_it_be(:merge_request) { table(:merge_requests).create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature') } - let_it_be(:timelog1) { create_timelog!(issue_id: issue.id) } - let_it_be(:timelog2) { create_timelog!(merge_request_id: merge_request.id) } - let_it_be(:timelog3) { create_timelog!(issue_id: issue.id, spent_at: previous_time) } - let_it_be(:timelog4) { create_timelog!(merge_request_id: merge_request.id, spent_at: previous_time) } + let!(:previous_time) { 10.days.ago } + let!(:namespace) { table(:namespaces).create!(name: 'namespace', path: 'namespace') } + let!(:project) { table(:projects).create!(namespace_id: namespace.id) } + let!(:issue) { table(:issues).create!(project_id: project.id) } + let!(:merge_request) { table(:merge_requests).create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature') } + let!(:timelog1) { create_timelog!(issue_id: issue.id) } + let!(:timelog2) { create_timelog!(merge_request_id: merge_request.id) } + let!(:timelog3) { create_timelog!(issue_id: issue.id, spent_at: previous_time) } + let!(:timelog4) { create_timelog!(merge_request_id: merge_request.id, spent_at: previous_time) } subject(:background_migration) { described_class.new } - before_all do + before do table(:timelogs).where.not(id: [timelog3.id, timelog4.id]).update_all(spent_at: nil) end diff --git a/spec/lib/gitlab/blob_helper_spec.rb b/spec/lib/gitlab/blob_helper_spec.rb index a2f20dcd4fc..e18277ba8d2 100644 --- a/spec/lib/gitlab/blob_helper_spec.rb +++ b/spec/lib/gitlab/blob_helper_spec.rb @@ -68,6 +68,7 @@ RSpec.describe Gitlab::BlobHelper do expect(blob.image?).to be_falsey end end + context 'with a .webp file' do it 'returns true' do expect(webp_blob.image?).to be_truthy diff --git a/spec/lib/gitlab/bullet_spec.rb b/spec/lib/gitlab/bullet_spec.rb index 1262a0b8bde..c575c656bb4 100644 --- a/spec/lib/gitlab/bullet_spec.rb +++ b/spec/lib/gitlab/bullet_spec.rb @@ -3,48 +3,55 @@ require 'spec_helper' RSpec.describe Gitlab::Bullet do - describe '#enabled?' do - it 'is enabled' do - stub_env('ENABLE_BULLET', true) - - expect(described_class.enabled?).to be(true) - end - - it 'is not enabled' do + context 'with bullet installed' do + before do stub_env('ENABLE_BULLET', nil) - - expect(described_class.enabled?).to be(false) + stub_const('::Bullet', double) end - it 'is correctly aliased for #extra_logging_enabled?' do - expect(described_class.method(:extra_logging_enabled?).original_name).to eq(:enabled?) - end - end + describe '#enabled?' do + context 'with env enabled' do + before do + stub_env('ENABLE_BULLET', true) + allow(Gitlab.config.bullet).to receive(:enabled).and_return(false) + end - describe '#configure_bullet?' do - context 'with ENABLE_BULLET true' do - before do - stub_env('ENABLE_BULLET', true) + it 'is enabled' do + expect(described_class.enabled?).to be(true) + end end - it 'is configurable' do - expect(described_class.configure_bullet?).to be(true) + context 'with env disabled' do + before do + stub_env('ENABLE_BULLET', false) + allow(Gitlab.config.bullet).to receive(:enabled).and_return(true) + end + + it 'is not enabled' do + expect(described_class.enabled?).to be(false) + end end end - context 'with ENABLE_BULLET falsey' do - before do - stub_env('ENABLE_BULLET', nil) - end + describe '#configure_bullet?' do + context 'with config enabled' do + before do + allow(Gitlab.config.bullet).to receive(:enabled).and_return(true) + end - it 'is not configurable' do - expect(described_class.configure_bullet?).to be(false) + it 'is configurable' do + expect(described_class.configure_bullet?).to be(true) + end end - it 'is configurable in development' do - allow(Rails).to receive_message_chain(:env, :development?).and_return(true) + context 'with config disabled' do + before do + allow(Gitlab.config.bullet).to receive(:enabled).and_return(false) + end - expect(described_class.configure_bullet?).to be(true) + it 'is not configurable' do + expect(described_class.configure_bullet?).to be(false) + end end end end diff --git a/spec/lib/gitlab/checks/timed_logger_spec.rb b/spec/lib/gitlab/checks/timed_logger_spec.rb index 6c488212eca..261fdd6c002 100644 --- a/spec/lib/gitlab/checks/timed_logger_spec.rb +++ b/spec/lib/gitlab/checks/timed_logger_spec.rb @@ -17,38 +17,44 @@ RSpec.describe Gitlab::Checks::TimedLogger do logger.append_message("Checking ref: #{ref}") end + around do |example| + freeze_time do + example.run + end + end + describe '#log_timed' do it 'logs message' do - Timecop.freeze(start + 30.seconds) do - logger.log_timed(log_messages[:foo], start) { bar_check } - end + travel_to(start + 30.seconds) + + logger.log_timed(log_messages[:foo], start) { bar_check } expect(logger.full_message).to eq("Checking ref: bar\nFoo message... (30000.0ms)") end context 'when time limit was reached' do it 'cancels action' do - Timecop.freeze(start + 50.seconds) do - expect do - logger.log_timed(log_messages[:foo], start) do - bar_check - end - end.to raise_error(described_class::TimeoutError) - end + travel_to(start + 50.seconds) + + expect do + logger.log_timed(log_messages[:foo], start) do + bar_check + end + end.to raise_error(described_class::TimeoutError) expect(logger.full_message).to eq("Checking ref: bar\nFoo message... (cancelled)") end it 'cancels action with time elapsed if work was performed' do - Timecop.freeze(start + 30.seconds) do - expect do - logger.log_timed(log_messages[:foo], start) do - grpc_check - end - end.to raise_error(described_class::TimeoutError) - - expect(logger.full_message).to eq("Checking ref: bar\nFoo message... (cancelled after 30000.0ms)") - end + travel_to(start + 30.seconds) + + expect do + logger.log_timed(log_messages[:foo], start) do + grpc_check + end + end.to raise_error(described_class::TimeoutError) + + expect(logger.full_message).to eq("Checking ref: bar\nFoo message... (cancelled after 30000.0ms)") end end end diff --git a/spec/lib/gitlab/ci/build/cache_spec.rb b/spec/lib/gitlab/ci/build/cache_spec.rb index 7477aedb994..a8fa14b4b4c 100644 --- a/spec/lib/gitlab/ci/build/cache_spec.rb +++ b/spec/lib/gitlab/ci/build/cache_spec.rb @@ -14,8 +14,8 @@ RSpec.describe Gitlab::Ci::Build::Cache do cache = described_class.new(cache_config, pipeline) - expect(Gitlab::Ci::Pipeline::Seed::Build::Cache).to have_received(:new).with(pipeline, { key: 'key-a' }) - expect(Gitlab::Ci::Pipeline::Seed::Build::Cache).to have_received(:new).with(pipeline, { key: 'key-b' }) + expect(Gitlab::Ci::Pipeline::Seed::Build::Cache).to have_received(:new).with(pipeline, { key: 'key-a' }, 0) + expect(Gitlab::Ci::Pipeline::Seed::Build::Cache).to have_received(:new).with(pipeline, { key: 'key-b' }, 1) expect(cache.instance_variable_get(:@cache)).to eq([cache_seed_a, cache_seed_b]) end end @@ -29,7 +29,7 @@ RSpec.describe Gitlab::Ci::Build::Cache do cache = described_class.new(cache_config, pipeline) - expect(Gitlab::Ci::Pipeline::Seed::Build::Cache).to have_received(:new).with(pipeline, cache_config) + expect(Gitlab::Ci::Pipeline::Seed::Build::Cache).to have_received(:new).with(pipeline, cache_config, 0) expect(cache.instance_variable_get(:@cache)).to eq([cache_seed]) end end diff --git a/spec/lib/gitlab/ci/build/context/build_spec.rb b/spec/lib/gitlab/ci/build/context/build_spec.rb index 7f862a3b80a..74739a67be0 100644 --- a/spec/lib/gitlab/ci/build/context/build_spec.rb +++ b/spec/lib/gitlab/ci/build/context/build_spec.rb @@ -2,11 +2,11 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Build::Context::Build do +RSpec.describe Gitlab::Ci::Build::Context::Build, feature_category: :pipeline_authoring do let(:pipeline) { create(:ci_pipeline) } let(:seed_attributes) { { 'name' => 'some-job' } } - let(:context) { described_class.new(pipeline, seed_attributes) } + subject(:context) { described_class.new(pipeline, seed_attributes) } shared_examples 'variables collection' do it { is_expected.to include('CI_COMMIT_REF_NAME' => 'master') } @@ -22,6 +22,12 @@ RSpec.describe Gitlab::Ci::Build::Context::Build do it { is_expected.to include('CI_BUILD_REF_NAME' => 'master') } it { is_expected.to include('CI_PROJECT_PATH' => pipeline.project.full_path) } end + + context 'when environment:name is provided' do + let(:seed_attributes) { { 'name' => 'some-job', 'environment' => 'test' } } + + it { is_expected.to include('CI_ENVIRONMENT_NAME' => 'test') } + end end describe '#variables' do diff --git a/spec/lib/gitlab/ci/build/hook_spec.rb b/spec/lib/gitlab/ci/build/hook_spec.rb new file mode 100644 index 00000000000..6ed40a44c97 --- /dev/null +++ b/spec/lib/gitlab/ci/build/hook_spec.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Build::Hook, feature_category: :pipeline_authoring do + let_it_be(:build1) do + FactoryBot.build(:ci_build, + options: { hooks: { pre_get_sources_script: ["echo 'hello pre_get_sources_script'"] } }) + end + + describe '.from_hooks' do + subject(:from_hooks) { described_class.from_hooks(build1) } + + it 'initializes and returns hooks' do + expect(from_hooks.size).to eq(1) + expect(from_hooks[0].name).to eq('pre_get_sources_script') + expect(from_hooks[0].script).to eq(["echo 'hello pre_get_sources_script'"]) + end + end +end diff --git a/spec/lib/gitlab/ci/config/entry/artifacts_spec.rb b/spec/lib/gitlab/ci/config/entry/artifacts_spec.rb index 7476fc6c25f..6264e0c8e33 100644 --- a/spec/lib/gitlab/ci/config/entry/artifacts_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/artifacts_spec.rb @@ -142,6 +142,26 @@ RSpec.describe Gitlab::Ci::Config::Entry::Artifacts do end end + context 'when the `when` keyword is not a string' do + context 'when it is an array' do + let(:config) { { paths: %w[results.txt], when: ['always'] } } + + it 'returns error' do + expect(entry).not_to be_valid + expect(entry.errors).to include 'artifacts when should be a string' + end + end + + context 'when it is a boolean' do + let(:config) { { paths: %w[results.txt], when: true } } + + it 'returns error' do + expect(entry).not_to be_valid + expect(entry.errors).to include 'artifacts when should be a string' + end + end + end + describe 'excluded artifacts' do context 'when configuration is valid' do let(:config) { { untracked: true, exclude: ['some/directory/'] } } diff --git a/spec/lib/gitlab/ci/config/entry/bridge_spec.rb b/spec/lib/gitlab/ci/config/entry/bridge_spec.rb index 8da46561b73..736c184a289 100644 --- a/spec/lib/gitlab/ci/config/entry/bridge_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/bridge_spec.rb @@ -13,7 +13,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Bridge do # that we know that we don't want to inherit # as they do not have sense in context of Bridge let(:ignored_inheritable_columns) do - %i[before_script after_script image services cache interruptible timeout + %i[before_script after_script hooks image services cache interruptible timeout retry tags artifacts] end end diff --git a/spec/lib/gitlab/ci/config/entry/cache_spec.rb b/spec/lib/gitlab/ci/config/entry/cache_spec.rb index 247f4b63910..414cbb169b9 100644 --- a/spec/lib/gitlab/ci/config/entry/cache_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/cache_spec.rb @@ -163,22 +163,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Cache do end end - context 'when policy is unknown' do - let(:config) { { policy: 'unknown' } } - - it 'reports error' do - is_expected.to include('cache policy should be pull-push, push, or pull') - end - end - - context 'when `when` is unknown' do - let(:config) { { when: 'unknown' } } - - it 'reports error' do - is_expected.to include('cache when should be on_success, on_failure or always') - end - end - context 'when descendants are invalid' do context 'with invalid keys' do let(:config) { { key: 1 } } @@ -228,6 +212,62 @@ RSpec.describe Gitlab::Ci::Config::Entry::Cache do is_expected.to include 'cache config contains unknown keys: invalid' end end + + context 'when the `when` keyword is not a valid string' do + context 'when `when` is unknown' do + let(:config) { { when: 'unknown' } } + + it 'returns error' do + is_expected.to include('cache when should be one of: on_success, on_failure, always') + end + end + + context 'when it is an array' do + let(:config) { { when: ['always'] } } + + it 'returns error' do + expect(entry).not_to be_valid + is_expected.to include('cache when should be a string') + end + end + + context 'when it is a boolean' do + let(:config) { { when: true } } + + it 'returns error' do + expect(entry).not_to be_valid + is_expected.to include('cache when should be a string') + end + end + end + + context 'when the `policy` keyword is not a valid string' do + context 'when `policy` is unknown' do + let(:config) { { policy: 'unknown' } } + + it 'returns error' do + is_expected.to include('cache policy should be one of: pull-push, push, pull') + end + end + + context 'when it is an array' do + let(:config) { { policy: ['pull-push'] } } + + it 'returns error' do + expect(entry).not_to be_valid + is_expected.to include('cache policy should be a string') + end + end + + context 'when it is a boolean' do + let(:config) { { policy: true } } + + it 'returns error' do + expect(entry).not_to be_valid + is_expected.to include('cache policy should be a string') + end + end + end end end end diff --git a/spec/lib/gitlab/ci/config/entry/default_spec.rb b/spec/lib/gitlab/ci/config/entry/default_spec.rb index 5613b0f09d1..46e96843ee3 100644 --- a/spec/lib/gitlab/ci/config/entry/default_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/default_spec.rb @@ -26,9 +26,8 @@ RSpec.describe Gitlab::Ci::Config::Entry::Default do context 'when filtering all the entry/node names' do it 'contains the expected node names' do expect(described_class.nodes.keys) - .to match_array(%i[before_script image services - after_script cache interruptible - timeout retry tags artifacts]) + .to match_array(%i[before_script after_script hooks cache image services + interruptible timeout retry tags artifacts]) end end end diff --git a/spec/lib/gitlab/ci/config/entry/hooks_spec.rb b/spec/lib/gitlab/ci/config/entry/hooks_spec.rb new file mode 100644 index 00000000000..7a5ff244e18 --- /dev/null +++ b/spec/lib/gitlab/ci/config/entry/hooks_spec.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +RSpec.describe Gitlab::Ci::Config::Entry::Hooks do + subject(:entry) { described_class.new(config) } + + before do + entry.compose! + end + + describe 'validations' do + context 'when passing a valid hook' do + let(:config) { { pre_get_sources_script: ['ls'] } } + + it { is_expected.to be_valid } + end + + context 'when passing an invalid hook' do + let(:config) { { x_get_something: ['ls'] } } + + it { is_expected.not_to be_valid } + end + + context 'when entry config is not a hash' do + let(:config) { 'ls' } + + it { is_expected.not_to be_valid } + end + end + + describe '#value' do + let(:config) { { pre_get_sources_script: ['ls'] } } + + it 'returns a hash' do + expect(entry.value).to eq(config) + end + end +end diff --git a/spec/lib/gitlab/ci/config/entry/id_token_spec.rb b/spec/lib/gitlab/ci/config/entry/id_token_spec.rb new file mode 100644 index 00000000000..12585d662ec --- /dev/null +++ b/spec/lib/gitlab/ci/config/entry/id_token_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::Entry::IdToken do + context 'when given `aud` as a string' do + it 'is valid' do + config = { aud: 'https://gitlab.com' } + id_token = described_class.new(config) + + id_token.compose! + + expect(id_token).to be_valid + expect(id_token.value).to eq(aud: 'https://gitlab.com') + end + end + + context 'when given `aud` as an array' do + it 'is valid and concatenates the values' do + config = { aud: ['https://gitlab.com', 'https://aws.com'] } + id_token = described_class.new(config) + + id_token.compose! + + expect(id_token).to be_valid + expect(id_token.value).to eq(aud: ['https://gitlab.com', 'https://aws.com']) + end + end + + context 'when not given an `aud`' do + it 'is invalid' do + config = {} + id_token = described_class.new(config) + + id_token.compose! + + expect(id_token).not_to be_valid + expect(id_token.errors).to match_array([ + 'id token config missing required keys: aud', + 'id token aud should be an array of strings or a string' + ]) + end + end + + context 'when given an unknown keyword' do + it 'is invalid' do + config = { aud: 'https://gitlab.com', unknown: 'test' } + id_token = described_class.new(config) + + id_token.compose! + + expect(id_token).not_to be_valid + expect(id_token.errors).to match_array([ + 'id token config contains unknown keys: unknown' + ]) + end + end +end diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index acf60a6cdda..becb46ac2e7 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -27,7 +27,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do subject { described_class.nodes.keys } let(:result) do - %i[before_script script stage after_script cache + %i[before_script script after_script hooks stage cache image services only except rules needs variables artifacts environment coverage retry interruptible timeout release tags inherit parallel] @@ -716,7 +716,9 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do let(:config) do { before_script: %w[ls pwd], script: 'rspec', - after_script: %w[cleanup] } + after_script: %w[cleanup], + id_tokens: { TEST_ID_TOKEN: { aud: 'https://gitlab.com' } }, + hooks: { pre_get_sources_script: 'echo hello' } } end it 'returns correct value' do @@ -727,10 +729,33 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do stage: 'test', ignore: false, after_script: %w[cleanup], + hooks: { pre_get_sources_script: ['echo hello'] }, only: { refs: %w[branches tags] }, job_variables: {}, root_variables_inheritance: true, - scheduling_type: :stage) + scheduling_type: :stage, + id_tokens: { TEST_ID_TOKEN: { aud: 'https://gitlab.com' } }) + end + + context 'when the FF ci_hooks_pre_get_sources_script is disabled' do + before do + stub_feature_flags(ci_hooks_pre_get_sources_script: false) + end + + it 'returns correct value' do + expect(entry.value) + .to eq(name: :rspec, + before_script: %w[ls pwd], + script: %w[rspec], + stage: 'test', + ignore: false, + after_script: %w[cleanup], + only: { refs: %w[branches tags] }, + job_variables: {}, + root_variables_inheritance: true, + scheduling_type: :stage, + id_tokens: { TEST_ID_TOKEN: { aud: 'https://gitlab.com' } }) + end end end end diff --git a/spec/lib/gitlab/ci/config/entry/root_spec.rb b/spec/lib/gitlab/ci/config/entry/root_spec.rb index 085293d7368..c40589104cd 100644 --- a/spec/lib/gitlab/ci/config/entry/root_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/root_spec.rb @@ -5,7 +5,8 @@ require 'spec_helper' RSpec.describe Gitlab::Ci::Config::Entry::Root do let(:user) {} let(:project) {} - let(:root) { described_class.new(hash, user: user, project: project) } + let(:logger) { Gitlab::Ci::Pipeline::Logger.new(project: project) } + let(:root) { described_class.new(hash, user: user, project: project, logger: logger) } describe '.nodes' do it 'returns a hash' do @@ -37,7 +38,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do variables: { VAR: 'root', VAR2: { value: 'val 2', description: 'this is var 2' }, - VAR3: { value: %w[val3 val3b], description: 'this is var 3' } + VAR3: { value: 'val3', options: %w[val3 val4 val5], description: 'this is var 3 and some options' } }, after_script: ['make clean'], stages: %w(build pages release), @@ -228,6 +229,14 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do ) end end + + it 'tracks log entries' do + expect(logger.observations_hash).to match( + a_hash_including( + 'config_root_compose_jobs_factory_duration_s' => a_kind_of(Numeric) + ) + ) + end end end @@ -317,6 +326,42 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do end end + context 'when variables have `options` data' do + before do + root.compose! + end + + context 'and the value is in the `options` array' do + let(:hash) do + { + variables: { 'VAR' => { value: 'val1', options: %w[val1 val2] } }, + rspec: { script: 'bin/rspec' } + } + end + + it 'returns correct value' do + expect(root.variables_entry.value_with_data).to eq( + 'VAR' => { value: 'val1' } + ) + + expect(root.variables_value).to eq('VAR' => 'val1') + end + end + + context 'and the value is not in the `options` array' do + let(:hash) do + { + variables: { 'VAR' => { value: 'val', options: %w[val1 val2] } }, + rspec: { script: 'bin/rspec' } + } + end + + it 'returns an error' do + expect(root.errors).to contain_exactly('variables:var config value must be present in options') + end + end + end + context 'when variables have "expand" data' do let(:hash) do { diff --git a/spec/lib/gitlab/ci/config/entry/trigger_spec.rb b/spec/lib/gitlab/ci/config/entry/trigger_spec.rb index d0116c961d7..f47923af45a 100644 --- a/spec/lib/gitlab/ci/config/entry/trigger_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/trigger_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::Trigger do +RSpec.describe Gitlab::Ci::Config::Entry::Trigger, feature_category: :pipeline_authoring do subject { described_class.new(config) } context 'when trigger config is a non-empty string' do @@ -35,6 +35,48 @@ RSpec.describe Gitlab::Ci::Config::Entry::Trigger do end context 'when trigger is a hash - cross-project' do + context 'when project is a string' do + context 'when project is a non-empty string' do + let(:config) { { project: 'some/project' } } + + it 'is valid' do + expect(subject).to be_valid + end + end + + context 'when project is an empty string' do + let(:config) { { project: '' } } + + it 'returns error' do + expect(subject).not_to be_valid + expect(subject.errors.first) + .to match /project can't be blank/ + end + end + end + + context 'when project is not a string' do + context 'when project is an array' do + let(:config) { { project: ['some/project'] } } + + it 'returns error' do + expect(subject).not_to be_valid + expect(subject.errors.first) + .to match /should be a string/ + end + end + + context 'when project is a boolean' do + let(:config) { { project: true } } + + it 'returns error' do + expect(subject).not_to be_valid + expect(subject.errors.first) + .to match /should be a string/ + end + end + end + context 'when branch is provided' do let(:config) { { project: 'some/project', branch: 'feature' } } diff --git a/spec/lib/gitlab/ci/config/entry/variable_spec.rb b/spec/lib/gitlab/ci/config/entry/variable_spec.rb index d7023072312..97b06c8b1a5 100644 --- a/spec/lib/gitlab/ci/config/entry/variable_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/variable_spec.rb @@ -306,48 +306,48 @@ RSpec.describe Gitlab::Ci::Config::Entry::Variable do end end end - end - describe 'ComplexArrayVariable' do - context 'when allow_array_value metadata is false' do - let(:config) { { value: %w[value value2], description: 'description' } } - let(:metadata) { { allow_array_value: false } } + context 'when config is a hash with options' do + context 'when there is no metadata' do + let(:config) { { value: 'value', options: %w[value value2] } } + let(:metadata) { {} } - describe '#valid?' do - it { is_expected.not_to be_valid } - end + describe '#valid?' do + it { is_expected.not_to be_valid } + end - describe '#errors' do - subject(:errors) { entry.errors } + describe '#errors' do + subject(:errors) { entry.errors } - it { is_expected.to include 'var1 config value must be an alphanumeric string' } + it { is_expected.to include 'var1 config must be a string' } + end end - end - context 'when allow_array_value metadata is true' do - let(:config) { { value: %w[value value2], description: 'description' } } - let(:metadata) { { allowed_value_data: %i[value description], allow_array_value: true } } + context 'when options are allowed' do + let(:config) { { value: 'value', options: %w[value value2] } } + let(:metadata) { { allowed_value_data: %i[value options] } } - describe '#valid?' do - it { is_expected.to be_valid } - end + describe '#valid?' do + it { is_expected.to be_valid } + end - describe '#value' do - subject(:value) { entry.value } + describe '#value' do + subject(:value) { entry.value } - it { is_expected.to eq('value') } - end + it { is_expected.to eq('value') } + end - describe '#value_with_data' do - subject(:value_with_data) { entry.value_with_data } + describe '#value_with_data' do + subject(:value_with_data) { entry.value_with_data } - it { is_expected.to eq(value: 'value') } - end + it { is_expected.to eq(value: 'value') } + end - describe '#value_with_prefill_data' do - subject(:value_with_prefill_data) { entry.value_with_prefill_data } + describe '#value_with_prefill_data' do + subject(:value_with_prefill_data) { entry.value_with_prefill_data } - it { is_expected.to eq(value: 'value', description: 'description', value_options: %w[value value2]) } + it { is_expected.to eq(value: 'value', options: %w[value value2]) } + end end end end diff --git a/spec/lib/gitlab/ci/config/entry/variables_spec.rb b/spec/lib/gitlab/ci/config/entry/variables_spec.rb index 609e4422d5c..e7dbc78729d 100644 --- a/spec/lib/gitlab/ci/config/entry/variables_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/variables_spec.rb @@ -116,8 +116,8 @@ RSpec.describe Gitlab::Ci::Config::Entry::Variables do it_behaves_like 'invalid config', /variable_1 config must be a string/ end - context 'when metadata has allow_array_value and allowed_value_data' do - let(:metadata) { { allowed_value_data: %i[value description], allow_array_value: true } } + context 'when metadata has the allowed_value_data key' do + let(:metadata) { { allowed_value_data: %i[value description options] } } let(:result) do { 'VARIABLE_1' => 'value' } @@ -143,17 +143,15 @@ RSpec.describe Gitlab::Ci::Config::Entry::Variables do end end - context 'when entry config value has key-value pair and value is an array' do + context 'when entry config value has options' do let(:config) do - { 'VARIABLE_1' => { value: %w[value1 value2], description: 'variable 1' } } + { 'VARIABLE_1' => { + value: 'value1', options: %w[value1 value2], description: 'variable 1' + } } end - context 'when there is no allowed_value_data metadata' do - it_behaves_like 'invalid config', /variable_1 config value must be an alphanumeric string/ - end - - context 'when metadata has allow_array_value and allowed_value_data' do - let(:metadata) { { allowed_value_data: %i[value description], allow_array_value: true } } + context 'when metadata has allowed_value_data' do + let(:metadata) { { allowed_value_data: %i[value description options] } } let(:result) do { 'VARIABLE_1' => 'value1' } @@ -172,7 +170,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Variables do describe '#value_with_prefill_data' do it 'returns variable with prefill data' do expect(entry.value_with_prefill_data).to eq( - 'VARIABLE_1' => { value: 'value1', value_options: %w[value1 value2], description: 'variable 1' } + 'VARIABLE_1' => { value: 'value1', options: %w[value1 value2], description: 'variable 1' } ) end end @@ -234,14 +232,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Variables do it_behaves_like 'invalid config', /variable_1 config uses invalid data keys: hello/ end - context 'when entry config value has hash with nil description' do - let(:config) do - { 'VARIABLE_1' => { value: 'value 1', description: nil } } - end - - it_behaves_like 'invalid config', /variable_1 config description must be an alphanumeric string/ - end - context 'when entry config value has hash without description' do let(:config) do { 'VARIABLE_1' => { value: 'value 1' } } diff --git a/spec/lib/gitlab/ci/config/external/file/remote_spec.rb b/spec/lib/gitlab/ci/config/external/file/remote_spec.rb index c22afb32756..8d93cdcf378 100644 --- a/spec/lib/gitlab/ci/config/external/file/remote_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/remote_spec.rb @@ -188,6 +188,19 @@ RSpec.describe Gitlab::Ci::Config::External::File::Remote do 'is blocked: Requests to localhost are not allowed!' end end + + context 'when connection refused error has been raised' do + let(:location) { 'http://127.0.0.1/some/path/to/config.yaml' } + let(:exception) { Errno::ECONNREFUSED.new } + + before do + stub_full_request(location).to_raise(exception) + end + + it 'returns details about connection failure' do + expect(subject).to eq "Remote file could not be fetched because Connection refused!" + end + end end describe '#expand_context' do diff --git a/spec/lib/gitlab/ci/config/external/mapper/base_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/base_spec.rb new file mode 100644 index 00000000000..0fdcc5e8ff7 --- /dev/null +++ b/spec/lib/gitlab/ci/config/external/mapper/base_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::External::Mapper::Base, feature_category: :pipeline_authoring do + let(:test_class) do + Class.new(described_class) do + def self.name + 'TestClass' + end + end + end + + let(:context) { Gitlab::Ci::Config::External::Context.new } + let(:mapper) { test_class.new(context) } + + describe '#process' do + subject(:process) { mapper.process } + + context 'when the method is not implemented' do + it 'raises NotImplementedError' do + expect { process }.to raise_error(NotImplementedError) + end + end + + context 'when the method is implemented' do + before do + test_class.class_eval do + def process_without_instrumentation + 'test' + end + end + end + + it 'calls the method' do + expect(process).to eq('test') + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/external/mapper/filter_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/filter_spec.rb new file mode 100644 index 00000000000..df2a2f0fd01 --- /dev/null +++ b/spec/lib/gitlab/ci/config/external/mapper/filter_spec.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::External::Mapper::Filter, feature_category: :pipeline_authoring do + let_it_be(:variables) do + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'VARIABLE1', value: 'hello') + end + end + + let_it_be(:context) do + Gitlab::Ci::Config::External::Context.new(variables: variables) + end + + subject(:filter) { described_class.new(context) } + + describe '#process' do + let(:locations) do + [{ local: 'config/.gitlab-ci.yml', rules: [{ if: '$VARIABLE1' }] }, + { remote: 'https://example.com/.gitlab-ci.yml', rules: [{ if: '$VARIABLE2' }] }] + end + + subject(:process) { filter.process(locations) } + + it 'filters locations according to rules' do + is_expected.to eq( + [{ local: 'config/.gitlab-ci.yml', rules: [{ if: '$VARIABLE1' }] }] + ) + end + end +end diff --git a/spec/lib/gitlab/ci/config/external/mapper/location_expander_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/location_expander_spec.rb new file mode 100644 index 00000000000..b14b6b0ca29 --- /dev/null +++ b/spec/lib/gitlab/ci/config/external/mapper/location_expander_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::External::Mapper::LocationExpander, feature_category: :pipeline_authoring do + include RepoHelpers + + let_it_be(:project) { create(:project, :repository) } + let_it_be(:user) { project.owner } + + let(:sha) { project.commit.sha } + + let(:context) do + Gitlab::Ci::Config::External::Context.new(project: project, user: user, sha: sha) + end + + subject(:location_expander) { described_class.new(context) } + + describe '#process' do + subject(:process) { location_expander.process(locations) } + + context 'when there are project files' do + let(:locations) do + [{ project: 'gitlab-org/gitlab-1', file: ['builds.yml', 'tests.yml'] }, + { project: 'gitlab-org/gitlab-2', file: 'deploy.yml' }] + end + + it 'returns expanded locations' do + is_expected.to eq( + [{ project: 'gitlab-org/gitlab-1', file: 'builds.yml' }, + { project: 'gitlab-org/gitlab-1', file: 'tests.yml' }, + { project: 'gitlab-org/gitlab-2', file: 'deploy.yml' }] + ) + end + end + + context 'when there are local files' do + let(:locations) do + [{ local: 'builds/*.yml' }, + { local: 'tests.yml' }] + end + + let(:project_files) do + { 'builds/1.yml' => 'a', 'builds/2.yml' => 'b', 'tests.yml' => 'c' } + end + + around do |example| + create_and_delete_files(project, project_files) do + example.run + end + end + + it 'returns expanded locations' do + is_expected.to eq( + [{ local: 'builds/1.yml' }, + { local: 'builds/2.yml' }, + { local: 'tests.yml' }] + ) + end + end + + context 'when there are other files' do + let(:locations) do + [{ remote: 'https://gitlab.com/gitlab-org/gitlab-ce/raw/master/.gitlab-ci.yml' }] + end + + it 'returns the same location' do + is_expected.to eq(locations) + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb new file mode 100644 index 00000000000..5f321a696c9 --- /dev/null +++ b/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::External::Mapper::Matcher, feature_category: :pipeline_authoring do + let_it_be(:variables) do + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'A_MASKED_VAR', value: 'this-is-secret', masked: true) + end + end + + let_it_be(:context) do + Gitlab::Ci::Config::External::Context.new(variables: variables) + end + + subject(:matcher) { described_class.new(context) } + + describe '#process' do + let(:locations) do + [{ local: 'file.yml' }, + { file: 'file.yml', project: 'namespace/project' }, + { remote: 'https://example.com/.gitlab-ci.yml' }, + { template: 'file.yml' }, + { artifact: 'generated.yml', job: 'test' }] + end + + subject(:process) { matcher.process(locations) } + + it 'returns an array of file objects' do + is_expected.to contain_exactly( + an_instance_of(Gitlab::Ci::Config::External::File::Local), + an_instance_of(Gitlab::Ci::Config::External::File::Project), + an_instance_of(Gitlab::Ci::Config::External::File::Remote), + an_instance_of(Gitlab::Ci::Config::External::File::Template), + an_instance_of(Gitlab::Ci::Config::External::File::Artifact) + ) + end + + context 'when a location is not valid' do + let(:locations) { [{ invalid: 'file.yml' }] } + + it 'raises an error' do + expect { process }.to raise_error( + Gitlab::Ci::Config::External::Mapper::AmbigiousSpecificationError, + '`{"invalid":"file.yml"}` does not have a valid subkey for include. ' \ + 'Valid subkeys are: `local`, `project`, `remote`, `template`, `artifact`' + ) + end + + context 'when the invalid location includes a masked variable' do + let(:locations) { [{ invalid: 'this-is-secret.yml' }] } + + it 'raises an error with a masked sentence' do + expect { process }.to raise_error( + Gitlab::Ci::Config::External::Mapper::AmbigiousSpecificationError, + '`{"invalid":"xxxxxxxxxxxxxx.yml"}` does not have a valid subkey for include. ' \ + 'Valid subkeys are: `local`, `project`, `remote`, `template`, `artifact`' + ) + end + end + end + + context 'when a location is ambiguous' do + let(:locations) { [{ local: 'file.yml', remote: 'https://example.com/.gitlab-ci.yml' }] } + + it 'raises an error' do + expect { process }.to raise_error( + Gitlab::Ci::Config::External::Mapper::AmbigiousSpecificationError, + "Each include must use only one of: `local`, `project`, `remote`, `template`, `artifact`" + ) + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/external/mapper/normalizer_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/normalizer_spec.rb new file mode 100644 index 00000000000..709c234253b --- /dev/null +++ b/spec/lib/gitlab/ci/config/external/mapper/normalizer_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::External::Mapper::Normalizer, feature_category: :pipeline_authoring do + let_it_be(:variables) do + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'VARIABLE1', value: 'config') + variables.append(key: 'VARIABLE2', value: 'https://example.com') + end + end + + let_it_be(:context) do + Gitlab::Ci::Config::External::Context.new(variables: variables) + end + + subject(:normalizer) { described_class.new(context) } + + describe '#process' do + let(:locations) do + ['https://example.com/.gitlab-ci.yml', + 'config/.gitlab-ci.yml', + { local: 'config/.gitlab-ci.yml' }, + { remote: 'https://example.com/.gitlab-ci.yml' }, + { template: 'Template.gitlab-ci.yml' }, + '$VARIABLE1/.gitlab-ci.yml', + '$VARIABLE2/.gitlab-ci.yml'] + end + + subject(:process) { normalizer.process(locations) } + + it 'converts locations to canonical form' do + is_expected.to eq( + [{ remote: 'https://example.com/.gitlab-ci.yml' }, + { local: 'config/.gitlab-ci.yml' }, + { local: 'config/.gitlab-ci.yml' }, + { remote: 'https://example.com/.gitlab-ci.yml' }, + { template: 'Template.gitlab-ci.yml' }, + { local: 'config/.gitlab-ci.yml' }, + { remote: 'https://example.com/.gitlab-ci.yml' }] + ) + end + end +end diff --git a/spec/lib/gitlab/ci/config/external/mapper/variables_expander_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/variables_expander_spec.rb new file mode 100644 index 00000000000..f7454dcd4be --- /dev/null +++ b/spec/lib/gitlab/ci/config/external/mapper/variables_expander_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::External::Mapper::VariablesExpander, feature_category: :pipeline_authoring do + let_it_be(:variables) do + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'VARIABLE1', value: 'hello') + end + end + + let_it_be(:context) do + Gitlab::Ci::Config::External::Context.new(variables: variables) + end + + subject(:variables_expander) { described_class.new(context) } + + describe '#process' do + subject(:process) { variables_expander.process(locations) } + + context 'when locations are strings' do + let(:locations) { ['$VARIABLE1.gitlab-ci.yml'] } + + it 'expands variables' do + is_expected.to eq(['hello.gitlab-ci.yml']) + end + end + + context 'when locations are hashes' do + let(:locations) { [{ local: '$VARIABLE1.gitlab-ci.yml' }] } + + it 'expands variables' do + is_expected.to eq([{ local: 'hello.gitlab-ci.yml' }]) + end + end + + context 'when locations are arrays' do + let(:locations) { [{ local: ['$VARIABLE1.gitlab-ci.yml'] }] } + + it 'expands variables' do + is_expected.to eq([{ local: ['hello.gitlab-ci.yml'] }]) + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb new file mode 100644 index 00000000000..7c7252c6b0e --- /dev/null +++ b/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb @@ -0,0 +1,137 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category: :pipeline_authoring do + include RepoHelpers + include StubRequests + + let_it_be(:project) { create(:project, :repository) } + let_it_be(:user) { project.owner } + + let(:context) do + Gitlab::Ci::Config::External::Context.new(project: project, user: user, sha: project.commit.id) + end + + let(:remote_url) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' } + + let(:project_files) do + { + 'myfolder/file1.yml' => <<~YAML, + my_build: + script: echo Hello World + YAML + 'myfolder/file2.yml' => <<~YAML, + my_test: + script: echo Hello World + YAML + 'nested_configs.yml' => <<~YAML + include: + - local: myfolder/file1.yml + - local: myfolder/file2.yml + - remote: #{remote_url} + YAML + } + end + + around(:all) do |example| + create_and_delete_files(project, project_files) do + example.run + end + end + + before do + stub_full_request(remote_url).to_return( + body: <<~YAML + remote_test: + script: echo Hello World + YAML + ) + end + + subject(:verifier) { described_class.new(context) } + + describe '#process' do + subject(:process) { verifier.process(files) } + + context 'when files are local' do + let(:files) do + [ + Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context), + Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file2.yml' }, context) + ] + end + + it 'returns an array of file objects' do + expect(process.map(&:location)).to contain_exactly('myfolder/file1.yml', 'myfolder/file2.yml') + end + + it 'adds files to the expandset' do + expect { process }.to change { context.expandset.count }.by(2) + end + end + + context 'when a file includes other files' do + let(:files) do + [ + Gitlab::Ci::Config::External::File::Local.new({ local: 'nested_configs.yml' }, context) + ] + end + + it 'returns an array of file objects with combined hash' do + expect(process.map(&:to_hash)).to contain_exactly( + { my_build: { script: 'echo Hello World' }, + my_test: { script: 'echo Hello World' }, + remote_test: { script: 'echo Hello World' } } + ) + end + end + + context 'when there is an invalid file' do + let(:files) do + [ + Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/invalid.yml' }, context) + ] + end + + it 'adds an error to the file' do + expect(process.first.errors).to include("Local file `myfolder/invalid.yml` does not exist!") + end + end + + context 'when max_includes is exceeded' do + context 'when files are nested' do + let(:files) do + [ + Gitlab::Ci::Config::External::File::Local.new({ local: 'nested_configs.yml' }, context) + ] + end + + before do + allow(context).to receive(:max_includes).and_return(1) + end + + it 'raises Processor::IncludeError' do + expect { process }.to raise_error(Gitlab::Ci::Config::External::Processor::IncludeError) + end + end + + context 'when files are not nested' do + let(:files) do + [ + Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context), + Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file2.yml' }, context) + ] + end + + before do + allow(context).to receive(:max_includes).and_return(1) + end + + it 'raises Mapper::TooManyIncludesError' do + expect { process }.to raise_error(Gitlab::Ci::Config::External::Mapper::TooManyIncludesError) + end + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/external/mapper_spec.rb b/spec/lib/gitlab/ci/config/external/mapper_spec.rb index d905568f01e..b7e58d4dfa1 100644 --- a/spec/lib/gitlab/ci/config/external/mapper_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper_spec.rb @@ -2,8 +2,10 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Mapper do +# This will be removed with FF ci_refactoring_external_mapper and moved to below. +RSpec.shared_context 'gitlab_ci_config_external_mapper' do include StubRequests + include RepoHelpers let_it_be(:project) { create(:project, :repository) } let_it_be(:user) { project.owner } @@ -12,13 +14,13 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do let(:remote_url) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' } let(:template_file) { 'Auto-DevOps.gitlab-ci.yml' } let(:variables) { project.predefined_variables } - let(:context_params) { { project: project, sha: '123456', user: user, variables: variables } } + let(:context_params) { { project: project, sha: project.commit.sha, user: user, variables: variables } } let(:context) { Gitlab::Ci::Config::External::Context.new(**context_params) } let(:file_content) do - <<~HEREDOC + <<~YAML image: 'image:1.0' - HEREDOC + YAML end subject(:mapper) { described_class.new(values, context) } @@ -38,7 +40,7 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do it 'propagates the pipeline logger' do process - fetch_content_log_count = mapper + fetch_content_log_count = context .logger .observations_hash .dig(key, 'count') @@ -231,7 +233,7 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do it 'has expanset with one' do process - expect(mapper.expandset.size).to eq(1) + expect(context.expandset.size).to eq(1) end end @@ -379,17 +381,28 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do end context 'when local file path has wildcard' do - let(:project) { create(:project, :repository) } + let_it_be(:project) { create(:project, :repository) } let(:values) do { include: 'myfolder/*.yml' } end - before do - allow_next_instance_of(Repository) do |repository| - allow(repository).to receive(:search_files_by_wildcard_path).with('myfolder/*.yml', '123456') do - ['myfolder/file1.yml', 'myfolder/file2.yml'] - end + let(:project_files) do + { + 'myfolder/file1.yml' => <<~YAML, + my_build: + script: echo Hello World + YAML + 'myfolder/file2.yml' => <<~YAML + my_test: + script: echo Hello World + YAML + } + end + + around do |example| + create_and_delete_files(project, project_files) do + example.run end end @@ -445,8 +458,20 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do it 'has expanset with two' do process - expect(mapper.expandset.size).to eq(2) + expect(context.expandset.size).to eq(2) end end end end + +RSpec.describe Gitlab::Ci::Config::External::Mapper, feature_category: :pipeline_authoring do + it_behaves_like 'gitlab_ci_config_external_mapper' + + context 'when the FF ci_refactoring_external_mapper is disabled' do + before do + stub_feature_flags(ci_refactoring_external_mapper: false) + end + + it_behaves_like 'gitlab_ci_config_external_mapper' + end +end diff --git a/spec/lib/gitlab/ci/config/external/processor_spec.rb b/spec/lib/gitlab/ci/config/external/processor_spec.rb index b1dff6f9723..c9efaf2e1af 100644 --- a/spec/lib/gitlab/ci/config/external/processor_spec.rb +++ b/spec/lib/gitlab/ci/config/external/processor_spec.rb @@ -2,17 +2,31 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Processor do +RSpec.describe Gitlab::Ci::Config::External::Processor, feature_category: :pipeline_authoring do include StubRequests + include RepoHelpers - let_it_be(:project) { create(:project, :repository) } - let_it_be_with_reload(:another_project) { create(:project, :repository) } let_it_be(:user) { create(:user) } - let(:sha) { '12345' } + let_it_be_with_reload(:project) { create(:project, :repository) } + let_it_be_with_reload(:another_project) { create(:project, :repository) } + + let(:project_files) { {} } + let(:other_project_files) { {} } + + let(:sha) { project.commit.sha } let(:context_params) { { project: project, sha: sha, user: user } } let(:context) { Gitlab::Ci::Config::External::Context.new(**context_params) } - let(:processor) { described_class.new(values, context) } + + subject(:processor) { described_class.new(values, context) } + + around do |example| + create_and_delete_files(project, project_files) do + create_and_delete_files(another_project, other_project_files) do + example.run + end + end + end before do project.add_developer(user) @@ -63,7 +77,7 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do let(:remote_file) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' } let(:values) { { include: remote_file, image: 'image:1.0' } } let(:external_file_content) do - <<-HEREDOC + <<-YAML before_script: - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs - ruby -v @@ -77,7 +91,7 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do rubocop: script: - bundle exec rubocop - HEREDOC + YAML end before do @@ -98,7 +112,7 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do let(:remote_file) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' } let(:values) { { include: remote_file, image: 'image:1.0' } } let(:external_file_content) do - <<-HEREDOC + <<-YAML include: - local: another-file.yml rules: @@ -107,7 +121,7 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do rspec: script: - bundle exec rspec - HEREDOC + YAML end before do @@ -127,19 +141,16 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do context 'with a valid local external file is defined' do let(:values) { { include: '/lib/gitlab/ci/templates/template.yml', image: 'image:1.0' } } let(:local_file_content) do - <<-HEREDOC + <<-YAML before_script: - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs - ruby -v - which ruby - bundle install --jobs $(nproc) "${FLAGS[@]}" - HEREDOC + YAML end - before do - allow_any_instance_of(Gitlab::Ci::Config::External::File::Local) - .to receive(:fetch_local_content).and_return(local_file_content) - end + let(:project_files) { { '/lib/gitlab/ci/templates/template.yml' => local_file_content } } it 'appends the file to the values' do output = processor.perform @@ -153,6 +164,11 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do context 'with multiple external files are defined' do let(:remote_file) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' } + + let(:local_file_content) do + File.read(Rails.root.join('spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml')) + end + let(:external_files) do [ '/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml', @@ -168,20 +184,21 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do end let(:remote_file_content) do - <<-HEREDOC + <<-YAML stages: - build - review - cleanup - HEREDOC + YAML end - before do - local_file_content = File.read(Rails.root.join('spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml')) - - allow_any_instance_of(Gitlab::Ci::Config::External::File::Local) - .to receive(:fetch_local_content).and_return(local_file_content) + let(:project_files) do + { + '/spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml' => local_file_content + } + end + before do stub_full_request(remote_file).to_return(body: remote_file_content) end @@ -199,10 +216,7 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do let(:local_file_content) { 'invalid content file ////' } - before do - allow_any_instance_of(Gitlab::Ci::Config::External::File::Local) - .to receive(:fetch_local_content).and_return(local_file_content) - end + let(:project_files) { { '/lib/gitlab/ci/templates/template.yml' => local_file_content } } it 'raises an error' do expect { processor.perform }.to raise_error( @@ -222,9 +236,9 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do end let(:remote_file_content) do - <<~HEREDOC + <<~YAML image: php:5-fpm-alpine - HEREDOC + YAML end it 'takes precedence' do @@ -244,31 +258,32 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do } end - before do - allow(project.repository).to receive(:blob_data_at).with('12345', '/local/file.yml') do - <<~HEREDOC - include: - - template: Ruby.gitlab-ci.yml - - remote: http://my.domain.com/config.yml - - project: #{another_project.full_path} - file: /templates/my-workflow.yml - HEREDOC - end - - allow_any_instance_of(Repository).to receive(:blob_data_at).with(another_project.commit.id, '/templates/my-workflow.yml') do - <<~HEREDOC - include: - - local: /templates/my-build.yml - HEREDOC - end + let(:project_files) do + { + '/local/file.yml' => <<~YAML + include: + - template: Ruby.gitlab-ci.yml + - remote: http://my.domain.com/config.yml + - project: #{another_project.full_path} + file: /templates/my-workflow.yml + YAML + } + end - allow_any_instance_of(Repository).to receive(:blob_data_at).with(another_project.commit.id, '/templates/my-build.yml') do - <<~HEREDOC - my_build: - script: echo Hello World - HEREDOC - end + let(:other_project_files) do + { + '/templates/my-workflow.yml' => <<~YAML, + include: + - local: /templates/my-build.yml + YAML + '/templates/my-build.yml' => <<~YAML + my_build: + script: echo Hello World + YAML + } + end + before do stub_full_request('http://my.domain.com/config.yml') .to_return(body: 'remote_build: { script: echo Hello World }') end @@ -299,32 +314,32 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do expect(context.includes).to contain_exactly( { type: :local, location: '/local/file.yml', - blob: "http://localhost/#{project.full_path}/-/blob/12345/local/file.yml", - raw: "http://localhost/#{project.full_path}/-/raw/12345/local/file.yml", + blob: "http://localhost/#{project.full_path}/-/blob/#{sha}/local/file.yml", + raw: "http://localhost/#{project.full_path}/-/raw/#{sha}/local/file.yml", extra: {}, context_project: project.full_path, - context_sha: '12345' }, + context_sha: sha }, { type: :template, location: 'Ruby.gitlab-ci.yml', blob: nil, raw: 'https://gitlab.com/gitlab-org/gitlab/-/raw/master/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml', extra: {}, context_project: project.full_path, - context_sha: '12345' }, + context_sha: sha }, { type: :remote, location: 'http://my.domain.com/config.yml', blob: nil, raw: "http://my.domain.com/config.yml", extra: {}, context_project: project.full_path, - context_sha: '12345' }, + context_sha: sha }, { type: :file, location: '/templates/my-workflow.yml', blob: "http://localhost/#{another_project.full_path}/-/blob/#{another_project.commit.sha}/templates/my-workflow.yml", raw: "http://localhost/#{another_project.full_path}/-/raw/#{another_project.commit.sha}/templates/my-workflow.yml", extra: { project: another_project.full_path, ref: 'HEAD' }, context_project: project.full_path, - context_sha: '12345' }, + context_sha: sha }, { type: :local, location: '/templates/my-build.yml', blob: "http://localhost/#{another_project.full_path}/-/blob/#{another_project.commit.sha}/templates/my-build.yml", @@ -393,17 +408,17 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do } end + let(:other_project_files) do + { + '/templates/my-build.yml' => <<~YAML + my_build: + script: echo Hello World + YAML + } + end + before do another_project.add_developer(user) - - allow_next_instance_of(Repository) do |repository| - allow(repository).to receive(:blob_data_at).with(another_project.commit.id, '/templates/my-build.yml') do - <<~HEREDOC - my_build: - script: echo Hello World - HEREDOC - end - end end it 'appends the file to the values' do @@ -423,24 +438,21 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do } end + let(:other_project_files) do + { + '/templates/my-build.yml' => <<~YAML, + my_build: + script: echo Hello World + YAML + '/templates/my-test.yml' => <<~YAML + my_test: + script: echo Hello World + YAML + } + end + before do another_project.add_developer(user) - - allow_next_instance_of(Repository) do |repository| - allow(repository).to receive(:blob_data_at).with(another_project.commit.id, '/templates/my-build.yml') do - <<~HEREDOC - my_build: - script: echo Hello World - HEREDOC - end - - allow(repository).to receive(:blob_data_at).with(another_project.commit.id, '/templates/my-test.yml') do - <<~HEREDOC - my_test: - script: echo Hello World - HEREDOC - end - end end it 'appends the file to the values' do @@ -458,45 +470,34 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do raw: "http://localhost/#{another_project.full_path}/-/raw/#{another_project.commit.sha}/templates/my-build.yml", extra: { project: another_project.full_path, ref: 'HEAD' }, context_project: project.full_path, - context_sha: '12345' }, + context_sha: sha }, { type: :file, blob: "http://localhost/#{another_project.full_path}/-/blob/#{another_project.commit.sha}/templates/my-test.yml", raw: "http://localhost/#{another_project.full_path}/-/raw/#{another_project.commit.sha}/templates/my-test.yml", location: '/templates/my-test.yml', extra: { project: another_project.full_path, ref: 'HEAD' }, context_project: project.full_path, - context_sha: '12345' } + context_sha: sha } ) end end context 'when local file path has wildcard' do - let(:project) { create(:project, :repository) } - let(:values) do { include: 'myfolder/*.yml', image: 'image:1.0' } end - before do - allow_next_instance_of(Repository) do |repository| - allow(repository).to receive(:search_files_by_wildcard_path).with('myfolder/*.yml', sha) do - ['myfolder/file1.yml', 'myfolder/file2.yml'] - end - - allow(repository).to receive(:blob_data_at).with(sha, 'myfolder/file1.yml') do - <<~HEREDOC - my_build: - script: echo Hello World - HEREDOC - end - - allow(repository).to receive(:blob_data_at).with(sha, 'myfolder/file2.yml') do - <<~HEREDOC - my_test: - script: echo Hello World - HEREDOC - end - end + let(:project_files) do + { + 'myfolder/file1.yml' => <<~YAML, + my_build: + script: echo Hello World + YAML + 'myfolder/file2.yml' => <<~YAML + my_test: + script: echo Hello World + YAML + } end it 'fetches the matched files' do @@ -510,18 +511,18 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do expect(context.includes).to contain_exactly( { type: :local, location: 'myfolder/file1.yml', - blob: "http://localhost/#{project.full_path}/-/blob/12345/myfolder/file1.yml", - raw: "http://localhost/#{project.full_path}/-/raw/12345/myfolder/file1.yml", + blob: "http://localhost/#{project.full_path}/-/blob/#{sha}/myfolder/file1.yml", + raw: "http://localhost/#{project.full_path}/-/raw/#{sha}/myfolder/file1.yml", extra: {}, context_project: project.full_path, - context_sha: '12345' }, + context_sha: sha }, { type: :local, - blob: "http://localhost/#{project.full_path}/-/blob/12345/myfolder/file2.yml", - raw: "http://localhost/#{project.full_path}/-/raw/12345/myfolder/file2.yml", + blob: "http://localhost/#{project.full_path}/-/blob/#{sha}/myfolder/file2.yml", + raw: "http://localhost/#{project.full_path}/-/raw/#{sha}/myfolder/file2.yml", location: 'myfolder/file2.yml', extra: {}, context_project: project.full_path, - context_sha: '12345' } + context_sha: sha } ) end end diff --git a/spec/lib/gitlab/ci/config_spec.rb b/spec/lib/gitlab/ci/config_spec.rb index c4a6641ff6b..b48a89059bf 100644 --- a/spec/lib/gitlab/ci/config_spec.rb +++ b/spec/lib/gitlab/ci/config_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config do +RSpec.describe Gitlab::Ci::Config, feature_category: :pipeline_authoring do include StubRequests let_it_be(:user) { create(:user) } @@ -305,7 +305,7 @@ RSpec.describe Gitlab::Ci::Config do it 'raises error' do expect { config }.to raise_error( described_class::ConfigError, - /\!reference \["job-2", "before_script"\] is part of a circular chain/ + /!reference \["job-2", "before_script"\] is part of a circular chain/ ) end end @@ -503,7 +503,7 @@ RSpec.describe Gitlab::Ci::Config do expect { config }.to raise_error( described_class::ConfigError, - 'Resolving config took longer than expected' + 'Request timed out when fetching configuration files.' ) end end diff --git a/spec/lib/gitlab/ci/cron_parser_spec.rb b/spec/lib/gitlab/ci/cron_parser_spec.rb index 33474865a93..4b750cf3bcf 100644 --- a/spec/lib/gitlab/ci/cron_parser_spec.rb +++ b/spec/lib/gitlab/ci/cron_parser_spec.rb @@ -358,4 +358,22 @@ RSpec.describe Gitlab::Ci::CronParser do end end end + + describe '#match?' do + let(:run_date) { Time.zone.local(2021, 3, 2, 1, 0) } + + subject(:matched) { described_class.new(cron, Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE).match?(run_date) } + + context 'when cron matches up' do + let(:cron) { '0 1 2 3 *' } + + it { is_expected.to eq(true) } + end + + context 'when cron does not match' do + let(:cron) { '5 4 3 2 1' } + + it { is_expected.to eq(false) } + end + end end diff --git a/spec/lib/gitlab/ci/environment_matcher_spec.rb b/spec/lib/gitlab/ci/environment_matcher_spec.rb new file mode 100644 index 00000000000..172ada1b764 --- /dev/null +++ b/spec/lib/gitlab/ci/environment_matcher_spec.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::EnvironmentMatcher, feature_category: :continuous_integration do + describe '#match?' do + context 'when given pattern is a normal string' do + subject { described_class.new('production') } + + it 'returns true on an exact match' do + expect(subject.match?('production')).to eq true + end + + it 'returns false if not an exact match' do + expect(subject.match?('productiom')).to eq false + end + end + + context 'when given pattern has a wildcard' do + it 'returns true on wildcard matches', :aggregate_failures do + expect(described_class.new('review/*').match?('review/123')).to eq true + expect(described_class.new('review/*/*').match?('review/123/456')).to eq true + expect(described_class.new('*-this-is-a-pattern-*').match?('abc123-this-is-a-pattern-abc123')).to eq true + end + + it 'returns false when not a wildcard match', :aggregate_failures do + expect(described_class.new('review/*').match?('review123')).to eq false + expect(described_class.new('review/*/*').match?('review/123')).to eq false + expect(described_class.new('*-this-is-a-pattern-*').match?('abc123-this-is-a-pattern')).to eq false + end + end + + context 'when given pattern is nil' do + subject { described_class.new(nil) } + + it 'always returns false' do + expect(subject.match?('production')).to eq false + expect(subject.match?('review/123')).to eq false + end + end + + context 'when given pattern is an empty string' do + subject { described_class.new('') } + + it 'always returns false' do + expect(subject.match?('production')).to eq false + expect(subject.match?('review/123')).to eq false + end + end + end +end diff --git a/spec/lib/gitlab/ci/lint_spec.rb b/spec/lib/gitlab/ci/lint_spec.rb index cf07e952f26..b836ca395fa 100644 --- a/spec/lib/gitlab/ci/lint_spec.rb +++ b/spec/lib/gitlab/ci/lint_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Lint do +RSpec.describe Gitlab::Ci::Lint, feature_category: :pipeline_authoring do let_it_be(:project) { create(:project, :repository) } let_it_be(:user) { create(:user) } @@ -337,35 +337,28 @@ RSpec.describe Gitlab::Ci::Lint do end end - context 'pipeline logger' do - let(:counters) do - { - 'count' => a_kind_of(Numeric), - 'avg' => a_kind_of(Numeric), - 'sum' => a_kind_of(Numeric), - 'max' => a_kind_of(Numeric), - 'min' => a_kind_of(Numeric) - } - end - - let(:loggable_data) do + describe 'pipeline logger' do + let(:expected_data) do { 'class' => 'Gitlab::Ci::Pipeline::Logger', - 'config_build_context_duration_s' => counters, - 'config_build_variables_duration_s' => counters, - 'config_compose_duration_s' => counters, - 'config_expand_duration_s' => counters, - 'config_external_process_duration_s' => counters, - 'config_stages_inject_duration_s' => counters, - 'config_tags_resolve_duration_s' => counters, - 'config_yaml_extend_duration_s' => counters, - 'config_yaml_load_duration_s' => counters, + 'config_build_context_duration_s' => a_kind_of(Numeric), + 'config_build_variables_duration_s' => a_kind_of(Numeric), + 'config_root_duration_s' => a_kind_of(Numeric), + 'config_root_compose_duration_s' => a_kind_of(Numeric), + 'config_root_compose_jobs_factory_duration_s' => a_kind_of(Numeric), + 'config_root_compose_jobs_create_duration_s' => a_kind_of(Numeric), + 'config_expand_duration_s' => a_kind_of(Numeric), + 'config_external_process_duration_s' => a_kind_of(Numeric), + 'config_stages_inject_duration_s' => a_kind_of(Numeric), + 'config_tags_resolve_duration_s' => a_kind_of(Numeric), + 'config_yaml_extend_duration_s' => a_kind_of(Numeric), + 'config_yaml_load_duration_s' => a_kind_of(Numeric), 'pipeline_creation_caller' => 'Gitlab::Ci::Lint', 'pipeline_creation_service_duration_s' => a_kind_of(Numeric), 'pipeline_persisted' => false, 'pipeline_source' => 'unknown', 'project_id' => project&.id, - 'yaml_process_duration_s' => counters + 'yaml_process_duration_s' => a_kind_of(Numeric) } end @@ -403,7 +396,7 @@ RSpec.describe Gitlab::Ci::Lint do end it 'creates a log entry' do - expect(Gitlab::AppJsonLogger).to receive(:info).with(loggable_data) + expect(Gitlab::AppJsonLogger).to receive(:info).with(a_hash_including(expected_data)) validate end @@ -424,11 +417,11 @@ RSpec.describe Gitlab::Ci::Lint do let(:project) { nil } let(:project_nil_loggable_data) do - loggable_data.except('project_id') + expected_data.except('project_id') end it 'creates a log entry without project_id' do - expect(Gitlab::AppJsonLogger).to receive(:info).with(project_nil_loggable_data) + expect(Gitlab::AppJsonLogger).to receive(:info).with(a_hash_including(project_nil_loggable_data)) validate end diff --git a/spec/lib/gitlab/ci/parsers/sbom/cyclonedx_properties_spec.rb b/spec/lib/gitlab/ci/parsers/sbom/cyclonedx_properties_spec.rb index f09b85aa2c7..dacbe07c8b3 100644 --- a/spec/lib/gitlab/ci/parsers/sbom/cyclonedx_properties_spec.rb +++ b/spec/lib/gitlab/ci/parsers/sbom/cyclonedx_properties_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::Ci::Parsers::Sbom::CyclonedxProperties do +RSpec.describe Gitlab::Ci::Parsers::Sbom::CyclonedxProperties, feature_category: :dependency_management do subject(:parse_source_from_properties) { described_class.parse_source(properties) } context 'when properties are nil' do diff --git a/spec/lib/gitlab/ci/parsers/sbom/cyclonedx_spec.rb b/spec/lib/gitlab/ci/parsers/sbom/cyclonedx_spec.rb index 0b094880f69..d06537ac330 100644 --- a/spec/lib/gitlab/ci/parsers/sbom/cyclonedx_spec.rb +++ b/spec/lib/gitlab/ci/parsers/sbom/cyclonedx_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Parsers::Sbom::Cyclonedx do +RSpec.describe Gitlab::Ci::Parsers::Sbom::Cyclonedx, feature_category: :dependency_management do let(:report) { instance_double('Gitlab::Ci::Reports::Sbom::Report') } let(:report_data) { base_report_data } let(:raw_report_data) { report_data.to_json } diff --git a/spec/lib/gitlab/ci/parsers/sbom/source/dependency_scanning_spec.rb b/spec/lib/gitlab/ci/parsers/sbom/source/dependency_scanning_spec.rb index e12fa380209..bc97eb2d950 100644 --- a/spec/lib/gitlab/ci/parsers/sbom/source/dependency_scanning_spec.rb +++ b/spec/lib/gitlab/ci/parsers/sbom/source/dependency_scanning_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::Ci::Parsers::Sbom::Source::DependencyScanning do +RSpec.describe Gitlab::Ci::Parsers::Sbom::Source::DependencyScanning, feature_category: :dependency_management do subject { described_class.source(property_data) } context 'when all property data is present' do diff --git a/spec/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator_spec.rb b/spec/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator_spec.rb index f58a463f047..712dc00ec7a 100644 --- a/spec/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator_spec.rb +++ b/spec/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator_spec.rb @@ -2,7 +2,8 @@ require "spec_helper" -RSpec.describe Gitlab::Ci::Parsers::Sbom::Validators::CyclonedxSchemaValidator do +RSpec.describe Gitlab::Ci::Parsers::Sbom::Validators::CyclonedxSchemaValidator, + feature_category: :dependency_management do # Reports should be valid or invalid according to the specification at # https://cyclonedx.org/docs/1.4/json/ diff --git a/spec/lib/gitlab/ci/parsers/security/validators/schema_validator_spec.rb b/spec/lib/gitlab/ci/parsers/security/validators/schema_validator_spec.rb index e730afc72b5..c94ed1f8d6d 100644 --- a/spec/lib/gitlab/ci/parsers/security/validators/schema_validator_spec.rb +++ b/spec/lib/gitlab/ci/parsers/security/validators/schema_validator_spec.rb @@ -95,7 +95,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator do context 'when all files under schema path are explicitly listed' do # We only care about the part that comes before report-format.json # https://rubular.com/r/N8Juz7r8hYDYgD - filename_regex = /(?<report_type>[-\w]*)\-report-format.json/ + filename_regex = /(?<report_type>[-\w]*)-report-format.json/ versions = Dir.glob(File.join(schema_path, "*", File::SEPARATOR)).map { |path| path.split("/").last } diff --git a/spec/lib/gitlab/ci/pipeline/chain/assign_partition_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/assign_partition_spec.rb index 15df5b2f68c..74a68f28f3e 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/assign_partition_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/assign_partition_spec.rb @@ -10,13 +10,15 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::AssignPartition do Gitlab::Ci::Pipeline::Chain::Command.new(project: project, current_user: user) end - let(:pipeline) { build(:ci_pipeline, project: project) } + let(:pipeline) { build(:ci_pipeline, project: project, partition_id: nil) } let(:step) { described_class.new(pipeline, command) } let(:current_partition_id) { 123 } describe '#perform!' do + include Ci::PartitioningHelpers + before do - allow(Ci::Pipeline).to receive(:current_partition_value) { current_partition_id } + stub_current_partition_id(current_partition_id) end subject { step.perform! } diff --git a/spec/lib/gitlab/ci/pipeline/chain/build/associations_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/build/associations_spec.rb index 32c92724f62..b2128f77960 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/build/associations_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/build/associations_spec.rb @@ -2,11 +2,12 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Pipeline::Chain::Build::Associations do +RSpec.describe Gitlab::Ci::Pipeline::Chain::Build::Associations, feature_category: :continuous_integration do let_it_be_with_reload(:project) { create(:project, :repository) } let_it_be(:user) { create(:user, developer_projects: [project]) } - let(:pipeline) { Ci::Pipeline.new } + # Assigning partition_id here to validate it is being propagated correctly + let(:pipeline) { Ci::Pipeline.new(partition_id: ci_testing_partition_id) } let(:bridge) { nil } let(:variables_attributes) do diff --git a/spec/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines_spec.rb index fc3de2a14cd..16deeb6916f 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines_spec.rb @@ -173,21 +173,6 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::CancelPendingPipelines do expect(build_statuses(prev_pipeline)).to contain_exactly('running', 'success', 'created') expect(build_statuses(parent_pipeline)).to contain_exactly('running', 'running') end - - context 'when feature flag ci_skip_auto_cancelation_on_child_pipelines is disabled' do - before do - stub_feature_flags(ci_skip_auto_cancelation_on_child_pipelines: false) - end - - it 'does not cancel the parent pipeline' do - expect(build_statuses(parent_pipeline)).to contain_exactly('running', 'running') - - perform - - expect(build_statuses(prev_pipeline)).to contain_exactly('success', 'canceled', 'canceled') - expect(build_statuses(parent_pipeline)).to contain_exactly('running', 'running') - end - end end context 'when the previous pipeline source is webide' do diff --git a/spec/lib/gitlab/ci/pipeline/chain/command_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/command_spec.rb index 9126c6dab21..68158503628 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/command_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/command_spec.rb @@ -374,21 +374,57 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Command do end end + describe '#observe_creation_duration' do + let(:histogram) { instance_double(Prometheus::Client::Histogram) } + let(:duration) { 1.hour } + let(:command) { described_class.new(project: project) } + + subject(:observe_creation_duration) do + command.observe_creation_duration(duration) + end + + it 'records the duration as histogram' do + expect(::Gitlab::Ci::Pipeline::Metrics).to receive(:pipeline_creation_duration_histogram) + .and_return(histogram) + expect(histogram).to receive(:observe) + .with({ gitlab: 'false' }, duration.seconds) + + observe_creation_duration + end + + context 'when project is gitlab-org/gitlab' do + before do + allow(project).to receive(:full_path).and_return('gitlab-org/gitlab') + end + + it 'tracks the duration with the expected label' do + expect(::Gitlab::Ci::Pipeline::Metrics).to receive(:pipeline_creation_duration_histogram) + .and_return(histogram) + expect(histogram).to receive(:observe) + .with({ gitlab: 'true' }, duration.seconds) + + observe_creation_duration + end + end + end + describe '#observe_step_duration' do + let(:histogram) { instance_double(Prometheus::Client::Histogram) } + let(:duration) { 1.hour } + let(:command) { described_class.new } + + subject(:observe_step_duration) do + command.observe_step_duration(Gitlab::Ci::Pipeline::Chain::Build, duration) + end + context 'when ci_pipeline_creation_step_duration_tracking is enabled' do it 'adds the duration to the step duration histogram' do - histogram = instance_double(Prometheus::Client::Histogram) - duration = 1.hour - expect(::Gitlab::Ci::Pipeline::Metrics).to receive(:pipeline_creation_step_duration_histogram) .and_return(histogram) expect(histogram).to receive(:observe) .with({ step: 'Gitlab::Ci::Pipeline::Chain::Build' }, duration.seconds) - described_class.new.observe_step_duration( - Gitlab::Ci::Pipeline::Chain::Build, - duration - ) + observe_step_duration end end @@ -398,14 +434,9 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Command do end it 'does nothing' do - duration = 1.hour - expect(::Gitlab::Ci::Pipeline::Metrics).not_to receive(:pipeline_creation_step_duration_histogram) - described_class.new.observe_step_duration( - Gitlab::Ci::Pipeline::Chain::Build, - duration - ) + observe_step_duration end end end diff --git a/spec/lib/gitlab/ci/pipeline/chain/ensure_environments_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/ensure_environments_spec.rb index 7fb5b0b4200..39520149032 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/ensure_environments_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/ensure_environments_spec.rb @@ -36,9 +36,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::EnsureEnvironments, :aggregate_failu end context 'and the pipeline is for a merge request' do - let(:command) do - Gitlab::Ci::Pipeline::Chain::Command.new(project: project, current_user: user, merge_request: merge_request) - end + let(:pipeline) { build(:ci_pipeline, project: project, stages: [stage], merge_request: merge_request) } it 'associates the environment with the merge request' do expect { subject }.to change { Environment.count }.by(1) @@ -62,9 +60,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::EnsureEnvironments, :aggregate_failu end context 'and the pipeline is for a merge request' do - let(:command) do - Gitlab::Ci::Pipeline::Chain::Command.new(project: project, current_user: user, merge_request: merge_request) - end + let(:pipeline) { build(:ci_pipeline, project: project, stages: [stage], merge_request: merge_request) } it 'does not associate the environment with the merge request' do expect { subject }.not_to change { Environment.count } diff --git a/spec/lib/gitlab/ci/pipeline/chain/validate/abilities_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/validate/abilities_spec.rb index 7aaeee32f49..9373888aada 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/validate/abilities_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/validate/abilities_spec.rb @@ -2,8 +2,8 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Pipeline::Chain::Validate::Abilities do - let_it_be(:project, reload: true) { create(:project, :repository) } +RSpec.describe Gitlab::Ci::Pipeline::Chain::Validate::Abilities, feature_category: :pipeline_execution do + let(:project) { create(:project, :test_repo) } let_it_be(:user) { create(:user) } let(:pipeline) do diff --git a/spec/lib/gitlab/ci/pipeline/logger_spec.rb b/spec/lib/gitlab/ci/pipeline/logger_spec.rb index 3af0ebe7484..1c285889d1b 100644 --- a/spec/lib/gitlab/ci/pipeline/logger_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/logger_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe ::Gitlab::Ci::Pipeline::Logger do +RSpec.describe ::Gitlab::Ci::Pipeline::Logger, feature_category: :continuous_integration do let_it_be(:project) { build_stubbed(:project) } let_it_be(:pipeline) { build_stubbed(:ci_pipeline, project: project) } @@ -22,61 +22,54 @@ RSpec.describe ::Gitlab::Ci::Pipeline::Logger do end it 'records durations of instrumented operations' do - loggable_data = { + logger.instrument(:expensive_operation) { 123 } + + expected_data = { 'expensive_operation_duration_s' => { 'count' => 1, - 'sum' => a_kind_of(Numeric), - 'avg' => a_kind_of(Numeric), 'max' => a_kind_of(Numeric), - 'min' => a_kind_of(Numeric) + 'sum' => a_kind_of(Numeric) } } - - logger.instrument(:expensive_operation) { 123 } - expect(logger.observations_hash).to match(a_hash_including(loggable_data)) + expect(logger.observations_hash).to match(a_hash_including(expected_data)) end it 'raises an error when block is not provided' do expect { logger.instrument(:expensive_operation) } .to raise_error(ArgumentError, 'block not given') end + + context 'when once: true' do + it 'logs only one observation' do + logger.instrument(:expensive_operation, once: true) { 123 } + logger.instrument(:expensive_operation, once: true) { 123 } + + expected_data = { + 'expensive_operation_duration_s' => a_kind_of(Numeric) + } + expect(logger.observations_hash).to match(a_hash_including(expected_data)) + end + end end - describe '#instrument_with_sql', :request_store do - subject(:instrument_with_sql) do - logger.instrument_with_sql(:expensive_operation, &operation) + describe '#instrument_once_with_sql', :request_store do + subject(:instrument_once_with_sql) do + logger.instrument_once_with_sql(:expensive_operation, &operation) end - def loggable_data(count:, db_count: nil) + def expected_data(count:, db_count: nil) database_name = Ci::ApplicationRecord.connection.pool.db_config.name - keys = %W[ - expensive_operation_duration_s - expensive_operation_db_count - expensive_operation_db_primary_count - expensive_operation_db_primary_duration_s - expensive_operation_db_#{database_name}_count - expensive_operation_db_#{database_name}_duration_s - ] - - data = keys.each.with_object({}) do |key, accumulator| - accumulator[key] = { - 'count' => count, - 'avg' => a_kind_of(Numeric), - 'sum' => a_kind_of(Numeric), - 'max' => a_kind_of(Numeric), - 'min' => a_kind_of(Numeric) - } - end - - if db_count - data['expensive_operation_db_count']['max'] = db_count - data['expensive_operation_db_count']['min'] = db_count - data['expensive_operation_db_count']['avg'] = db_count - data['expensive_operation_db_count']['sum'] = count * db_count - end + total_db_count = count * db_count if db_count - data + { + "expensive_operation_duration_s" => a_kind_of(Numeric), + "expensive_operation_db_count" => total_db_count || a_kind_of(Numeric), + "expensive_operation_db_primary_count" => a_kind_of(Numeric), + "expensive_operation_db_primary_duration_s" => a_kind_of(Numeric), + "expensive_operation_db_#{database_name}_count" => a_kind_of(Numeric), + "expensive_operation_db_#{database_name}_duration_s" => a_kind_of(Numeric) + } end context 'with a single query' do @@ -85,10 +78,10 @@ RSpec.describe ::Gitlab::Ci::Pipeline::Logger do it { is_expected.to eq(operation.call) } it 'includes SQL metrics' do - instrument_with_sql + instrument_once_with_sql expect(logger.observations_hash) - .to match(a_hash_including(loggable_data(count: 1, db_count: 1))) + .to match(a_hash_including(expected_data(count: 1, db_count: 1))) end end @@ -98,21 +91,10 @@ RSpec.describe ::Gitlab::Ci::Pipeline::Logger do it { is_expected.to eq(operation.call) } it 'includes SQL metrics' do - instrument_with_sql - - expect(logger.observations_hash) - .to match(a_hash_including(loggable_data(count: 1, db_count: 2))) - end - end - - context 'with multiple observations' do - let(:operation) { -> { Ci::Build.count + Ci::Bridge.count } } - - it 'includes SQL metrics' do - 2.times { logger.instrument_with_sql(:expensive_operation, &operation) } + instrument_once_with_sql expect(logger.observations_hash) - .to match(a_hash_including(loggable_data(count: 2, db_count: 2))) + .to match(a_hash_including(expected_data(count: 1, db_count: 2))) end end @@ -122,7 +104,7 @@ RSpec.describe ::Gitlab::Ci::Pipeline::Logger do it { is_expected.to eq(operation.call) } it 'does not include SQL metrics' do - instrument_with_sql + instrument_once_with_sql expect(logger.observations_hash.keys) .to match_array(['expensive_operation_duration_s']) @@ -132,14 +114,40 @@ RSpec.describe ::Gitlab::Ci::Pipeline::Logger do describe '#observe' do it 'records durations of observed operations' do - loggable_data = { + expect(logger.observe(:pipeline_creation_duration_s, 30)).to be_truthy + + expected_data = { 'pipeline_creation_duration_s' => { - 'avg' => 30, 'sum' => 30, 'count' => 1, 'max' => 30, 'min' => 30 + 'sum' => 30, 'count' => 1, 'max' => 30 } } + expect(logger.observations_hash).to match(a_hash_including(expected_data)) + end - expect(logger.observe(:pipeline_creation_duration_s, 30)).to be_truthy - expect(logger.observations_hash).to match(a_hash_including(loggable_data)) + context 'when once: true' do + it 'records the latest observation' do + expect(logger.observe(:pipeline_creation_duration_s, 20, once: true)).to be_truthy + expect(logger.observe(:pipeline_creation_duration_s, 30, once: true)).to be_truthy + + expected_data = { + 'pipeline_creation_duration_s' => 30 + } + expect(logger.observations_hash).to match(a_hash_including(expected_data)) + end + + it 'logs data as expected' do + expect(logger.observe(:pipeline_creation_duration_s, 30, once: true)).to be_truthy + expect(logger.observe(:pipeline_operation_x_duration_s, 20)).to be_truthy + expect(logger.observe(:pipeline_operation_x_duration_s, 20)).to be_truthy + + expected_data = { + 'pipeline_creation_duration_s' => 30, + 'pipeline_operation_x_duration_s' => { + 'sum' => 40, 'count' => 2, 'max' => 20 + } + } + expect(logger.observations_hash).to match(a_hash_including(expected_data)) + end end end @@ -158,8 +166,11 @@ RSpec.describe ::Gitlab::Ci::Pipeline::Logger do context 'when the feature flag is enabled' do let(:flag) { true } - let(:loggable_data) do + let(:expected_data) do { + 'correlation_id' => a_kind_of(String), + 'meta.project' => project.full_path, + 'meta.root_namespace' => project.root_namespace.full_path, 'class' => described_class.name.to_s, 'pipeline_id' => pipeline.id, 'pipeline_persisted' => true, @@ -168,10 +179,10 @@ RSpec.describe ::Gitlab::Ci::Pipeline::Logger do 'pipeline_creation_caller' => 'source', 'pipeline_source' => pipeline.source, 'pipeline_save_duration_s' => { - 'avg' => 60, 'sum' => 60, 'count' => 1, 'max' => 60, 'min' => 60 + 'sum' => 60, 'count' => 1, 'max' => 60 }, 'pipeline_creation_duration_s' => { - 'avg' => 20, 'sum' => 40, 'count' => 2, 'max' => 30, 'min' => 10 + 'sum' => 40, 'count' => 2, 'max' => 30 } } end @@ -179,7 +190,7 @@ RSpec.describe ::Gitlab::Ci::Pipeline::Logger do it 'logs to application.json' do expect(Gitlab::AppJsonLogger) .to receive(:info) - .with(a_hash_including(loggable_data)) + .with(a_hash_including(expected_data)) .and_call_original expect(commit).to be_truthy @@ -200,28 +211,43 @@ RSpec.describe ::Gitlab::Ci::Pipeline::Logger do expect(Gitlab::AppJsonLogger) .to receive(:info) - .with(a_hash_including(loggable_data)) + .with(a_hash_including(expected_data)) .and_call_original expect(commit).to be_truthy end + + context 'with unexistent observations in condition' do + it 'does not commit the log' do + logger.log_when do |observations| + value = observations['non_existent_value'] + next false unless value + + value > 0 + end + + expect(Gitlab::AppJsonLogger).not_to receive(:info) + + expect(commit).to be_falsey + end + end end context 'when project is not passed and pipeline is not persisted' do let(:project) {} let(:pipeline) { build(:ci_pipeline) } - let(:loggable_data) do + let(:expected_data) do { 'class' => described_class.name.to_s, 'pipeline_persisted' => false, 'pipeline_creation_service_duration_s' => a_kind_of(Numeric), 'pipeline_creation_caller' => 'source', 'pipeline_save_duration_s' => { - 'avg' => 60, 'sum' => 60, 'count' => 1, 'max' => 60, 'min' => 60 + 'sum' => 60, 'count' => 1, 'max' => 60 }, 'pipeline_creation_duration_s' => { - 'avg' => 20, 'sum' => 40, 'count' => 2, 'max' => 30, 'min' => 10 + 'sum' => 40, 'count' => 2, 'max' => 30 } } end @@ -229,7 +255,7 @@ RSpec.describe ::Gitlab::Ci::Pipeline::Logger do it 'logs to application.json' do expect(Gitlab::AppJsonLogger) .to receive(:info) - .with(a_hash_including(loggable_data)) + .with(a_hash_including(expected_data)) .and_call_original expect(commit).to be_truthy diff --git a/spec/lib/gitlab/ci/pipeline/seed/build/cache_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build/cache_spec.rb index 910c12389c3..fb8020bf43e 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build/cache_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build/cache_spec.rb @@ -6,8 +6,9 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do let_it_be(:project) { create(:project, :repository) } let_it_be(:head_sha) { project.repository.head_commit.id } let_it_be(:pipeline) { create(:ci_pipeline, project: project, sha: head_sha) } + let(:index) { 1 } - let(:processor) { described_class.new(pipeline, config) } + let(:processor) { described_class.new(pipeline, config, index) } describe '#attributes' do subject { processor.attributes } @@ -40,10 +41,12 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do { key: { files: files } } end - it 'uses default key' do - expected = { key: 'default' } + context 'without a prefix' do + it 'uses default key with an index as a prefix' do + expected = { key: '1-default' } - is_expected.to include(expected) + is_expected.to include(expected) + end end end @@ -57,13 +60,15 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do } end - it 'builds a string key' do - expected = { - key: '703ecc8fef1635427a1f86a8a1a308831c122392', - paths: ['vendor/ruby'] - } + context 'without a prefix' do + it 'builds a string key with an index as a prefix' do + expected = { + key: '1-703ecc8fef1635427a1f86a8a1a308831c122392', + paths: ['vendor/ruby'] + } - is_expected.to include(expected) + is_expected.to include(expected) + end end end @@ -107,10 +112,12 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do } end - it 'builds a string key' do - expected = { key: '74bf43fb1090f161bdd4e265802775dbda2f03d1' } + context 'without a prefix' do + it 'builds a string key with an index as a prefix' do + expected = { key: '1-74bf43fb1090f161bdd4e265802775dbda2f03d1' } - is_expected.to include(expected) + is_expected.to include(expected) + end end end diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb index 75f6a773c2d..1f7f800e238 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do +RSpec.describe Gitlab::Ci::Pipeline::Seed::Build, feature_category: :pipeline_authoring do let_it_be_with_reload(:project) { create(:project, :repository) } let_it_be(:head_sha) { project.repository.head_commit.id } @@ -11,861 +11,954 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do let(:seed_context) { Gitlab::Ci::Pipeline::Seed::Context.new(pipeline, root_variables: root_variables) } let(:attributes) { { name: 'rspec', ref: 'master', scheduling_type: :stage, when: 'on_success' } } let(:previous_stages) { [] } - let(:current_stage) { double(seeds_names: [attributes[:name]]) } + let(:current_stage) { instance_double(Gitlab::Ci::Pipeline::Seed::Stage, seeds_names: [attributes[:name]]) } + let(:current_ci_stage) { build(:ci_stage, pipeline: pipeline) } - let(:seed_build) { described_class.new(seed_context, attributes, previous_stages + [current_stage]) } + let(:seed_build) { described_class.new(seed_context, attributes, previous_stages + [current_stage], current_ci_stage) } - describe '#attributes' do - subject { seed_build.attributes } + shared_examples 'build seed' do + describe '#attributes' do + subject { seed_build.attributes } - it { is_expected.to be_a(Hash) } - it { is_expected.to include(:name, :project, :ref) } + it { is_expected.to be_a(Hash) } + it { is_expected.to include(:name, :project, :ref) } - context 'with job:when' do - let(:attributes) { { name: 'rspec', ref: 'master', when: 'on_failure' } } + context 'with job:when' do + let(:attributes) { { name: 'rspec', ref: 'master', when: 'on_failure' } } - it { is_expected.to include(when: 'on_failure') } - end - - context 'with job:when:delayed' do - let(:attributes) { { name: 'rspec', ref: 'master', when: 'delayed', start_in: '3 hours' } } - - it { is_expected.to include(when: 'delayed', start_in: '3 hours') } - end - - context 'with job:rules:[when:]' do - context 'is matched' do - let(:attributes) { { name: 'rspec', ref: 'master', rules: [{ if: '$VAR == null', when: 'always' }] } } - - it { is_expected.to include(when: 'always') } + it { is_expected.to include(when: 'on_failure') } end - context 'is not matched' do - let(:attributes) { { name: 'rspec', ref: 'master', rules: [{ if: '$VAR != null', when: 'always' }] } } - - it { is_expected.to include(when: 'never') } - end - end - - context 'with job:rules:[when:delayed]' do - context 'is matched' do - let(:attributes) { { name: 'rspec', ref: 'master', rules: [{ if: '$VAR == null', when: 'delayed', start_in: '3 hours' }] } } + context 'with job:when:delayed' do + let(:attributes) { { name: 'rspec', ref: 'master', when: 'delayed', options: { start_in: '3 hours' } } } it { is_expected.to include(when: 'delayed', options: { start_in: '3 hours' }) } end - context 'is not matched' do - let(:attributes) { { name: 'rspec', ref: 'master', rules: [{ if: '$VAR != null', when: 'delayed', start_in: '3 hours' }] } } - - it { is_expected.to include(when: 'never') } - end - end - - context 'with job: rules but no explicit when:' do - let(:base_attributes) { { name: 'rspec', ref: 'master' } } - - context 'with a manual job' do - context 'with a matched rule' do - let(:attributes) { base_attributes.merge(when: 'manual', rules: [{ if: '$VAR == null' }]) } + context 'with job:rules:[when:]' do + context 'is matched' do + let(:attributes) { { name: 'rspec', ref: 'master', rules: [{ if: '$VAR == null', when: 'always' }] } } - it { is_expected.to include(when: 'manual') } + it { is_expected.to include(when: 'always') } end context 'is not matched' do - let(:attributes) { base_attributes.merge(when: 'manual', rules: [{ if: '$VAR != null' }]) } + let(:attributes) { { name: 'rspec', ref: 'master', rules: [{ if: '$VAR != null', when: 'always' }] } } it { is_expected.to include(when: 'never') } end end - context 'with an automatic job' do + context 'with job:rules:[when:delayed]' do context 'is matched' do - let(:attributes) { base_attributes.merge(when: 'on_success', rules: [{ if: '$VAR == null' }]) } + let(:attributes) { { name: 'rspec', ref: 'master', rules: [{ if: '$VAR == null', when: 'delayed', start_in: '3 hours' }] } } - it { is_expected.to include(when: 'on_success') } + it { is_expected.to include(when: 'delayed', options: { start_in: '3 hours' }) } end context 'is not matched' do - let(:attributes) { base_attributes.merge(when: 'on_success', rules: [{ if: '$VAR != null' }]) } + let(:attributes) { { name: 'rspec', ref: 'master', rules: [{ if: '$VAR != null', when: 'delayed', start_in: '3 hours' }] } } it { is_expected.to include(when: 'never') } end end - end - context 'with job:rules:[variables:]' do - let(:attributes) do - { name: 'rspec', - ref: 'master', - job_variables: [{ key: 'VAR1', value: 'var 1' }, - { key: 'VAR2', value: 'var 2' }], - rules: [{ if: '$VAR == null', variables: { VAR1: 'new var 1', VAR3: 'var 3' } }] } - end + context 'with job: rules but no explicit when:' do + let(:base_attributes) { { name: 'rspec', ref: 'master' } } - it do - is_expected.to include(yaml_variables: [{ key: 'VAR1', value: 'new var 1' }, - { key: 'VAR3', value: 'var 3' }, - { key: 'VAR2', value: 'var 2' }]) - end - end + context 'with a manual job' do + context 'with a matched rule' do + let(:attributes) { base_attributes.merge(when: 'manual', rules: [{ if: '$VAR == null' }]) } - context 'with job:tags' do - let(:attributes) do - { - name: 'rspec', - ref: 'master', - job_variables: [{ key: 'VARIABLE', value: 'value' }], - tag_list: ['static-tag', '$VARIABLE', '$NO_VARIABLE'] - } - end + it { is_expected.to include(when: 'manual') } + end - it { is_expected.to include(tag_list: ['static-tag', 'value', '$NO_VARIABLE']) } - it { is_expected.to include(yaml_variables: [{ key: 'VARIABLE', value: 'value' }]) } - end + context 'is not matched' do + let(:attributes) { base_attributes.merge(when: 'manual', rules: [{ if: '$VAR != null' }]) } - context 'with cache:key' do - let(:attributes) do - { - name: 'rspec', - ref: 'master', - cache: [{ - key: 'a-value' - }] - } - end + it { is_expected.to include(when: 'never') } + end + end + + context 'with an automatic job' do + context 'is matched' do + let(:attributes) { base_attributes.merge(when: 'on_success', rules: [{ if: '$VAR == null' }]) } + + it { is_expected.to include(when: 'on_success') } + end - it { is_expected.to include(options: { cache: [a_hash_including(key: 'a-value')] }) } + context 'is not matched' do + let(:attributes) { base_attributes.merge(when: 'on_success', rules: [{ if: '$VAR != null' }]) } + + it { is_expected.to include(when: 'never') } + end + end + end - context 'with cache:key:files' do + context 'with job:rules:[variables:]' do let(:attributes) do - { - name: 'rspec', + { name: 'rspec', ref: 'master', - cache: [{ - key: { - files: ['VERSION'] - } - }] - } + job_variables: [{ key: 'VAR1', value: 'var 1' }, + { key: 'VAR2', value: 'var 2' }], + rules: [{ if: '$VAR == null', variables: { VAR1: 'new var 1', VAR3: 'var 3' } }] } end - it 'includes cache options' do - cache_options = { - options: { - cache: [a_hash_including(key: 'f155568ad0933d8358f66b846133614f76dd0ca4')] - } - } + it do + is_expected.to include(yaml_variables: [{ key: 'VAR1', value: 'new var 1' }, + { key: 'VAR3', value: 'var 3' }, + { key: 'VAR2', value: 'var 2' }]) + end - is_expected.to include(cache_options) + it 'expects the same results on to_resource' do + expect(seed_build.to_resource.yaml_variables).to include({ key: 'VAR1', value: 'new var 1' }, + { key: 'VAR3', value: 'var 3' }, + { key: 'VAR2', value: 'var 2' }) end end - context 'with cache:key:prefix' do + context 'with job:tags' do let(:attributes) do { name: 'rspec', ref: 'master', - cache: [{ - key: { - prefix: 'something' - } - }] + job_variables: [{ key: 'VARIABLE', value: 'value' }], + tag_list: ['static-tag', '$VARIABLE', '$NO_VARIABLE'] } end - it { is_expected.to include(options: { cache: [a_hash_including( key: 'something-default' )] }) } + it { is_expected.to include(tag_list: ['static-tag', 'value', '$NO_VARIABLE']) } + it { is_expected.to include(yaml_variables: [{ key: 'VARIABLE', value: 'value' }]) } end - context 'with cache:key:files and prefix' do + context 'with cache:key' do let(:attributes) do { name: 'rspec', ref: 'master', cache: [{ - key: { - files: ['VERSION'], - prefix: 'something' - } + key: 'a-value' }] } end - it 'includes cache options' do - cache_options = { - options: { - cache: [a_hash_including(key: 'something-f155568ad0933d8358f66b846133614f76dd0ca4')] + it { is_expected.to include(options: { cache: [a_hash_including(key: 'a-value')] }) } + + context 'with cache:key:files' do + let(:attributes) do + { + name: 'rspec', + ref: 'master', + cache: [{ + key: { + files: ['VERSION'] + } + }] } - } + end - is_expected.to include(cache_options) - end - end - end + it 'includes cache options' do + cache_options = { + options: { + cache: [a_hash_including(key: '0-f155568ad0933d8358f66b846133614f76dd0ca4')] + } + } - context 'with empty cache' do - let(:attributes) do - { - name: 'rspec', - ref: 'master', - cache: {} - } - end + is_expected.to include(cache_options) + end + end - it { is_expected.to include({}) } - end + context 'with cache:key:prefix' do + let(:attributes) do + { + name: 'rspec', + ref: 'master', + cache: [{ + key: { + prefix: 'something' + } + }] + } + end - context 'with allow_failure' do - let(:options) do - { allow_failure_criteria: { exit_codes: [42] } } - end + it { is_expected.to include(options: { cache: [a_hash_including( key: 'something-default' )] }) } + end - let(:rules) do - [{ if: '$VAR == null', when: 'always' }] - end + context 'with cache:key:files and prefix' do + let(:attributes) do + { + name: 'rspec', + ref: 'master', + cache: [{ + key: { + files: ['VERSION'], + prefix: 'something' + } + }] + } + end - let(:attributes) do - { - name: 'rspec', - ref: 'master', - options: options, - rules: rules - } - end + it 'includes cache options' do + cache_options = { + options: { + cache: [a_hash_including(key: 'something-f155568ad0933d8358f66b846133614f76dd0ca4')] + } + } - context 'when rules does not override allow_failure' do - it { is_expected.to match a_hash_including(options: options) } + is_expected.to include(cache_options) + end + end end - context 'when rules set allow_failure to true' do - let(:rules) do - [{ if: '$VAR == null', when: 'always', allow_failure: true }] + context 'with empty cache' do + let(:attributes) do + { + name: 'rspec', + ref: 'master', + cache: {} + } end - it { is_expected.to match a_hash_including(options: { allow_failure_criteria: nil }) } + it { is_expected.to include({}) } end - context 'when rules set allow_failure to false' do - let(:rules) do - [{ if: '$VAR == null', when: 'always', allow_failure: false }] + context 'with allow_failure' do + let(:options) do + { allow_failure_criteria: { exit_codes: [42] } } end - it { is_expected.to match a_hash_including(options: { allow_failure_criteria: nil }) } - end - end - - context 'with workflow:rules:[variables:]' do - let(:attributes) do - { name: 'rspec', - ref: 'master', - yaml_variables: [{ key: 'VAR2', value: 'var 2' }, - { key: 'VAR3', value: 'var 3' }], - job_variables: [{ key: 'VAR2', value: 'var 2' }, - { key: 'VAR3', value: 'var 3' }], - root_variables_inheritance: root_variables_inheritance } - end + let(:rules) do + [{ if: '$VAR == null', when: 'always' }] + end - context 'when the pipeline has variables' do - let(:root_variables) do - [{ key: 'VAR1', value: 'var overridden pipeline 1' }, - { key: 'VAR2', value: 'var pipeline 2' }, - { key: 'VAR3', value: 'var pipeline 3' }, - { key: 'VAR4', value: 'new var pipeline 4' }] + let(:attributes) do + { + name: 'rspec', + ref: 'master', + options: options, + rules: rules + } end - context 'when root_variables_inheritance is true' do - let(:root_variables_inheritance) { true } + context 'when rules does not override allow_failure' do + it { is_expected.to match a_hash_including(options: options) } + end - it 'returns calculated yaml variables' do - expect(subject[:yaml_variables]).to match_array( - [{ key: 'VAR1', value: 'var overridden pipeline 1' }, - { key: 'VAR2', value: 'var 2' }, - { key: 'VAR3', value: 'var 3' }, - { key: 'VAR4', value: 'new var pipeline 4' }] - ) + context 'when rules set allow_failure to true' do + let(:rules) do + [{ if: '$VAR == null', when: 'always', allow_failure: true }] end - end - context 'when root_variables_inheritance is false' do - let(:root_variables_inheritance) { false } + it { is_expected.to match a_hash_including(options: { allow_failure_criteria: nil }) } - it 'returns job variables' do - expect(subject[:yaml_variables]).to match_array( - [{ key: 'VAR2', value: 'var 2' }, - { key: 'VAR3', value: 'var 3' }] - ) - end - end + context 'when options contain other static values' do + let(:options) do + { image: 'busybox', allow_failure_criteria: { exit_codes: [42] } } + end - context 'when root_variables_inheritance is an array' do - let(:root_variables_inheritance) { %w(VAR1 VAR2 VAR3) } + it { is_expected.to match a_hash_including(options: { image: 'busybox', allow_failure_criteria: nil }) } - it 'returns calculated yaml variables' do - expect(subject[:yaml_variables]).to match_array( - [{ key: 'VAR1', value: 'var overridden pipeline 1' }, - { key: 'VAR2', value: 'var 2' }, - { key: 'VAR3', value: 'var 3' }] - ) + it 'deep merges options when exporting to_resource' do + expect(seed_build.to_resource.options).to match a_hash_including( + image: 'busybox', allow_failure_criteria: nil + ) + end end end - end - context 'when the pipeline has not a variable' do - let(:root_variables_inheritance) { true } + context 'when rules set allow_failure to false' do + let(:rules) do + [{ if: '$VAR == null', when: 'always', allow_failure: false }] + end - it 'returns seed yaml variables' do - expect(subject[:yaml_variables]).to match_array( - [{ key: 'VAR2', value: 'var 2' }, - { key: 'VAR3', value: 'var 3' }]) + it { is_expected.to match a_hash_including(options: { allow_failure_criteria: nil }) } end end - end - context 'when the job rule depends on variables' do - let(:attributes) do - { name: 'rspec', - ref: 'master', - yaml_variables: [{ key: 'VAR1', value: 'var 1' }], - job_variables: [{ key: 'VAR1', value: 'var 1' }], - root_variables_inheritance: root_variables_inheritance, - rules: rules } - end + context 'with workflow:rules:[variables:]' do + let(:attributes) do + { name: 'rspec', + ref: 'master', + yaml_variables: [{ key: 'VAR2', value: 'var 2' }, + { key: 'VAR3', value: 'var 3' }], + job_variables: [{ key: 'VAR2', value: 'var 2' }, + { key: 'VAR3', value: 'var 3' }], + root_variables_inheritance: root_variables_inheritance } + end + + context 'when the pipeline has variables' do + let(:root_variables) do + [{ key: 'VAR1', value: 'var overridden pipeline 1' }, + { key: 'VAR2', value: 'var pipeline 2' }, + { key: 'VAR3', value: 'var pipeline 3' }, + { key: 'VAR4', value: 'new var pipeline 4' }] + end - let(:root_variables_inheritance) { true } + context 'when root_variables_inheritance is true' do + let(:root_variables_inheritance) { true } - context 'when the rules use job variables' do - let(:rules) do - [{ if: '$VAR1 == "var 1"', variables: { VAR1: 'overridden var 1', VAR2: 'new var 2' } }] + it 'returns calculated yaml variables' do + expect(subject[:yaml_variables]).to match_array( + [{ key: 'VAR1', value: 'var overridden pipeline 1' }, + { key: 'VAR2', value: 'var 2' }, + { key: 'VAR3', value: 'var 3' }, + { key: 'VAR4', value: 'new var pipeline 4' }] + ) + end + end + + context 'when root_variables_inheritance is false' do + let(:root_variables_inheritance) { false } + + it 'returns job variables' do + expect(subject[:yaml_variables]).to match_array( + [{ key: 'VAR2', value: 'var 2' }, + { key: 'VAR3', value: 'var 3' }] + ) + end + end + + context 'when root_variables_inheritance is an array' do + let(:root_variables_inheritance) { %w(VAR1 VAR2 VAR3) } + + it 'returns calculated yaml variables' do + expect(subject[:yaml_variables]).to match_array( + [{ key: 'VAR1', value: 'var overridden pipeline 1' }, + { key: 'VAR2', value: 'var 2' }, + { key: 'VAR3', value: 'var 3' }] + ) + end + end end - it 'recalculates the variables' do - expect(subject[:yaml_variables]).to contain_exactly({ key: 'VAR1', value: 'overridden var 1' }, - { key: 'VAR2', value: 'new var 2' }) + context 'when the pipeline has not a variable' do + let(:root_variables_inheritance) { true } + + it 'returns seed yaml variables' do + expect(subject[:yaml_variables]).to match_array( + [{ key: 'VAR2', value: 'var 2' }, + { key: 'VAR3', value: 'var 3' }]) + end end end - context 'when the rules use root variables' do - let(:root_variables) do - [{ key: 'VAR2', value: 'var pipeline 2' }] + context 'when the job rule depends on variables' do + let(:attributes) do + { name: 'rspec', + ref: 'master', + yaml_variables: [{ key: 'VAR1', value: 'var 1' }], + job_variables: [{ key: 'VAR1', value: 'var 1' }], + root_variables_inheritance: root_variables_inheritance, + rules: rules } end - let(:rules) do - [{ if: '$VAR2 == "var pipeline 2"', variables: { VAR1: 'overridden var 1', VAR2: 'overridden var 2' } }] - end + let(:root_variables_inheritance) { true } + + context 'when the rules use job variables' do + let(:rules) do + [{ if: '$VAR1 == "var 1"', variables: { VAR1: 'overridden var 1', VAR2: 'new var 2' } }] + end - it 'recalculates the variables' do - expect(subject[:yaml_variables]).to contain_exactly({ key: 'VAR1', value: 'overridden var 1' }, - { key: 'VAR2', value: 'overridden var 2' }) + it 'recalculates the variables' do + expect(subject[:yaml_variables]).to contain_exactly({ key: 'VAR1', value: 'overridden var 1' }, + { key: 'VAR2', value: 'new var 2' }) + end end - context 'when the root_variables_inheritance is false' do - let(:root_variables_inheritance) { false } + context 'when the rules use root variables' do + let(:root_variables) do + [{ key: 'VAR2', value: 'var pipeline 2' }] + end - it 'does not recalculate the variables' do - expect(subject[:yaml_variables]).to contain_exactly({ key: 'VAR1', value: 'var 1' }) + let(:rules) do + [{ if: '$VAR2 == "var pipeline 2"', variables: { VAR1: 'overridden var 1', VAR2: 'overridden var 2' } }] + end + + it 'recalculates the variables' do + expect(subject[:yaml_variables]).to contain_exactly({ key: 'VAR1', value: 'overridden var 1' }, + { key: 'VAR2', value: 'overridden var 2' }) end - end - end - end - end - describe '#bridge?' do - subject { seed_build.bridge? } + context 'when the root_variables_inheritance is false' do + let(:root_variables_inheritance) { false } - context 'when job is a downstream bridge' do - let(:attributes) do - { name: 'rspec', ref: 'master', options: { trigger: 'my/project' } } + it 'does not recalculate the variables' do + expect(subject[:yaml_variables]).to contain_exactly({ key: 'VAR1', value: 'var 1' }) + end + end + end end + end - it { is_expected.to be_truthy } + describe '#bridge?' do + subject { seed_build.bridge? } - context 'when trigger definition is empty' do + context 'when job is a downstream bridge' do let(:attributes) do - { name: 'rspec', ref: 'master', options: { trigger: '' } } + { name: 'rspec', ref: 'master', options: { trigger: 'my/project' } } end - it { is_expected.to be_falsey } - end - end + it { is_expected.to be_truthy } - context 'when job is an upstream bridge' do - let(:attributes) do - { name: 'rspec', ref: 'master', options: { bridge_needs: { pipeline: 'my/project' } } } - end + context 'when trigger definition is empty' do + let(:attributes) do + { name: 'rspec', ref: 'master', options: { trigger: '' } } + end - it { is_expected.to be_truthy } + it { is_expected.to be_falsey } + end + end - context 'when upstream definition is empty' do + context 'when job is an upstream bridge' do let(:attributes) do - { name: 'rspec', ref: 'master', options: { bridge_needs: { pipeline: '' } } } + { name: 'rspec', ref: 'master', options: { bridge_needs: { pipeline: 'my/project' } } } end - it { is_expected.to be_falsey } - end - end + it { is_expected.to be_truthy } - context 'when job is not a bridge' do - it { is_expected.to be_falsey } - end - end + context 'when upstream definition is empty' do + let(:attributes) do + { name: 'rspec', ref: 'master', options: { bridge_needs: { pipeline: '' } } } + end - describe '#to_resource' do - subject { seed_build.to_resource } + it { is_expected.to be_falsey } + end + end - it 'memoizes a resource object' do - expect(subject.object_id).to eq seed_build.to_resource.object_id + context 'when job is not a bridge' do + it { is_expected.to be_falsey } + end end - it 'can not be persisted without explicit assignment' do - pipeline.save! + describe '#to_resource' do + subject { seed_build.to_resource } - expect(subject).not_to be_persisted - end - end + it 'memoizes a resource object' do + expect(subject.object_id).to eq seed_build.to_resource.object_id + end - describe 'applying job inclusion policies' do - subject { seed_build } + it 'can not be persisted without explicit assignment' do + pipeline.save! - context 'when no branch policy is specified' do - let(:attributes) do - { name: 'rspec' } + expect(subject).not_to be_persisted end - - it { is_expected.to be_included } end - context 'when branch policy does not match' do - context 'when using only' do - let(:attributes) do - { name: 'rspec', only: { refs: ['deploy'] } } - end - - it { is_expected.not_to be_included } - end + describe 'applying job inclusion policies' do + subject { seed_build } - context 'when using except' do + context 'when no branch policy is specified' do let(:attributes) do - { name: 'rspec', except: { refs: ['deploy'] } } + { name: 'rspec' } end it { is_expected.to be_included } end - context 'with both only and except policies' do - let(:attributes) do - { - name: 'rspec', - only: { refs: %w[deploy] }, - except: { refs: %w[deploy] } - } + context 'when branch policy does not match' do + context 'when using only' do + let(:attributes) do + { name: 'rspec', only: { refs: ['deploy'] } } + end + + it { is_expected.not_to be_included } end - it { is_expected.not_to be_included } - end - end + context 'when using except' do + let(:attributes) do + { name: 'rspec', except: { refs: ['deploy'] } } + end - context 'when branch regexp policy does not match' do - context 'when using only' do - let(:attributes) do - { name: 'rspec', only: { refs: %w[/^deploy$/] } } + it { is_expected.to be_included } end - it { is_expected.not_to be_included } - end + context 'with both only and except policies' do + let(:attributes) do + { + name: 'rspec', + only: { refs: %w[deploy] }, + except: { refs: %w[deploy] } + } + end - context 'when using except' do - let(:attributes) do - { name: 'rspec', except: { refs: %w[/^deploy$/] } } + it { is_expected.not_to be_included } end - - it { is_expected.to be_included } end - context 'with both only and except policies' do - let(:attributes) do - { - name: 'rspec', - only: { refs: %w[/^deploy$/] }, - except: { refs: %w[/^deploy$/] } - } + context 'when branch regexp policy does not match' do + context 'when using only' do + let(:attributes) do + { name: 'rspec', only: { refs: %w[/^deploy$/] } } + end + + it { is_expected.not_to be_included } end - it { is_expected.not_to be_included } - end - end + context 'when using except' do + let(:attributes) do + { name: 'rspec', except: { refs: %w[/^deploy$/] } } + end - context 'when branch policy matches' do - context 'when using only' do - let(:attributes) do - { name: 'rspec', only: { refs: %w[deploy master] } } + it { is_expected.to be_included } end - it { is_expected.to be_included } - end + context 'with both only and except policies' do + let(:attributes) do + { + name: 'rspec', + only: { refs: %w[/^deploy$/] }, + except: { refs: %w[/^deploy$/] } + } + end - context 'when using except' do - let(:attributes) do - { name: 'rspec', except: { refs: %w[deploy master] } } + it { is_expected.not_to be_included } end - - it { is_expected.not_to be_included } end - context 'when using both only and except policies' do - let(:attributes) do - { - name: 'rspec', - only: { refs: %w[deploy master] }, - except: { refs: %w[deploy master] } - } + context 'when branch policy matches' do + context 'when using only' do + let(:attributes) do + { name: 'rspec', only: { refs: %w[deploy master] } } + end + + it { is_expected.to be_included } end - it { is_expected.not_to be_included } - end - end + context 'when using except' do + let(:attributes) do + { name: 'rspec', except: { refs: %w[deploy master] } } + end - context 'when keyword policy matches' do - context 'when using only' do - let(:attributes) do - { name: 'rspec', only: { refs: %w[branches] } } + it { is_expected.not_to be_included } end - it { is_expected.to be_included } - end + context 'when using both only and except policies' do + let(:attributes) do + { + name: 'rspec', + only: { refs: %w[deploy master] }, + except: { refs: %w[deploy master] } + } + end - context 'when using except' do - let(:attributes) do - { name: 'rspec', except: { refs: %w[branches] } } + it { is_expected.not_to be_included } end - - it { is_expected.not_to be_included } end - context 'when using both only and except policies' do - let(:attributes) do - { - name: 'rspec', - only: { refs: %w[branches] }, - except: { refs: %w[branches] } - } + context 'when keyword policy matches' do + context 'when using only' do + let(:attributes) do + { name: 'rspec', only: { refs: %w[branches] } } + end + + it { is_expected.to be_included } end - it { is_expected.not_to be_included } - end - end + context 'when using except' do + let(:attributes) do + { name: 'rspec', except: { refs: %w[branches] } } + end - context 'when keyword policy does not match' do - context 'when using only' do - let(:attributes) do - { name: 'rspec', only: { refs: %w[tags] } } + it { is_expected.not_to be_included } end - it { is_expected.not_to be_included } - end + context 'when using both only and except policies' do + let(:attributes) do + { + name: 'rspec', + only: { refs: %w[branches] }, + except: { refs: %w[branches] } + } + end - context 'when using except' do - let(:attributes) do - { name: 'rspec', except: { refs: %w[tags] } } + it { is_expected.not_to be_included } end - - it { is_expected.to be_included } end - context 'when using both only and except policies' do - let(:attributes) do - { - name: 'rspec', - only: { refs: %w[tags] }, - except: { refs: %w[tags] } - } + context 'when keyword policy does not match' do + context 'when using only' do + let(:attributes) do + { name: 'rspec', only: { refs: %w[tags] } } + end + + it { is_expected.not_to be_included } end - it { is_expected.not_to be_included } - end - end + context 'when using except' do + let(:attributes) do + { name: 'rspec', except: { refs: %w[tags] } } + end - context 'with source-keyword policy' do - using RSpec::Parameterized + it { is_expected.to be_included } + end - let(:pipeline) do - build(:ci_empty_pipeline, ref: 'deploy', tag: false, source: source, project: project) - end + context 'when using both only and except policies' do + let(:attributes) do + { + name: 'rspec', + only: { refs: %w[tags] }, + except: { refs: %w[tags] } + } + end - context 'matches' do - where(:keyword, :source) do - [ - %w[pushes push], - %w[web web], - %w[triggers trigger], - %w[schedules schedule], - %w[api api], - %w[external external] - ] + it { is_expected.not_to be_included } end + end - with_them do - context 'using an only policy' do - let(:attributes) do - { name: 'rspec', only: { refs: [keyword] } } - end + context 'with source-keyword policy' do + using RSpec::Parameterized - it { is_expected.to be_included } + let(:pipeline) do + build(:ci_empty_pipeline, ref: 'deploy', tag: false, source: source, project: project) + end + + context 'matches' do + where(:keyword, :source) do + [ + %w[pushes push], + %w[web web], + %w[triggers trigger], + %w[schedules schedule], + %w[api api], + %w[external external] + ] end - context 'using an except policy' do - let(:attributes) do - { name: 'rspec', except: { refs: [keyword] } } + with_them do + context 'using an only policy' do + let(:attributes) do + { name: 'rspec', only: { refs: [keyword] } } + end + + it { is_expected.to be_included } end - it { is_expected.not_to be_included } - end + context 'using an except policy' do + let(:attributes) do + { name: 'rspec', except: { refs: [keyword] } } + end - context 'using both only and except policies' do - let(:attributes) do - { - name: 'rspec', - only: { refs: [keyword] }, - except: { refs: [keyword] } - } + it { is_expected.not_to be_included } end - it { is_expected.not_to be_included } + context 'using both only and except policies' do + let(:attributes) do + { + name: 'rspec', + only: { refs: [keyword] }, + except: { refs: [keyword] } + } + end + + it { is_expected.not_to be_included } + end end end - end - context 'non-matches' do - where(:keyword, :source) do - %w[web trigger schedule api external].map { |source| ['pushes', source] } + - %w[push trigger schedule api external].map { |source| ['web', source] } + - %w[push web schedule api external].map { |source| ['triggers', source] } + - %w[push web trigger api external].map { |source| ['schedules', source] } + - %w[push web trigger schedule external].map { |source| ['api', source] } + - %w[push web trigger schedule api].map { |source| ['external', source] } - end + context 'non-matches' do + where(:keyword, :source) do + %w[web trigger schedule api external].map { |source| ['pushes', source] } + + %w[push trigger schedule api external].map { |source| ['web', source] } + + %w[push web schedule api external].map { |source| ['triggers', source] } + + %w[push web trigger api external].map { |source| ['schedules', source] } + + %w[push web trigger schedule external].map { |source| ['api', source] } + + %w[push web trigger schedule api].map { |source| ['external', source] } + end - with_them do - context 'using an only policy' do - let(:attributes) do - { name: 'rspec', only: { refs: [keyword] } } + with_them do + context 'using an only policy' do + let(:attributes) do + { name: 'rspec', only: { refs: [keyword] } } + end + + it { is_expected.not_to be_included } end - it { is_expected.not_to be_included } - end + context 'using an except policy' do + let(:attributes) do + { name: 'rspec', except: { refs: [keyword] } } + end - context 'using an except policy' do - let(:attributes) do - { name: 'rspec', except: { refs: [keyword] } } + it { is_expected.to be_included } end - it { is_expected.to be_included } - end + context 'using both only and except policies' do + let(:attributes) do + { + name: 'rspec', + only: { refs: [keyword] }, + except: { refs: [keyword] } + } + end - context 'using both only and except policies' do - let(:attributes) do - { - name: 'rspec', - only: { refs: [keyword] }, - except: { refs: [keyword] } - } + it { is_expected.not_to be_included } end - - it { is_expected.not_to be_included } end end end - end - context 'when repository path matches' do - context 'when using only' do - let(:attributes) do - { name: 'rspec', only: { refs: ["branches@#{pipeline.project_full_path}"] } } + context 'when repository path matches' do + context 'when using only' do + let(:attributes) do + { name: 'rspec', only: { refs: ["branches@#{pipeline.project_full_path}"] } } + end + + it { is_expected.to be_included } end - it { is_expected.to be_included } - end + context 'when using except' do + let(:attributes) do + { name: 'rspec', except: { refs: ["branches@#{pipeline.project_full_path}"] } } + end - context 'when using except' do - let(:attributes) do - { name: 'rspec', except: { refs: ["branches@#{pipeline.project_full_path}"] } } + it { is_expected.not_to be_included } end - it { is_expected.not_to be_included } - end + context 'when using both only and except policies' do + let(:attributes) do + { + name: 'rspec', + only: { refs: ["branches@#{pipeline.project_full_path}"] }, + except: { refs: ["branches@#{pipeline.project_full_path}"] } + } + end - context 'when using both only and except policies' do - let(:attributes) do - { - name: 'rspec', - only: { refs: ["branches@#{pipeline.project_full_path}"] }, - except: { refs: ["branches@#{pipeline.project_full_path}"] } - } + it { is_expected.not_to be_included } end - it { is_expected.not_to be_included } - end - - context 'when using both only and except policies' do - let(:attributes) do - { - name: 'rspec', - only: { - refs: ["branches@#{pipeline.project_full_path}"] - }, - except: { - refs: ["branches@#{pipeline.project_full_path}"] + context 'when using both only and except policies' do + let(:attributes) do + { + name: 'rspec', + only: { + refs: ["branches@#{pipeline.project_full_path}"] + }, + except: { + refs: ["branches@#{pipeline.project_full_path}"] + } } - } - end + end - it { is_expected.not_to be_included } + it { is_expected.not_to be_included } + end end - end - context 'when repository path does not match' do - context 'when using only' do - let(:attributes) do - { name: 'rspec', only: { refs: %w[branches@fork] } } + context 'when repository path does not match' do + context 'when using only' do + let(:attributes) do + { name: 'rspec', only: { refs: %w[branches@fork] } } + end + + it { is_expected.not_to be_included } end - it { is_expected.not_to be_included } - end + context 'when using except' do + let(:attributes) do + { name: 'rspec', except: { refs: %w[branches@fork] } } + end - context 'when using except' do - let(:attributes) do - { name: 'rspec', except: { refs: %w[branches@fork] } } + it { is_expected.to be_included } end - it { is_expected.to be_included } - end + context 'when using both only and except policies' do + let(:attributes) do + { + name: 'rspec', + only: { refs: %w[branches@fork] }, + except: { refs: %w[branches@fork] } + } + end - context 'when using both only and except policies' do - let(:attributes) do - { - name: 'rspec', - only: { refs: %w[branches@fork] }, - except: { refs: %w[branches@fork] } - } + it { is_expected.not_to be_included } end - - it { is_expected.not_to be_included } end - end - context 'using rules:' do - using RSpec::Parameterized + context 'using rules:' do + using RSpec::Parameterized - let(:attributes) { { name: 'rspec', rules: rule_set, when: 'on_success' } } + let(:attributes) { { name: 'rspec', rules: rule_set, when: 'on_success' } } - context 'with a matching if: rule' do - context 'with an explicit `when: never`' do - where(:rule_set) do - [ - [[{ if: '$VARIABLE == null', when: 'never' }]], - [[{ if: '$VARIABLE == null', when: 'never' }, { if: '$VARIABLE == null', when: 'always' }]], - [[{ if: '$VARIABLE != "the wrong value"', when: 'never' }, { if: '$VARIABLE == null', when: 'always' }]] - ] - end + context 'with a matching if: rule' do + context 'with an explicit `when: never`' do + where(:rule_set) do + [ + [[{ if: '$VARIABLE == null', when: 'never' }]], + [[{ if: '$VARIABLE == null', when: 'never' }, { if: '$VARIABLE == null', when: 'always' }]], + [[{ if: '$VARIABLE != "the wrong value"', when: 'never' }, { if: '$VARIABLE == null', when: 'always' }]] + ] + end - with_them do - it { is_expected.not_to be_included } + with_them do + it { is_expected.not_to be_included } - it 'still correctly populates when:' do - expect(seed_build.attributes).to include(when: 'never') + it 'still correctly populates when:' do + expect(seed_build.attributes).to include(when: 'never') + end end end - end - context 'with an explicit `when: always`' do - where(:rule_set) do - [ - [[{ if: '$VARIABLE == null', when: 'always' }]], - [[{ if: '$VARIABLE == null', when: 'always' }, { if: '$VARIABLE == null', when: 'never' }]], - [[{ if: '$VARIABLE != "the wrong value"', when: 'always' }, { if: '$VARIABLE == null', when: 'never' }]] - ] + context 'with an explicit `when: always`' do + where(:rule_set) do + [ + [[{ if: '$VARIABLE == null', when: 'always' }]], + [[{ if: '$VARIABLE == null', when: 'always' }, { if: '$VARIABLE == null', when: 'never' }]], + [[{ if: '$VARIABLE != "the wrong value"', when: 'always' }, { if: '$VARIABLE == null', when: 'never' }]] + ] + end + + with_them do + it { is_expected.to be_included } + + it 'correctly populates when:' do + expect(seed_build.attributes).to include(when: 'always') + end + end end - with_them do - it { is_expected.to be_included } + context 'with an explicit `when: on_failure`' do + where(:rule_set) do + [ + [[{ if: '$CI_JOB_NAME == "rspec" && $VAR == null', when: 'on_failure' }]], + [[{ if: '$VARIABLE != null', when: 'delayed', start_in: '1 day' }, { if: '$CI_JOB_NAME == "rspec"', when: 'on_failure' }]], + [[{ if: '$VARIABLE == "the wrong value"', when: 'delayed', start_in: '1 day' }, { if: '$CI_BUILD_NAME == "rspec"', when: 'on_failure' }]] + ] + end - it 'correctly populates when:' do - expect(seed_build.attributes).to include(when: 'always') + with_them do + it { is_expected.to be_included } + + it 'correctly populates when:' do + expect(seed_build.attributes).to include(when: 'on_failure') + end end end - end - context 'with an explicit `when: on_failure`' do - where(:rule_set) do - [ - [[{ if: '$CI_JOB_NAME == "rspec" && $VAR == null', when: 'on_failure' }]], - [[{ if: '$VARIABLE != null', when: 'delayed', start_in: '1 day' }, { if: '$CI_JOB_NAME == "rspec"', when: 'on_failure' }]], - [[{ if: '$VARIABLE == "the wrong value"', when: 'delayed', start_in: '1 day' }, { if: '$CI_BUILD_NAME == "rspec"', when: 'on_failure' }]] - ] + context 'with an explicit `when: delayed`' do + where(:rule_set) do + [ + [[{ if: '$VARIABLE == null', when: 'delayed', start_in: '1 day' }]], + [[{ if: '$VARIABLE == null', when: 'delayed', start_in: '1 day' }, { if: '$VARIABLE == null', when: 'never' }]], + [[{ if: '$VARIABLE != "the wrong value"', when: 'delayed', start_in: '1 day' }, { if: '$VARIABLE == null', when: 'never' }]] + ] + end + + with_them do + it { is_expected.to be_included } + + it 'correctly populates when:' do + expect(seed_build.attributes).to include(when: 'delayed', options: { start_in: '1 day' }) + end + end end - with_them do - it { is_expected.to be_included } + context 'without an explicit when: value' do + where(:rule_set) do + [ + [[{ if: '$VARIABLE == null' }]], + [[{ if: '$VARIABLE == null' }, { if: '$VARIABLE == null' }]], + [[{ if: '$VARIABLE != "the wrong value"' }, { if: '$VARIABLE == null' }]] + ] + end - it 'correctly populates when:' do - expect(seed_build.attributes).to include(when: 'on_failure') + with_them do + it { is_expected.to be_included } + + it 'correctly populates when:' do + expect(seed_build.attributes).to include(when: 'on_success') + end end end end - context 'with an explicit `when: delayed`' do - where(:rule_set) do - [ - [[{ if: '$VARIABLE == null', when: 'delayed', start_in: '1 day' }]], - [[{ if: '$VARIABLE == null', when: 'delayed', start_in: '1 day' }, { if: '$VARIABLE == null', when: 'never' }]], - [[{ if: '$VARIABLE != "the wrong value"', when: 'delayed', start_in: '1 day' }, { if: '$VARIABLE == null', when: 'never' }]] - ] + context 'with a matching changes: rule' do + let(:pipeline) do + build(:ci_pipeline, project: project).tap do |pipeline| + stub_pipeline_modified_paths(pipeline, %w[app/models/ci/pipeline.rb spec/models/ci/pipeline_spec.rb .gitlab-ci.yml]) + end end - with_them do - it { is_expected.to be_included } + context 'with an explicit `when: never`' do + where(:rule_set) do + [ + [[{ changes: { paths: %w[*/**/*.rb] }, when: 'never' }, { changes: { paths: %w[*/**/*.rb] }, when: 'always' }]], + [[{ changes: { paths: %w[app/models/ci/pipeline.rb] }, when: 'never' }, { changes: { paths: %w[app/models/ci/pipeline.rb] }, when: 'always' }]], + [[{ changes: { paths: %w[spec/**/*.rb] }, when: 'never' }, { changes: { paths: %w[spec/**/*.rb] }, when: 'always' }]], + [[{ changes: { paths: %w[*.yml] }, when: 'never' }, { changes: { paths: %w[*.yml] }, when: 'always' }]], + [[{ changes: { paths: %w[.*.yml] }, when: 'never' }, { changes: { paths: %w[.*.yml] }, when: 'always' }]], + [[{ changes: { paths: %w[**/*] }, when: 'never' }, { changes: { paths: %w[**/*] }, when: 'always' }]], + [[{ changes: { paths: %w[*/**/*.rb *.yml] }, when: 'never' }, { changes: { paths: %w[*/**/*.rb *.yml] }, when: 'always' }]], + [[{ changes: { paths: %w[.*.yml **/*] }, when: 'never' }, { changes: { paths: %w[.*.yml **/*] }, when: 'always' }]] + ] + end - it 'correctly populates when:' do - expect(seed_build.attributes).to include(when: 'delayed', options: { start_in: '1 day' }) + with_them do + it { is_expected.not_to be_included } + + it 'correctly populates when:' do + expect(seed_build.attributes).to include(when: 'never') + end end end - end - context 'without an explicit when: value' do - where(:rule_set) do - [ - [[{ if: '$VARIABLE == null' }]], - [[{ if: '$VARIABLE == null' }, { if: '$VARIABLE == null' }]], - [[{ if: '$VARIABLE != "the wrong value"' }, { if: '$VARIABLE == null' }]] - ] - end + context 'with an explicit `when: always`' do + where(:rule_set) do + [ + [[{ changes: { paths: %w[*/**/*.rb] }, when: 'always' }, { changes: { paths: %w[*/**/*.rb] }, when: 'never' }]], + [[{ changes: { paths: %w[app/models/ci/pipeline.rb] }, when: 'always' }, { changes: { paths: %w[app/models/ci/pipeline.rb] }, when: 'never' }]], + [[{ changes: { paths: %w[spec/**/*.rb] }, when: 'always' }, { changes: { paths: %w[spec/**/*.rb] }, when: 'never' }]], + [[{ changes: { paths: %w[*.yml] }, when: 'always' }, { changes: { paths: %w[*.yml] }, when: 'never' }]], + [[{ changes: { paths: %w[.*.yml] }, when: 'always' }, { changes: { paths: %w[.*.yml] }, when: 'never' }]], + [[{ changes: { paths: %w[**/*] }, when: 'always' }, { changes: { paths: %w[**/*] }, when: 'never' }]], + [[{ changes: { paths: %w[*/**/*.rb *.yml] }, when: 'always' }, { changes: { paths: %w[*/**/*.rb *.yml] }, when: 'never' }]], + [[{ changes: { paths: %w[.*.yml **/*] }, when: 'always' }, { changes: { paths: %w[.*.yml **/*] }, when: 'never' }]] + ] + end - with_them do - it { is_expected.to be_included } + with_them do + it { is_expected.to be_included } - it 'correctly populates when:' do - expect(seed_build.attributes).to include(when: 'on_success') + it 'correctly populates when:' do + expect(seed_build.attributes).to include(when: 'always') + end end end - end - end - context 'with a matching changes: rule' do - let(:pipeline) do - build(:ci_pipeline, project: project).tap do |pipeline| - stub_pipeline_modified_paths(pipeline, %w[app/models/ci/pipeline.rb spec/models/ci/pipeline_spec.rb .gitlab-ci.yml]) + context 'without an explicit when: value' do + where(:rule_set) do + [ + [[{ changes: { paths: %w[*/**/*.rb] } }]], + [[{ changes: { paths: %w[app/models/ci/pipeline.rb] } }]], + [[{ changes: { paths: %w[spec/**/*.rb] } }]], + [[{ changes: { paths: %w[*.yml] } }]], + [[{ changes: { paths: %w[.*.yml] } }]], + [[{ changes: { paths: %w[**/*] } }]], + [[{ changes: { paths: %w[*/**/*.rb *.yml] } }]], + [[{ changes: { paths: %w[.*.yml **/*] } }]] + ] + end + + with_them do + it { is_expected.to be_included } + + it 'correctly populates when:' do + expect(seed_build.attributes).to include(when: 'on_success') + end + end end end - context 'with an explicit `when: never`' do + context 'with no matching rule' do where(:rule_set) do [ - [[{ changes: { paths: %w[*/**/*.rb] }, when: 'never' }, { changes: { paths: %w[*/**/*.rb] }, when: 'always' }]], - [[{ changes: { paths: %w[app/models/ci/pipeline.rb] }, when: 'never' }, { changes: { paths: %w[app/models/ci/pipeline.rb] }, when: 'always' }]], - [[{ changes: { paths: %w[spec/**/*.rb] }, when: 'never' }, { changes: { paths: %w[spec/**/*.rb] }, when: 'always' }]], - [[{ changes: { paths: %w[*.yml] }, when: 'never' }, { changes: { paths: %w[*.yml] }, when: 'always' }]], - [[{ changes: { paths: %w[.*.yml] }, when: 'never' }, { changes: { paths: %w[.*.yml] }, when: 'always' }]], - [[{ changes: { paths: %w[**/*] }, when: 'never' }, { changes: { paths: %w[**/*] }, when: 'always' }]], - [[{ changes: { paths: %w[*/**/*.rb *.yml] }, when: 'never' }, { changes: { paths: %w[*/**/*.rb *.yml] }, when: 'always' }]], - [[{ changes: { paths: %w[.*.yml **/*] }, when: 'never' }, { changes: { paths: %w[.*.yml **/*] }, when: 'always' }]] + [[{ if: '$VARIABLE != null', when: 'never' }]], + [[{ if: '$VARIABLE != null', when: 'never' }, { if: '$VARIABLE != null', when: 'always' }]], + [[{ if: '$VARIABLE == "the wrong value"', when: 'never' }, { if: '$VARIABLE != null', when: 'always' }]], + [[{ if: '$VARIABLE != null', when: 'always' }]], + [[{ if: '$VARIABLE != null', when: 'always' }, { if: '$VARIABLE != null', when: 'never' }]], + [[{ if: '$VARIABLE == "the wrong value"', when: 'always' }, { if: '$VARIABLE != null', when: 'never' }]], + [[{ if: '$VARIABLE != null' }]], + [[{ if: '$VARIABLE != null' }, { if: '$VARIABLE != null' }]], + [[{ if: '$VARIABLE == "the wrong value"' }, { if: '$VARIABLE != null' }]] ] end @@ -878,257 +971,249 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do end end - context 'with an explicit `when: always`' do - where(:rule_set) do - [ - [[{ changes: { paths: %w[*/**/*.rb] }, when: 'always' }, { changes: { paths: %w[*/**/*.rb] }, when: 'never' }]], - [[{ changes: { paths: %w[app/models/ci/pipeline.rb] }, when: 'always' }, { changes: { paths: %w[app/models/ci/pipeline.rb] }, when: 'never' }]], - [[{ changes: { paths: %w[spec/**/*.rb] }, when: 'always' }, { changes: { paths: %w[spec/**/*.rb] }, when: 'never' }]], - [[{ changes: { paths: %w[*.yml] }, when: 'always' }, { changes: { paths: %w[*.yml] }, when: 'never' }]], - [[{ changes: { paths: %w[.*.yml] }, when: 'always' }, { changes: { paths: %w[.*.yml] }, when: 'never' }]], - [[{ changes: { paths: %w[**/*] }, when: 'always' }, { changes: { paths: %w[**/*] }, when: 'never' }]], - [[{ changes: { paths: %w[*/**/*.rb *.yml] }, when: 'always' }, { changes: { paths: %w[*/**/*.rb *.yml] }, when: 'never' }]], - [[{ changes: { paths: %w[.*.yml **/*] }, when: 'always' }, { changes: { paths: %w[.*.yml **/*] }, when: 'never' }]] - ] + context 'with a rule using CI_ENVIRONMENT_NAME variable' do + let(:rule_set) do + [{ if: '$CI_ENVIRONMENT_NAME == "test"' }] end - with_them do + context 'when environment:name satisfies the rule' do + let(:attributes) { { name: 'rspec', rules: rule_set, environment: 'test', when: 'on_success' } } + it { is_expected.to be_included } it 'correctly populates when:' do - expect(seed_build.attributes).to include(when: 'always') + expect(seed_build.attributes).to include(when: 'on_success') end end - end - context 'without an explicit when: value' do - where(:rule_set) do - [ - [[{ changes: { paths: %w[*/**/*.rb] } }]], - [[{ changes: { paths: %w[app/models/ci/pipeline.rb] } }]], - [[{ changes: { paths: %w[spec/**/*.rb] } }]], - [[{ changes: { paths: %w[*.yml] } }]], - [[{ changes: { paths: %w[.*.yml] } }]], - [[{ changes: { paths: %w[**/*] } }]], - [[{ changes: { paths: %w[*/**/*.rb *.yml] } }]], - [[{ changes: { paths: %w[.*.yml **/*] } }]] - ] + context 'when environment:name does not satisfy rule' do + let(:attributes) { { name: 'rspec', rules: rule_set, environment: 'dev', when: 'on_success' } } + + it { is_expected.not_to be_included } + + it 'correctly populates when:' do + expect(seed_build.attributes).to include(when: 'never') + end end - with_them do - it { is_expected.to be_included } + context 'when environment:name is not set' do + it { is_expected.not_to be_included } it 'correctly populates when:' do - expect(seed_build.attributes).to include(when: 'on_success') + expect(seed_build.attributes).to include(when: 'never') end end end - end - context 'with no matching rule' do - where(:rule_set) do - [ - [[{ if: '$VARIABLE != null', when: 'never' }]], - [[{ if: '$VARIABLE != null', when: 'never' }, { if: '$VARIABLE != null', when: 'always' }]], - [[{ if: '$VARIABLE == "the wrong value"', when: 'never' }, { if: '$VARIABLE != null', when: 'always' }]], - [[{ if: '$VARIABLE != null', when: 'always' }]], - [[{ if: '$VARIABLE != null', when: 'always' }, { if: '$VARIABLE != null', when: 'never' }]], - [[{ if: '$VARIABLE == "the wrong value"', when: 'always' }, { if: '$VARIABLE != null', when: 'never' }]], - [[{ if: '$VARIABLE != null' }]], - [[{ if: '$VARIABLE != null' }, { if: '$VARIABLE != null' }]], - [[{ if: '$VARIABLE == "the wrong value"' }, { if: '$VARIABLE != null' }]] - ] + context 'with no rules' do + let(:rule_set) { [] } + + it { is_expected.not_to be_included } + + it 'correctly populates when:' do + expect(seed_build.attributes).to include(when: 'never') + end end - with_them do + context 'with invalid rules raising error' do + let(:rule_set) do + [ + { changes: { paths: ['README.md'], compare_to: 'invalid-ref' }, when: 'never' } + ] + end + it { is_expected.not_to be_included } it 'correctly populates when:' do expect(seed_build.attributes).to include(when: 'never') end + + it 'returns an error' do + expect(seed_build.errors).to contain_exactly( + 'Failed to parse rule for rspec: rules:changes:compare_to is not a valid ref' + ) + end end end + end - context 'with no rules' do - let(:rule_set) { [] } + describe 'applying needs: dependency' do + subject { seed_build } - it { is_expected.not_to be_included } + let(:needs_count) { 1 } - it 'correctly populates when:' do - expect(seed_build.attributes).to include(when: 'never') - end + let(:needs_attributes) do + Array.new(needs_count, name: 'build') end - context 'with invalid rules raising error' do - let(:rule_set) do - [ - { changes: { paths: ['README.md'], compare_to: 'invalid-ref' }, when: 'never' } - ] - end + let(:attributes) do + { + name: 'rspec', + needs_attributes: needs_attributes + } + end - it { is_expected.not_to be_included } + context 'when build job is not present in prior stages' do + it "is included" do + is_expected.to be_included + end - it 'correctly populates when:' do - expect(seed_build.attributes).to include(when: 'never') + it "returns an error" do + expect(subject.errors).to contain_exactly( + "'rspec' job needs 'build' job, but 'build' is not in any previous stage") end - it 'returns an error' do - expect(seed_build.errors).to contain_exactly( - 'Failed to parse rule for rspec: rules:changes:compare_to is not a valid ref' - ) + context 'when the needed job is optional' do + let(:needs_attributes) { [{ name: 'build', optional: true }] } + + it "does not return an error" do + expect(subject.errors).to be_empty + end end end - end - end - describe 'applying needs: dependency' do - subject { seed_build } + context 'when build job is part of prior stages' do + let(:stage_attributes) do + { + name: 'build', + index: 0, + builds: [{ name: 'build' }] + } + end - let(:needs_count) { 1 } + let(:stage_seed) do + Gitlab::Ci::Pipeline::Seed::Stage.new(seed_context, stage_attributes, []) + end - let(:needs_attributes) do - Array.new(needs_count, name: 'build') - end + let(:previous_stages) { [stage_seed] } - let(:attributes) do - { - name: 'rspec', - needs_attributes: needs_attributes - } - end + it "is included" do + is_expected.to be_included + end - context 'when build job is not present in prior stages' do - it "is included" do - is_expected.to be_included + it "does not have errors" do + expect(subject.errors).to be_empty + end end - it "returns an error" do - expect(subject.errors).to contain_exactly( - "'rspec' job needs 'build' job, but 'build' is not in any previous stage") - end + context 'when build job is part of the same stage' do + let(:current_stage) { double(seeds_names: [attributes[:name], 'build']) } - context 'when the needed job is optional' do - let(:needs_attributes) { [{ name: 'build', optional: true }] } + it 'is included' do + is_expected.to be_included + end - it "does not return an error" do + it 'does not have errors' do expect(subject.errors).to be_empty end end - end - - context 'when build job is part of prior stages' do - let(:stage_attributes) do - { - name: 'build', - index: 0, - builds: [{ name: 'build' }] - } - end - - let(:stage_seed) do - Gitlab::Ci::Pipeline::Seed::Stage.new(seed_context, stage_attributes, []) - end - let(:previous_stages) { [stage_seed] } + context 'when using 101 needs' do + let(:needs_count) { 101 } - it "is included" do - is_expected.to be_included - end + it "returns an error" do + expect(subject.errors).to contain_exactly( + "rspec: one job can only need 50 others, but you have listed 101. See needs keyword documentation for more details") + end - it "does not have errors" do - expect(subject.errors).to be_empty - end - end + context 'when ci_needs_size_limit is set to 100' do + before do + project.actual_limits.update!(ci_needs_size_limit: 100) + end - context 'when build job is part of the same stage' do - let(:current_stage) { double(seeds_names: [attributes[:name], 'build']) } + it "returns an error" do + expect(subject.errors).to contain_exactly( + "rspec: one job can only need 100 others, but you have listed 101. See needs keyword documentation for more details") + end + end - it 'is included' do - is_expected.to be_included - end + context 'when ci_needs_size_limit is set to 0' do + before do + project.actual_limits.update!(ci_needs_size_limit: 0) + end - it 'does not have errors' do - expect(subject.errors).to be_empty + it "returns an error" do + expect(subject.errors).to contain_exactly( + "rspec: one job can only need 0 others, but you have listed 101. See needs keyword documentation for more details") + end + end end end - context 'when using 101 needs' do - let(:needs_count) { 101 } + describe 'applying pipeline variables' do + subject { seed_build } - it "returns an error" do - expect(subject.errors).to contain_exactly( - "rspec: one job can only need 50 others, but you have listed 101. See needs keyword documentation for more details") + let(:pipeline_variables) { [] } + let(:pipeline) do + build(:ci_empty_pipeline, project: project, sha: head_sha, variables: pipeline_variables) end - context 'when ci_needs_size_limit is set to 100' do - before do - project.actual_limits.update!(ci_needs_size_limit: 100) + context 'containing variable references' do + let(:pipeline_variables) do + [ + build(:ci_pipeline_variable, key: 'A', value: '$B'), + build(:ci_pipeline_variable, key: 'B', value: '$C') + ] end - it "returns an error" do - expect(subject.errors).to contain_exactly( - "rspec: one job can only need 100 others, but you have listed 101. See needs keyword documentation for more details") + it "does not have errors" do + expect(subject.errors).to be_empty end end - context 'when ci_needs_size_limit is set to 0' do - before do - project.actual_limits.update!(ci_needs_size_limit: 0) + context 'containing cyclic reference' do + let(:pipeline_variables) do + [ + build(:ci_pipeline_variable, key: 'A', value: '$B'), + build(:ci_pipeline_variable, key: 'B', value: '$C'), + build(:ci_pipeline_variable, key: 'C', value: '$A') + ] end it "returns an error" do expect(subject.errors).to contain_exactly( - "rspec: one job can only need 0 others, but you have listed 101. See needs keyword documentation for more details") + 'rspec: circular variable reference detected: ["A", "B", "C"]') + end + + context 'with job:rules:[if:]' do + let(:attributes) { { name: 'rspec', ref: 'master', rules: [{ if: '$C != null', when: 'always' }] } } + + it "included? does not raise" do + expect { subject.included? }.not_to raise_error + end + + it "included? returns true" do + expect(subject.included?).to eq(true) + end end end end end - describe 'applying pipeline variables' do - subject { seed_build } - - let(:pipeline_variables) { [] } - let(:pipeline) do - build(:ci_empty_pipeline, project: project, sha: head_sha, variables: pipeline_variables) + describe 'feature flag ci_reuse_build_in_seed_context' do + let(:attributes) do + { name: 'rspec', rules: [{ if: '$VARIABLE == null' }], when: 'on_success' } end - context 'containing variable references' do - let(:pipeline_variables) do - [ - build(:ci_pipeline_variable, key: 'A', value: '$B'), - build(:ci_pipeline_variable, key: 'B', value: '$C') - ] - end + context 'when enabled' do + it_behaves_like 'build seed' - it "does not have errors" do - expect(subject.errors).to be_empty + it 'initializes the build once' do + expect(Ci::Build).to receive(:new).once.and_call_original + seed_build.to_resource end end - context 'containing cyclic reference' do - let(:pipeline_variables) do - [ - build(:ci_pipeline_variable, key: 'A', value: '$B'), - build(:ci_pipeline_variable, key: 'B', value: '$C'), - build(:ci_pipeline_variable, key: 'C', value: '$A') - ] - end - - it "returns an error" do - expect(subject.errors).to contain_exactly( - 'rspec: circular variable reference detected: ["A", "B", "C"]') + context 'when disabled' do + before do + stub_feature_flags(ci_reuse_build_in_seed_context: false) end - context 'with job:rules:[if:]' do - let(:attributes) { { name: 'rspec', ref: 'master', rules: [{ if: '$C != null', when: 'always' }] } } - - it "included? does not raise" do - expect { subject.included? }.not_to raise_error - end + it_behaves_like 'build seed' - it "included? returns true" do - expect(subject.included?).to eq(true) - end + it 'initializes the build twice' do + expect(Ci::Build).to receive(:new).twice.and_call_original + seed_build.to_resource end end end diff --git a/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb index a632b5dedcf..288ac3f3854 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Pipeline::Seed::Stage do +RSpec.describe Gitlab::Ci::Pipeline::Seed::Stage, feature_category: :pipeline_authoring do let(:project) { create(:project, :repository) } let(:pipeline) { create(:ci_empty_pipeline, project: project) } let(:previous_stages) { [] } diff --git a/spec/lib/gitlab/ci/reports/sbom/component_spec.rb b/spec/lib/gitlab/ci/reports/sbom/component_spec.rb index cdaf9354104..5dbcc1991d4 100644 --- a/spec/lib/gitlab/ci/reports/sbom/component_spec.rb +++ b/spec/lib/gitlab/ci/reports/sbom/component_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Reports::Sbom::Component do +RSpec.describe Gitlab::Ci::Reports::Sbom::Component, feature_category: :dependency_management do let(:component_type) { 'library' } let(:name) { 'component-name' } let(:purl_type) { 'npm' } diff --git a/spec/lib/gitlab/ci/reports/sbom/report_spec.rb b/spec/lib/gitlab/ci/reports/sbom/report_spec.rb index f9a83378f46..5d281f6ed76 100644 --- a/spec/lib/gitlab/ci/reports/sbom/report_spec.rb +++ b/spec/lib/gitlab/ci/reports/sbom/report_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Reports::Sbom::Report do +RSpec.describe Gitlab::Ci::Reports::Sbom::Report, feature_category: :dependency_management do subject(:report) { described_class.new } describe '#valid?' do diff --git a/spec/lib/gitlab/ci/reports/sbom/reports_spec.rb b/spec/lib/gitlab/ci/reports/sbom/reports_spec.rb index 75ea91251eb..4fb766d7d38 100644 --- a/spec/lib/gitlab/ci/reports/sbom/reports_spec.rb +++ b/spec/lib/gitlab/ci/reports/sbom/reports_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::Ci::Reports::Sbom::Reports do +RSpec.describe Gitlab::Ci::Reports::Sbom::Reports, feature_category: :dependency_management do subject(:reports_list) { described_class.new } describe '#add_report' do diff --git a/spec/lib/gitlab/ci/reports/sbom/source_spec.rb b/spec/lib/gitlab/ci/reports/sbom/source_spec.rb index 343c0d8c15c..63b8e5fdf01 100644 --- a/spec/lib/gitlab/ci/reports/sbom/source_spec.rb +++ b/spec/lib/gitlab/ci/reports/sbom/source_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::Ci::Reports::Sbom::Source do +RSpec.describe Gitlab::Ci::Reports::Sbom::Source, feature_category: :dependency_management do let(:attributes) do { type: :dependency_scanning, diff --git a/spec/lib/gitlab/ci/reports/security/reports_spec.rb b/spec/lib/gitlab/ci/reports/security/reports_spec.rb index 33f3317c655..cb6a91655ed 100644 --- a/spec/lib/gitlab/ci/reports/security/reports_spec.rb +++ b/spec/lib/gitlab/ci/reports/security/reports_spec.rb @@ -52,105 +52,4 @@ RSpec.describe Gitlab::Ci::Reports::Security::Reports do it { is_expected.to match_array(expected_findings) } end - - describe "#violates_default_policy_against?" do - let(:high_severity_dast) { build(:ci_reports_security_finding, severity: 'high', report_type: 'dast') } - let(:vulnerabilities_allowed) { 0 } - let(:severity_levels) { %w(critical high) } - let(:vulnerability_states) { %w(newly_detected) } - - subject { security_reports.violates_default_policy_against?(target_reports, vulnerabilities_allowed, severity_levels, vulnerability_states) } - - before do - security_reports.get_report('sast', artifact).add_finding(high_severity_dast) - end - - context 'when the target_reports is `nil`' do - let(:target_reports) { nil } - - context 'with severity levels matching the existing vulnerabilities' do - it { is_expected.to be(true) } - end - - context "without any severity levels matching the existing vulnerabilities" do - let(:severity_levels) { %w(critical) } - - it { is_expected.to be(false) } - end - end - - context 'when the target_reports is not `nil`' do - let(:target_reports) { described_class.new(pipeline) } - - context "when a report has a new unsafe vulnerability" do - context 'with severity levels matching the existing vulnerabilities' do - it { is_expected.to be(true) } - end - - it { is_expected.to be(true) } - - context 'with vulnerabilities_allowed higher than the number of new vulnerabilities' do - let(:vulnerabilities_allowed) { 10000 } - - it { is_expected.to be(false) } - end - - context "without any severity levels matching the existing vulnerabilities" do - let(:severity_levels) { %w(critical) } - - it { is_expected.to be(false) } - end - end - - context "when none of the reports have a new unsafe vulnerability" do - before do - target_reports.get_report('sast', artifact).add_finding(high_severity_dast) - end - - it { is_expected.to be(false) } - end - - context 'with related report_types' do - let(:report_types) { %w(dast sast) } - - subject { security_reports.violates_default_policy_against?(target_reports, vulnerabilities_allowed, severity_levels, vulnerability_states, report_types) } - - it { is_expected.to be(true) } - end - - context 'with unrelated report_types' do - let(:report_types) { %w(dependency_scanning sast) } - - subject { security_reports.violates_default_policy_against?(target_reports, vulnerabilities_allowed, severity_levels, vulnerability_states, report_types) } - - it { is_expected.to be(false) } - end - - context 'when target_reports is not nil and reports is empty' do - let(:without_reports) { described_class.new(pipeline) } - - subject { without_reports.violates_default_policy_against?(target_reports, vulnerabilities_allowed, severity_levels, vulnerability_states) } - - before do - target_reports.get_report('sast', artifact).add_finding(high_severity_dast) - end - - context 'when require_approval_on_scan_removal feature is enabled' do - before do - stub_feature_flags(require_approval_on_scan_removal: true) - end - - it { is_expected.to be(true) } - end - - context 'when require_approval_on_scan_removal feature is disabled' do - before do - stub_feature_flags(require_approval_on_scan_removal: false) - end - - it { is_expected.to be(false) } - end - end - end - end end diff --git a/spec/lib/gitlab/ci/runner_instructions_spec.rb b/spec/lib/gitlab/ci/runner_instructions_spec.rb index f872c631a50..56f69720b87 100644 --- a/spec/lib/gitlab/ci/runner_instructions_spec.rb +++ b/spec/lib/gitlab/ci/runner_instructions_spec.rb @@ -69,6 +69,7 @@ RSpec.describe Gitlab::Ci::RunnerInstructions do 'windows' | 'amd64' 'windows' | '386' 'osx' | 'amd64' + 'osx' | 'arm64' end with_them do diff --git a/spec/lib/gitlab/ci/trace/archive_spec.rb b/spec/lib/gitlab/ci/trace/archive_spec.rb index f91cb03883a..582c4ad343f 100644 --- a/spec/lib/gitlab/ci/trace/archive_spec.rb +++ b/spec/lib/gitlab/ci/trace/archive_spec.rb @@ -75,15 +75,6 @@ RSpec.describe Gitlab::Ci::Trace::Archive do include_context 'with FIPS' end - context 'with background_upload enabled' do - before do - stub_artifacts_object_storage(background_upload: true) - end - - it_behaves_like 'skips validations' - include_context 'with FIPS' - end - context 'with direct_upload enabled' do before do stub_artifacts_object_storage(direct_upload: true) diff --git a/spec/lib/gitlab/ci/variables/builder_spec.rb b/spec/lib/gitlab/ci/variables/builder_spec.rb index 52ba85d2df1..5aa752ee429 100644 --- a/spec/lib/gitlab/ci/variables/builder_spec.rb +++ b/spec/lib/gitlab/ci/variables/builder_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache do +RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache, feature_category: :pipeline_authoring do include Ci::TemplateHelpers let_it_be(:group) { create(:group) } let_it_be(:project) { create(:project, :repository, namespace: group) } @@ -13,7 +13,8 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache do name: 'rspec:test 1', pipeline: pipeline, user: user, - yaml_variables: [{ key: 'YAML_VARIABLE', value: 'value' }] + yaml_variables: [{ key: 'YAML_VARIABLE', value: 'value' }], + environment: 'test' ) end @@ -32,6 +33,8 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache do value: job.stage_name }, { key: 'CI_NODE_TOTAL', value: '1' }, + { key: 'CI_ENVIRONMENT_NAME', + value: 'test' }, { key: 'CI_BUILD_NAME', value: 'rspec:test 1' }, { key: 'CI_BUILD_STAGE', @@ -76,6 +79,8 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache do value: project.full_path_slug }, { key: 'CI_PROJECT_NAMESPACE', value: project.namespace.full_path }, + { key: 'CI_PROJECT_NAMESPACE_ID', + value: project.namespace.id.to_s }, { key: 'CI_PROJECT_ROOT_NAMESPACE', value: project.namespace.root_ancestor.path }, { key: 'CI_PROJECT_URL', @@ -276,11 +281,17 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache do subject { builder.kubernetes_variables(environment: nil, job: job) } before do - allow(Ci::GenerateKubeconfigService).to receive(:new).with(job.pipeline, token: job.token).and_return(service) + allow(Ci::GenerateKubeconfigService).to receive(:new).with(job.pipeline, token: job.token, environment: anything).and_return(service) end it { is_expected.to include(key: 'KUBECONFIG', value: 'example-kubeconfig', public: false, file: true) } + it 'calls the GenerateKubeconfigService with the correct arguments' do + expect(Ci::GenerateKubeconfigService).to receive(:new).with(job.pipeline, token: job.token, environment: nil) + + subject + end + context 'generated config is invalid' do let(:template_valid) { false } @@ -297,6 +308,16 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache do expect(subject['KUBECONFIG'].value).to eq('example-kubeconfig') expect(subject['OTHER'].value).to eq('some value') end + + context 'when environment is not nil' do + subject { builder.kubernetes_variables(environment: 'production', job: job) } + + it 'passes the environment when generating the KUBECONFIG' do + expect(Ci::GenerateKubeconfigService).to receive(:new).with(job.pipeline, token: job.token, environment: 'production') + + subject + end + end end describe '#deployment_variables' do diff --git a/spec/lib/gitlab/ci/yaml_processor/result_spec.rb b/spec/lib/gitlab/ci/yaml_processor/result_spec.rb index 7f203168706..5c9f156e054 100644 --- a/spec/lib/gitlab/ci/yaml_processor/result_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor/result_spec.rb @@ -12,6 +12,20 @@ module Gitlab let(:ci_config) { Gitlab::Ci::Config.new(config_content, user: user) } let(:result) { described_class.new(ci_config: ci_config, warnings: ci_config&.warnings) } + describe '#builds' do + context 'when a job has ID tokens' do + let(:config_content) do + YAML.dump( + test: { stage: 'test', script: 'echo', id_tokens: { TEST_ID_TOKEN: { aud: 'https://gitlab.com' } } } + ) + end + + it 'includes `id_tokens`' do + expect(result.builds.first[:id_tokens]).to eq({ TEST_ID_TOKEN: { aud: 'https://gitlab.com' } }) + end + end + end + describe '#config_metadata' do subject(:config_metadata) { result.config_metadata } diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index 5de813f7739..ae98d2e0cad 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -870,6 +870,69 @@ module Gitlab end end end + + describe "hooks" do + context 'when it is a simple script' do + let(:config) do + { + test: { script: ["script"], + hooks: { pre_get_sources_script: ["echo 1", "echo 2", "pwd"] } } + } + end + + it "returns hooks in options" do + expect(subject[:options][:hooks]).to eq( + { pre_get_sources_script: ["echo 1", "echo 2", "pwd"] } + ) + end + end + + context 'when it is nested arrays of strings' do + let(:config) do + { + test: { script: ["script"], + hooks: { pre_get_sources_script: [[["global script"], "echo 1"], "echo 2", ["ls"], "pwd"] } } + } + end + + it "returns hooks in options" do + expect(subject[:options][:hooks]).to eq( + { pre_get_sources_script: ["global script", "echo 1", "echo 2", "ls", "pwd"] } + ) + end + end + + context 'when receiving from the default' do + let(:config) do + { + default: { hooks: { pre_get_sources_script: ["echo 1", "echo 2", "pwd"] } }, + test: { script: ["script"] } + } + end + + it "inherits hooks" do + expect(subject[:options][:hooks]).to eq( + { pre_get_sources_script: ["echo 1", "echo 2", "pwd"] } + ) + end + end + + context 'when overriding the default' do + let(:config) do + { + default: { hooks: { pre_get_sources_script: ["echo 1", "echo 2", "pwd"] } }, + test: { script: ["script"], + hooks: { pre_get_sources_script: ["echo 3", "echo 4", "pwd"] } } + } + end + + it "overrides hooks" do + expect(subject[:options][:hooks]).to eq( + { pre_get_sources_script: ["echo 3", "echo 4", "pwd"] } + ) + end + end + end end describe "Image and service handling" do @@ -2883,7 +2946,7 @@ module Gitlab context 'returns errors if job artifacts:when is not an a predefined value' do let(:config) { YAML.dump({ stages: %w(build test), rspec: { script: "test", artifacts: { when: 1 } } }) } - it_behaves_like 'returns errors', 'jobs:rspec:artifacts when should be on_success, on_failure or always' + it_behaves_like 'returns errors', 'jobs:rspec:artifacts when should be one of: on_success, on_failure, always' end context 'returns errors if job artifacts:expire_in is not an a string' do diff --git a/spec/lib/gitlab/cluster/rack_timeout_observer_spec.rb b/spec/lib/gitlab/cluster/rack_timeout_observer_spec.rb index 05df4089075..2d4a7a3b170 100644 --- a/spec/lib/gitlab/cluster/rack_timeout_observer_spec.rb +++ b/spec/lib/gitlab/cluster/rack_timeout_observer_spec.rb @@ -73,5 +73,28 @@ RSpec.describe Gitlab::Cluster::RackTimeoutObserver do subject.callback.call(env) end end + + context 'when request contains invalid string' do + let(:env) do + { + ::Rack::Timeout::ENV_INFO_KEY => double(state: :timed_out), + 'action_dispatch.request.parameters' => { + 'controller' => 'foo', + 'action' => '\u003c', + 'route' => '?8\u003c/x' + } + } + end + + subject { described_class.new } + + it 'sanitizes string' do + expect(counter) + .to receive(:increment) + .with({ controller: 'foo', action: '\\u003c', route: '?8\\u003c/x', state: :timed_out }) + + subject.callback.call(env) + end + end end end diff --git a/spec/lib/gitlab/config/entry/attributable_spec.rb b/spec/lib/gitlab/config/entry/attributable_spec.rb index 8a207bddaae..0a2f8ac2c3a 100644 --- a/spec/lib/gitlab/config/entry/attributable_spec.rb +++ b/spec/lib/gitlab/config/entry/attributable_spec.rb @@ -10,10 +10,11 @@ RSpec.describe Gitlab::Config::Entry::Attributable do end let(:instance) { node.new } + let(:prefix) { nil } before do - node.class_eval do - attributes :name, :test + node.class_exec(prefix) do |pre| + attributes :name, :test, prefix: pre end end @@ -24,6 +25,17 @@ RSpec.describe Gitlab::Config::Entry::Attributable do .and_return({ name: 'some name', test: 'some test' }) end + context 'and is provided a prefix' do + let(:prefix) { :pre } + + it 'returns the value of config' do + expect(instance).to have_pre_name + expect(instance.pre_name).to eq 'some name' + expect(instance).to have_pre_test + expect(instance.pre_test).to eq 'some test' + end + end + it 'returns the value of config' do expect(instance).to have_name expect(instance.name).to eq 'some name' diff --git a/spec/lib/gitlab/conflict/file_spec.rb b/spec/lib/gitlab/conflict/file_spec.rb index 165305476d2..6ea8e6c6706 100644 --- a/spec/lib/gitlab/conflict/file_spec.rb +++ b/spec/lib/gitlab/conflict/file_spec.rb @@ -30,7 +30,7 @@ RSpec.describe Gitlab::Conflict::File do let(:section_keys) { conflict_file.sections.map { |section| section[:id] }.compact } context 'when resolving everything to the same side' do - let(:resolution_hash) { section_keys.to_h { |key| [key, 'head'] } } + let(:resolution_hash) { section_keys.index_with { 'head' } } let(:resolved_lines) { conflict_file.resolve_lines(resolution_hash) } let(:expected_lines) { conflict_file.lines.reject { |line| line.type == 'old' } } @@ -63,8 +63,8 @@ RSpec.describe Gitlab::Conflict::File do end it 'raises ResolutionError when passed a hash without resolutions for all sections' do - empty_hash = section_keys.to_h { |key| [key, nil] } - invalid_hash = section_keys.to_h { |key| [key, 'invalid'] } + empty_hash = section_keys.index_with { nil } + invalid_hash = section_keys.index_with { 'invalid' } expect { conflict_file.resolve_lines({}) } .to raise_error(Gitlab::Git::Conflict::Resolver::ResolutionError) diff --git a/spec/lib/gitlab/content_security_policy/config_loader_spec.rb b/spec/lib/gitlab/content_security_policy/config_loader_spec.rb index aadfb41a46e..88bffd41947 100644 --- a/spec/lib/gitlab/content_security_policy/config_loader_spec.rb +++ b/spec/lib/gitlab/content_security_policy/config_loader_spec.rb @@ -102,13 +102,65 @@ RSpec.describe Gitlab::ContentSecurityPolicy::ConfigLoader do end context 'when sentry is configured' do + let(:legacy_dsn) { 'dummy://abc@legacy-sentry.example.com/1' } + let(:dsn) { 'dummy://def@sentry.example.com/2' } + before do - stub_sentry_settings stub_config_setting(host: 'gitlab.example.com') end - it 'adds sentry path to CSP without user' do - expect(directives['connect_src']).to eq("'self' ws://gitlab.example.com dummy://example.com") + context 'when legacy sentry is configured' do + before do + allow(Gitlab.config.sentry).to receive(:enabled).and_return(true) + allow(Gitlab.config.sentry).to receive(:clientside_dsn).and_return(legacy_dsn) + allow(Gitlab::CurrentSettings).to receive(:sentry_enabled).and_return(false) + end + + it 'adds legacy sentry path to CSP' do + expect(directives['connect_src']).to eq("'self' ws://gitlab.example.com dummy://legacy-sentry.example.com") + end + end + + context 'when sentry is configured' do + before do + allow(Gitlab.config.sentry).to receive(:enabled).and_return(false) + allow(Gitlab::CurrentSettings).to receive(:sentry_enabled).and_return(true) + allow(Gitlab::CurrentSettings).to receive(:sentry_clientside_dsn).and_return(dsn) + end + + it 'adds new sentry path to CSP' do + expect(directives['connect_src']).to eq("'self' ws://gitlab.example.com dummy://sentry.example.com") + end + end + + context 'when sentry settings are from older schemas and sentry setting are missing' do + before do + allow(Gitlab.config.sentry).to receive(:enabled).and_return(false) + + allow(Gitlab::CurrentSettings).to receive(:respond_to?).with(:sentry_enabled).and_return(false) + allow(Gitlab::CurrentSettings).to receive(:sentry_enabled).and_raise(NoMethodError) + + allow(Gitlab::CurrentSettings).to receive(:respond_to?).with(:sentry_clientside_dsn).and_return(false) + allow(Gitlab::CurrentSettings).to receive(:sentry_clientside_dsn).and_raise(NoMethodError) + end + + it 'config is backwards compatible, does not add sentry path to CSP' do + expect(directives['connect_src']).to eq("'self' ws://gitlab.example.com") + end + end + + context 'when legacy sentry and sentry are both configured' do + before do + allow(Gitlab.config.sentry).to receive(:enabled).and_return(true) + allow(Gitlab.config.sentry).to receive(:clientside_dsn).and_return(legacy_dsn) + + allow(Gitlab::CurrentSettings).to receive(:sentry_enabled).and_return(true) + allow(Gitlab::CurrentSettings).to receive(:sentry_clientside_dsn).and_return(dsn) + end + + it 'adds both sentry paths to CSP' do + expect(directives['connect_src']).to eq("'self' ws://gitlab.example.com dummy://legacy-sentry.example.com dummy://sentry.example.com") + end end end diff --git a/spec/lib/gitlab/contributions_calendar_spec.rb b/spec/lib/gitlab/contributions_calendar_spec.rb index 8a9ab736d46..3736914669a 100644 --- a/spec/lib/gitlab/contributions_calendar_spec.rb +++ b/spec/lib/gitlab/contributions_calendar_spec.rb @@ -2,24 +2,24 @@ require 'spec_helper' -RSpec.describe Gitlab::ContributionsCalendar do - let(:contributor) { create(:user) } - let(:user) { create(:user) } +RSpec.describe Gitlab::ContributionsCalendar, feature_category: :users do + let_it_be_with_reload(:contributor) { create(:user) } + let_it_be_with_reload(:user) { create(:user) } let(:travel_time) { nil } - let(:private_project) do + let_it_be_with_reload(:private_project) do create(:project, :private) do |project| create(:project_member, user: contributor, project: project) end end - let(:public_project) do + let_it_be(:public_project) do create(:project, :public, :repository) do |project| create(:project_member, user: contributor, project: project) end end - let(:feature_project) do + let_it_be(:feature_project) do create(:project, :public, :issues_private) do |project| create(:project_member, user: contributor, project: project).project end @@ -30,6 +30,7 @@ RSpec.describe Gitlab::ContributionsCalendar do let(:tomorrow) { today + 1.day } let(:last_week) { today - 7.days } let(:last_year) { today - 1.year } + let(:targets) { {} } before do travel_to travel_time || Time.now.utc.end_of_day @@ -44,26 +45,28 @@ RSpec.describe Gitlab::ContributionsCalendar do end def create_event(project, day, hour = 0, action = :created, target_symbol = :issue) - @targets ||= {} - @targets[project] ||= create(target_symbol, project: project, author: contributor) + targets[project] ||= create(target_symbol, project: project, author: contributor) Event.create!( project: project, action: action, - target_type: @targets[project].class.name, - target_id: @targets[project].id, + target_type: targets[project].class.name, + target_id: targets[project].id, author: contributor, created_at: DateTime.new(day.year, day.month, day.day, hour) ) end - describe '#activity_dates' do + describe '#activity_dates', :aggregate_failures do it "returns a hash of date => count" do create_event(public_project, last_week) create_event(public_project, last_week) create_event(public_project, today) + work_item_event = create_event(private_project, today, 0, :created, :work_item) - expect(calendar.activity_dates).to eq(last_week => 2, today => 1) + # make sure the target is a work item as we want to include those in the count + expect(work_item_event.target_type).to eq('WorkItem') + expect(calendar(contributor).activity_dates).to eq(last_week => 2, today => 2) end context "when the user has opted-in for private contributions" do @@ -176,9 +179,11 @@ RSpec.describe Gitlab::ContributionsCalendar do it "returns all events for a given date" do e1 = create_event(public_project, today) e2 = create_event(public_project, today) + e3 = create_event(private_project, today, 0, :created, :work_item) create_event(public_project, last_week) - expect(calendar.events_by_date(today)).to contain_exactly(e1, e2) + expect([e1, e2, e3].map(&:target_type)).to contain_exactly('WorkItem', 'Issue', 'Issue') + expect(calendar(contributor).events_by_date(today)).to contain_exactly(e1, e2, e3) end it "only shows private events to authorized users" do diff --git a/spec/lib/gitlab/counters/buffered_counter_spec.rb b/spec/lib/gitlab/counters/buffered_counter_spec.rb new file mode 100644 index 00000000000..a1fd97768ea --- /dev/null +++ b/spec/lib/gitlab/counters/buffered_counter_spec.rb @@ -0,0 +1,233 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Counters::BufferedCounter, :clean_gitlab_redis_shared_state do + using RSpec::Parameterized::TableSyntax + + subject(:counter) { described_class.new(counter_record, attribute) } + + let(:counter_record) { create(:project_statistics) } + let(:attribute) { :build_artifacts_size } + + describe '#get' do + it 'returns the value when there is an existing value stored in the counter' do + Gitlab::Redis::SharedState.with do |redis| + redis.set(counter.key, 456) + end + + expect(counter.get).to eq(456) + end + + it 'returns 0 when there is no existing value' do + expect(counter.get).to eq(0) + end + end + + describe '#increment' do + it 'sets a new key by the given value' do + counter.increment(123) + + expect(counter.get).to eq(123) + end + + it 'increments an existing key by the given value' do + counter.increment(100) + counter.increment(123) + + expect(counter.get).to eq(100 + 123) + end + + it 'returns the new value' do + counter.increment(123) + + expect(counter.increment(23)).to eq(146) + end + + it 'schedules a worker to commit the counter into database' do + expect(FlushCounterIncrementsWorker).to receive(:perform_in) + .with(described_class::WORKER_DELAY, counter_record.class.to_s, counter_record.id, attribute) + + counter.increment(123) + end + end + + describe '#reset!' do + before do + allow(counter_record).to receive(:update!) + + counter.increment(123) + end + + it 'removes the key from Redis' do + counter.reset! + + Gitlab::Redis::SharedState.with do |redis| + expect(redis.exists?(counter.key)).to eq(false) + end + end + + it 'resets the counter to 0' do + counter.reset! + + expect(counter.get).to eq(0) + end + + it 'resets the record to 0' do + expect(counter_record).to receive(:update!).with(attribute => 0) + + counter.reset! + end + end + + describe '#commit_increment!' do + it 'obtains an exclusive lease during processing' do + expect(counter).to receive(:with_exclusive_lease).and_call_original + + counter.commit_increment! + end + + context 'when there is an amount to commit' do + let(:increments) { [10, -3] } + + before do + increments.each { |i| counter.increment(i) } + end + + it 'commits the increment into the database' do + expect { counter.commit_increment! } + .to change { counter_record.reset.read_attribute(attribute) }.by(increments.sum) + end + + it 'removes the increment entry from Redis' do + Gitlab::Redis::SharedState.with do |redis| + key_exists = redis.exists?(counter.key) + expect(key_exists).to be_truthy + end + + counter.commit_increment! + + Gitlab::Redis::SharedState.with do |redis| + key_exists = redis.exists?(counter.key) + expect(key_exists).to be_falsey + end + end + end + + context 'when there are no counters to flush' do + context 'when there are no counters in the relative :flushed key' do + it 'does not change the record' do + expect { counter.commit_increment! }.not_to change { counter_record.reset.attributes } + end + end + + # This can be the case where updating counters in the database fails with error + # and retrying the worker will retry flushing the counters but the main key has + # disappeared and the increment has been moved to the "<...>:flushed" key. + context 'when there are counters in the relative :flushed key' do + let(:flushed_amount) { 10 } + + before do + Gitlab::Redis::SharedState.with do |redis| + redis.incrby(counter.flushed_key, flushed_amount) + end + end + + it 'updates the record' do + expect { counter.commit_increment! } + .to change { counter_record.reset.read_attribute(attribute) }.by(flushed_amount) + end + + it 'deletes the relative :flushed key' do + counter.commit_increment! + + Gitlab::Redis::SharedState.with do |redis| + key_exists = redis.exists?(counter.flushed_key) + expect(key_exists).to be_falsey + end + end + end + + context 'when deleting :flushed key fails' do + before do + Gitlab::Redis::SharedState.with do |redis| + redis.incrby(counter.flushed_key, 10) + + allow(redis).to receive(:del).and_raise('could not delete key') + end + end + + it 'does a rollback of the counter update' do + expect { counter.commit_increment! }.to raise_error('could not delete key') + + expect(counter_record.reset.read_attribute(attribute)).to eq(0) + end + end + + context 'when the counter record has after_commit callbacks' do + it 'has registered callbacks' do + expect(counter_record.class.after_commit_callbacks.size).to eq(1) + end + + context 'when there are increments to flush' do + before do + counter.increment(10) + end + + it 'executes the callbacks' do + expect(counter_record).to receive(:execute_after_commit_callbacks).and_call_original + + counter.commit_increment! + end + end + + context 'when there are no increments to flush' do + it 'does not execute the callbacks' do + expect(counter_record).not_to receive(:execute_after_commit_callbacks).and_call_original + + counter.commit_increment! + end + end + end + end + end + + describe '#amount_to_be_flushed' do + let(:increment_key) { counter.key } + let(:flushed_key) { counter.flushed_key } + + where(:increment, :flushed, :result, :flushed_key_present) do + nil | nil | 0 | false + nil | 0 | 0 | false + 0 | 0 | 0 | false + 1 | 0 | 1 | true + 1 | nil | 1 | true + 1 | 1 | 2 | true + 1 | -2 | -1 | true + -1 | 1 | 0 | false + end + + with_them do + before do + Gitlab::Redis::SharedState.with do |redis| + redis.set(increment_key, increment) if increment + redis.set(flushed_key, flushed) if flushed + end + end + + it 'returns the current value to be flushed' do + value = counter.amount_to_be_flushed + expect(value).to eq(result) + end + + it 'drops the increment key and creates the flushed key if it does not exist' do + counter.amount_to_be_flushed + + Gitlab::Redis::SharedState.with do |redis| + expect(redis.exists?(increment_key)).to eq(false) + expect(redis.exists?(flushed_key)).to eq(flushed_key_present) + end + end + end + end +end diff --git a/spec/lib/gitlab/counters/legacy_counter_spec.rb b/spec/lib/gitlab/counters/legacy_counter_spec.rb new file mode 100644 index 00000000000..e66b1ce08c4 --- /dev/null +++ b/spec/lib/gitlab/counters/legacy_counter_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Counters::LegacyCounter do + subject(:counter) { described_class.new(counter_record, attribute) } + + let(:counter_record) { create(:project_statistics) } + let(:attribute) { :snippets_size } + let(:amount) { 123 } + + describe '#increment' do + it 'increments the attribute in the counter record' do + expect { counter.increment(amount) }.to change { counter_record.reload.method(attribute).call }.by(amount) + end + + it 'returns the value after the increment' do + counter.increment(100) + + expect(counter.increment(amount)).to eq(100 + amount) + end + + it 'executes after counter_record after commit callback' do + expect(counter_record).to receive(:execute_after_commit_callbacks).and_call_original + + counter.increment(amount) + end + end + + describe '#reset!' do + before do + allow(counter_record).to receive(:update!) + end + + it 'resets the record to 0' do + expect(counter_record).to receive(:update!).with(attribute => 0) + + counter.reset! + end + end +end diff --git a/spec/lib/gitlab/cycle_analytics/stage_summary_spec.rb b/spec/lib/gitlab/cycle_analytics/stage_summary_spec.rb index 0e7d7f1efda..92ffeee8509 100644 --- a/spec/lib/gitlab/cycle_analytics/stage_summary_spec.rb +++ b/spec/lib/gitlab/cycle_analytics/stage_summary_spec.rb @@ -29,8 +29,8 @@ RSpec.describe Gitlab::CycleAnalytics::StageSummary do context 'when from date is given' do before do - Timecop.freeze(5.days.ago) { create(:issue, project: project) } - Timecop.freeze(5.days.from_now) { create(:issue, project: project) } + travel_to(5.days.ago) { create(:issue, project: project) } + travel_to(5.days.from_now) { create(:issue, project: project) } end it "finds the number of issues created after the 'from date'" do @@ -45,15 +45,15 @@ RSpec.describe Gitlab::CycleAnalytics::StageSummary do end it "doesn't find issues from other projects" do - Timecop.freeze(5.days.from_now) { create(:issue, project: create(:project)) } + travel_to(5.days.from_now) { create(:issue, project: create(:project)) } expect(subject[:value]).to eq('-') end context 'when `to` parameter is given' do before do - Timecop.freeze(5.days.ago) { create(:issue, project: project) } - Timecop.freeze(5.days.from_now) { create(:issue, project: project) } + travel_to(5.days.ago) { create(:issue, project: project) } + travel_to(5.days.from_now) { create(:issue, project: project) } end it "doesn't find any record" do @@ -78,8 +78,8 @@ RSpec.describe Gitlab::CycleAnalytics::StageSummary do context 'when from date is given' do before do - Timecop.freeze(5.days.ago) { create_commit("Test message", project, user, 'master') } - Timecop.freeze(5.days.from_now) { create_commit("Test message", project, user, 'master') } + travel_to(5.days.ago) { create_commit("Test message", project, user, 'master') } + travel_to(5.days.from_now) { create_commit("Test message", project, user, 'master') } end it "finds the number of commits created after the 'from date'" do @@ -94,21 +94,21 @@ RSpec.describe Gitlab::CycleAnalytics::StageSummary do end it "doesn't find commits from other projects" do - Timecop.freeze(5.days.from_now) { create_commit("Test message", create(:project, :repository), user, 'master') } + travel_to(5.days.from_now) { create_commit("Test message", create(:project, :repository), user, 'master') } expect(subject[:value]).to eq('-') end it "finds a large (> 100) number of commits if present" do - Timecop.freeze(5.days.from_now) { create_commit("Test message", project, user, 'master', count: 100) } + travel_to(5.days.from_now) { create_commit("Test message", project, user, 'master', count: 100) } expect(subject[:value]).to eq('100') end context 'when `to` parameter is given' do before do - Timecop.freeze(5.days.ago) { create_commit("Test message", project, user, 'master') } - Timecop.freeze(5.days.from_now) { create_commit("Test message", project, user, 'master') } + travel_to(5.days.ago) { create_commit("Test message", project, user, 'master') } + travel_to(5.days.from_now) { create_commit("Test message", project, user, 'master') } end it "doesn't find any record" do diff --git a/spec/lib/gitlab/data_builder/deployment_spec.rb b/spec/lib/gitlab/data_builder/deployment_spec.rb index 8ee57542d43..bf08e782035 100644 --- a/spec/lib/gitlab/data_builder/deployment_spec.rb +++ b/spec/lib/gitlab/data_builder/deployment_spec.rb @@ -12,8 +12,8 @@ RSpec.describe Gitlab::DataBuilder::Deployment do expect(data[:object_kind]).to eq('deployment') end - it 'returns data for the given build' do - environment = create(:environment, name: "somewhere") + it 'returns data for the given build', :aggregate_failures do + environment = create(:environment, name: 'somewhere/1', external_url: 'https://test.com') project = create(:project, :repository, name: 'myproj') commit = project.commit('HEAD') deployment = create(:deployment, status: :failed, environment: environment, sha: commit.sha, project: project) @@ -30,7 +30,9 @@ RSpec.describe Gitlab::DataBuilder::Deployment do expect(data[:deployment_id]).to eq(deployment.id) expect(data[:deployable_id]).to eq(deployable.id) expect(data[:deployable_url]).to eq(expected_deployable_url) - expect(data[:environment]).to eq("somewhere") + expect(data[:environment]).to eq('somewhere/1') + expect(data[:environment_slug]).to eq('somewhere-1-78avk6') + expect(data[:environment_external_url]).to eq('https://test.com') expect(data[:project]).to eq(project.hook_attrs) expect(data[:short_sha]).to eq(deployment.short_sha) expect(data[:user]).to eq(deployment.deployed_by.hook_attrs) diff --git a/spec/lib/gitlab/database/gitlab_schema_spec.rb b/spec/lib/gitlab/database/gitlab_schema_spec.rb index 72950895022..4b37cbda047 100644 --- a/spec/lib/gitlab/database/gitlab_schema_spec.rb +++ b/spec/lib/gitlab/database/gitlab_schema_spec.rb @@ -1,42 +1,86 @@ # frozen_string_literal: true require 'spec_helper' +RSpec.shared_examples 'validate path globs' do |path_globs| + it 'returns an array of path globs' do + expect(path_globs).to be_an(Array) + expect(path_globs).to all(be_an(Pathname)) + end +end + RSpec.describe Gitlab::Database::GitlabSchema do - describe '.tables_to_schema' do - it 'all tables have assigned a known gitlab_schema' do - expect(described_class.tables_to_schema).to all( + describe '.views_and_tables_to_schema' do + it 'all tables and views have assigned a known gitlab_schema' do + expect(described_class.views_and_tables_to_schema).to all( match([be_a(String), be_in(Gitlab::Database.schemas_to_base_models.keys.map(&:to_sym))]) ) end # This being run across different databases indirectly also tests # a general consistency of structure across databases - Gitlab::Database.database_base_models.select { |k, _| k != 'geo' }.each do |db_config_name, db_class| + Gitlab::Database.database_base_models.except(:geo).each do |db_config_name, db_class| context "for #{db_config_name} using #{db_class}" do let(:db_data_sources) { db_class.connection.data_sources } # The Geo database does not share the same structure as all decomposed databases - subject { described_class.tables_to_schema.select { |_, v| v != :gitlab_geo } } + subject { described_class.views_and_tables_to_schema.select { |_, v| v != :gitlab_geo } } it 'new data sources are added' do - missing_tables = db_data_sources.to_set - subject.keys + missing_data_sources = db_data_sources.to_set - subject.keys - expect(missing_tables).to be_empty, \ - "Missing table(s) #{missing_tables.to_a} not found in #{described_class}.tables_to_schema. " \ - "Any new tables must be added to #{described_class::GITLAB_SCHEMAS_FILE}." + expect(missing_data_sources).to be_empty, \ + "Missing table/view(s) #{missing_data_sources.to_a} not found in " \ + "#{described_class}.views_and_tables_to_schema. " \ + "Any new tables or views must be added to the database dictionary. " \ + "More info: https://docs.gitlab.com/ee/development/database/database_dictionary.html" end it 'non-existing data sources are removed' do - extra_tables = subject.keys.to_set - db_data_sources + extra_data_sources = subject.keys.to_set - db_data_sources - expect(extra_tables).to be_empty, \ - "Extra table(s) #{extra_tables.to_a} found in #{described_class}.tables_to_schema. " \ - "Any removed or renamed tables must be removed from #{described_class::GITLAB_SCHEMAS_FILE}." + expect(extra_data_sources).to be_empty, \ + "Extra table/view(s) #{extra_data_sources.to_a} found in #{described_class}.views_and_tables_to_schema. " \ + "Any removed or renamed tables or views must be removed from the database dictionary. " \ + "More info: https://docs.gitlab.com/ee/development/database/database_dictionary.html" end end end end + describe '.dictionary_path_globs' do + include_examples 'validate path globs', described_class.dictionary_path_globs + end + + describe '.view_path_globs' do + include_examples 'validate path globs', described_class.view_path_globs + end + + describe '.tables_to_schema' do + let(:database_models) { Gitlab::Database.database_base_models.except(:geo) } + let(:views) { database_models.flat_map { |_, m| m.connection.views }.sort.uniq } + + subject { described_class.tables_to_schema } + + it 'returns only tables' do + tables = subject.keys + + expect(tables).not_to include(views.to_set) + end + end + + describe '.views_to_schema' do + let(:database_models) { Gitlab::Database.database_base_models.except(:geo) } + let(:tables) { database_models.flat_map { |_, m| m.connection.tables }.sort.uniq } + + subject { described_class.views_to_schema } + + it 'returns only views' do + views = subject.keys + + expect(views).not_to include(tables.to_set) + end + end + describe '.table_schema' do using RSpec::Parameterized::TableSyntax diff --git a/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb b/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb index b7915e6cf69..7eb20f77417 100644 --- a/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::LoadBalancing::SidekiqClientMiddleware do +RSpec.describe Gitlab::Database::LoadBalancing::SidekiqClientMiddleware, feature_category: :database do let(:middleware) { described_class.new } let(:worker_class) { 'TestDataConsistencyWorker' } @@ -34,8 +34,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqClientMiddleware do data_consistency data_consistency, feature_flag: feature_flag - def perform(*args) - end + def perform(*args); end end end @@ -83,21 +82,41 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqClientMiddleware do allow(Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary?).and_return(false) end - it 'passes database_replica_location' do - expected_location = {} + context 'when replica hosts are available' do + it 'passes database_replica_location' do + expected_location = {} - Gitlab::Database::LoadBalancing.each_load_balancer do |lb| - expect(lb.host) - .to receive(:database_replica_location) - .and_return(location) + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + expect(lb.host) + .to receive(:database_replica_location) + .and_return(location) - expected_location[lb.name] = location + expected_location[lb.name] = location + end + + run_middleware + + expect(job['wal_locations']).to eq(expected_location) + expect(job['wal_location_source']).to eq(:replica) end + end - run_middleware + context 'when no replica hosts are available' do + it 'passes primary_write_location' do + expected_location = {} - expect(job['wal_locations']).to eq(expected_location) - expect(job['wal_location_source']).to eq(:replica) + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + expect(lb).to receive(:host).and_return(nil) + expect(lb).to receive(:primary_write_location).and_return(location) + + expected_location[lb.name] = location + end + + run_middleware + + expect(job['wal_locations']).to eq(expected_location) + expect(job['wal_location_source']).to eq(:replica) + end end include_examples 'job data consistency' diff --git a/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb b/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb index 61b63016f1a..abf10456d0a 100644 --- a/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb @@ -33,8 +33,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ data_consistency data_consistency, feature_flag: feature_flag - def perform(*args) - end + def perform(*args); end end end @@ -332,28 +331,6 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ expect(middleware.send(:databases_in_sync?, locations)) .to eq(false) end - - context 'when "indifferent_wal_location_keys" FF is off' do - before do - stub_feature_flags(indifferent_wal_location_keys: false) - end - - it 'returns true when the load balancers are not in sync' do - locations = {} - - Gitlab::Database::LoadBalancing.each_load_balancer do |lb| - locations[lb.name.to_s] = 'foo' - - allow(lb) - .to receive(:select_up_to_date_host) - .with('foo') - .and_return(false) - end - - expect(middleware.send(:databases_in_sync?, locations)) - .to eq(true) - end - end end end diff --git a/spec/lib/gitlab/database/lock_writes_manager_spec.rb b/spec/lib/gitlab/database/lock_writes_manager_spec.rb index b1cc8add55a..242b2040eaa 100644 --- a/spec/lib/gitlab/database/lock_writes_manager_spec.rb +++ b/spec/lib/gitlab/database/lock_writes_manager_spec.rb @@ -37,6 +37,14 @@ RSpec.describe Gitlab::Database::LockWritesManager do it 'returns true for a table that is locked for writes' do expect { subject.lock_writes }.to change { subject.table_locked_for_writes?(test_table) }.from(false).to(true) end + + context 'for detached partition tables in another schema' do + let(:test_table) { 'gitlab_partitions_dynamic._test_table_20220101' } + + it 'returns true for a table that is locked for writes' do + expect { subject.lock_writes }.to change { subject.table_locked_for_writes?(test_table) }.from(false).to(true) + end + end end describe '#lock_writes' do diff --git a/spec/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables_spec.rb b/spec/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables_spec.rb new file mode 100644 index 00000000000..9fd49b312eb --- /dev/null +++ b/spec/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables_spec.rb @@ -0,0 +1,334 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, + :reestablished_active_record_base, query_analyzers: false do + using RSpec::Parameterized::TableSyntax + + let(:schema_class) { Class.new(Gitlab::Database::Migration[2.1]) } + let(:gitlab_main_table_name) { :_test_gitlab_main_table } + let(:gitlab_ci_table_name) { :_test_gitlab_ci_table } + let(:gitlab_geo_table_name) { :_test_gitlab_geo_table } + let(:gitlab_shared_table_name) { :_test_table } + + before do + stub_feature_flags(automatic_lock_writes_on_table: true) + reconfigure_db_connection(model: ActiveRecord::Base, config_model: config_model) + end + + shared_examples 'does not lock writes on table' do |config_model| + let(:config_model) { config_model } + + it 'allows deleting records from the table' do + allow_next_instance_of(Gitlab::Database::LockWritesManager) do |instance| + expect(instance).not_to receive(:lock_writes) + end + + run_migration + + expect do + migration_class.connection.execute("DELETE FROM #{table_name}") + end.not_to raise_error + end + end + + shared_examples 'locks writes on table' do |config_model| + let(:config_model) { config_model } + + it 'errors on deleting' do + allow_next_instance_of(Gitlab::Database::LockWritesManager) do |instance| + expect(instance).to receive(:lock_writes).and_call_original + end + + run_migration + + expect do + migration_class.connection.execute("DELETE FROM #{table_name}") + end.to raise_error(ActiveRecord::StatementInvalid, /is write protected/) + end + end + + context 'when executing create_table migrations' do + let(:create_gitlab_main_table_migration_class) { create_table_migration(gitlab_main_table_name) } + let(:create_gitlab_ci_table_migration_class) { create_table_migration(gitlab_ci_table_name) } + let(:create_gitlab_shared_table_migration_class) { create_table_migration(gitlab_shared_table_name) } + + context 'when single database' do + let(:config_model) { Gitlab::Database.database_base_models[:main] } + + before do + skip_if_multiple_databases_are_setup + end + + it 'does not lock any newly created tables' do + allow_next_instance_of(Gitlab::Database::LockWritesManager) do |instance| + expect(instance).not_to receive(:lock_writes) + end + + create_gitlab_main_table_migration_class.migrate(:up) + create_gitlab_ci_table_migration_class.migrate(:up) + create_gitlab_shared_table_migration_class.migrate(:up) + + expect do + create_gitlab_main_table_migration_class.connection.execute("DELETE FROM #{gitlab_main_table_name}") + create_gitlab_ci_table_migration_class.connection.execute("DELETE FROM #{gitlab_ci_table_name}") + create_gitlab_shared_table_migration_class.connection.execute("DELETE FROM #{gitlab_shared_table_name}") + end.not_to raise_error + end + end + + context 'when multiple databases' do + before do + skip_if_multiple_databases_not_setup + end + + let(:skip_automatic_lock_on_writes) { false } + let(:migration_class) { create_table_migration(table_name, skip_automatic_lock_on_writes) } + let(:run_migration) { migration_class.migrate(:up) } + + context 'for creating a gitlab_main table' do + let(:table_name) { gitlab_main_table_name } + + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:main] + it_behaves_like 'locks writes on table', Gitlab::Database.database_base_models[:ci] + + context 'when table listed as a deleted table' do + before do + stub_const("Gitlab::Database::GitlabSchema::DELETED_TABLES", { table_name.to_s => :gitlab_main }) + end + + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:ci] + end + + context 'when the migration skips automatic locking of tables' do + let(:skip_automatic_lock_on_writes) { true } + + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:ci] + end + + context 'when the SKIP_AUTOMATIC_LOCK_ON_WRITES feature flag is set' do + before do + stub_env('SKIP_AUTOMATIC_LOCK_ON_WRITES' => 'true') + end + + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:ci] + end + + context 'when the automatic_lock_writes_on_table feature flag is disabled' do + before do + stub_feature_flags(automatic_lock_writes_on_table: false) + end + + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:ci] + end + end + + context 'for creating a gitlab_ci table' do + let(:table_name) { gitlab_ci_table_name } + + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:ci] + it_behaves_like 'locks writes on table', Gitlab::Database.database_base_models[:main] + + context 'when table listed as a deleted table' do + before do + stub_const("Gitlab::Database::GitlabSchema::DELETED_TABLES", { table_name.to_s => :gitlab_ci }) + end + + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:main] + end + + context 'when the migration skips automatic locking of tables' do + let(:skip_automatic_lock_on_writes) { true } + + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:main] + end + + context 'when the SKIP_AUTOMATIC_LOCK_ON_WRITES feature flag is set' do + before do + stub_env('SKIP_AUTOMATIC_LOCK_ON_WRITES' => 'true') + end + + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:main] + end + + context 'when the automatic_lock_writes_on_table feature flag is disabled' do + before do + stub_feature_flags(automatic_lock_writes_on_table: false) + end + + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:main] + end + end + + context 'for creating gitlab_shared table' do + let(:table_name) { gitlab_shared_table_name } + + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:main] + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:ci] + end + + context 'for creating a gitlab_geo table' do + before do + skip unless geo_configured? + end + + let(:table_name) { gitlab_geo_table_name } + + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:geo] + end + + context 'for creating an unknown gitlab_schema table' do + let(:table_name) { :foobar } # no gitlab_schema defined + let(:config_model) { Gitlab::Database.database_base_models[:main] } + + it "raises an error about undefined gitlab_schema" do + expected_error_message = <<~ERROR + No gitlab_schema is defined for the table #{table_name}. Please consider + adding it to the database dictionary. + More info: https://docs.gitlab.com/ee/development/database/database_dictionary.html + ERROR + + expect { run_migration }.to raise_error(expected_error_message) + end + end + end + end + + context 'when renaming a table' do + before do + skip_if_multiple_databases_not_setup + create_table_migration(old_table_name).migrate(:up) # create the table first before renaming it + end + + let(:migration_class) { rename_table_migration(old_table_name, table_name) } + let(:run_migration) { migration_class.migrate(:up) } + + context 'when a gitlab_main table' do + let(:old_table_name) { gitlab_main_table_name } + let(:table_name) { :_test_gitlab_main_new_table } + let(:database_base_model) { Gitlab::Database.database_base_models[:main] } + + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:main] + it_behaves_like 'locks writes on table', Gitlab::Database.database_base_models[:ci] + end + + context 'when a gitlab_ci table' do + let(:old_table_name) { gitlab_ci_table_name } + let(:table_name) { :_test_gitlab_ci_new_table } + let(:database_base_model) { Gitlab::Database.database_base_models[:ci] } + + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:ci] + it_behaves_like 'locks writes on table', Gitlab::Database.database_base_models[:main] + end + end + + context 'when reversing drop_table migrations' do + let(:drop_gitlab_main_table_migration_class) { drop_table_migration(gitlab_main_table_name) } + let(:drop_gitlab_ci_table_migration_class) { drop_table_migration(gitlab_ci_table_name) } + let(:drop_gitlab_shared_table_migration_class) { drop_table_migration(gitlab_shared_table_name) } + + context 'when single database' do + let(:config_model) { Gitlab::Database.database_base_models[:main] } + + before do + skip_if_multiple_databases_are_setup + end + + it 'does not lock any newly created tables' do + allow_next_instance_of(Gitlab::Database::LockWritesManager) do |instance| + expect(instance).not_to receive(:lock_writes) + end + + drop_gitlab_main_table_migration_class.connection.execute("CREATE TABLE #{gitlab_main_table_name}()") + drop_gitlab_ci_table_migration_class.connection.execute("CREATE TABLE #{gitlab_ci_table_name}()") + drop_gitlab_shared_table_migration_class.connection.execute("CREATE TABLE #{gitlab_shared_table_name}()") + + drop_gitlab_main_table_migration_class.migrate(:up) + drop_gitlab_ci_table_migration_class.migrate(:up) + drop_gitlab_shared_table_migration_class.migrate(:up) + + drop_gitlab_main_table_migration_class.migrate(:down) + drop_gitlab_ci_table_migration_class.migrate(:down) + drop_gitlab_shared_table_migration_class.migrate(:down) + + expect do + drop_gitlab_main_table_migration_class.connection.execute("DELETE FROM #{gitlab_main_table_name}") + drop_gitlab_ci_table_migration_class.connection.execute("DELETE FROM #{gitlab_ci_table_name}") + drop_gitlab_shared_table_migration_class.connection.execute("DELETE FROM #{gitlab_shared_table_name}") + end.not_to raise_error + end + end + + context 'when multiple databases' do + before do + skip_if_multiple_databases_not_setup + migration_class.connection.execute("CREATE TABLE #{table_name}()") + migration_class.migrate(:up) + end + + let(:migration_class) { drop_table_migration(table_name) } + let(:run_migration) { migration_class.migrate(:down) } + + context 'for re-creating a gitlab_main table' do + let(:table_name) { gitlab_main_table_name } + + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:main] + it_behaves_like 'locks writes on table', Gitlab::Database.database_base_models[:ci] + end + + context 'for re-creating a gitlab_ci table' do + let(:table_name) { gitlab_ci_table_name } + + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:ci] + it_behaves_like 'locks writes on table', Gitlab::Database.database_base_models[:main] + end + + context 'for re-creating a gitlab_shared table' do + let(:table_name) { gitlab_shared_table_name } + + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:main] + it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:ci] + end + end + end + + def create_table_migration(table_name, skip_lock_on_writes = false) + migration_class = Class.new(schema_class) do + class << self; attr_accessor :table_name; end + def change + create_table self.class.table_name + end + end + migration_class.skip_automatic_lock_on_writes = skip_lock_on_writes + migration_class.tap { |klass| klass.table_name = table_name } + end + + def rename_table_migration(old_table_name, new_table_name) + migration_class = Class.new(schema_class) do + class << self; attr_accessor :old_table_name, :new_table_name; end + def change + rename_table self.class.old_table_name, self.class.new_table_name + end + end + + migration_class.tap do |klass| + klass.old_table_name = old_table_name + klass.new_table_name = new_table_name + end + end + + def drop_table_migration(table_name) + migration_class = Class.new(schema_class) do + class << self; attr_accessor :table_name; end + def change + drop_table(self.class.table_name) {} + end + end + migration_class.tap { |klass| klass.table_name = table_name } + end + + def geo_configured? + !!ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'geo') + end +end diff --git a/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb b/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb index e43cfe0814e..e8045f5afec 100644 --- a/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb @@ -14,7 +14,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_a describe '#restrict_gitlab_migration' do it 'invalid schema raises exception' do - expect { schema_class.restrict_gitlab_migration gitlab_schema: :gitlab_non_exisiting } + expect { schema_class.restrict_gitlab_migration gitlab_schema: :gitlab_non_existing } .to raise_error /Unknown 'gitlab_schema:/ end @@ -102,7 +102,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_a "does add index to projects in gitlab_main and gitlab_ci" => { migration: ->(klass) do def change - # Due to running in transactin we cannot use `add_concurrent_index` + # Due to running in transaction we cannot use `add_concurrent_index` add_index :projects, :hidden end end, @@ -185,8 +185,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_a execute("create schema __test_schema") end - def down - end + def down; end end, query_matcher: /create schema __test_schema/, expected: { @@ -306,8 +305,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_a detached_partitions_class.create!(drop_after: Time.current, table_name: '_test_table') end - def down - end + def down; end def detached_partitions_class Class.new(Gitlab::Database::Migration[2.0]::MigrationRecord) do @@ -450,8 +448,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_a ApplicationSetting.last end - def down - end + def down; end end, query_matcher: /FROM "application_settings"/, expected: { @@ -475,8 +472,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_a Feature.enabled?(:redis_hll_tracking, type: :ops) end - def down - end + def down; end end, query_matcher: /FROM "features"/, expected: { @@ -505,8 +501,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_a end end - def down - end + def down; end end, query_matcher: /FROM ci_builds/, setup: -> (_) { skip_if_multiple_databases_not_setup }, diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index 65fbc8d9935..30eeff31326 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -1199,18 +1199,6 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end end - describe '#add_column_with_default' do - let(:column) { Project.columns.find { |c| c.name == "id" } } - - it 'delegates to #add_column' do - expect(model).to receive(:add_column).with(:projects, :foo, :integer, default: 10, limit: nil, null: true) - - model.add_column_with_default(:projects, :foo, :integer, - default: 10, - allow_null: true) - end - end - describe '#rename_column_concurrently' do context 'in a transaction' do it 'raises RuntimeError' do @@ -2006,170 +1994,6 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end end - describe 'sidekiq migration helpers', :redis do - let(:worker) do - Class.new do - include Sidekiq::Worker - - sidekiq_options queue: 'test' - - def self.name - 'WorkerClass' - end - end - end - - let(:same_queue_different_worker) do - Class.new do - include Sidekiq::Worker - - sidekiq_options queue: 'test' - - def self.name - 'SameQueueDifferentWorkerClass' - end - end - end - - let(:unrelated_worker) do - Class.new do - include Sidekiq::Worker - - sidekiq_options queue: 'unrelated' - - def self.name - 'UnrelatedWorkerClass' - end - end - end - - before do - stub_const(worker.name, worker) - stub_const(unrelated_worker.name, unrelated_worker) - stub_const(same_queue_different_worker.name, same_queue_different_worker) - end - - describe '#sidekiq_remove_jobs', :clean_gitlab_redis_queues do - def clear_queues - Sidekiq::Queue.new('test').clear - Sidekiq::Queue.new('unrelated').clear - Sidekiq::RetrySet.new.clear - Sidekiq::ScheduledSet.new.clear - end - - around do |example| - clear_queues - Sidekiq::Testing.disable!(&example) - clear_queues - end - - it "removes all related job instances from the job class's queue" do - worker.perform_async - same_queue_different_worker.perform_async - unrelated_worker.perform_async - - queue_we_care_about = Sidekiq::Queue.new(worker.queue) - unrelated_queue = Sidekiq::Queue.new(unrelated_worker.queue) - - expect(queue_we_care_about.size).to eq(2) - expect(unrelated_queue.size).to eq(1) - - model.sidekiq_remove_jobs(job_klass: worker) - - expect(queue_we_care_about.size).to eq(1) - expect(queue_we_care_about.map(&:klass)).not_to include(worker.name) - expect(queue_we_care_about.map(&:klass)).to include( - same_queue_different_worker.name - ) - expect(unrelated_queue.size).to eq(1) - end - - context 'when job instances are in the scheduled set' do - it 'removes all related job instances from the scheduled set' do - worker.perform_in(1.hour) - unrelated_worker.perform_in(1.hour) - - scheduled = Sidekiq::ScheduledSet.new - - expect(scheduled.size).to eq(2) - expect(scheduled.map(&:klass)).to include( - worker.name, - unrelated_worker.name - ) - - model.sidekiq_remove_jobs(job_klass: worker) - - expect(scheduled.size).to eq(1) - expect(scheduled.map(&:klass)).not_to include(worker.name) - expect(scheduled.map(&:klass)).to include(unrelated_worker.name) - end - end - - context 'when job instances are in the retry set' do - include_context 'when handling retried jobs' - - it 'removes all related job instances from the retry set' do - retry_in(worker, 1.hour) - retry_in(worker, 2.hours) - retry_in(worker, 3.hours) - retry_in(unrelated_worker, 4.hours) - - retries = Sidekiq::RetrySet.new - - expect(retries.size).to eq(4) - expect(retries.map(&:klass)).to include( - worker.name, - unrelated_worker.name - ) - - model.sidekiq_remove_jobs(job_klass: worker) - - expect(retries.size).to eq(1) - expect(retries.map(&:klass)).not_to include(worker.name) - expect(retries.map(&:klass)).to include(unrelated_worker.name) - end - end - end - - describe '#sidekiq_queue_length' do - context 'when queue is empty' do - it 'returns zero' do - Sidekiq::Testing.disable! do - expect(model.sidekiq_queue_length('test')).to eq 0 - end - end - end - - context 'when queue contains jobs' do - it 'returns correct size of the queue' do - Sidekiq::Testing.disable! do - worker.perform_async('Something', [1]) - worker.perform_async('Something', [2]) - - expect(model.sidekiq_queue_length('test')).to eq 2 - end - end - end - end - - describe '#sidekiq_queue_migrate' do - it 'migrates jobs from one sidekiq queue to another' do - Sidekiq::Testing.disable! do - worker.perform_async('Something', [1]) - worker.perform_async('Something', [2]) - - expect(model.sidekiq_queue_length('test')).to eq 2 - expect(model.sidekiq_queue_length('new_test')).to eq 0 - - model.sidekiq_queue_migrate('test', to: 'new_test') - - expect(model.sidekiq_queue_length('test')).to eq 0 - expect(model.sidekiq_queue_length('new_test')).to eq 2 - end - end - end - end - describe '#check_trigger_permissions!' do it 'does nothing when the user has the correct permissions' do expect { model.check_trigger_permissions!('users') } @@ -2790,18 +2614,18 @@ RSpec.describe Gitlab::Database::MigrationHelpers do model.backfill_iids('issues') - issue = issue_class.create!(project_id: project.id) + issue = issue_class.create!(project_id: project.id, namespace_id: project.project_namespace_id) expect(issue.iid).to eq(1) end it 'generates iids properly for models created after the migration when iids are backfilled' do project = setup - issue_a = issues.create!(project_id: project.id, work_item_type_id: issue_type.id) + issue_a = issues.create!(project_id: project.id, namespace_id: project.project_namespace_id, work_item_type_id: issue_type.id) model.backfill_iids('issues') - issue_b = issue_class.create!(project_id: project.id) + issue_b = issue_class.create!(project_id: project.id, namespace_id: project.project_namespace_id) expect(issue_a.reload.iid).to eq(1) expect(issue_b.iid).to eq(2) @@ -2810,14 +2634,14 @@ RSpec.describe Gitlab::Database::MigrationHelpers do it 'generates iids properly for models created after the migration across multiple projects' do project_a = setup project_b = setup - issues.create!(project_id: project_a.id, work_item_type_id: issue_type.id) - issues.create!(project_id: project_b.id, work_item_type_id: issue_type.id) - issues.create!(project_id: project_b.id, work_item_type_id: issue_type.id) + issues.create!(project_id: project_a.id, namespace_id: project_a.project_namespace_id, work_item_type_id: issue_type.id) + issues.create!(project_id: project_b.id, namespace_id: project_b.project_namespace_id, work_item_type_id: issue_type.id) + issues.create!(project_id: project_b.id, namespace_id: project_b.project_namespace_id, work_item_type_id: issue_type.id) model.backfill_iids('issues') - issue_a = issue_class.create!(project_id: project_a.id, work_item_type_id: issue_type.id) - issue_b = issue_class.create!(project_id: project_b.id, work_item_type_id: issue_type.id) + issue_a = issue_class.create!(project_id: project_a.id, namespace_id: project_a.project_namespace_id, work_item_type_id: issue_type.id) + issue_b = issue_class.create!(project_id: project_b.id, namespace_id: project_b.project_namespace_id, work_item_type_id: issue_type.id) expect(issue_a.iid).to eq(2) expect(issue_b.iid).to eq(3) @@ -2827,11 +2651,11 @@ RSpec.describe Gitlab::Database::MigrationHelpers do it 'generates an iid' do project_a = setup project_b = setup - issue_a = issues.create!(project_id: project_a.id, work_item_type_id: issue_type.id) + issue_a = issues.create!(project_id: project_a.id, namespace_id: project_a.project_namespace_id, work_item_type_id: issue_type.id) model.backfill_iids('issues') - issue_b = issue_class.create!(project_id: project_b.id) + issue_b = issue_class.create!(project_id: project_b.id, namespace_id: project_b.project_namespace_id) expect(issue_a.reload.iid).to eq(1) expect(issue_b.reload.iid).to eq(1) @@ -2841,8 +2665,8 @@ RSpec.describe Gitlab::Database::MigrationHelpers do context 'when a row already has an iid set in the database' do it 'backfills iids' do project = setup - issue_a = issues.create!(project_id: project.id, work_item_type_id: issue_type.id, iid: 1) - issue_b = issues.create!(project_id: project.id, work_item_type_id: issue_type.id, iid: 2) + issue_a = issues.create!(project_id: project.id, namespace_id: project.project_namespace_id, work_item_type_id: issue_type.id, iid: 1) + issue_b = issues.create!(project_id: project.id, namespace_id: project.project_namespace_id, work_item_type_id: issue_type.id, iid: 2) model.backfill_iids('issues') @@ -2853,9 +2677,9 @@ RSpec.describe Gitlab::Database::MigrationHelpers do it 'backfills for multiple projects' do project_a = setup project_b = setup - issue_a = issues.create!(project_id: project_a.id, work_item_type_id: issue_type.id, iid: 1) - issue_b = issues.create!(project_id: project_b.id, work_item_type_id: issue_type.id, iid: 1) - issue_c = issues.create!(project_id: project_a.id, work_item_type_id: issue_type.id, iid: 2) + issue_a = issues.create!(project_id: project_a.id, namespace_id: project_a.project_namespace_id, work_item_type_id: issue_type.id, iid: 1) + issue_b = issues.create!(project_id: project_b.id, namespace_id: project_b.project_namespace_id, work_item_type_id: issue_type.id, iid: 1) + issue_c = issues.create!(project_id: project_a.id, namespace_id: project_a.project_namespace_id, work_item_type_id: issue_type.id, iid: 2) model.backfill_iids('issues') diff --git a/spec/lib/gitlab/database/migrations/batched_migration_last_id_spec.rb b/spec/lib/gitlab/database/migrations/batched_migration_last_id_spec.rb new file mode 100644 index 00000000000..97b432406eb --- /dev/null +++ b/spec/lib/gitlab/database/migrations/batched_migration_last_id_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Migrations::BatchedMigrationLastId, feature_category: :pipeline_insights do + subject(:test_sampling) { described_class.new(connection, base_dir) } + + let(:base_dir) { Pathname.new(Dir.mktmpdir) } + let(:file_name) { 'last-batched-background-migration-id.txt' } + let(:file_path) { base_dir.join(file_name) } + let(:file_contents) { nil } + + where(:base_model) do + [ + [ApplicationRecord], [Ci::ApplicationRecord] + ] + end + + with_them do + let(:connection) { base_model.connection } + + after do + FileUtils.rm_rf(file_path) + end + + describe '#read' do + before do + File.write(file_path, file_contents) + end + + context 'when the file exists and have content' do + let(:file_contents) { 99 } + + it { expect(test_sampling.read).to eq(file_contents) } + end + + context 'when the file exists and is blank' do + it { expect(test_sampling.read).to be_nil } + end + + context "when the file doesn't exists" do + before do + FileUtils.rm_rf(file_path) + end + + it { expect(test_sampling.read).to be_nil } + end + end + + describe '#store' do + let(:file_contents) { File.read(file_path) } + let(:migration) do + Gitlab::Database::SharedModel.using_connection(connection) do + create(:batched_background_migration) + end + end + + it 'creates the file properly' do + test_sampling.store + + expect(File).to exist(file_path) + end + + it 'stores the proper id in the file' do + migration + test_sampling.store + + expect(file_contents).to eq(migration.id.to_s) + end + end + end +end diff --git a/spec/lib/gitlab/database/migrations/runner_spec.rb b/spec/lib/gitlab/database/migrations/runner_spec.rb index bd382547689..66eb5a5de51 100644 --- a/spec/lib/gitlab/database/migrations/runner_spec.rb +++ b/spec/lib/gitlab/database/migrations/runner_spec.rb @@ -230,5 +230,13 @@ RSpec.describe Gitlab::Database::Migrations::Runner, :reestablished_active_recor end end end + + describe '.batched_migrations_last_id' do + let(:runner_class) { Gitlab::Database::Migrations::BatchedMigrationLastId } + + it 'matches the expected runner class' do + expect(described_class.batched_migrations_last_id(database)).to be_a(runner_class) + end + end end end diff --git a/spec/lib/gitlab/database/migrations/sidekiq_helpers_spec.rb b/spec/lib/gitlab/database/migrations/sidekiq_helpers_spec.rb new file mode 100644 index 00000000000..fb1cb46171f --- /dev/null +++ b/spec/lib/gitlab/database/migrations/sidekiq_helpers_spec.rb @@ -0,0 +1,276 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe Gitlab::Database::Migrations::SidekiqHelpers do + let(:model) do + ActiveRecord::Migration.new.extend(described_class) + end + + describe "sidekiq migration helpers", :redis do + let(:worker) do + Class.new do + include Sidekiq::Worker + + sidekiq_options queue: "test" + + def self.name + "WorkerClass" + end + end + end + + let(:worker_two) do + Class.new do + include Sidekiq::Worker + + sidekiq_options queue: "test_two" + + def self.name + "WorkerTwoClass" + end + end + end + + let(:same_queue_different_worker) do + Class.new do + include Sidekiq::Worker + + sidekiq_options queue: "test" + + def self.name + "SameQueueDifferentWorkerClass" + end + end + end + + let(:unrelated_worker) do + Class.new do + include Sidekiq::Worker + + sidekiq_options queue: "unrelated" + + def self.name + "UnrelatedWorkerClass" + end + end + end + + before do + stub_const(worker.name, worker) + stub_const(worker_two.name, worker_two) + stub_const(unrelated_worker.name, unrelated_worker) + stub_const(same_queue_different_worker.name, same_queue_different_worker) + end + + describe "#sidekiq_remove_jobs", :clean_gitlab_redis_queues do + def clear_queues + Sidekiq::Queue.new("test").clear + Sidekiq::Queue.new("test_two").clear + Sidekiq::Queue.new("unrelated").clear + Sidekiq::RetrySet.new.clear + Sidekiq::ScheduledSet.new.clear + end + + around do |example| + clear_queues + Sidekiq::Testing.disable!(&example) + clear_queues + end + + context "when the constant is not defined" do + it "doesn't try to delete it" do + my_non_constant = +"SomeThingThatIsNotAConstant" + + expect(Sidekiq::Queue).not_to receive(:new).with(any_args) + model.sidekiq_remove_jobs(job_klasses: [my_non_constant]) + end + end + + context "when the constant is defined" do + it "will use it find job instances to delete" do + my_constant = worker.name + expect(Sidekiq::Queue) + .to receive(:new) + .with(worker.queue) + .and_call_original + model.sidekiq_remove_jobs(job_klasses: [my_constant]) + end + end + + it "removes all related job instances from the job classes' queues" do + worker.perform_async + worker_two.perform_async + same_queue_different_worker.perform_async + unrelated_worker.perform_async + + worker_queue = Sidekiq::Queue.new(worker.queue) + worker_two_queue = Sidekiq::Queue.new(worker_two.queue) + unrelated_queue = Sidekiq::Queue.new(unrelated_worker.queue) + + expect(worker_queue.size).to eq(2) + expect(worker_two_queue.size).to eq(1) + expect(unrelated_queue.size).to eq(1) + + model.sidekiq_remove_jobs(job_klasses: [worker.name, worker_two.name]) + + expect(worker_queue.size).to eq(1) + expect(worker_two_queue.size).to eq(0) + expect(worker_queue.map(&:klass)).not_to include(worker.name) + expect(worker_queue.map(&:klass)).to include( + same_queue_different_worker.name + ) + expect(worker_two_queue.map(&:klass)).not_to include(worker_two.name) + expect(unrelated_queue.size).to eq(1) + end + + context "when job instances are in the scheduled set" do + it "removes all related job instances from the scheduled set" do + worker.perform_in(1.hour) + worker_two.perform_in(1.hour) + unrelated_worker.perform_in(1.hour) + + scheduled = Sidekiq::ScheduledSet.new + + expect(scheduled.size).to eq(3) + expect(scheduled.map(&:klass)).to include( + worker.name, + worker_two.name, + unrelated_worker.name + ) + + model.sidekiq_remove_jobs(job_klasses: [worker.name, worker_two.name]) + + expect(scheduled.size).to eq(1) + expect(scheduled.map(&:klass)).not_to include(worker.name) + expect(scheduled.map(&:klass)).not_to include(worker_two.name) + expect(scheduled.map(&:klass)).to include(unrelated_worker.name) + end + end + + context "when job instances are in the retry set" do + include_context "when handling retried jobs" + + it "removes all related job instances from the retry set" do + retry_in(worker, 1.hour) + retry_in(worker, 2.hours) + retry_in(worker, 3.hours) + retry_in(worker_two, 4.hours) + retry_in(unrelated_worker, 5.hours) + + retries = Sidekiq::RetrySet.new + + expect(retries.size).to eq(5) + expect(retries.map(&:klass)).to include( + worker.name, + worker_two.name, + unrelated_worker.name + ) + + model.sidekiq_remove_jobs(job_klasses: [worker.name, worker_two.name]) + + expect(retries.size).to eq(1) + expect(retries.map(&:klass)).not_to include(worker.name) + expect(retries.map(&:klass)).not_to include(worker_two.name) + expect(retries.map(&:klass)).to include(unrelated_worker.name) + end + end + + # Imitate job deletion returning zero and then non zero. + context "when job fails to be deleted" do + let(:job_double) do + instance_double( + "Sidekiq::JobRecord", + klass: worker.name + ) + end + + context "and does not work enough times in a row before max attempts" do + it "tries the max attempts without succeeding" do + worker.perform_async + + allow(job_double).to receive(:delete).and_return(true) + + # Scheduled set runs last so only need to stub out its values. + allow(Sidekiq::ScheduledSet) + .to receive(:new) + .and_return([job_double]) + + expect(model.sidekiq_remove_jobs(job_klasses: [worker.name])) + .to eq( + { + attempts: 5, + success: false + } + ) + end + end + + context "and then it works enough times in a row before max attempts" do + it "succeeds" do + worker.perform_async + + # attempt 1: false will increment the streak once to 1 + # attempt 2: true resets it back to 0 + # attempt 3: false will increment the streak once to 1 + # attempt 4: false will increment the streak once to 2, loop breaks + allow(job_double).to receive(:delete).and_return(false, true, false) + + worker.perform_async + + # Scheduled set runs last so only need to stub out its values. + allow(Sidekiq::ScheduledSet) + .to receive(:new) + .and_return([job_double]) + + expect(model.sidekiq_remove_jobs(job_klasses: [worker.name])) + .to eq( + { + attempts: 4, + success: true + } + ) + end + end + end + end + + describe "#sidekiq_queue_length" do + context "when queue is empty" do + it "returns zero" do + Sidekiq::Testing.disable! do + expect(model.sidekiq_queue_length("test")).to eq 0 + end + end + end + + context "when queue contains jobs" do + it "returns correct size of the queue" do + Sidekiq::Testing.disable! do + worker.perform_async("Something", [1]) + worker.perform_async("Something", [2]) + + expect(model.sidekiq_queue_length("test")).to eq 2 + end + end + end + end + + describe "#sidekiq_queue_migrate" do + it "migrates jobs from one sidekiq queue to another" do + Sidekiq::Testing.disable! do + worker.perform_async("Something", [1]) + worker.perform_async("Something", [2]) + + expect(model.sidekiq_queue_length("test")).to eq 2 + expect(model.sidekiq_queue_length("new_test")).to eq 0 + + model.sidekiq_queue_migrate("test", to: "new_test") + + expect(model.sidekiq_queue_length("test")).to eq 0 + expect(model.sidekiq_queue_length("new_test")).to eq 2 + end + end + end + end +end diff --git a/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb b/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb index 07226f3d025..73d69d55e5a 100644 --- a/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb +++ b/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb @@ -55,6 +55,8 @@ RSpec.describe Gitlab::Database::Migrations::TestBatchedBackgroundRunner, :freez let(:table_name) { "_test_column_copying" } + let(:from_id) { 0 } + before do connection.execute(<<~SQL) CREATE TABLE #{table_name} ( @@ -76,7 +78,8 @@ RSpec.describe Gitlab::Database::Migrations::TestBatchedBackgroundRunner, :freez end subject(:sample_migration) do - described_class.new(result_dir: result_dir, connection: connection).run_jobs(for_duration: 1.minute) + described_class.new(result_dir: result_dir, connection: connection, + from_id: from_id).run_jobs(for_duration: 1.minute) end it 'runs sampled jobs from the batched background migration' do @@ -111,20 +114,26 @@ RSpec.describe Gitlab::Database::Migrations::TestBatchedBackgroundRunner, :freez job_interval: 5.minutes, batch_size: 100) - described_class.new(result_dir: result_dir, connection: connection).run_jobs(for_duration: 3.minutes) + described_class.new(result_dir: result_dir, connection: connection, + from_id: from_id).run_jobs(for_duration: 3.minutes) expect(calls).not_to be_empty end context 'with multiple jobs to run' do - it 'runs all jobs created within the last 3 hours' do + let(:last_id) do + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::BackgroundMigration::BatchedMigration.maximum(:id) + end + end + + it 'runs all pending jobs based on the last migration id' do old_migration = define_background_migration(migration_name) queue_migration(migration_name, table_name, :id, job_interval: 5.minutes, batch_size: 100) - travel 4.hours - + last_id new_migration = define_background_migration('NewMigration') { travel 1.second } queue_migration('NewMigration', table_name, :id, job_interval: 5.minutes, @@ -138,14 +147,15 @@ RSpec.describe Gitlab::Database::Migrations::TestBatchedBackgroundRunner, :freez sub_batch_size: 5) expect_migration_runs(new_migration => 3, other_new_migration => 2, old_migration => 0) do - described_class.new(result_dir: result_dir, connection: connection).run_jobs(for_duration: 5.seconds) + described_class.new(result_dir: result_dir, connection: connection, + from_id: last_id).run_jobs(for_duration: 5.seconds) end end end end context 'choosing uniform batches to run' do - subject { described_class.new(result_dir: result_dir, connection: connection) } + subject { described_class.new(result_dir: result_dir, connection: connection, from_id: from_id) } describe '#uniform_fractions' do it 'generates evenly distributed sequences of fractions' do diff --git a/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb b/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb index 336dec3a8a0..646ae50fb44 100644 --- a/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb +++ b/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb @@ -41,7 +41,7 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do SQL end - def create_partition(name:, table: 'parent_table', from:, to:, attached:, drop_after:) + def create_partition(name:, from:, to:, attached:, drop_after:, table: 'parent_table') from = from.beginning_of_month to = to.beginning_of_month full_name = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{name}" diff --git a/spec/lib/gitlab/database/partitioning/sliding_list_strategy_spec.rb b/spec/lib/gitlab/database/partitioning/sliding_list_strategy_spec.rb index 550f254c4da..e6014f81b74 100644 --- a/spec/lib/gitlab/database/partitioning/sliding_list_strategy_spec.rb +++ b/spec/lib/gitlab/database/partitioning/sliding_list_strategy_spec.rb @@ -229,11 +229,9 @@ RSpec.describe Gitlab::Database::Partitioning::SlidingListStrategy do next_partition_if: method(:next_partition_if_wrapper), detach_partition_if: method(:detach_partition_if_wrapper) - def self.next_partition?(current_partition) - end + def self.next_partition?(current_partition); end - def self.detach_partition?(partition) - end + def self.detach_partition?(partition); end end end diff --git a/spec/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin_spec.rb b/spec/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin_spec.rb index 8b06f068503..884c4f625dd 100644 --- a/spec/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin_spec.rb +++ b/spec/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin_spec.rb @@ -9,8 +9,7 @@ RSpec.describe Gitlab::Database::PostgresqlAdapter::DumpSchemaVersionsMixin do original_dump_schema_information end - def original_dump_schema_information - end + def original_dump_schema_information; end end klass.prepend(described_class) diff --git a/spec/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin_spec.rb b/spec/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin_spec.rb index 3e675a85999..3bb206c6627 100644 --- a/spec/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin_spec.rb +++ b/spec/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin_spec.rb @@ -9,8 +9,7 @@ RSpec.describe Gitlab::Database::PostgresqlDatabaseTasks::LoadSchemaVersionsMixi original_structure_load end - def original_structure_load - end + def original_structure_load; end end klass.prepend(described_class) diff --git a/spec/lib/gitlab/database/query_analyzers/query_recorder_spec.rb b/spec/lib/gitlab/database/query_analyzers/query_recorder_spec.rb index ec01ae623ae..bcc39c0c3db 100644 --- a/spec/lib/gitlab/database/query_analyzers/query_recorder_spec.rb +++ b/spec/lib/gitlab/database/query_analyzers/query_recorder_spec.rb @@ -10,29 +10,76 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::QueryRecorder, query_analyzers: end end - context 'when analyzer is enabled for tests' do + context 'with query analyzer' do let(:query) { 'SELECT 1 FROM projects' } - let(:log_path) { Rails.root.join(described_class::LOG_FILE) } + let(:log_path) { Rails.root.join(described_class::LOG_PATH) } + let(:log_file) { described_class.log_file } - before do - stub_env('CI', 'true') + after do + ::Gitlab::Database::QueryAnalyzer.instance.end!([described_class]) + end - # This is needed to be able to stub_env the CI variable - ::Gitlab::Database::QueryAnalyzer.instance.begin!([described_class]) + shared_examples_for 'an enabled query recorder' do + it 'logs queries to a file' do + allow(FileUtils).to receive(:mkdir_p) + .with(log_path) + expect(File).to receive(:write) + .with(log_file, /^{"sql":"#{query}/, mode: 'a') + expect(described_class).to receive(:analyze).with(/^#{query}/).and_call_original + + expect { ApplicationRecord.connection.execute(query) }.not_to raise_error + end end - after do - ::Gitlab::Database::QueryAnalyzer.instance.end!([described_class]) + context 'on default branch' do + before do + stub_env('CI_MERGE_REQUEST_LABELS', nil) + stub_env('CI_DEFAULT_BRANCH', 'default_branch_name') + stub_env('CI_COMMIT_REF_NAME', 'default_branch_name') + + # This is needed to be able to stub_env the CI variable + ::Gitlab::Database::QueryAnalyzer.instance.begin!([described_class]) + end + + it_behaves_like 'an enabled query recorder' + end + + context 'on database merge requests' do + before do + stub_env('CI_MERGE_REQUEST_LABELS', 'database') + + # This is needed to be able to stub_env the CI variable + ::Gitlab::Database::QueryAnalyzer.instance.begin!([described_class]) + end + + it_behaves_like 'an enabled query recorder' + end + end + + describe '.log_file' do + let(:folder) { 'query_recorder' } + let(:extension) { 'ndjson' } + let(:default_name) { 'rspec' } + let(:job_name) { 'test-job-1' } + + subject { described_class.log_file.to_s } + + context 'when in CI' do + before do + stub_env('CI_JOB_NAME_SLUG', job_name) + end + + it { is_expected.to include("#{folder}/#{job_name}.#{extension}") } + it { is_expected.not_to include("#{folder}/#{default_name}.#{extension}") } end - it 'logs queries to a file' do - allow(FileUtils).to receive(:mkdir_p) - .with(File.dirname(log_path)) - expect(File).to receive(:write) - .with(log_path, /^{"sql":"#{query}/, mode: 'a') - expect(described_class).to receive(:analyze).with(/^#{query}/).and_call_original + context 'when not in CI' do + before do + stub_env('CI_JOB_NAME_SLUG', nil) + end - expect { ApplicationRecord.connection.execute(query) }.not_to raise_error + it { is_expected.to include("#{folder}/#{default_name}.#{extension}") } + it { is_expected.not_to include("#{folder}/#{job_name}.#{extension}") } end end end diff --git a/spec/lib/gitlab/database/reindexing_spec.rb b/spec/lib/gitlab/database/reindexing_spec.rb index 4c98185e780..fa26aa59329 100644 --- a/spec/lib/gitlab/database/reindexing_spec.rb +++ b/spec/lib/gitlab/database/reindexing_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::Reindexing do +RSpec.describe Gitlab::Database::Reindexing, feature_category: :database do include ExclusiveLeaseHelpers include Database::DatabaseHelpers diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb index ac2de43b7c6..c507bce634e 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb @@ -97,39 +97,48 @@ RSpec.describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespa let(:namespace) { create(:group, name: 'hello-group') } it 'moves a project for a namespace' do - create(:project, :repository, :legacy_storage, namespace: namespace, path: 'hello-project') - expected_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - File.join(TestEnv.repos_path, 'bye-group', 'hello-project.git') - end + project = create(:project, :repository, :legacy_storage, namespace: namespace, path: 'hello-project') + expected_repository = Gitlab::Git::Repository.new( + project.repository_storage, + 'bye-group/hello-project.git', + nil, + nil + ) subject.move_repositories(namespace, 'hello-group', 'bye-group') - expect(File.directory?(expected_path)).to be(true) + expect(expected_repository).to exist end it 'moves a namespace in a subdirectory correctly' do child_namespace = create(:group, name: 'sub-group', parent: namespace) - create(:project, :repository, :legacy_storage, namespace: child_namespace, path: 'hello-project') + project = create(:project, :repository, :legacy_storage, namespace: child_namespace, path: 'hello-project') - expected_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - File.join(TestEnv.repos_path, 'hello-group', 'renamed-sub-group', 'hello-project.git') - end + expected_repository = Gitlab::Git::Repository.new( + project.repository_storage, + 'hello-group/renamed-sub-group/hello-project.git', + nil, + nil + ) subject.move_repositories(child_namespace, 'hello-group/sub-group', 'hello-group/renamed-sub-group') - expect(File.directory?(expected_path)).to be(true) + expect(expected_repository).to exist end it 'moves a parent namespace with subdirectories' do child_namespace = create(:group, name: 'sub-group', parent: namespace) - create(:project, :repository, :legacy_storage, namespace: child_namespace, path: 'hello-project') - expected_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - File.join(TestEnv.repos_path, 'renamed-group', 'sub-group', 'hello-project.git') - end + project = create(:project, :repository, :legacy_storage, namespace: child_namespace, path: 'hello-project') + expected_repository = Gitlab::Git::Repository.new( + project.repository_storage, + 'renamed-group/sub-group/hello-project.git', + nil, + nil + ) subject.move_repositories(child_namespace, 'hello-group', 'renamed-group') - expect(File.directory?(expected_path)).to be(true) + expect(expected_repository).to exist end end @@ -175,14 +184,17 @@ RSpec.describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespa describe '#rename_namespace_dependencies' do it "moves the repository for a project in the namespace" do - create(:project, :repository, :legacy_storage, namespace: namespace, path: "the-path-project") - expected_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - File.join(TestEnv.repos_path, "the-path0", "the-path-project.git") - end + project = create(:project, :repository, :legacy_storage, namespace: namespace, path: "the-path-project") + expected_repository = Gitlab::Git::Repository.new( + project.repository_storage, + "the-path0/the-path-project.git", + nil, + nil + ) subject.rename_namespace_dependencies(namespace, 'the-path', 'the-path0') - expect(File.directory?(expected_repo)).to be(true) + expect(expected_repository).to exist end it "moves the uploads for the namespace" do @@ -276,9 +288,7 @@ RSpec.describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespa project.create_repository subject.rename_namespace(namespace) - expected_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - File.join(TestEnv.repos_path, 'the-path', 'a-project.git') - end + expected_repository = Gitlab::Git::Repository.new(project.repository_storage, 'the-path/a-project.git', nil, nil) expect(subject).to receive(:rename_namespace_dependencies) .with( @@ -289,7 +299,7 @@ RSpec.describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespa subject.revert_renames - expect(File.directory?(expected_path)).to be_truthy + expect(expected_repository).to exist end it "doesn't break when the namespace was renamed" do diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb index 6292f0246f7..aa2a3329477 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb @@ -126,13 +126,16 @@ RSpec.describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProject let(:project) { create(:project, :repository, :legacy_storage, path: 'the-path', namespace: known_parent) } it 'moves the repository for a project' do - expected_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - File.join(TestEnv.repos_path, 'known-parent', 'new-repo.git') - end + expected_repository = Gitlab::Git::Repository.new( + project.repository_storage, + 'known-parent/new-repo.git', + nil, + nil + ) subject.move_repository(project, 'known-parent/the-path', 'known-parent/new-repo') - expect(File.directory?(expected_path)).to be(true) + expect(expected_repository).to exist end end @@ -157,9 +160,12 @@ RSpec.describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProject project.create_repository subject.rename_project(project) - expected_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - File.join(TestEnv.repos_path, 'known-parent', 'the-path.git') - end + expected_repository = Gitlab::Git::Repository.new( + project.repository_storage, + 'known-parent/the-path.git', + nil, + nil + ) expect(subject).to receive(:move_project_folders) .with( @@ -170,7 +176,7 @@ RSpec.describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProject subject.revert_renames - expect(File.directory?(expected_path)).to be_truthy + expect(expected_repository).to exist end it "doesn't break when the project was renamed" do diff --git a/spec/lib/gitlab/database/schema_cleaner_spec.rb b/spec/lib/gitlab/database/schema_cleaner_spec.rb index 950759c7f96..5283b34ca86 100644 --- a/spec/lib/gitlab/database/schema_cleaner_spec.rb +++ b/spec/lib/gitlab/database/schema_cleaner_spec.rb @@ -19,6 +19,15 @@ RSpec.describe Gitlab::Database::SchemaCleaner do expect(subject).not_to match(/public\.\w+/) end + it 'cleans up all the gitlab_schema_prevent_write table triggers' do + expect(subject).not_to match(/CREATE TRIGGER gitlab_schema_write_trigger_for_\w+/) + expect(subject).not_to match(/FOR EACH STATEMENT EXECUTE FUNCTION gitlab_schema_prevent_write/) + end + + it 'keeps the lock_writes trigger functions' do + expect(subject).to match(/CREATE FUNCTION gitlab_schema_prevent_write/) + end + it 'cleans up the full schema as expected (blackbox test with example)' do expected_schema = fixture_file(File.join('gitlab', 'database', 'structure_example_cleaned.sql')) diff --git a/spec/lib/gitlab/database/tables_sorted_by_foreign_keys_spec.rb b/spec/lib/gitlab/database/tables_sorted_by_foreign_keys_spec.rb index 97abd6d23bd..aa25590ed58 100644 --- a/spec/lib/gitlab/database/tables_sorted_by_foreign_keys_spec.rb +++ b/spec/lib/gitlab/database/tables_sorted_by_foreign_keys_spec.rb @@ -4,7 +4,10 @@ require 'spec_helper' RSpec.describe Gitlab::Database::TablesSortedByForeignKeys do let(:connection) { ApplicationRecord.connection } - let(:tables) { %w[_test_gitlab_main_items _test_gitlab_main_references] } + let(:tables) do + %w[_test_gitlab_main_items _test_gitlab_main_references _test_gitlab_partition_parent + gitlab_partitions_dynamic._test_gitlab_partition_20220101] + end subject do described_class.new(connection, tables).execute @@ -19,13 +22,33 @@ RSpec.describe Gitlab::Database::TablesSortedByForeignKeys do item_id BIGINT NOT NULL, CONSTRAINT fk_constrained_1 FOREIGN KEY(item_id) REFERENCES _test_gitlab_main_items(id) ); + + CREATE TABLE _test_gitlab_partition_parent ( + id bigserial not null, + created_at timestamptz not null, + item_id BIGINT NOT NULL, + primary key (id, created_at), + CONSTRAINT fk_constrained_1 FOREIGN KEY(item_id) REFERENCES _test_gitlab_main_items(id) + ) PARTITION BY RANGE(created_at); + + CREATE TABLE gitlab_partitions_dynamic._test_gitlab_partition_20220101 + PARTITION OF _test_gitlab_partition_parent + FOR VALUES FROM ('20220101') TO ('20220131'); + + ALTER TABLE _test_gitlab_partition_parent DETACH PARTITION gitlab_partitions_dynamic._test_gitlab_partition_20220101; SQL connection.execute(statement) end describe '#execute' do it 'returns the tables sorted by the foreign keys dependency' do - expect(subject).to eq([['_test_gitlab_main_references'], ['_test_gitlab_main_items']]) + expect(subject).to eq( + [ + ['_test_gitlab_main_references'], + ['_test_gitlab_partition_parent'], + ['gitlab_partitions_dynamic._test_gitlab_partition_20220101'], + ['_test_gitlab_main_items'] + ]) end it 'returns both tables together if they are strongly connected' do @@ -35,7 +58,12 @@ RSpec.describe Gitlab::Database::TablesSortedByForeignKeys do SQL connection.execute(statement) - expect(subject).to eq([tables]) + expect(subject).to eq( + [ + ['_test_gitlab_partition_parent'], + ['gitlab_partitions_dynamic._test_gitlab_partition_20220101'], + %w[_test_gitlab_main_items _test_gitlab_main_references] + ]) end end end diff --git a/spec/lib/gitlab/database/tables_truncate_spec.rb b/spec/lib/gitlab/database/tables_truncate_spec.rb index 4f68cd93a8e..4d04bd67a1e 100644 --- a/spec/lib/gitlab/database/tables_truncate_spec.rb +++ b/spec/lib/gitlab/database/tables_truncate_spec.rb @@ -6,14 +6,9 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba :suppress_gitlab_schemas_validate_connection do include MigrationsHelpers - let(:logger) { instance_double(Logger) } - let(:dry_run) { false } - let(:until_table) { nil } let(:min_batch_size) { 1 } let(:main_connection) { ApplicationRecord.connection } let(:ci_connection) { Ci::ApplicationRecord.connection } - let(:test_gitlab_main_table) { '_test_gitlab_main_table' } - let(:test_gitlab_ci_table) { '_test_gitlab_ci_table' } # Main Database let(:main_db_main_item_model) { table("_test_gitlab_main_items", database: "main") } @@ -21,24 +16,37 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba let(:main_db_ci_item_model) { table("_test_gitlab_ci_items", database: "main") } let(:main_db_ci_reference_model) { table("_test_gitlab_ci_references", database: "main") } let(:main_db_shared_item_model) { table("_test_gitlab_shared_items", database: "main") } + let(:main_db_partitioned_item) { table("_test_gitlab_hook_logs", database: "main") } + let(:main_db_partitioned_item_detached) do + table("gitlab_partitions_dynamic._test_gitlab_hook_logs_20220101", database: "main") + end + # CI Database let(:ci_db_main_item_model) { table("_test_gitlab_main_items", database: "ci") } let(:ci_db_main_reference_model) { table("_test_gitlab_main_references", database: "ci") } let(:ci_db_ci_item_model) { table("_test_gitlab_ci_items", database: "ci") } let(:ci_db_ci_reference_model) { table("_test_gitlab_ci_references", database: "ci") } let(:ci_db_shared_item_model) { table("_test_gitlab_shared_items", database: "ci") } - - subject(:truncate_legacy_tables) do - described_class.new( - database_name: database_name, - min_batch_size: min_batch_size, - logger: logger, - dry_run: dry_run, - until_table: until_table - ).execute + let(:ci_db_partitioned_item) { table("_test_gitlab_hook_logs", database: "ci") } + let(:ci_db_partitioned_item_detached) do + table("gitlab_partitions_dynamic._test_gitlab_hook_logs_20220101", database: "ci") end shared_examples 'truncating legacy tables on a database' do + let(:logger) { instance_double(Logger) } + let(:dry_run) { false } + let(:until_table) { nil } + + subject(:truncate_legacy_tables) do + described_class.new( + database_name: connection.pool.db_config.name, + min_batch_size: min_batch_size, + logger: logger, + dry_run: dry_run, + until_table: until_table + ).execute + end + before do skip_if_multiple_databases_not_setup @@ -51,6 +59,24 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba item_id BIGINT NOT NULL, CONSTRAINT fk_constrained_1 FOREIGN KEY(item_id) REFERENCES _test_gitlab_main_items(id) ); + + CREATE TABLE _test_gitlab_hook_logs ( + id bigserial not null, + created_at timestamptz not null, + item_id BIGINT NOT NULL, + primary key (id, created_at), + CONSTRAINT fk_constrained_1 FOREIGN KEY(item_id) REFERENCES _test_gitlab_main_items(id) + ) PARTITION BY RANGE(created_at); + + CREATE TABLE gitlab_partitions_dynamic._test_gitlab_hook_logs_20220101 + PARTITION OF _test_gitlab_hook_logs + FOR VALUES FROM ('20220101') TO ('20220131'); + + CREATE TABLE gitlab_partitions_dynamic._test_gitlab_hook_logs_20220201 + PARTITION OF _test_gitlab_hook_logs + FOR VALUES FROM ('20220201') TO ('20220228'); + + ALTER TABLE _test_gitlab_hook_logs DETACH PARTITION gitlab_partitions_dynamic._test_gitlab_hook_logs_20220101; SQL main_connection.execute(main_tables_sql) @@ -84,18 +110,49 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba main_db_ci_item_model.create!(id: i) main_db_ci_reference_model.create!(item_id: i) main_db_shared_item_model.create!(id: i) + main_db_partitioned_item.create!(item_id: i, created_at: '2022-02-02 02:00') + main_db_partitioned_item_detached.create!(item_id: i, created_at: '2022-01-01 01:00') # CI Database ci_db_main_item_model.create!(id: i) ci_db_main_reference_model.create!(item_id: i) ci_db_ci_item_model.create!(id: i) ci_db_ci_reference_model.create!(item_id: i) ci_db_shared_item_model.create!(id: i) + ci_db_partitioned_item.create!(item_id: i, created_at: '2022-02-02 02:00') + ci_db_partitioned_item_detached.create!(item_id: i, created_at: '2022-01-01 01:00') + end + + Gitlab::Database::SharedModel.using_connection(main_connection) do + Postgresql::DetachedPartition.create!( + table_name: '_test_gitlab_hook_logs_20220101', + drop_after: Time.current + ) + end + + Gitlab::Database::SharedModel.using_connection(ci_connection) do + Postgresql::DetachedPartition.create!( + table_name: '_test_gitlab_hook_logs_20220101', + drop_after: Time.current + ) end allow(Gitlab::Database::GitlabSchema).to receive(:tables_to_schema).and_return( { "_test_gitlab_main_items" => :gitlab_main, "_test_gitlab_main_references" => :gitlab_main, + "_test_gitlab_hook_logs" => :gitlab_main, + "_test_gitlab_ci_items" => :gitlab_ci, + "_test_gitlab_ci_references" => :gitlab_ci, + "_test_gitlab_shared_items" => :gitlab_shared, + "_test_gitlab_geo_items" => :gitlab_geo + } + ) + + allow(Gitlab::Database::GitlabSchema).to receive(:views_and_tables_to_schema).and_return( + { + "_test_gitlab_main_items" => :gitlab_main, + "_test_gitlab_main_references" => :gitlab_main, + "_test_gitlab_hook_logs" => :gitlab_main, "_test_gitlab_ci_items" => :gitlab_ci, "_test_gitlab_ci_references" => :gitlab_ci, "_test_gitlab_shared_items" => :gitlab_shared, @@ -119,7 +176,7 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba Gitlab::Database::LockWritesManager.new( table_name: table, connection: connection, - database_name: database_name + database_name: connection.pool.db_config.name ).lock_writes end end @@ -199,7 +256,6 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba context 'when truncating gitlab_ci tables on the main database' do let(:connection) { ApplicationRecord.connection } - let(:database_name) { "main" } let(:legacy_tables_models) { [main_db_ci_item_model, main_db_ci_reference_model] } let(:referencing_table_model) { main_db_ci_reference_model } let(:referenced_table_model) { main_db_ci_item_model } @@ -217,8 +273,10 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba context 'when truncating gitlab_main tables on the ci database' do let(:connection) { Ci::ApplicationRecord.connection } - let(:database_name) { "ci" } - let(:legacy_tables_models) { [ci_db_main_item_model, ci_db_main_reference_model] } + let(:legacy_tables_models) do + [ci_db_main_item_model, ci_db_main_reference_model, ci_db_partitioned_item, ci_db_partitioned_item_detached] + end + let(:referencing_table_model) { ci_db_main_reference_model } let(:referenced_table_model) { ci_db_main_item_model } let(:other_tables_models) do diff --git a/spec/lib/gitlab/database/transaction/context_spec.rb b/spec/lib/gitlab/database/transaction/context_spec.rb index 33a47150060..1681098e20c 100644 --- a/spec/lib/gitlab/database/transaction/context_spec.rb +++ b/spec/lib/gitlab/database/transaction/context_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::Transaction::Context do +RSpec.describe Gitlab::Database::Transaction::Context, feature_category: :database do subject { described_class.new } let(:data) { subject.context } diff --git a/spec/lib/gitlab/database/type/indifferent_jsonb_spec.rb b/spec/lib/gitlab/database/type/indifferent_jsonb_spec.rb new file mode 100644 index 00000000000..6d27cbe180d --- /dev/null +++ b/spec/lib/gitlab/database/type/indifferent_jsonb_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Type::IndifferentJsonb do + let(:type) { described_class.new } + + describe '#deserialize' do + using RSpec::Parameterized::TableSyntax + + subject { type.deserialize(json) } + + where(:json, :value) do + nil | nil + '{"key":"value"}' | { key: 'value' } + '{"key":[1,2,3]}' | { key: [1, 2, 3] } + '{"key":{"subkey":"value"}}' | { key: { subkey: 'value' } } + '{"key":{"a":[{"b":"c"},{"d":"e"}]}}' | { key: { a: [{ b: 'c' }, { d: 'e' }] } } + end + + with_them do + it { is_expected.to match(value) } + it { is_expected.to match(value&.deep_stringify_keys) } + end + end + + context 'when used by a model' do + let(:model) do + Class.new(ApplicationRecord) do + self.table_name = :_test_indifferent_jsonb + + attribute :options, :ind_jsonb + end + end + + let(:record) do + model.create!(name: 'test', options: { key: 'value' }) + end + + before do + model.connection.execute(<<~SQL) + CREATE TABLE _test_indifferent_jsonb( + id serial NOT NULL PRIMARY KEY, + name text, + options jsonb); + SQL + + model.reset_column_information + end + + it { expect(record.options).to match({ key: 'value' }) } + it { expect(record.options).to match({ 'key' => 'value' }) } + + it 'ignores changes to other attributes' do + record.name = 'other test' + + expect(record.changes).to match('name' => ['test', 'other test']) + end + + it 'tracks changes to options' do + record.options = { key: 'other value' } + + expect(record.changes).to match('options' => [{ 'key' => 'value' }, { 'key' => 'other value' }]) + end + end +end diff --git a/spec/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer_spec.rb b/spec/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer_spec.rb new file mode 100644 index 00000000000..d8173794b3f --- /dev/null +++ b/spec/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer_spec.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::DatabaseImporters::WorkItems::HierarchyRestrictionsImporter, + feature_category: :portfolio_management do + subject { described_class.upsert_restrictions } + + it_behaves_like 'work item hierarchy restrictions importer' +end diff --git a/spec/lib/gitlab/database_spec.rb b/spec/lib/gitlab/database_spec.rb index c788022bd3a..1a482b33a92 100644 --- a/spec/lib/gitlab/database_spec.rb +++ b/spec/lib/gitlab/database_spec.rb @@ -139,7 +139,7 @@ RSpec.describe Gitlab::Database do describe '.db_config_for_connection' do context 'when the regular connection is used' do it 'returns db_config' do - connection = ActiveRecord::Base.retrieve_connection + connection = ApplicationRecord.retrieve_connection expect(described_class.db_config_for_connection(connection)).to eq(connection.pool.db_config) end @@ -147,12 +147,15 @@ RSpec.describe Gitlab::Database do context 'when the connection is LoadBalancing::ConnectionProxy', :database_replica do it 'returns primary db config even if ambiguous queries default to replica' do - Gitlab::Database::LoadBalancing::Session.current.use_primary! - primary_config = described_class.db_config_for_connection(ActiveRecord::Base.connection) - - Gitlab::Database::LoadBalancing::Session.clear_session - Gitlab::Database::LoadBalancing::Session.current.fallback_to_replicas_for_ambiguous_queries do - expect(described_class.db_config_for_connection(ActiveRecord::Base.connection)).to eq(primary_config) + Gitlab::Database.database_base_models_using_load_balancing.each_value do |database_base_model| + connection = database_base_model.connection + Gitlab::Database::LoadBalancing::Session.current.use_primary! + primary_config = described_class.db_config_for_connection(connection) + + Gitlab::Database::LoadBalancing::Session.clear_session + Gitlab::Database::LoadBalancing::Session.current.fallback_to_replicas_for_ambiguous_queries do + expect(described_class.db_config_for_connection(connection)).to eq(primary_config) + end end end end @@ -180,11 +183,16 @@ RSpec.describe Gitlab::Database do end context 'when replicas are configured', :database_replica do - it 'returns the name for a replica' do - replica = ActiveRecord::Base.load_balancer.host - + it 'returns the main_replica for a main database replica' do + replica = ApplicationRecord.load_balancer.host expect(described_class.db_config_name(replica)).to eq('main_replica') end + + it 'returns the ci_replica for a ci database replica' do + skip_if_multiple_databases_not_setup + replica = Ci::ApplicationRecord.load_balancer.host + expect(described_class.db_config_name(replica)).to eq('ci_replica') + end end end @@ -214,13 +222,17 @@ RSpec.describe Gitlab::Database do expect(described_class.gitlab_schemas_for_connection(Ci::Build.connection)).to include(:gitlab_ci, :gitlab_shared) end + # rubocop:disable Database/MultipleDatabases it 'does return gitlab_ci when a ActiveRecord::Base is using CI connection' do with_reestablished_active_record_base do reconfigure_db_connection(model: ActiveRecord::Base, config_model: Ci::Build) - expect(described_class.gitlab_schemas_for_connection(ActiveRecord::Base.connection)).to include(:gitlab_ci, :gitlab_shared) + expect( + described_class.gitlab_schemas_for_connection(ActiveRecord::Base.connection) + ).to include(:gitlab_ci, :gitlab_shared) end end + # rubocop:enable Database/MultipleDatabases it 'does return a valid schema for a replica connection' do with_replica_pool_for(ActiveRecord::Base) do |main_replica_pool| @@ -281,7 +293,8 @@ RSpec.describe Gitlab::Database do it 'does return empty for non-adopted connections' do new_connection = ActiveRecord::Base.postgresql_connection( - ActiveRecord::Base.connection_db_config.configuration_hash) + ActiveRecord::Base.connection_db_config.configuration_hash # rubocop:disable Database/MultipleDatabases + ) expect(described_class.gitlab_schemas_for_connection(new_connection)).to be_nil ensure @@ -405,7 +418,7 @@ RSpec.describe Gitlab::Database do context 'within a transaction block' do it 'publishes a transaction event' do events = subscribe_events do - ActiveRecord::Base.transaction do + ApplicationRecord.transaction do User.first end end @@ -424,10 +437,11 @@ RSpec.describe Gitlab::Database do context 'within an empty transaction block' do it 'publishes a transaction event' do events = subscribe_events do - ActiveRecord::Base.transaction {} + ApplicationRecord.transaction {} + Ci::ApplicationRecord.transaction {} end - expect(events.length).to be(1) + expect(events.length).to be(2) event = events.first expect(event).not_to be_nil @@ -441,9 +455,9 @@ RSpec.describe Gitlab::Database do context 'within a nested transaction block' do it 'publishes multiple transaction events' do events = subscribe_events do - ActiveRecord::Base.transaction do - ActiveRecord::Base.transaction do - ActiveRecord::Base.transaction do + ApplicationRecord.transaction do + ApplicationRecord.transaction do + ApplicationRecord.transaction do User.first end end @@ -465,7 +479,7 @@ RSpec.describe Gitlab::Database do context 'within a cancelled transaction block' do it 'publishes multiple transaction events' do events = subscribe_events do - ActiveRecord::Base.transaction do + ApplicationRecord.transaction do User.first raise ActiveRecord::Rollback end diff --git a/spec/lib/gitlab/diff/file_collection/compare_spec.rb b/spec/lib/gitlab/diff/file_collection/compare_spec.rb index ce70903a480..c3f768db7f0 100644 --- a/spec/lib/gitlab/diff/file_collection/compare_spec.rb +++ b/spec/lib/gitlab/diff/file_collection/compare_spec.rb @@ -16,10 +16,11 @@ RSpec.describe Gitlab::Diff::FileCollection::Compare do end let(:diffable) { Compare.new(raw_compare, project) } + let(:diff_options) { {} } let(:collection_default_args) do { project: diffable.project, - diff_options: {}, + diff_options: diff_options, diff_refs: diffable.diff_refs } end @@ -65,4 +66,32 @@ RSpec.describe Gitlab::Diff::FileCollection::Compare do expect(cache_key).to eq ['compare', head_commit.id, start_commit.id] end end + + describe 'pagination methods' do + subject(:compare) { described_class.new(diffable, **collection_default_args) } + + context 'when pagination options are not present' do + it 'returns default values' do + expect(compare.limit_value).to eq(Kaminari.config.default_per_page) + expect(compare.current_page).to eq(1) + expect(compare.next_page).to be_nil + expect(compare.prev_page).to be_nil + expect(compare.total_count).to be_nil + expect(compare.total_pages).to eq(0) + end + end + + context 'when pagination options are present' do + let(:diff_options) { { page: 1, per_page: 10, count: 20 } } + + it 'returns values based on options' do + expect(compare.limit_value).to eq(10) + expect(compare.current_page).to eq(1) + expect(compare.next_page).to eq(2) + expect(compare.prev_page).to be_nil + expect(compare.total_count).to eq(20) + expect(compare.total_pages).to eq(2) + end + end + end end diff --git a/spec/lib/gitlab/diff/file_collection/merge_request_diff_batch_spec.rb b/spec/lib/gitlab/diff/file_collection/merge_request_diff_batch_spec.rb index beb85d383a0..9ac242459bf 100644 --- a/spec/lib/gitlab/diff/file_collection/merge_request_diff_batch_spec.rb +++ b/spec/lib/gitlab/diff/file_collection/merge_request_diff_batch_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Diff::FileCollection::MergeRequestDiffBatch do +RSpec.describe Gitlab::Diff::FileCollection::MergeRequestDiffBatch, feature_category: :code_review do let(:merge_request) { create(:merge_request) } let(:batch_page) { 0 } let(:batch_size) { 10 } diff --git a/spec/lib/gitlab/diff/file_collection/paginated_merge_request_diff_spec.rb b/spec/lib/gitlab/diff/file_collection/paginated_merge_request_diff_spec.rb new file mode 100644 index 00000000000..74e5e667702 --- /dev/null +++ b/spec/lib/gitlab/diff/file_collection/paginated_merge_request_diff_spec.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Diff::FileCollection::PaginatedMergeRequestDiff, feature_category: :code_review do + let(:merge_request) { create(:merge_request) } + let(:page) { 1 } + let(:per_page) { 10 } + let(:diffable) { merge_request.merge_request_diff } + let(:diff_files_relation) { diffable.merge_request_diff_files } + let(:diff_files) { subject.diff_files } + + subject do + described_class.new(diffable, + page, + per_page) + end + + describe '#diff_files' do + let(:per_page) { 3 } + let(:paginated_rel) { diff_files_relation.page(page).per(per_page) } + + let(:expected_batch_files) do + paginated_rel.map(&:new_path) + end + + it 'returns paginated diff files' do + expect(diff_files.size).to eq(3) + end + + it 'returns a valid instance of a DiffCollection' do + expect(diff_files).to be_a(Gitlab::Git::DiffCollection) + end + + context 'when first page' do + it 'returns correct diff files' do + expect(diff_files.map(&:new_path)).to eq(expected_batch_files) + end + end + + context 'when another page' do + let(:page) { 2 } + + it 'returns correct diff files' do + expect(diff_files.map(&:new_path)).to eq(expected_batch_files) + end + end + + context 'when page is nil' do + let(:page) { nil } + + it 'returns correct diff files' do + expected_batch_files = + diff_files_relation.page(described_class::DEFAULT_PAGE).per(per_page).map(&:new_path) + + expect(diff_files.map(&:new_path)).to eq(expected_batch_files) + end + end + + context 'when per_page is nil' do + let(:per_page) { nil } + + it 'returns correct diff files' do + expected_batch_files = + diff_files_relation.page(page).per(described_class::DEFAULT_PER_PAGE).map(&:new_path) + + expect(diff_files.map(&:new_path)).to eq(expected_batch_files) + end + end + + context 'when invalid page' do + let(:page) { 999 } + + it 'returns correct diff files' do + expect(diff_files.map(&:new_path)).to be_empty + end + end + + context 'when last page' do + it 'returns correct diff files' do + last_page = diff_files_relation.count - per_page + collection = described_class.new(diffable, + last_page, + per_page) + + expected_batch_files = diff_files_relation.page(last_page).per(per_page).map(&:new_path) + + expect(collection.diff_files.map(&:new_path)).to eq(expected_batch_files) + end + end + end + + it_behaves_like 'unfoldable diff' do + subject do + described_class.new(merge_request.merge_request_diff, + page, + per_page) + end + end + + it_behaves_like 'cacheable diff collection' do + let(:cacheable_files_count) { per_page } + end + + it_behaves_like 'unsortable diff files' do + let(:diffable) { merge_request.merge_request_diff } + + subject do + described_class.new(merge_request.merge_request_diff, + page, + per_page) + end + end +end diff --git a/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb b/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb index 75538baf07f..8ff8de2379a 100644 --- a/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb +++ b/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Email::Handler::CreateIssueHandler do - include_context :email_shared_context + include_context 'email shared context' let!(:user) do create( :user, @@ -16,13 +16,13 @@ RSpec.describe Gitlab::Email::Handler::CreateIssueHandler do let(:namespace) { create(:namespace, path: 'gitlabhq') } let(:email_raw) { email_fixture('emails/valid_new_issue.eml') } - it_behaves_like :reply_processing_shared_examples - before do stub_incoming_email_setting(enabled: true, address: "incoming+%{key}@appmail.adventuretime.ooo") stub_config_setting(host: 'localhost') end + it_behaves_like 'reply processing shared examples' + context "when email key" do let(:mail) { Mail::Message.new(email_raw) } diff --git a/spec/lib/gitlab/email/handler/create_merge_request_handler_spec.rb b/spec/lib/gitlab/email/handler/create_merge_request_handler_spec.rb index 37ee4591db0..f5b44d30c50 100644 --- a/spec/lib/gitlab/email/handler/create_merge_request_handler_spec.rb +++ b/spec/lib/gitlab/email/handler/create_merge_request_handler_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Email::Handler::CreateMergeRequestHandler do - include_context :email_shared_context + include_context 'email shared context' let!(:user) do create( :user, @@ -16,16 +16,16 @@ RSpec.describe Gitlab::Email::Handler::CreateMergeRequestHandler do let(:namespace) { create(:namespace, path: 'gitlabhq') } let(:email_raw) { email_fixture('emails/valid_new_merge_request.eml') } - it_behaves_like :reply_processing_shared_examples + after do + TestEnv.clean_test_path + end before do stub_incoming_email_setting(enabled: true, address: "incoming+%{key}@appmail.adventuretime.ooo") stub_config_setting(host: 'localhost') end - after do - TestEnv.clean_test_path - end + it_behaves_like 'reply processing shared examples' context "when email key" do let(:mail) { Mail::Message.new(email_raw) } diff --git a/spec/lib/gitlab/email/handler/create_note_handler_spec.rb b/spec/lib/gitlab/email/handler/create_note_handler_spec.rb index 585dce331ed..f70645a8272 100644 --- a/spec/lib/gitlab/email/handler/create_note_handler_spec.rb +++ b/spec/lib/gitlab/email/handler/create_note_handler_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Email::Handler::CreateNoteHandler do - include_context :email_shared_context + include_context 'email shared context' let_it_be(:user) { create(:user, email: 'jake@adventuretime.ooo') } let_it_be(:project) { create(:project, :public, :repository) } @@ -15,9 +15,14 @@ RSpec.describe Gitlab::Email::Handler::CreateNoteHandler do SentNotification.record_note(note, user.id, mail_key) end - it_behaves_like :reply_processing_shared_examples + before do + stub_incoming_email_setting(enabled: true, address: "reply+%{key}@appmail.adventuretime.ooo") + stub_config_setting(host: 'localhost') + end + + it_behaves_like 'reply processing shared examples' - it_behaves_like :note_handler_shared_examples do + it_behaves_like 'note handler shared examples' do let(:recipient) { sent_notification.recipient } let(:update_commands_only) { fixture_file('emails/update_commands_only_reply.eml') } @@ -26,11 +31,6 @@ RSpec.describe Gitlab::Email::Handler::CreateNoteHandler do let(:with_quick_actions) { fixture_file('emails/valid_reply_with_quick_actions.eml') } end - before do - stub_incoming_email_setting(enabled: true, address: "reply+%{key}@appmail.adventuretime.ooo") - stub_config_setting(host: 'localhost') - end - context 'when the recipient address does not include a mail key' do let(:email_raw) { fixture_file('emails/valid_reply.eml').gsub(mail_key, '') } @@ -92,7 +92,7 @@ RSpec.describe Gitlab::Email::Handler::CreateNoteHandler do issue.update_attribute(:confidential, true) end - it_behaves_like :checks_permissions_on_noteable_examples + it_behaves_like 'checks permissions on noteable examples' end shared_examples 'a reply to existing comment' do diff --git a/spec/lib/gitlab/email/handler/create_note_on_issuable_handler_spec.rb b/spec/lib/gitlab/email/handler/create_note_on_issuable_handler_spec.rb index d3535fa9bd3..6e83c06c1b4 100644 --- a/spec/lib/gitlab/email/handler/create_note_on_issuable_handler_spec.rb +++ b/spec/lib/gitlab/email/handler/create_note_on_issuable_handler_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Email::Handler::CreateNoteOnIssuableHandler do - include_context :email_shared_context + include_context 'email shared context' let_it_be(:user) { create(:user, email: 'jake@adventuretime.ooo', incoming_email_token: 'auth_token') } let_it_be(:namespace) { create(:namespace, path: 'gitlabhq') } @@ -17,9 +17,9 @@ RSpec.describe Gitlab::Email::Handler::CreateNoteOnIssuableHandler do stub_config_setting(host: 'localhost') end - it_behaves_like :reply_processing_shared_examples + it_behaves_like 'reply processing shared examples' - it_behaves_like :note_handler_shared_examples, true do + it_behaves_like 'note handler shared examples', true do let_it_be(:recipient) { user } let(:update_commands_only) { email_reply_fixture('emails/update_commands_only.eml') } @@ -42,7 +42,7 @@ RSpec.describe Gitlab::Email::Handler::CreateNoteOnIssuableHandler do noteable.update_attribute(:confidential, true) end - it_behaves_like :checks_permissions_on_noteable_examples + it_behaves_like 'checks permissions on noteable examples' end def email_fixture(path) diff --git a/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb b/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb index 08a7383700b..7bba0775668 100644 --- a/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb +++ b/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb @@ -3,7 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do - include_context :email_shared_context + include ServiceDeskHelper + include_context 'email shared context' before do stub_incoming_email_setting(enabled: true, address: "incoming+%{key}@appmail.adventuretime.ooo") @@ -184,12 +185,6 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do context 'and template is present' do let_it_be(:settings) { create(:service_desk_setting, project: project) } - def set_template_file(file_name, content) - file_path = ".gitlab/issue_templates/#{file_name}.md" - project.repository.create_file(user, file_path, content, message: 'message', branch_name: 'master') - settings.update!(issue_template_key: file_name) - end - it 'appends template text to issue description' do set_template_file('service_desk', 'text from template') diff --git a/spec/lib/gitlab/email/handler/unsubscribe_handler_spec.rb b/spec/lib/gitlab/email/handler/unsubscribe_handler_spec.rb index 2bc3cd81b48..f33e9eba5c6 100644 --- a/spec/lib/gitlab/email/handler/unsubscribe_handler_spec.rb +++ b/spec/lib/gitlab/email/handler/unsubscribe_handler_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Email::Handler::UnsubscribeHandler do - include_context :email_shared_context + include_context 'email shared context' before do stub_incoming_email_setting(enabled: true, address: 'reply+%{key}@appmail.adventuretime.ooo') diff --git a/spec/lib/gitlab/email/hook/disable_email_interceptor_spec.rb b/spec/lib/gitlab/email/hook/disable_email_interceptor_spec.rb index 47f6015c6f8..b22c55208f0 100644 --- a/spec/lib/gitlab/email/hook/disable_email_interceptor_spec.rb +++ b/spec/lib/gitlab/email/hook/disable_email_interceptor_spec.rb @@ -7,15 +7,15 @@ RSpec.describe Gitlab::Email::Hook::DisableEmailInterceptor do Mail.register_interceptor(described_class) end + after do + Mail.unregister_interceptor(described_class) + end + it 'does not send emails' do allow(Gitlab.config.gitlab).to receive(:email_enabled).and_return(false) expect { deliver_mail }.not_to change(ActionMailer::Base.deliveries, :count) end - after do - Mail.unregister_interceptor(described_class) - end - def deliver_mail key = create :personal_key Notify.new_ssh_key_email(key.id) diff --git a/spec/lib/gitlab/email/receiver_spec.rb b/spec/lib/gitlab/email/receiver_spec.rb index 9240d07fd59..865e40d4ecb 100644 --- a/spec/lib/gitlab/email/receiver_spec.rb +++ b/spec/lib/gitlab/email/receiver_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Email::Receiver do - include_context :email_shared_context + include_context 'email shared context' let_it_be(:project) { create(:project) } let(:metric_transaction) { instance_double(Gitlab::Metrics::WebTransaction) } diff --git a/spec/lib/gitlab/file_type_detection_spec.rb b/spec/lib/gitlab/file_type_detection_spec.rb index c435d3f6097..1be0f7d53fa 100644 --- a/spec/lib/gitlab/file_type_detection_spec.rb +++ b/spec/lib/gitlab/file_type_detection_spec.rb @@ -31,6 +31,7 @@ RSpec.describe Gitlab::FileTypeDetection do expect(described_class.extension_match?('my/file.foo', extensions)).to eq(true) end end + context 'when class is an uploader' do let(:uploader) do example_uploader = Class.new(CarrierWave::Uploader::Base) do diff --git a/spec/lib/gitlab/gfm/uploads_rewriter_spec.rb b/spec/lib/gitlab/gfm/uploads_rewriter_spec.rb index b1bff242f33..e1c0da69317 100644 --- a/spec/lib/gitlab/gfm/uploads_rewriter_spec.rb +++ b/spec/lib/gitlab/gfm/uploads_rewriter_spec.rb @@ -31,7 +31,7 @@ RSpec.describe Gitlab::Gfm::UploadsRewriter do referenced_files.compact.select(&:exists?) end - shared_examples "files are accessible" do + shared_examples 'files are accessible' do describe '#rewrite' do subject(:rewrite) { new_text } @@ -82,6 +82,18 @@ RSpec.describe Gitlab::Gfm::UploadsRewriter do rewrite expect(new_files).to be_empty + expect(new_text).to eq(text) + end + + it 'skips non-existant files' do + allow_next_instance_of(FileUploader) do |file| + allow(file).to receive(:exists?).and_return(false) + end + + rewrite + + expect(new_files).to be_empty + expect(new_text).to eq(text) end end end @@ -107,11 +119,11 @@ RSpec.describe Gitlab::Gfm::UploadsRewriter do end end - context "file are stored locally" do - include_examples "files are accessible" + context 'file are stored locally' do + include_examples 'files are accessible' end - context "files are stored remotely" do + context 'files are stored remotely' do before do stub_uploads_object_storage(FileUploader) @@ -120,7 +132,7 @@ RSpec.describe Gitlab::Gfm::UploadsRewriter do end end - include_examples "files are accessible" + include_examples 'files are accessible' end describe '#needs_rewrite?' do diff --git a/spec/lib/gitlab/git/base_error_spec.rb b/spec/lib/gitlab/git/base_error_spec.rb index 851cfa16512..d4db7cf2430 100644 --- a/spec/lib/gitlab/git/base_error_spec.rb +++ b/spec/lib/gitlab/git/base_error_spec.rb @@ -20,4 +20,15 @@ RSpec.describe Gitlab::Git::BaseError do with_them do it { is_expected.to eq(result) } end + + describe "When initialized with GRPC errors" do + let(:grpc_error) { GRPC::DeadlineExceeded.new } + let(:git_error) { described_class.new grpc_error } + + it "has status and code fields" do + expect(git_error.service).to eq('git') + expect(git_error.status).to eq(4) + expect(git_error.code).to eq('deadline_exceeded') + end + end end diff --git a/spec/lib/gitlab/git/cross_repo_comparer_spec.rb b/spec/lib/gitlab/git/cross_repo_comparer_spec.rb deleted file mode 100644 index 7888e224d59..00000000000 --- a/spec/lib/gitlab/git/cross_repo_comparer_spec.rb +++ /dev/null @@ -1,117 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Git::CrossRepoComparer do - let(:source_project) { create(:project, :repository) } - let(:target_project) { create(:project, :repository) } - - let(:source_repo) { source_project.repository.raw_repository } - let(:target_repo) { target_project.repository.raw_repository } - - let(:source_branch) { 'feature' } - let(:target_branch) { 'master' } - let(:straight) { false } - - let(:source_commit) { source_repo.commit(source_branch) } - let(:target_commit) { source_repo.commit(target_branch) } - - subject(:result) { described_class.new(source_repo, target_repo).compare(source_branch, target_branch, straight: straight) } - - describe '#compare' do - context 'within a single repository' do - let(:target_project) { source_project } - - context 'a non-straight comparison' do - it 'compares without fetching from another repo' do - expect(source_repo).not_to receive(:fetch_source_branch!) - - expect_compare(result, from: source_commit, to: target_commit) - expect(result.straight).to eq(false) - end - end - - context 'a straight comparison' do - let(:straight) { true } - - it 'compares without fetching from another repo' do - expect(source_repo).not_to receive(:fetch_source_branch!) - - expect_compare(result, from: source_commit, to: target_commit) - expect(result.straight).to eq(true) - end - end - end - - context 'across two repositories' do - context 'target ref exists in source repo' do - it 'compares without fetching from another repo' do - expect(source_repo).not_to receive(:fetch_source_branch!) - expect(source_repo).not_to receive(:delete_refs) - - expect_compare(result, from: source_commit, to: target_commit) - end - end - - context 'target ref does not exist in source repo' do - it 'compares in the source repo by fetching from the target to a temporary ref' do - new_commit_id = create_commit(target_project.owner, target_repo, target_branch) - new_commit = target_repo.commit(new_commit_id) - - # This is how the temporary ref is generated - expect(SecureRandom).to receive(:hex).at_least(:once).and_return('foo') - - expect(source_repo) - .to receive(:fetch_source_branch!) - .with(target_repo, new_commit_id, 'refs/tmp/foo') - .and_call_original - - expect(source_repo).to receive(:delete_refs).with('refs/tmp/foo').and_call_original - - expect_compare(result, from: source_commit, to: new_commit) - end - end - - context 'source ref does not exist in source repo' do - let(:source_branch) { 'does-not-exist' } - - it 'returns an empty comparison' do - expect(source_repo).not_to receive(:fetch_source_branch!) - expect(source_repo).not_to receive(:delete_refs) - - expect(result).to be_a(::Gitlab::Git::Compare) - expect(result.commits.size).to eq(0) - end - end - - context 'target ref does not exist in target repo' do - let(:target_branch) { 'does-not-exist' } - - it 'returns nil' do - expect(source_repo).not_to receive(:fetch_source_branch!) - expect(source_repo).not_to receive(:delete_refs) - - is_expected.to be_nil - end - end - end - end - - def expect_compare(of, from:, to:) - expect(of).to be_a(::Gitlab::Git::Compare) - expect(from).to be_a(::Gitlab::Git::Commit) - expect(to).to be_a(::Gitlab::Git::Commit) - - expect(of.commits).not_to be_empty - expect(of.head).to eq(from) - expect(of.base).to eq(to) - end - - def create_commit(user, repo, branch) - action = { action: :create, file_path: '/FILE', content: 'content' } - - result = repo.commit_files(user, branch_name: branch, message: 'Commit', actions: [action]) - - result.newrev - end -end diff --git a/spec/lib/gitlab/git/cross_repo_spec.rb b/spec/lib/gitlab/git/cross_repo_spec.rb new file mode 100644 index 00000000000..09a28c144a4 --- /dev/null +++ b/spec/lib/gitlab/git/cross_repo_spec.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Git::CrossRepo do + let_it_be(:source_project) { create(:project, :repository) } + let_it_be(:target_project) { create(:project, :repository) } + + let(:source_repo) { source_project.repository.raw_repository } + let(:target_repo) { target_project.repository.raw_repository } + + let(:source_branch) { 'feature' } + let(:target_branch) { target_repo.root_ref } + + let(:source_commit) { source_repo.commit(source_branch) } + let(:target_commit) { source_repo.commit(target_branch) } + + def execute(&block) + described_class.new(source_repo, target_repo).execute(target_branch, &block) + end + + describe '#execute' do + context 'when executed within a single repository' do + let(:target_project) { source_project } + + it 'does not fetch from another repo' do + expect(source_repo).not_to receive(:fetch_source_branch!) + + expect { |block| execute(&block) }.to yield_with_args(target_branch) + end + end + + context 'when executed across two repositories' do + context 'and target ref exists in source repo' do + it 'does not fetch from another repo' do + expect(source_repo).not_to receive(:fetch_source_branch!) + expect(source_repo).not_to receive(:delete_refs) + + expect { |block| execute(&block) }.to yield_with_args(target_commit.id) + end + end + + context 'and target ref does not exist in source repo' do + let_it_be(:target_project) { create(:project, :repository) } + + it 'fetches from the target to a temporary ref' do + new_commit_id = create_commit(target_project.owner, target_repo, target_branch) + + # This is how the temporary ref is generated + expect(SecureRandom).to receive(:hex).at_least(:once).and_return('foo') + + expect(source_repo) + .to receive(:fetch_source_branch!) + .with(target_repo, new_commit_id, 'refs/tmp/foo') + .and_call_original + + expect(source_repo).to receive(:delete_refs).with('refs/tmp/foo').and_call_original + + expect { |block| execute(&block) }.to yield_with_args(new_commit_id) + end + end + + context 'and target ref does not exist in target repo' do + let(:target_branch) { 'does-not-exist' } + + it 'returns nil' do + expect(source_repo).not_to receive(:fetch_source_branch!) + expect(source_repo).not_to receive(:delete_refs) + + expect { |block| execute(&block) }.not_to yield_control + end + end + end + end + + def create_commit(user, repo, branch) + action = { action: :create, file_path: '/FILE', content: 'content' } + + result = repo.commit_files(user, branch_name: branch, message: 'Commit', actions: [action]) + + result.newrev + end +end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 197662943a0..6cff39c1167 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -2,7 +2,7 @@ require "spec_helper" -RSpec.describe Gitlab::Git::Repository do +RSpec.describe Gitlab::Git::Repository, feature_category: :source_code_management do include Gitlab::EncodingHelper include RepoHelpers using RSpec::Parameterized::TableSyntax @@ -70,12 +70,7 @@ RSpec.describe Gitlab::Git::Repository do it { is_expected.to include("master") } it { is_expected.not_to include("branch-from-space") } - it 'gets the branch names from GitalyClient' do - expect_any_instance_of(Gitlab::GitalyClient::RefService).to receive(:branch_names) - subject - end - - it_behaves_like 'wrapping gRPC errors', Gitlab::GitalyClient::RefService, :branch_names + it_behaves_like 'wrapping gRPC errors', Gitlab::GitalyClient::RefService, :list_refs end describe '#tag_names' do @@ -100,7 +95,7 @@ RSpec.describe Gitlab::Git::Repository do it { is_expected.to include("v1.0.0") } it { is_expected.not_to include("v5.0.0") } - it_behaves_like 'wrapping gRPC errors', Gitlab::GitalyClient::RefService, :tag_names + it_behaves_like 'wrapping gRPC errors', Gitlab::GitalyClient::RefService, :list_refs end describe '#tags' do @@ -1353,7 +1348,7 @@ RSpec.describe Gitlab::Git::Repository do it "returns the number of commits in the whole repository" do options = { all: true } - expect(repository.count_commits(options)).to eq(314) + expect(repository.count_commits(options)).to eq(315) end end @@ -1378,6 +1373,24 @@ RSpec.describe Gitlab::Git::Repository do expect(branch).to eq(nil) end + + context 'when branch is ambiguous' do + let(:ambiguous_branch) { 'prefix' } + let(:branch_with_prefix) { 'prefix/branch' } + + before do + repository.create_branch(branch_with_prefix) + end + + after do + repository.delete_branch(branch_with_prefix) + end + + it 'returns nil for ambiguous branch' do + expect(repository.find_branch(branch_with_prefix)).to be_a_kind_of(Gitlab::Git::Branch) + expect(repository.find_branch(ambiguous_branch)).to eq(nil) + end + end end describe '#branches' do @@ -1416,16 +1429,6 @@ RSpec.describe Gitlab::Git::Repository do it 'returns the count of local branches' do expect(repository.branch_count).to eq(repository.local_branches.count) end - - context 'with Gitaly disabled' do - before do - allow(Gitlab::GitalyClient).to receive(:feature_enabled?).and_return(false) - end - - it 'returns the count of local branches' do - expect(repository.branch_count).to eq(repository.local_branches.count) - end - end end end @@ -2212,15 +2215,49 @@ RSpec.describe Gitlab::Git::Repository do end describe '#compare_source_branch' do - it 'delegates to Gitlab::Git::CrossRepoComparer' do - expect_next_instance_of(::Gitlab::Git::CrossRepoComparer) do |instance| - expect(instance.source_repo).to eq(:source_repository) - expect(instance.target_repo).to eq(repository) + it 'compares two branches cross repo' do + mutable_repository.commit_files( + user, + branch_name: mutable_repository.root_ref, message: 'Committing something', + actions: [{ action: :create, file_path: 'encoding/CHANGELOG', content: 'New file' }] + ) + + repository.commit_files( + user, + branch_name: repository.root_ref, message: 'Commit to root ref', + actions: [{ action: :create, file_path: 'encoding/CHANGELOG', content: 'One more' }] + ) - expect(instance).to receive(:compare).with('feature', 'master', straight: :straight) + [ + [repository, mutable_repository, true], + [repository, mutable_repository, false], + [mutable_repository, repository, true], + [mutable_repository, repository, false] + ].each do |source_repo, target_repo, straight| + raw_compare = target_repo.compare_source_branch( + target_repo.root_ref, source_repo, source_repo.root_ref, straight: straight) + + expect(raw_compare).to be_a(::Gitlab::Git::Compare) + + expect(raw_compare.commits).to eq([source_repo.commit]) + expect(raw_compare.head).to eq(source_repo.commit) + expect(raw_compare.base).to eq(target_repo.commit) + expect(raw_compare.straight).to eq(straight) end + end + + context 'source ref does not exist in source repo' do + it 'returns an empty comparison' do + expect_next_instance_of(::Gitlab::Git::CrossRepo) do |instance| + expect(instance).not_to receive(:fetch_source_branch!) + end + + raw_compare = repository.compare_source_branch( + repository.root_ref, mutable_repository, 'does-not-exist', straight: true) - repository.compare_source_branch('master', :source_repository, 'feature', straight: :straight) + expect(raw_compare).to be_a(::Gitlab::Git::Compare) + expect(raw_compare.commits.size).to eq(0) + end end end @@ -2517,4 +2554,30 @@ RSpec.describe Gitlab::Git::Repository do end end end + + describe '#check_objects_exist' do + it 'returns hash specifying which object exists in repo' do + refs_exist = %w( + b83d6e391c22777fca1ed3012fce84f633d7fed0 + 498214de67004b1da3d820901307bed2a68a8ef6 + 1b12f15a11fc6e62177bef08f47bc7b5ce50b141 + ) + refs_dont_exist = %w( + 1111111111111111111111111111111111111111 + 2222222222222222222222222222222222222222 + ) + object_existence_map = repository.check_objects_exist(refs_exist + refs_dont_exist) + expect(object_existence_map).to eq({ + 'b83d6e391c22777fca1ed3012fce84f633d7fed0' => true, + '498214de67004b1da3d820901307bed2a68a8ef6' => true, + '1b12f15a11fc6e62177bef08f47bc7b5ce50b141' => true, + '1111111111111111111111111111111111111111' => false, + '2222222222222222222222222222222222222222' => false + }) + expect(object_existence_map.keys).to eq(refs_exist + refs_dont_exist) + + single_sha = 'b83d6e391c22777fca1ed3012fce84f633d7fed0' + expect(repository.check_objects_exist(single_sha)).to eq({ single_sha => true }) + end + end end diff --git a/spec/lib/gitlab/git/rugged_impl/use_rugged_spec.rb b/spec/lib/gitlab/git/rugged_impl/use_rugged_spec.rb index 524b373a5b7..1b8da0b380b 100644 --- a/spec/lib/gitlab/git/rugged_impl/use_rugged_spec.rb +++ b/spec/lib/gitlab/git/rugged_impl/use_rugged_spec.rb @@ -4,7 +4,7 @@ require 'spec_helper' require 'json' require 'tempfile' -RSpec.describe Gitlab::Git::RuggedImpl::UseRugged do +RSpec.describe Gitlab::Git::RuggedImpl::UseRugged, feature_category: :gitlay do let(:project) { create(:project, :repository) } let(:repository) { project.repository } let(:feature_flag_name) { wrapper.rugged_feature_keys.first } @@ -18,8 +18,7 @@ RSpec.describe Gitlab::Git::RuggedImpl::UseRugged do klazz = Class.new do include Gitlab::Git::RuggedImpl::UseRugged - def rugged_test(ref, test_number) - end + def rugged_test(ref, test_number); end end klazz.new diff --git a/spec/lib/gitlab/git/tree_spec.rb b/spec/lib/gitlab/git/tree_spec.rb index 17f802b9f66..2a68fa66b18 100644 --- a/spec/lib/gitlab/git/tree_spec.rb +++ b/spec/lib/gitlab/git/tree_spec.rb @@ -8,7 +8,7 @@ RSpec.describe Gitlab::Git::Tree do let(:project) { create(:project, :repository) } let(:repository) { project.repository.raw } - shared_examples :repo do + shared_examples 'repo' do subject(:tree) { Gitlab::Git::Tree.where(repository, sha, path, recursive, skip_flat_paths, pagination_params) } let(:sha) { SeedRepo::Commit::ID } @@ -151,7 +151,7 @@ RSpec.describe Gitlab::Git::Tree do end describe '.where with Gitaly enabled' do - it_behaves_like :repo do + it_behaves_like 'repo' do context 'with pagination parameters' do let(:pagination_params) { { limit: 3, page_token: nil } } @@ -172,7 +172,7 @@ RSpec.describe Gitlab::Git::Tree do described_class.where(repository, SeedRepo::Commit::ID, 'files', false, false) end - it_behaves_like :repo do + it_behaves_like 'repo' do describe 'Pagination' do context 'with restrictive limit' do let(:pagination_params) { { limit: 3, page_token: nil } } diff --git a/spec/lib/gitlab/gitaly_client/operation_service_spec.rb b/spec/lib/gitlab/gitaly_client/operation_service_spec.rb index 604feeea325..82d5d0f292b 100644 --- a/spec/lib/gitlab/gitaly_client/operation_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/operation_service_spec.rb @@ -898,7 +898,7 @@ RSpec.describe Gitlab::GitalyClient::OperationService do end shared_examples '#user_commit_files failure' do - it 'raises a PreReceiveError' do + it 'raises an IndexError' do expect_any_instance_of(Gitaly::OperationService::Stub) .to receive(:user_commit_files).with(kind_of(Enumerator), kind_of(Hash)) .and_raise(structured_error) @@ -912,7 +912,7 @@ RSpec.describe Gitlab::GitalyClient::OperationService do context 'with missing file' do let(:status_code) { GRPC::Core::StatusCodes::NOT_FOUND } - let(:expected_message) { "File not found: README.md" } + let(:expected_message) { "A file with this name doesn't exist" } let(:expected_error) do Gitaly::UserCommitFilesError.new( index_update: Gitaly::IndexError.new( @@ -926,7 +926,7 @@ RSpec.describe Gitlab::GitalyClient::OperationService do context 'with existing directory' do let(:status_code) { GRPC::Core::StatusCodes::ALREADY_EXISTS } - let(:expected_message) { "Directory already exists: dir1" } + let(:expected_message) { "A directory with this name already exists" } let(:expected_error) do Gitaly::UserCommitFilesError.new( index_update: Gitaly::IndexError.new( @@ -940,7 +940,7 @@ RSpec.describe Gitlab::GitalyClient::OperationService do context 'with existing file' do let(:status_code) { GRPC::Core::StatusCodes::ALREADY_EXISTS } - let(:expected_message) { "File already exists: README.md" } + let(:expected_message) { "A file with this name already exists" } let(:expected_error) do Gitaly::UserCommitFilesError.new( index_update: Gitaly::IndexError.new( @@ -954,7 +954,7 @@ RSpec.describe Gitlab::GitalyClient::OperationService do context 'with invalid path' do let(:status_code) { GRPC::Core::StatusCodes::INVALID_ARGUMENT } - let(:expected_message) { "Invalid path: invalid://file/name" } + let(:expected_message) { "invalid path: 'invalid://file/name'" } let(:expected_error) do Gitaly::UserCommitFilesError.new( index_update: Gitaly::IndexError.new( @@ -968,7 +968,7 @@ RSpec.describe Gitlab::GitalyClient::OperationService do context 'with directory traversal' do let(:status_code) { GRPC::Core::StatusCodes::INVALID_ARGUMENT } - let(:expected_message) { "Directory traversal in path escapes repository: ../../../../etc/shadow" } + let(:expected_message) { "Path cannot include directory traversal" } let(:expected_error) do Gitaly::UserCommitFilesError.new( index_update: Gitaly::IndexError.new( @@ -982,7 +982,7 @@ RSpec.describe Gitlab::GitalyClient::OperationService do context 'with empty path' do let(:status_code) { GRPC::Core::StatusCodes::INVALID_ARGUMENT } - let(:expected_message) { "Received empty path" } + let(:expected_message) { "You must provide a file path" } let(:expected_error) do Gitaly::UserCommitFilesError.new( index_update: Gitaly::IndexError.new( @@ -1009,16 +1009,33 @@ RSpec.describe Gitlab::GitalyClient::OperationService do end context 'with an exception without the detailed error' do - let(:permission_error) do - GRPC::PermissionDenied.new - end - - it 'raises PermissionDenied' do + before do expect_any_instance_of(Gitaly::OperationService::Stub) .to receive(:user_commit_files).with(kind_of(Enumerator), kind_of(Hash)) - .and_raise(permission_error) + .and_raise(raised_error) + end - expect { subject }.to raise_error(GRPC::PermissionDenied) + context 'with an index error from libgit2' do + let(:raised_error) do + GRPC::Internal.new('invalid path: .git/foo') + end + + it 'raises IndexError' do + expect { subject }.to raise_error do |error| + expect(error).to be_a(Gitlab::Git::Index::IndexError) + expect(error.message).to eq('invalid path: .git/foo') + end + end + end + + context 'with a generic error' do + let(:raised_error) do + GRPC::PermissionDenied.new + end + + it 'raises PermissionDenied' do + expect { subject }.to raise_error(GRPC::PermissionDenied) + end end end end diff --git a/spec/lib/gitlab/gitaly_client/ref_service_spec.rb b/spec/lib/gitlab/gitaly_client/ref_service_spec.rb index bd96e9baf1d..ae2e343377d 100644 --- a/spec/lib/gitlab/gitaly_client/ref_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/ref_service_spec.rb @@ -71,28 +71,6 @@ RSpec.describe Gitlab::GitalyClient::RefService do end end - describe '#branch_names' do - it 'sends a find_all_branch_names message' do - expect_any_instance_of(Gitaly::RefService::Stub) - .to receive(:find_all_branch_names) - .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) - .and_return([]) - - client.branch_names - end - end - - describe '#tag_names' do - it 'sends a find_all_tag_names message' do - expect_any_instance_of(Gitaly::RefService::Stub) - .to receive(:find_all_tag_names) - .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) - .and_return([]) - - client.tag_names - end - end - describe '#find_branch' do it 'sends a find_branch message' do expect_any_instance_of(Gitaly::RefService::Stub) @@ -102,6 +80,16 @@ RSpec.describe Gitlab::GitalyClient::RefService do client.find_branch('name') end + + context 'when Gitaly returns a ambiguios reference error' do + it 'raises an UnknownRef error' do + expect_any_instance_of(Gitaly::RefService::Stub) + .to receive(:find_branch) + .and_raise(GRPC::BadStatus.new(2, 'reference is ambiguous')) + + expect { client.find_branch('name') }.to raise_error(Gitlab::Git::AmbiguousRef, 'branch is ambiguous: name') + end + end end describe '#find_tag' do diff --git a/spec/lib/gitlab/gitaly_client/with_feature_flag_actors_spec.rb b/spec/lib/gitlab/gitaly_client/with_feature_flag_actors_spec.rb index 41dce5d76dd..61945cc06b8 100644 --- a/spec/lib/gitlab/gitaly_client/with_feature_flag_actors_spec.rb +++ b/spec/lib/gitlab/gitaly_client/with_feature_flag_actors_spec.rb @@ -157,70 +157,47 @@ RSpec.describe Gitlab::GitalyClient::WithFeatureFlagActors do let(:call_arg_2) { double } let(:call_arg_3) { double } let(:call_result) { double } + let(:repository_actor) { instance_double(::Repository) } + let(:user_actor) { instance_double(::User) } + let(:project_actor) { instance_double(Project) } + let(:group_actor) { instance_double(Group) } before do + allow(service).to receive(:user_actor).and_return(user_actor) + allow(service).to receive(:repository_actor).and_return(repository_actor) + allow(service).to receive(:project_actor).and_return(project_actor) + allow(service).to receive(:group_actor).and_return(group_actor) + allow(Gitlab::GitalyClient).to receive(:with_feature_flag_actors).and_call_original allow(Gitlab::GitalyClient).to receive(:call).and_return(call_result) end - context 'when actors_aware_gitaly_calls flag is enabled' do - let(:repository_actor) { instance_double(::Repository) } - let(:user_actor) { instance_double(::User) } - let(:project_actor) { instance_double(Project) } - let(:group_actor) { instance_double(Group) } - - before do - stub_feature_flags(actors_aware_gitaly_calls: true) - - allow(service).to receive(:user_actor).and_return(user_actor) - allow(service).to receive(:repository_actor).and_return(repository_actor) - allow(service).to receive(:project_actor).and_return(project_actor) - allow(service).to receive(:group_actor).and_return(group_actor) - allow(Gitlab::GitalyClient).to receive(:with_feature_flag_actors).and_call_original - end - - it 'triggers client call with feature flag actors' do - result = service.gitaly_client_call(call_arg_1, call_arg_2, karg: call_arg_3) - - expect(Gitlab::GitalyClient).to have_received(:call).with(call_arg_1, call_arg_2, karg: call_arg_3) - expect(Gitlab::GitalyClient).to have_received(:with_feature_flag_actors).with( - repository: repository_actor, - user: user_actor, - project: project_actor, - group: group_actor - ) - expect(result).to be(call_result) - end - - context 'when call without repository_actor' do - before do - allow(service).to receive(:repository_actor).and_return(nil) - allow(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).and_call_original - end - - it 'calls error tracking track_and_raise_for_dev_exception' do - expect do - service.gitaly_client_call(call_arg_1, call_arg_2, karg: call_arg_3) - end.to raise_error /gitaly_client_call called without setting repository_actor/ - - expect(Gitlab::ErrorTracking).to have_received(:track_and_raise_for_dev_exception).with( - be_a(Feature::InvalidFeatureFlagError) - ) - end - end + it 'triggers client call with feature flag actors' do + result = service.gitaly_client_call(call_arg_1, call_arg_2, karg: call_arg_3) + + expect(Gitlab::GitalyClient).to have_received(:call).with(call_arg_1, call_arg_2, karg: call_arg_3) + expect(Gitlab::GitalyClient).to have_received(:with_feature_flag_actors).with( + repository: repository_actor, + user: user_actor, + project: project_actor, + group: group_actor + ) + expect(result).to be(call_result) end - context 'when actors_aware_gitaly_calls not enabled' do + context 'when call without repository_actor' do before do - stub_feature_flags(actors_aware_gitaly_calls: false) + allow(service).to receive(:repository_actor).and_return(nil) + allow(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).and_call_original end - it 'triggers client call without feature flag actors' do - expect(Gitlab::GitalyClient).not_to receive(:with_feature_flag_actors) + it 'calls error tracking track_and_raise_for_dev_exception' do + expect do + service.gitaly_client_call(call_arg_1, call_arg_2, karg: call_arg_3) + end.to raise_error /gitaly_client_call called without setting repository_actor/ - result = service.gitaly_client_call(call_arg_1, call_arg_2, karg: call_arg_3) - - expect(Gitlab::GitalyClient).to have_received(:call).with(call_arg_1, call_arg_2, karg: call_arg_3) - expect(result).to be(call_result) + expect(Gitlab::ErrorTracking).to have_received(:track_and_raise_for_dev_exception).with( + be_a(Feature::InvalidFeatureFlagError) + ) end end @@ -228,47 +205,28 @@ RSpec.describe Gitlab::GitalyClient::WithFeatureFlagActors do let_it_be(:project) { create(:project) } let(:repository_actor) { project.repository } - context 'when actors_aware_gitaly_calls flag is enabled' do - let(:user_actor) { instance_double(::User) } - let(:project_actor) { instance_double(Project) } - let(:group_actor) { instance_double(Group) } - - before do - stub_feature_flags(actors_aware_gitaly_calls: true) - - allow(Feature::Gitaly).to receive(:user_actor).and_return(user_actor) - allow(Feature::Gitaly).to receive(:project_actor).with(project).and_return(project_actor) - allow(Feature::Gitaly).to receive(:group_actor).with(project).and_return(group_actor) - end - - it 'returns a hash with collected feature flag actors' do - result = service.gitaly_feature_flag_actors(repository_actor) - expect(result).to eql( - repository: repository_actor, - user: user_actor, - project: project_actor, - group: group_actor - ) - - expect(Feature::Gitaly).to have_received(:user_actor).with(no_args) - expect(Feature::Gitaly).to have_received(:project_actor).with(project) - expect(Feature::Gitaly).to have_received(:group_actor).with(project) - end - end + let(:user_actor) { instance_double(::User) } + let(:project_actor) { instance_double(Project) } + let(:group_actor) { instance_double(Group) } - context 'when actors_aware_gitaly_calls not enabled' do - before do - stub_feature_flags(actors_aware_gitaly_calls: false) - end + before do + allow(Feature::Gitaly).to receive(:user_actor).and_return(user_actor) + allow(Feature::Gitaly).to receive(:project_actor).with(project).and_return(project_actor) + allow(Feature::Gitaly).to receive(:group_actor).with(project).and_return(group_actor) + end - it 'returns an empty hash' do - expect(Feature::Gitaly).not_to receive(:user_actor) - expect(Feature::Gitaly).not_to receive(:project_actor) - expect(Feature::Gitaly).not_to receive(:group_actor) + it 'returns a hash with collected feature flag actors' do + result = service.gitaly_feature_flag_actors(repository_actor) + expect(result).to eql( + repository: repository_actor, + user: user_actor, + project: project_actor, + group: group_actor + ) - result = service.gitaly_feature_flag_actors(repository_actor) - expect(result).to eql({}) - end + expect(Feature::Gitaly).to have_received(:user_actor).with(no_args) + expect(Feature::Gitaly).to have_received(:project_actor).with(project) + expect(Feature::Gitaly).to have_received(:group_actor).with(project) end end end diff --git a/spec/lib/gitlab/github_gists_import/importer/gist_importer_spec.rb b/spec/lib/gitlab/github_gists_import/importer/gist_importer_spec.rb new file mode 100644 index 00000000000..69a4d646562 --- /dev/null +++ b/spec/lib/gitlab/github_gists_import/importer/gist_importer_spec.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubGistsImport::Importer::GistImporter, feature_category: :importer do + subject { described_class.new(gist_object, user.id).execute } + + let_it_be(:user) { create(:user) } + let(:created_at) { Time.utc(2022, 1, 9, 12, 15) } + let(:updated_at) { Time.utc(2022, 5, 9, 12, 17) } + let(:gist_file) { { file_name: '_Summary.md', file_content: 'File content' } } + let(:url) { 'https://host.com/gistid.git' } + let(:gist_object) do + instance_double('Gitlab::GithubGistsImport::Representation::Gist', + truncated_title: 'My Gist', + visibility_level: 0, + files: { '_Summary.md': gist_file }, + first_file: gist_file, + git_pull_url: url, + created_at: created_at, + updated_at: updated_at + ) + end + + let(:expected_snippet_attrs) do + { + title: 'My Gist', + visibility_level: 0, + content: 'File content', + file_name: '_Summary.md', + author_id: user.id, + created_at: gist_object.created_at, + updated_at: gist_object.updated_at + }.stringify_keys + end + + describe '#execute' do + context 'when success' do + it 'creates expected snippet and snippet repository' do + expect_next_instance_of(Repository) do |repository| + expect(repository).to receive(:fetch_as_mirror) + end + + expect { subject }.to change { user.snippets.count }.by(1) + expect(user.snippets[0].attributes).to include expected_snippet_attrs + end + end + + context 'when file size limit exeeded' do + before do + files = [].tap { |array| 11.times { |n| array << ["file#{n}.txt", {}] } }.to_h + + allow(gist_object).to receive(:files).and_return(files) + allow_next_instance_of(Repository) do |repository| + allow(repository).to receive(:fetch_as_mirror) + allow(repository).to receive(:empty?).and_return(false) + allow(repository).to receive(:ls_files).and_return(files.keys) + end + end + + it 'returns error' do + result = subject + + expect(user.snippets.count).to eq(0) + expect(result.error?).to eq(true) + expect(result.errors).to match_array(['Snippet max file count exceeded']) + end + end + + context 'when invalid attributes' do + let(:gist_file) { { file_name: '_Summary.md', file_content: nil } } + + it 'raises an error' do + expect { subject }.to raise_error(ActiveRecord::RecordInvalid, "Validation failed: Content can't be blank") + end + end + + context 'when repository cloning fails' do + it 'returns error' do + expect_next_instance_of(Repository) do |repository| + expect(repository).to receive(:fetch_as_mirror).and_raise(Gitlab::Shell::Error) + expect(repository).to receive(:remove) + end + + expect { subject }.to raise_error(Gitlab::Shell::Error) + expect(user.snippets.count).to eq(0) + end + end + + context 'when url is invalid' do + let(:url) { 'invalid' } + + context 'when local network is allowed' do + before do + allow(::Gitlab::CurrentSettings) + .to receive(:allow_local_requests_from_web_hooks_and_services?).and_return(true) + end + + it 'raises error' do + expect(Gitlab::UrlBlocker) + .to receive(:validate!) + .with(url, ports: [80, 443], schemes: %w[http https git], + allow_localhost: true, allow_local_network: true) + .and_raise(Gitlab::UrlBlocker::BlockedUrlError) + + expect { subject }.to raise_error(Gitlab::UrlBlocker::BlockedUrlError) + end + end + + context 'when local network is not allowed' do + before do + allow(::Gitlab::CurrentSettings) + .to receive(:allow_local_requests_from_web_hooks_and_services?).and_return(false) + end + + it 'raises error' do + expect(Gitlab::UrlBlocker) + .to receive(:validate!) + .with(url, ports: [80, 443], schemes: %w[http https git], + allow_localhost: false, allow_local_network: false) + .and_raise(Gitlab::UrlBlocker::BlockedUrlError) + + expect { subject }.to raise_error(Gitlab::UrlBlocker::BlockedUrlError) + end + end + end + end +end diff --git a/spec/lib/gitlab/github_gists_import/importer/gists_importer_spec.rb b/spec/lib/gitlab/github_gists_import/importer/gists_importer_spec.rb new file mode 100644 index 00000000000..704999a99a9 --- /dev/null +++ b/spec/lib/gitlab/github_gists_import/importer/gists_importer_spec.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubGistsImport::Importer::GistsImporter, feature_category: :importer do + subject(:result) { described_class.new(user, token).execute } + + let_it_be(:user) { create(:user) } + let(:client) { instance_double('Gitlab::GithubImport::Client', rate_limit_resets_in: 5) } + let(:token) { 'token' } + let(:page_counter) { instance_double('Gitlab::GithubImport::PageCounter', current: 1, set: true, expire!: true) } + let(:page) { instance_double('Gitlab::GithubImport::Client::Page', objects: [gist], number: 1) } + let(:url) { 'https://gist.github.com/foo/bar.git' } + let(:waiter) { Gitlab::JobWaiter.new(0, 'some-job-key') } + + let(:gist) do + { + id: '055b70', + git_pull_url: url, + files: { + 'random.txt': { + filename: 'random.txt', + type: 'text/plain', + language: 'Text', + raw_url: 'https://gist.githubusercontent.com/user_name/055b70/raw/66a7be0d/random.txt', + size: 166903 + } + }, + public: false, + created_at: '2022-09-06T11:38:18Z', + updated_at: '2022-09-06T11:38:18Z', + description: 'random text' + } + end + + let(:gist_hash) do + { + id: '055b70', + import_url: url, + files: { + 'random.txt': { + filename: 'random.txt', + type: 'text/plain', + language: 'Text', + raw_url: 'https://gist.githubusercontent.com/user_name/055b70/raw/66a7be0d/random.txt', + size: 166903 + } + }, + public: false, + created_at: '2022-09-06T11:38:18Z', + updated_at: '2022-09-06T11:38:18Z', + title: 'random text' + } + end + + let(:gist_represent) { instance_double('Gitlab::GithubGistsImport::Representation::Gist', to_hash: gist_hash) } + + describe '#execute' do + before do + allow(Gitlab::GithubImport::Client) + .to receive(:new) + .with(token, parallel: true) + .and_return(client) + + allow(Gitlab::GithubImport::PageCounter) + .to receive(:new) + .with(user, :gists, 'github-gists-importer') + .and_return(page_counter) + + allow(client) + .to receive(:each_page) + .with(:gists, nil, { page: 1 }) + .and_yield(page) + + allow(Gitlab::GithubGistsImport::Representation::Gist) + .to receive(:from_api_response) + .with(gist) + .and_return(gist_represent) + + allow(Gitlab::JobWaiter) + .to receive(:new) + .and_return(waiter) + end + + context 'when success' do + it 'spread parallel import' do + expect(Gitlab::GithubGistsImport::ImportGistWorker) + .to receive(:bulk_perform_in) + .with( + 1.second, + [[user.id, gist_hash, waiter.key]], + batch_delay: 1.minute, + batch_size: 1000 + ) + + expect(result.waiter).to be_an_instance_of(Gitlab::JobWaiter) + expect(result.waiter.jobs_remaining).to eq(1) + end + end + + context 'when failure' do + it 'returns an error' do + expect(Gitlab::GithubGistsImport::ImportGistWorker) + .to receive(:bulk_perform_in) + .and_raise(StandardError, 'Error Message') + + expect(result.error).to be_an_instance_of(StandardError) + end + end + + context 'when rate limit reached' do + it 'returns an error' do + expect(Gitlab::GithubGistsImport::ImportGistWorker) + .to receive(:bulk_perform_in) + .and_raise(Gitlab::GithubImport::RateLimitError) + + expect(result.error).to be_an_instance_of(Gitlab::GithubImport::RateLimitError) + end + end + end +end diff --git a/spec/lib/gitlab/github_gists_import/representation/gist_spec.rb b/spec/lib/gitlab/github_gists_import/representation/gist_spec.rb new file mode 100644 index 00000000000..480aefb2c74 --- /dev/null +++ b/spec/lib/gitlab/github_gists_import/representation/gist_spec.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubGistsImport::Representation::Gist, feature_category: :importer do + shared_examples 'a Gist' do + it 'returns an instance of Gist' do + expect(gist).to be_an_instance_of(described_class) + end + + context 'with Gist' do + it 'includes gist attributes' do + expect(gist).to have_attributes( + id: '1', + description: 'Gist title', + is_public: true, + files: { '_Summary.md': { filename: '_Summary.md', raw_url: 'https://some_url' } }, + git_pull_url: 'https://gist.github.com/gistid.git' + ) + end + end + end + + describe '.from_api_response' do + let(:response) do + { + id: '1', + description: 'Gist title', + public: true, + created_at: '2022-04-26 18:30:53 UTC', + updated_at: '2022-04-26 18:30:53 UTC', + files: { '_Summary.md': { filename: '_Summary.md', raw_url: 'https://some_url' } }, + git_pull_url: 'https://gist.github.com/gistid.git' + } + end + + it_behaves_like 'a Gist' do + let(:gist) { described_class.from_api_response(response) } + end + end + + describe '.from_json_hash' do + it_behaves_like 'a Gist' do + let(:hash) do + { + 'id' => '1', + 'description' => 'Gist title', + 'is_public' => true, + 'files' => { '_Summary.md': { filename: '_Summary.md', raw_url: 'https://some_url' } }, + 'git_pull_url' => 'https://gist.github.com/gistid.git' + } + end + + let(:gist) { described_class.from_json_hash(hash) } + end + end + + describe '#truncated_title' do + it 'truncates the title to 255 characters' do + object = described_class.new(description: 'm' * 300) + + expect(object.truncated_title.length).to eq(255) + end + + it 'does not truncate the title if it is shorter than 255 characters' do + object = described_class.new(description: 'foo') + + expect(object.truncated_title).to eq('foo') + end + end + + describe '#github_identifiers' do + it 'returns a hash with needed identifiers' do + github_identifiers = { id: 1 } + gist = described_class.new(github_identifiers.merge(something_else: '_something_else_')) + + expect(gist.github_identifiers).to eq(github_identifiers) + end + end + + describe '#visibility_level' do + it 'returns 20 when public' do + visibility = { is_public: true } + gist = described_class.new(visibility.merge(something_else: '_something_else_')) + + expect(gist.visibility_level).to eq(20) + end + + it 'returns 0 when private' do + visibility = { is_public: false } + gist = described_class.new(visibility.merge(something_else: '_something_else_')) + + expect(gist.visibility_level).to eq(0) + end + end + + describe '#first_file' do + let(:http_response) { instance_double('HTTParty::Response', body: 'File content') } + + before do + allow(Gitlab::HTTP).to receive(:try_get).and_return(http_response) + end + + it 'returns a hash with needed identifiers' do + files = { files: { '_Summary.md': { filename: '_Summary.md', raw_url: 'https://some_url' } } } + gist = described_class.new(files.merge(something_else: '_something_else_')) + + expect(gist.first_file).to eq(file_name: '_Summary.md', file_content: 'File content') + end + end +end diff --git a/spec/lib/gitlab/github_gists_import/status_spec.rb b/spec/lib/gitlab/github_gists_import/status_spec.rb new file mode 100644 index 00000000000..4cbbbd430eb --- /dev/null +++ b/spec/lib/gitlab/github_gists_import/status_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubGistsImport::Status, :clean_gitlab_redis_cache, feature_category: :importer do + subject(:import_status) { described_class.new(user.id) } + + let_it_be(:user) { create(:user) } + let(:key) { "gitlab:github-gists-import:#{user.id}" } + + describe '#start!' do + it 'expires the key' do + import_status.start! + + Gitlab::Redis::SharedState.with do |redis| + expect(redis.get(key)).to eq('started') + end + end + end + + describe '#fail!' do + it 'sets failed status' do + import_status.fail! + + Gitlab::Redis::SharedState.with do |redis| + expect(redis.get(key)).to eq('failed') + end + end + end + + describe '#finish!' do + it 'sets finished status' do + import_status.finish! + + Gitlab::Redis::SharedState.with do |redis| + expect(redis.get(key)).to eq('finished') + end + end + end + + describe '#started?' do + before do + Gitlab::Redis::SharedState.with { |redis| redis.set(key, 'started') } + end + + it 'checks if status is started' do + expect(import_status.started?).to eq(true) + end + end +end diff --git a/spec/lib/gitlab/github_import/bulk_importing_spec.rb b/spec/lib/gitlab/github_import/bulk_importing_spec.rb index e170496ff7b..af31cb6c873 100644 --- a/spec/lib/gitlab/github_import/bulk_importing_spec.rb +++ b/spec/lib/gitlab/github_import/bulk_importing_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::BulkImporting do +RSpec.describe Gitlab::GithubImport::BulkImporting, feature_category: :importer do let(:project) { instance_double(Project, id: 1) } let(:importer) { MyImporter.new(project, double) } let(:importer_class) do @@ -12,22 +12,33 @@ RSpec.describe Gitlab::GithubImport::BulkImporting do def object_type :object_type end + + def model + Label + end end end + let(:label) { instance_double('Label', invalid?: false) } + before do stub_const 'MyImporter', importer_class end describe '#build_database_rows' do - it 'returns an Array containing the rows to insert' do + it 'returns an Array containing the rows to insert and validation errors if object invalid' do object = double(:object, title: 'Foo') expect(importer) - .to receive(:build) + .to receive(:build_attributes) .with(object) .and_return({ title: 'Foo' }) + expect(Label) + .to receive(:new) + .with({ title: 'Foo' }) + .and_return(label) + expect(importer) .to receive(:already_imported?) .with(object) @@ -53,14 +64,17 @@ RSpec.describe Gitlab::GithubImport::BulkImporting do enum = [[object, 1]].to_enum - expect(importer.build_database_rows(enum)).to eq([{ title: 'Foo' }]) + rows, errors = importer.build_database_rows(enum) + + expect(rows).to match_array([{ title: 'Foo' }]) + expect(errors).to be_empty end it 'does not import objects that have already been imported' do object = double(:object, title: 'Foo') expect(importer) - .not_to receive(:build) + .not_to receive(:build_attributes) expect(importer) .to receive(:already_imported?) @@ -87,14 +101,16 @@ RSpec.describe Gitlab::GithubImport::BulkImporting do enum = [[object, 1]].to_enum - expect(importer.build_database_rows(enum)).to be_empty + rows, errors = importer.build_database_rows(enum) + + expect(rows).to be_empty + expect(errors).to be_empty end end describe '#bulk_insert' do it 'bulk inserts rows into the database' do rows = [{ title: 'Foo' }] * 10 - model = double(:model, table_name: 'kittens') expect(Gitlab::Import::Logger) .to receive(:info) @@ -119,14 +135,43 @@ RSpec.describe Gitlab::GithubImport::BulkImporting do expect(ApplicationRecord) .to receive(:legacy_bulk_insert) .ordered - .with('kittens', rows.first(5)) + .with('labels', rows.first(5)) expect(ApplicationRecord) .to receive(:legacy_bulk_insert) .ordered - .with('kittens', rows.last(5)) + .with('labels', rows.last(5)) + + importer.bulk_insert(rows, batch_size: 5) + end + end + + describe '#bulk_insert_failures', :timecop do + let(:import_failures) { instance_double('ImportFailure::ActiveRecord_Associations_CollectionProxy') } + let(:label) { Label.new(title: 'invalid,title') } + let(:validation_errors) { ActiveModel::Errors.new(label) } + let(:formatted_errors) do + [{ + source: 'MyImporter', + exception_class: 'ActiveRecord::RecordInvalid', + exception_message: 'Title invalid', + correlation_id_value: 'cid', + retry_count: nil, + created_at: Time.zone.now + }] + end - importer.bulk_insert(model, rows, batch_size: 5) + it 'bulk inserts validation errors into import_failures' do + error = ActiveModel::Errors.new(label) + error.add(:base, 'Title invalid') + + freeze_time do + expect(project).to receive(:import_failures).and_return(import_failures) + expect(import_failures).to receive(:insert_all).with(formatted_errors) + expect(Labkit::Correlation::CorrelationId).to receive(:current_or_new_id).and_return('cid') + + importer.bulk_insert_failures([error]) + end end end end diff --git a/spec/lib/gitlab/github_import/client_spec.rb b/spec/lib/gitlab/github_import/client_spec.rb index 95f7933fbc5..526a8721ff3 100644 --- a/spec/lib/gitlab/github_import/client_spec.rb +++ b/spec/lib/gitlab/github_import/client_spec.rb @@ -579,13 +579,69 @@ RSpec.describe Gitlab::GithubImport::Client do allow(client.octokit).to receive(:user).and_return(user) end - describe '#search_repos_by_name' do + describe '#search_repos_by_name_graphql' do + let(:expected_query) { 'test in:name is:public,private user:user repo:repo1 repo:repo2 org:org1 org:org2' } + let(:expected_graphql_params) { "type: REPOSITORY, query: \"#{expected_query}\"" } + let(:expected_graphql) do + <<-TEXT + { + search(#{expected_graphql_params}) { + nodes { + __typename + ... on Repository { + id: databaseId + name + full_name: nameWithOwner + owner { login } + } + } + pageInfo { + startCursor + endCursor + hasNextPage + hasPreviousPage + } + } + } + TEXT + end + it 'searches for repositories based on name' do - expected_search_query = 'test in:name is:public,private user:user repo:repo1 repo:repo2 org:org1 org:org2' + expect(client.octokit).to receive(:post).with( + '/graphql', { query: expected_graphql }.to_json + ) - expect(client.octokit).to receive(:search_repositories).with(expected_search_query, {}) + client.search_repos_by_name_graphql('test') + end - client.search_repos_by_name('test') + context 'when pagination options present' do + context 'with "first" option' do + let(:expected_graphql_params) do + "type: REPOSITORY, query: \"#{expected_query}\", first: 25" + end + + it 'searches for repositories via expected query' do + expect(client.octokit).to receive(:post).with( + '/graphql', { query: expected_graphql }.to_json + ) + + client.search_repos_by_name_graphql('test', { first: 25 }) + end + end + + context 'with "after" option' do + let(:expected_graphql_params) do + "type: REPOSITORY, query: \"#{expected_query}\", after: \"Y3Vyc29yOjE=\"" + end + + it 'searches for repositories via expected query' do + expect(client.octokit).to receive(:post).with( + '/graphql', { query: expected_graphql }.to_json + ) + + client.search_repos_by_name_graphql('test', { after: 'Y3Vyc29yOjE=' }) + end + end end context 'when Faraday error received from octokit', :aggregate_failures do @@ -593,41 +649,62 @@ RSpec.describe Gitlab::GithubImport::Client do let(:info_params) { { 'error.class': error_class } } it 'retries on error and succeeds' do - allow_retry(:search_repositories) + allow_retry(:post) expect(Gitlab::Import::Logger).to receive(:info).with(hash_including(info_params)).once - expect(client.search_repos_by_name('test')).to eq({}) + expect(client.search_repos_by_name_graphql('test')).to eq({}) end it 'retries and does not succeed' do - allow(client.octokit).to receive(:search_repositories).and_raise(error_class, 'execution expired') + allow(client.octokit) + .to receive(:post) + .with('/graphql', { query: expected_graphql }.to_json) + .and_raise(error_class, 'execution expired') - expect { client.search_repos_by_name('test') }.to raise_error(error_class, 'execution expired') + expect { client.search_repos_by_name_graphql('test') }.to raise_error(error_class, 'execution expired') end end end - describe '#search_query' do - it 'returns base search query' do - result = client.search_query(str: 'test', type: :test, include_collaborations: false, include_orgs: false) + describe '#search_repos_by_name' do + let(:expected_query) { 'test in:name is:public,private user:user repo:repo1 repo:repo2 org:org1 org:org2' } - expect(result).to eq('test in:test is:public,private user:user') + it 'searches for repositories based on name' do + expect(client.octokit).to receive(:search_repositories).with(expected_query, {}) + + client.search_repos_by_name('test') end - context 'when include_collaborations is true' do - it 'returns search query including users' do - result = client.search_query(str: 'test', type: :test, include_collaborations: true, include_orgs: false) + context 'when pagination options present' do + it 'searches for repositories via expected query' do + expect(client.octokit).to receive(:search_repositories).with( + expected_query, { page: 2, per_page: 25 } + ) - expect(result).to eq('test in:test is:public,private user:user repo:repo1 repo:repo2') + client.search_repos_by_name('test', { page: 2, per_page: 25 }) end end - context 'when include_orgs is true' do - it 'returns search query including users' do - result = client.search_query(str: 'test', type: :test, include_collaborations: false, include_orgs: true) + context 'when Faraday error received from octokit', :aggregate_failures do + let(:error_class) { described_class::CLIENT_CONNECTION_ERROR } + let(:info_params) { { 'error.class': error_class } } + + it 'retries on error and succeeds' do + allow_retry(:search_repositories) + + expect(Gitlab::Import::Logger).to receive(:info).with(hash_including(info_params)).once + + expect(client.search_repos_by_name('test')).to eq({}) + end + + it 'retries and does not succeed' do + allow(client.octokit) + .to receive(:search_repositories) + .with(expected_query, {}) + .and_raise(error_class, 'execution expired') - expect(result).to eq('test in:test is:public,private user:user org:org1 org:org2') + expect { client.search_repos_by_name('test') }.to raise_error(error_class, 'execution expired') end end end diff --git a/spec/lib/gitlab/github_import/clients/proxy_spec.rb b/spec/lib/gitlab/github_import/clients/proxy_spec.rb new file mode 100644 index 00000000000..9fef57f2a38 --- /dev/null +++ b/spec/lib/gitlab/github_import/clients/proxy_spec.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Clients::Proxy, :manage, feature_category: :import do + subject(:client) { described_class.new(access_token, client_options) } + + let(:access_token) { 'test_token' } + let(:client_options) { { foo: :bar } } + + describe '#repos' do + let(:search_text) { 'search text' } + let(:pagination_options) { { limit: 10 } } + + context 'when remove_legacy_github_client FF is enabled' do + let(:client_stub) { instance_double(Gitlab::GithubImport::Client) } + + context 'with github_client_fetch_repos_via_graphql FF enabled' do + let(:client_response) do + { + data: { + search: { + nodes: [{ name: 'foo' }, { name: 'bar' }], + pageInfo: { startCursor: 'foo', endCursor: 'bar' } + } + } + } + end + + it 'fetches repos with Gitlab::GithubImport::Client (GraphQL API)' do + expect(Gitlab::GithubImport::Client) + .to receive(:new).with(access_token).and_return(client_stub) + expect(client_stub) + .to receive(:search_repos_by_name_graphql) + .with(search_text, pagination_options).and_return(client_response) + + expect(client.repos(search_text, pagination_options)).to eq( + { + repos: [{ name: 'foo' }, { name: 'bar' }], + page_info: { startCursor: 'foo', endCursor: 'bar' } + } + ) + end + end + + context 'with github_client_fetch_repos_via_graphql FF disabled' do + let(:client_response) do + { items: [{ name: 'foo' }, { name: 'bar' }] } + end + + before do + stub_feature_flags(github_client_fetch_repos_via_graphql: false) + end + + it 'fetches repos with Gitlab::GithubImport::Client (REST API)' do + expect(Gitlab::GithubImport::Client) + .to receive(:new).with(access_token).and_return(client_stub) + expect(client_stub) + .to receive(:search_repos_by_name) + .with(search_text, pagination_options).and_return(client_response) + + expect(client.repos(search_text, pagination_options)).to eq( + { repos: [{ name: 'foo' }, { name: 'bar' }] } + ) + end + end + end + + context 'when remove_legacy_github_client FF is disabled' do + let(:client_stub) { instance_double(Gitlab::LegacyGithubImport::Client) } + let(:search_text) { nil } + + before do + stub_feature_flags(remove_legacy_github_client: false) + end + + it 'fetches repos with Gitlab::LegacyGithubImport::Client' do + expect(Gitlab::LegacyGithubImport::Client) + .to receive(:new).with(access_token, client_options).and_return(client_stub) + expect(client_stub).to receive(:repos) + .and_return([{ name: 'foo' }, { name: 'bar' }]) + + expect(client.repos(search_text, pagination_options)) + .to eq({ repos: [{ name: 'foo' }, { name: 'bar' }] }) + end + + context 'with filter params' do + let(:search_text) { 'fo' } + + it 'fetches repos with Gitlab::LegacyGithubImport::Client' do + expect(Gitlab::LegacyGithubImport::Client) + .to receive(:new).with(access_token, client_options).and_return(client_stub) + expect(client_stub).to receive(:repos) + .and_return([{ name: 'FOO' }, { name: 'bAr' }]) + + expect(client.repos(search_text, pagination_options)) + .to eq({ repos: [{ name: 'FOO' }] }) + end + end + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb b/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb index 8eeb2332131..73ba49bf4ed 100644 --- a/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb @@ -35,7 +35,8 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNoteImporter, :aggregate_fail end_line: end_line, github_id: 1, diff_hunk: diff_hunk, - side: 'RIGHT' + side: 'RIGHT', + discussion_id: discussion_id ) end @@ -114,10 +115,6 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNoteImporter, :aggregate_fail .to receive(:database_id) .and_return(merge_request.id) end - - expect(Discussion) - .to receive(:discussion_id) - .and_return(discussion_id) end it_behaves_like 'diff notes without suggestion' @@ -218,6 +215,16 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNoteImporter, :aggregate_fail end end end + + context 'when diff note is invalid' do + it 'fails validation' do + stub_user_finder(user.id, true) + + expect(note_representation).to receive(:line_code).and_return(nil) + + expect { subject.execute }.to raise_error(ActiveRecord::RecordInvalid) + end + end end end diff --git a/spec/lib/gitlab/github_import/importer/issues_importer_spec.rb b/spec/lib/gitlab/github_import/importer/issues_importer_spec.rb index 308b8185589..4a5525c250e 100644 --- a/spec/lib/gitlab/github_import/importer/issues_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/issues_importer_spec.rb @@ -90,9 +90,13 @@ RSpec.describe Gitlab::GithubImport::Importer::IssuesImporter do .to receive(:each_object_to_import) .and_yield(github_issue) - expect(Gitlab::GithubImport::ImportIssueWorker).to receive(:bulk_perform_in).with(1.second, [ - [project.id, an_instance_of(Hash), an_instance_of(String)] - ], batch_size: 1000, batch_delay: 1.minute) + expect(Gitlab::GithubImport::ImportIssueWorker) + .to receive(:bulk_perform_in) + .with(1.second, + [[project.id, an_instance_of(Hash), an_instance_of(String)]], + batch_size: 1000, + batch_delay: 1.minute + ) waiter = importer.parallel_import diff --git a/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb b/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb index e68849755b2..e005d8eda84 100644 --- a/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb @@ -36,26 +36,11 @@ RSpec.describe Gitlab::GithubImport::Importer::LabelLinksImporter do expect(importer) .to receive(:find_target_id) - .and_return(1) + .and_return(4) - freeze_time do - expect(ApplicationRecord) - .to receive(:legacy_bulk_insert) - .with( - LabelLink.table_name, - [ - { - label_id: 2, - target_id: 1, - target_type: Issue, - created_at: Time.zone.now, - updated_at: Time.zone.now - } - ] - ) + expect(LabelLink).to receive(:bulk_insert!) - importer.create_labels - end + importer.create_labels end it 'does not insert label links for non-existing labels' do @@ -64,9 +49,9 @@ RSpec.describe Gitlab::GithubImport::Importer::LabelLinksImporter do .with('bug') .and_return(nil) - expect(ApplicationRecord) - .to receive(:legacy_bulk_insert) - .with(LabelLink.table_name, []) + expect(LabelLink) + .to receive(:bulk_insert!) + .with([]) importer.create_labels end diff --git a/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb b/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb index 81d534c566f..ad9ef4afddd 100644 --- a/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::Importer::LabelsImporter, :clean_gitlab_redis_cache do +RSpec.describe Gitlab::GithubImport::Importer::LabelsImporter, :clean_gitlab_redis_cache, feature_category: :importer do let(:project) { create(:project, import_source: 'foo/bar') } let(:client) { double(:client) } let(:importer) { described_class.new(project, client) } @@ -11,40 +11,58 @@ RSpec.describe Gitlab::GithubImport::Importer::LabelsImporter, :clean_gitlab_red it 'imports the labels in bulk' do label_hash = { title: 'bug', color: '#fffaaa' } - expect(importer) - .to receive(:build_labels) - .and_return([label_hash]) - - expect(importer) - .to receive(:bulk_insert) - .with(Label, [label_hash]) - - expect(importer) - .to receive(:build_labels_cache) + expect(importer).to receive(:build_labels).and_return([[label_hash], []]) + expect(importer).to receive(:bulk_insert).with([label_hash]) + expect(importer).to receive(:build_labels_cache) importer.execute end end describe '#build_labels' do - it 'returns an Array containnig label rows' do + it 'returns an Array containing label rows' do label = { name: 'bug', color: 'ffffff' } expect(importer).to receive(:each_label).and_return([label]) - rows = importer.build_labels + rows, errors = importer.build_labels expect(rows.length).to eq(1) expect(rows[0][:title]).to eq('bug') + expect(errors).to be_blank end - it 'does not create labels that already exist' do + it 'does not build labels that already exist' do create(:label, project: project, title: 'bug') label = { name: 'bug', color: 'ffffff' } expect(importer).to receive(:each_label).and_return([label]) - expect(importer.build_labels).to be_empty + + rows, errors = importer.build_labels + + expect(rows).to be_empty + expect(errors).to be_empty + end + + it 'does not build labels that are invalid' do + label = { id: 1, name: 'bug,bug', color: 'ffffff' } + + expect(importer).to receive(:each_label).and_return([label]) + expect(Gitlab::Import::Logger).to receive(:error) + .with( + import_type: :github, + project_id: project.id, + importer: described_class.name, + message: ['Title is invalid'], + github_identifier: 1 + ) + + rows, errors = importer.build_labels + + expect(rows).to be_empty + expect(errors.length).to eq(1) + expect(errors[0].full_messages).to match_array(['Title is invalid']) end end @@ -58,9 +76,9 @@ RSpec.describe Gitlab::GithubImport::Importer::LabelsImporter, :clean_gitlab_red end end - describe '#build' do + describe '#build_attributes' do let(:label_hash) do - importer.build({ name: 'bug', color: 'ffffff' }) + importer.build_attributes({ name: 'bug', color: 'ffffff' }) end it 'returns the attributes of the label as a Hash' do diff --git a/spec/lib/gitlab/github_import/importer/lfs_objects_importer_spec.rb b/spec/lib/gitlab/github_import/importer/lfs_objects_importer_spec.rb index 99536588718..678aa705b6c 100644 --- a/spec/lib/gitlab/github_import/importer/lfs_objects_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/lfs_objects_importer_spec.rb @@ -58,7 +58,7 @@ RSpec.describe Gitlab::GithubImport::Importer::LfsObjectsImporter do expect_next_instance_of(Projects::LfsPointers::LfsObjectDownloadListService) do |service| expect(service) - .to receive(:execute) + .to receive(:each_list_item) .and_raise(exception) end @@ -79,7 +79,7 @@ RSpec.describe Gitlab::GithubImport::Importer::LfsObjectsImporter do expect_next_instance_of(Projects::LfsPointers::LfsObjectDownloadListService) do |service| expect(service) - .to receive(:execute) + .to receive(:each_list_item) .and_raise(exception) end @@ -94,7 +94,7 @@ RSpec.describe Gitlab::GithubImport::Importer::LfsObjectsImporter do lfs_object_importer = double(:lfs_object_importer) expect_next_instance_of(Projects::LfsPointers::LfsObjectDownloadListService) do |service| - expect(service).to receive(:execute).and_return([lfs_download_object]) + expect(service).to receive(:each_list_item).and_yield(lfs_download_object) end expect(Gitlab::GithubImport::Importer::LfsObjectImporter) @@ -115,7 +115,7 @@ RSpec.describe Gitlab::GithubImport::Importer::LfsObjectsImporter do importer = described_class.new(project, client) expect_next_instance_of(Projects::LfsPointers::LfsObjectDownloadListService) do |service| - expect(service).to receive(:execute).and_return([lfs_download_object]) + expect(service).to receive(:each_list_item).and_yield(lfs_download_object) end expect(Gitlab::GithubImport::ImportLfsObjectWorker).to receive(:bulk_perform_in) diff --git a/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb b/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb index 04d76bd1f06..8667729d79b 100644 --- a/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::Importer::MilestonesImporter, :clean_gitlab_redis_cache do +RSpec.describe Gitlab::GithubImport::Importer::MilestonesImporter, :clean_gitlab_redis_cache, + feature_category: :importer do let(:project) { create(:project, import_source: 'foo/bar') } let(:client) { double(:client) } let(:importer) { described_class.new(project, client) } @@ -38,41 +39,61 @@ RSpec.describe Gitlab::GithubImport::Importer::MilestonesImporter, :clean_gitlab it 'imports the milestones in bulk' do milestone_hash = { number: 1, title: '1.0' } - expect(importer) - .to receive(:build_milestones) - .and_return([milestone_hash]) - - expect(importer) - .to receive(:bulk_insert) - .with(Milestone, [milestone_hash]) - - expect(importer) - .to receive(:build_milestones_cache) + expect(importer).to receive(:build_milestones).and_return([[milestone_hash], []]) + expect(importer).to receive(:bulk_insert).with([milestone_hash]) + expect(importer).to receive(:build_milestones_cache) importer.execute end end describe '#build_milestones' do - it 'returns an Array containnig milestone rows' do + it 'returns an Array containing milestone rows' do expect(importer) .to receive(:each_milestone) .and_return([milestone]) - rows = importer.build_milestones + rows, errors = importer.build_milestones expect(rows.length).to eq(1) expect(rows[0][:title]).to eq('1.0') + expect(errors).to be_empty end - it 'does not create milestones that already exist' do + it 'does not build milestones that already exist' do create(:milestone, project: project, title: '1.0', iid: 1) expect(importer) .to receive(:each_milestone) .and_return([milestone]) - expect(importer.build_milestones).to be_empty + rows, errors = importer.build_milestones + + expect(rows).to be_empty + expect(errors).to be_empty + end + + it 'does not build milestones that are invalid' do + milestone = { id: 1, title: nil } + + expect(importer) + .to receive(:each_milestone) + .and_return([milestone]) + + expect(Gitlab::Import::Logger).to receive(:error) + .with( + import_type: :github, + project_id: project.id, + importer: described_class.name, + message: ["Title can't be blank"], + github_identifier: 1 + ) + + rows, errors = importer.build_milestones + + expect(rows).to be_empty + expect(errors.length).to eq(1) + expect(errors[0].full_messages).to match_array(["Title can't be blank"]) end end @@ -86,9 +107,9 @@ RSpec.describe Gitlab::GithubImport::Importer::MilestonesImporter, :clean_gitlab end end - describe '#build' do - let(:milestone_hash) { importer.build(milestone) } - let(:milestone_hash2) { importer.build(milestone2) } + describe '#build_attributes' do + let(:milestone_hash) { importer.build_attributes(milestone) } + let(:milestone_hash2) { importer.build_attributes(milestone2) } it 'returns the attributes of the milestone as a Hash' do expect(milestone_hash).to be_an_instance_of(Hash) diff --git a/spec/lib/gitlab/github_import/importer/note_importer_spec.rb b/spec/lib/gitlab/github_import/importer/note_importer_spec.rb index c60ecd85e92..5ac50578b6a 100644 --- a/spec/lib/gitlab/github_import/importer/note_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/note_importer_spec.rb @@ -113,6 +113,19 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteImporter do .to eq('There were an invalid char "" <= right here') end end + + context 'when note is invalid' do + it 'fails validation' do + expect(importer.user_finder) + .to receive(:author_id_for) + .with(github_note) + .and_return([user.id, true]) + + expect(github_note).to receive(:discussion_id).and_return('invalid') + + expect { importer.execute }.to raise_error(ActiveRecord::RecordInvalid) + end + end end context 'when the noteable does not exist' do diff --git a/spec/lib/gitlab/github_import/importer/pull_request_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_request_importer_spec.rb index c7388314253..dd73b6879e0 100644 --- a/spec/lib/gitlab/github_import/importer/pull_request_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/pull_request_importer_spec.rb @@ -202,6 +202,20 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestImporter, :clean_gitla importer.create_merge_request end end + + context 'when merge request is invalid' do + before do + allow(pull_request).to receive(:formatted_source_branch).and_return(nil) + allow(importer.user_finder) + .to receive(:author_id_for) + .with(pull_request) + .and_return([project.creator_id, false]) + end + + it 'fails validation' do + expect { importer.create_merge_request }.to raise_error(ActiveRecord::RecordInvalid) + end + end end describe '#set_merge_request_assignees' do @@ -292,6 +306,16 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestImporter, :clean_gitla expect(project.repository.branch_exists?(mr.source_branch)).to be_falsey expect(project.repository.branch_exists?(mr.target_branch)).to be_truthy end + + it 'ignores Git PreReceive errors when creating a branch' do + expect(project.repository).to receive(:add_branch).and_raise(Gitlab::Git::PreReceiveError) + expect(Gitlab::ErrorTracking).to receive(:track_exception).and_call_original + + mr = insert_git_data + + expect(project.repository.branch_exists?(mr.source_branch)).to be_falsey + expect(project.repository.branch_exists?(mr.target_branch)).to be_truthy + end end it 'creates a merge request diff and sets it as the latest' do diff --git a/spec/lib/gitlab/github_import/importer/pull_request_merged_by_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_request_merged_by_importer_spec.rb index f3a9bbac785..01d706beea2 100644 --- a/spec/lib/gitlab/github_import/importer/pull_request_merged_by_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/pull_request_merged_by_importer_spec.rb @@ -22,6 +22,23 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestMergedByImporter, :cle subject { described_class.new(pull_request, project, client_double) } + shared_examples 'adds a note referencing the merger user' do + it 'adds a note referencing the merger user' do + expect { subject.execute } + .to change(Note, :count).by(1) + .and not_change(merge_request, :updated_at) + + metrics = merge_request.metrics.reload + expect(metrics.merged_by).to be_nil + expect(metrics.merged_at).to eq(merged_at) + + last_note = merge_request.notes.last + expect(last_note.created_at).to eq(merged_at) + expect(last_note.author).to eq(project.creator) + expect(last_note.note).to eq("*Merged by: merger at #{merged_at}*") + end + end + context 'when the merger user can be mapped' do it 'assigns the merged by user when mapped' do merge_user = create(:user, email: 'merger@email.com') @@ -35,19 +52,14 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestMergedByImporter, :cle end context 'when the merger user cannot be mapped to a gitlab user' do - it 'adds a note referencing the merger user' do - expect { subject.execute } - .to change(Note, :count).by(1) - .and not_change(merge_request, :updated_at) + it_behaves_like 'adds a note referencing the merger user' - metrics = merge_request.metrics.reload - expect(metrics.merged_by).to be_nil - expect(metrics.merged_at).to eq(merged_at) + context 'when original user cannot be found on github' do + before do + allow(client_double).to receive(:user).and_raise(Octokit::NotFound) + end - last_note = merge_request.notes.last - expect(last_note.note).to eq("*Merged by: merger at 2017-01-01 12:00:00 UTC*") - expect(last_note.created_at).to eq(merged_at) - expect(last_note.author).to eq(project.creator) + it_behaves_like 'adds a note referencing the merger user' end end @@ -64,9 +76,9 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestMergedByImporter, :cle expect(metrics.merged_at).to eq(merged_at) last_note = merge_request.notes.last - expect(last_note.note).to eq("*Merged by: ghost at 2017-01-01 12:00:00 UTC*") expect(last_note.created_at).to eq(merged_at) expect(last_note.author).to eq(project.creator) + expect(last_note.note).to eq("*Merged by: ghost at #{merged_at}*") end end end diff --git a/spec/lib/gitlab/github_import/importer/pull_request_review_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_request_review_importer_spec.rb index 49794eceb5a..2e1a3c496cc 100644 --- a/spec/lib/gitlab/github_import/importer/pull_request_review_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/pull_request_review_importer_spec.rb @@ -208,6 +208,23 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, :clean end end + context 'when original author cannot be found on github' do + before do + allow(client_double).to receive(:user).and_raise(Octokit::NotFound) + end + + let(:review) { create_review(type: 'APPROVED', note: '') } + + it 'creates a note for the review with the author username' do + expect { subject.execute } + .to change(Note, :count).by(1) + last_note = merge_request.notes.last + expect(last_note.note).to eq("*Created by: author*\n\n**Review:** Approved") + expect(last_note.author).to eq(project.creator) + expect(last_note.created_at).to eq(submitted_at) + end + end + context 'when the submitted_at is not provided' do let(:review) { create_review(type: 'APPROVED', note: '', submitted_at: nil) } diff --git a/spec/lib/gitlab/github_import/importer/pull_requests/review_requests_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_requests/review_requests_importer_spec.rb index 6c7fc4d5b15..3bf1976ee10 100644 --- a/spec/lib/gitlab/github_import/importer/pull_requests/review_requests_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/pull_requests/review_requests_importer_spec.rb @@ -109,7 +109,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequests::ReviewRequestsImpor expect(Gitlab::GithubImport::PullRequests::ImportReviewRequestWorker) .to receive(:bulk_perform_in).with( 1.second, - expected_worker_payload, + match_array(expected_worker_payload), batch_size: 1000, batch_delay: 1.minute ) diff --git a/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb b/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb index 84d639a09ef..ccbe5b5fc50 100644 --- a/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::Importer::ReleasesImporter do +RSpec.describe Gitlab::GithubImport::Importer::ReleasesImporter, feature_category: :importer do let(:project) { create(:project) } let(:client) { double(:client) } let(:importer) { described_class.new(project, client) } @@ -48,8 +48,8 @@ RSpec.describe Gitlab::GithubImport::Importer::ReleasesImporter do released_at: released_at } - expect(importer).to receive(:build_releases).and_return([release_hash]) - expect(importer).to receive(:bulk_insert).with(Release, [release_hash]) + expect(importer).to receive(:build_releases).and_return([[release_hash], []]) + expect(importer).to receive(:bulk_insert).with([release_hash]) importer.execute end @@ -86,24 +86,29 @@ RSpec.describe Gitlab::GithubImport::Importer::ReleasesImporter do it 'returns an Array containing release rows' do expect(importer).to receive(:each_release).and_return([github_release]) - rows = importer.build_releases + rows, errors = importer.build_releases expect(rows.length).to eq(1) expect(rows[0][:tag]).to eq('1.0') + expect(errors).to be_empty end it 'does not create releases that already exist' do create(:release, project: project, tag: '1.0', description: '1.0') expect(importer).to receive(:each_release).and_return([github_release]) - expect(importer.build_releases).to be_empty + + rows, errors = importer.build_releases + + expect(rows).to be_empty + expect(errors).to be_empty end it 'uses a default release description if none is provided' do github_release[:body] = nil expect(importer).to receive(:each_release).and_return([github_release]) - release = importer.build_releases.first + release, _ = importer.build_releases.first expect(release[:description]).to eq('Release for tag 1.0') end @@ -115,20 +120,36 @@ RSpec.describe Gitlab::GithubImport::Importer::ReleasesImporter do } expect(importer).to receive(:each_release).and_return([null_tag_release]) - expect(importer.build_releases).to be_empty + + rows, errors = importer.build_releases + + expect(rows).to be_empty + expect(errors).to be_empty end it 'does not create duplicate release tags' do expect(importer).to receive(:each_release).and_return([github_release, github_release]) - releases = importer.build_releases + releases, _ = importer.build_releases expect(releases.length).to eq(1) expect(releases[0][:description]).to eq('This is my release') end + + it 'does not create invalid release' do + github_release[:body] = SecureRandom.alphanumeric(Gitlab::Database::MAX_TEXT_SIZE_LIMIT + 1) + + expect(importer).to receive(:each_release).and_return([github_release]) + + releases, errors = importer.build_releases + + expect(releases).to be_empty + expect(errors.length).to eq(1) + expect(errors[0].full_messages).to match_array(['Description is too long (maximum is 1000000 characters)']) + end end - describe '#build' do - let(:release_hash) { importer.build(github_release) } + describe '#build_attributes' do + let(:release_hash) { importer.build_attributes(github_release) } context 'the returned Hash' do before do diff --git a/spec/lib/gitlab/github_import/markdown/attachment_spec.rb b/spec/lib/gitlab/github_import/markdown/attachment_spec.rb index 5d29de34141..588a3076f59 100644 --- a/spec/lib/gitlab/github_import/markdown/attachment_spec.rb +++ b/spec/lib/gitlab/github_import/markdown/attachment_spec.rb @@ -35,6 +35,12 @@ RSpec.describe Gitlab::GithubImport::Markdown::Attachment do it { expect(described_class.from_markdown(markdown_node)).to eq nil } end + + context 'when URL is blank' do + let(:url) { nil } + + it { expect(described_class.from_markdown(markdown_node)).to eq nil } + end end context "when it's an image attachment" do @@ -63,6 +69,12 @@ RSpec.describe Gitlab::GithubImport::Markdown::Attachment do it { expect(described_class.from_markdown(markdown_node)).to eq nil } end + + context 'when URL is blank' do + let(:url) { nil } + + it { expect(described_class.from_markdown(markdown_node)).to eq nil } + end end context "when it's an inline html node" do @@ -80,6 +92,12 @@ RSpec.describe Gitlab::GithubImport::Markdown::Attachment do expect(attachment.name).to eq name expect(attachment.url).to eq url end + + context 'when image src is not present' do + let(:img) { "<img width=\"248\" alt=\"#{name}\">" } + + it { expect(described_class.from_markdown(markdown_node)).to eq nil } + end end end diff --git a/spec/lib/gitlab/github_import/page_counter_spec.rb b/spec/lib/gitlab/github_import/page_counter_spec.rb index 568bc8cbbef..511b19c00e5 100644 --- a/spec/lib/gitlab/github_import/page_counter_spec.rb +++ b/spec/lib/gitlab/github_import/page_counter_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::PageCounter, :clean_gitlab_redis_cache do +RSpec.describe Gitlab::GithubImport::PageCounter, :clean_gitlab_redis_cache, feature_category: :importer do let(:project) { double(:project, id: 1) } let(:counter) { described_class.new(project, :issues) } @@ -16,6 +16,16 @@ RSpec.describe Gitlab::GithubImport::PageCounter, :clean_gitlab_redis_cache do expect(described_class.new(project, :issues).current).to eq(2) end + + context 'when gists import' do + let(:user) { instance_double('User', id: 2) } + + it 'uses gists specific key' do + result = described_class.new(user, :gists, 'github-gists-importer') + + expect(result.cache_key).to eq('github-gists-importer/page-counter/2/gists') + end + end end describe '#set' do diff --git a/spec/lib/gitlab/github_import/representation/diff_note_spec.rb b/spec/lib/gitlab/github_import/representation/diff_note_spec.rb index a656cd0d056..56fabe854f9 100644 --- a/spec/lib/gitlab/github_import/representation/diff_note_spec.rb +++ b/spec/lib/gitlab/github_import/representation/diff_note_spec.rb @@ -128,64 +128,6 @@ RSpec.describe Gitlab::GithubImport::Representation::DiffNote, :clean_gitlab_red end end - describe '#discussion_id' do - before do - note.project = project - note.merge_request = merge_request - end - - context 'when the note is a reply to a discussion' do - it 'uses the cached value as the discussion_id only when responding an existing discussion' do - expect(Discussion) - .to receive(:discussion_id) - .and_return('FIRST_DISCUSSION_ID', 'SECOND_DISCUSSION_ID') - - # Creates the first discussion id and caches its value - expect(note.discussion_id) - .to eq('FIRST_DISCUSSION_ID') - - reply_note = described_class.from_json_hash( - 'note_id' => note.note_id + 1, - 'in_reply_to_id' => note.note_id - ) - reply_note.project = project - reply_note.merge_request = merge_request - - # Reading from the cached value - expect(reply_note.discussion_id) - .to eq('FIRST_DISCUSSION_ID') - - new_discussion_note = described_class.from_json_hash( - 'note_id' => note.note_id + 2, - 'in_reply_to_id' => nil - ) - new_discussion_note.project = project - new_discussion_note.merge_request = merge_request - - # Because it's a new discussion, it must not use the cached value - expect(new_discussion_note.discussion_id) - .to eq('SECOND_DISCUSSION_ID') - end - - context 'when cached value does not exist' do - it 'falls back to generating a new discussion_id' do - expect(Discussion) - .to receive(:discussion_id) - .and_return('NEW_DISCUSSION_ID') - - reply_note = described_class.from_json_hash( - 'note_id' => note.note_id + 1, - 'in_reply_to_id' => note.note_id - ) - reply_note.project = project - reply_note.merge_request = merge_request - - expect(reply_note.discussion_id).to eq('NEW_DISCUSSION_ID') - end - end - end - end - describe '#github_identifiers' do it 'returns a hash with needed identifiers' do expect(note.github_identifiers).to eq( @@ -273,27 +215,40 @@ RSpec.describe Gitlab::GithubImport::Representation::DiffNote, :clean_gitlab_red end describe '.from_api_response' do - it_behaves_like 'a DiffNote representation' do - let(:response) do - { - id: note_id, - html_url: 'https://github.com/foo/bar/pull/42', - path: 'README.md', - commit_id: '123abc', - original_commit_id: 'original123abc', - side: side, - user: user_data, - diff_hunk: hunk, - body: note_body, - created_at: created_at, - updated_at: updated_at, - line: end_line, - start_line: start_line, - in_reply_to_id: in_reply_to_id - } - end + let(:response) do + { + id: note_id, + html_url: 'https://github.com/foo/bar/pull/42', + path: 'README.md', + commit_id: '123abc', + original_commit_id: 'original123abc', + side: side, + user: user_data, + diff_hunk: hunk, + body: note_body, + created_at: created_at, + updated_at: updated_at, + line: end_line, + start_line: start_line, + in_reply_to_id: in_reply_to_id + } + end + + subject(:note) { described_class.from_api_response(response) } + + it_behaves_like 'a DiffNote representation' + + describe '#discussion_id' do + it 'finds or generates discussion_id value' do + discussion_id = 'discussion_id' + discussion_id_class = Gitlab::GithubImport::Representation::DiffNotes::DiscussionId - subject(:note) { described_class.from_api_response(response) } + expect_next_instance_of(discussion_id_class, response) do |discussion_id_object| + expect(discussion_id_object).to receive(:find_or_generate).and_return(discussion_id) + end + + expect(note.discussion_id).to eq(discussion_id) + end end end @@ -302,6 +257,7 @@ RSpec.describe Gitlab::GithubImport::Representation::DiffNote, :clean_gitlab_red let(:hash) do { 'note_id' => note_id, + 'html_url' => 'https://github.com/foo/bar/pull/42', 'noteable_type' => 'MergeRequest', 'noteable_id' => 42, 'file_path' => 'README.md', @@ -315,7 +271,8 @@ RSpec.describe Gitlab::GithubImport::Representation::DiffNote, :clean_gitlab_red 'updated_at' => updated_at.to_s, 'end_line' => end_line, 'start_line' => start_line, - 'in_reply_to_id' => in_reply_to_id + 'in_reply_to_id' => in_reply_to_id, + 'discussion_id' => 'FIRST_DISCUSSION_ID' } end diff --git a/spec/lib/gitlab/github_import/representation/diff_notes/discussion_id_spec.rb b/spec/lib/gitlab/github_import/representation/diff_notes/discussion_id_spec.rb new file mode 100644 index 00000000000..64a16516e07 --- /dev/null +++ b/spec/lib/gitlab/github_import/representation/diff_notes/discussion_id_spec.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Representation::DiffNotes::DiscussionId, :clean_gitlab_redis_cache, + feature_category: :importers do + describe '#discussion_id' do + let(:hunk) do + '@@ -1 +1 @@ + -Hello + +Hello world' + end + + let(:note_id) { 1 } + let(:html_url) { 'https://github.com/foo/project_name/pull/42' } + let(:note) do + { + id: note_id, + html_url: html_url, + path: 'README.md', + commit_id: '123abc', + original_commit_id: 'original123abc', + side: 'RIGHT', + user: { id: 4, login: 'alice' }, + diff_hunk: hunk, + body: 'Hello world', + created_at: Time.new(2017, 1, 1, 12, 10).utc, + updated_at: Time.new(2017, 1, 1, 12, 15).utc, + line: 23, + start_line: nil, + in_reply_to_id: nil + } + end + + context 'when the note is not a reply to a discussion' do + subject(:discussion_id) { described_class.new(note).find_or_generate } + + it 'generates and caches new discussion_id' do + expect(Discussion) + .to receive(:discussion_id) + .and_return('FIRST_DISCUSSION_ID') + + expect(Gitlab::Cache::Import::Caching).to receive(:write).with( + "github-importer/discussion-id-map/project_name/42/#{note_id}", + 'FIRST_DISCUSSION_ID' + ).and_return('FIRST_DISCUSSION_ID') + + expect(discussion_id).to eq('FIRST_DISCUSSION_ID') + end + end + + context 'when the note is a reply to a discussion' do + let(:reply_note) do + { + note_id: note_id + 1, + in_reply_to_id: note_id, + html_url: html_url + } + end + + subject(:discussion_id) { described_class.new(reply_note).find_or_generate } + + it 'uses the cached value as the discussion_id' do + expect(Discussion) + .to receive(:discussion_id) + .and_return('FIRST_DISCUSSION_ID') + + described_class.new(note).find_or_generate + + expect(discussion_id).to eq('FIRST_DISCUSSION_ID') + end + + context 'when cached value does not exist' do + it 'falls back to generating a new discussion_id' do + expect(Discussion) + .to receive(:discussion_id) + .and_return('NEW_DISCUSSION_ID') + + expect(discussion_id).to eq('NEW_DISCUSSION_ID') + end + end + end + end +end diff --git a/spec/lib/gitlab/gon_helper_spec.rb b/spec/lib/gitlab/gon_helper_spec.rb index 5a1fcc5e2dc..6e8997d51c3 100644 --- a/spec/lib/gitlab/gon_helper_spec.rb +++ b/spec/lib/gitlab/gon_helper_spec.rb @@ -40,11 +40,11 @@ RSpec.describe Gitlab::GonHelper do end end - describe 'sentry configuration' do + context 'when sentry is configured' do let(:clientside_dsn) { 'https://xxx@sentry.example.com/1' } let(:environment) { 'staging' } - describe 'sentry integration' do + context 'with legacy sentry configuration' do before do stub_config(sentry: { enabled: true, clientside_dsn: clientside_dsn, environment: environment }) end @@ -57,7 +57,7 @@ RSpec.describe Gitlab::GonHelper do end end - describe 'new sentry integration' do + context 'with sentry settings' do before do stub_application_setting(sentry_enabled: true) stub_application_setting(sentry_clientside_dsn: clientside_dsn) @@ -104,6 +104,7 @@ RSpec.describe Gitlab::GonHelper do thing = stub_feature_flag_gate('thing') stub_feature_flags(my_feature_flag: thing) + stub_feature_flag_definition(:my_feature_flag) allow(helper) .to receive(:gon) diff --git a/spec/lib/gitlab/graphql/limit/field_call_count_spec.rb b/spec/lib/gitlab/graphql/limit/field_call_count_spec.rb index 5858986dfc8..afcfb063af3 100644 --- a/spec/lib/gitlab/graphql/limit/field_call_count_spec.rb +++ b/spec/lib/gitlab/graphql/limit/field_call_count_spec.rb @@ -36,6 +36,15 @@ RSpec.describe Gitlab::Graphql::Limit::FieldCallCount do expect(resolve_value).to be_an_instance_of(Gitlab::Graphql::Errors::LimitError) end + it 'does not return an error when the field is called multiple times in separte queries' do + query_1 = GraphQL::Query.new(GitlabSchema) + query_2 = GraphQL::Query.new(GitlabSchema) + + resolve_field(field, { value: 'foo' }, object_type: owner, query: query_1) + + expect { resolve_field(field, { value: 'foo' }, object_type: owner, query: query_2) }.not_to raise_error + end + context 'when limit is not specified' do let(:field) do ::Types::BaseField.new(name: 'value', type: GraphQL::Types::String, null: true, owner: owner) do diff --git a/spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb b/spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb index 1124868bdae..773df9b20ee 100644 --- a/spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb +++ b/spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb @@ -50,17 +50,16 @@ RSpec.describe Gitlab::Graphql::Pagination::Keyset::Connection do end before do - stub_feature_flags(graphql_keyset_pagination_without_next_page_query: false) allow(GitlabSchema).to receive(:default_max_page_size).and_return(2) end - it 'invokes an extra query for the next page check' do + it 'invokes no an extra query for the next page check' do arguments[:first] = 1 subject.nodes count = ActiveRecord::QueryRecorder.new { subject.has_next_page }.count - expect(count).to eq(1) + expect(count).to eq(0) end context 'when the relation is loaded' do @@ -438,382 +437,4 @@ RSpec.describe Gitlab::Graphql::Pagination::Keyset::Connection do end end end - - # duplicated tests, remove with the removal of the graphql_keyset_pagination_without_next_page_query FF - context 'when the graphql_keyset_pagination_without_next_page_query is on' do - let(:nodes) { Project.all.order(Gitlab::Pagination::Keyset::Order.build([column_order_id])) } - - before do - stub_feature_flags(graphql_keyset_pagination_without_next_page_query: true) - end - - it 'does not invoke an extra query for the next page check' do - arguments[:first] = 1 - - subject.nodes - - count = ActiveRecord::QueryRecorder.new { subject.has_next_page }.count - expect(count).to eq(0) - end - - it_behaves_like 'a connection with collection methods' - - it_behaves_like 'a redactable connection' do - let_it_be(:projects) { create_list(:project, 2) } - let(:unwanted) { projects.second } - end - - describe '#cursor_for' do - let(:project) { create(:project) } - let(:cursor) { connection.cursor_for(project) } - - it 'returns an encoded ID' do - expect(decoded_cursor(cursor)).to eq('id' => project.id.to_s) - end - - context 'when an order is specified' do - let(:nodes) { Project.all.order(Gitlab::Pagination::Keyset::Order.build([column_order_id])) } - - it 'returns the encoded value of the order' do - expect(decoded_cursor(cursor)).to include('id' => project.id.to_s) - end - end - - context 'when multiple orders are specified' do - let(:nodes) { Project.all.order(Gitlab::Pagination::Keyset::Order.build([column_order_updated_at, column_order_created_at, column_order_id])) } - - it 'returns the encoded value of the order' do - expect(decoded_cursor(cursor)).to include('updated_at' => project.updated_at.to_s(:inspect)) - end - end - end - - describe '#sliced_nodes' do - let(:projects) { create_list(:project, 4) } - - context 'when before is passed' do - let(:arguments) { { before: encoded_cursor(projects[1]) } } - - it 'only returns the project before the selected one' do - expect(subject.sliced_nodes).to contain_exactly(projects.first) - end - - context 'when the sort order is descending' do - let(:nodes) { Project.all.order(Gitlab::Pagination::Keyset::Order.build([column_order_id_desc])) } - - it 'returns the correct nodes' do - expect(subject.sliced_nodes).to contain_exactly(*projects[2..]) - end - end - end - - context 'when after is passed' do - let(:arguments) { { after: encoded_cursor(projects[1]) } } - - it 'only returns the project before the selected one' do - expect(subject.sliced_nodes).to contain_exactly(*projects[2..]) - end - - context 'when the sort order is descending' do - let(:nodes) { Project.all.order(Gitlab::Pagination::Keyset::Order.build([column_order_id_desc])) } - - it 'returns the correct nodes' do - expect(subject.sliced_nodes).to contain_exactly(projects.first) - end - end - end - - context 'when both before and after are passed' do - let(:arguments) do - { - after: encoded_cursor(projects[1]), - before: encoded_cursor(projects[3]) - } - end - - it 'returns the expected set' do - expect(subject.sliced_nodes).to contain_exactly(projects[2]) - end - end - - shared_examples 'nodes are in ascending order' do - context 'when no cursor is passed' do - let(:arguments) { {} } - - it 'returns projects in ascending order' do - expect(subject.sliced_nodes).to eq(ascending_nodes) - end - end - - context 'when before cursor value is not NULL' do - let(:arguments) { { before: encoded_cursor(ascending_nodes[2]) } } - - it 'returns all projects before the cursor' do - expect(subject.sliced_nodes).to eq(ascending_nodes.first(2)) - end - end - - context 'when after cursor value is not NULL' do - let(:arguments) { { after: encoded_cursor(ascending_nodes[1]) } } - - it 'returns all projects after the cursor' do - expect(subject.sliced_nodes).to eq(ascending_nodes.last(3)) - end - end - - context 'when before and after cursor' do - let(:arguments) { { before: encoded_cursor(ascending_nodes.last), after: encoded_cursor(ascending_nodes.first) } } - - it 'returns all projects after the cursor' do - expect(subject.sliced_nodes).to eq(ascending_nodes[1..3]) - end - end - end - - shared_examples 'nodes are in descending order' do - context 'when no cursor is passed' do - let(:arguments) { {} } - - it 'only returns projects in descending order' do - expect(subject.sliced_nodes).to eq(descending_nodes) - end - end - - context 'when before cursor value is not NULL' do - let(:arguments) { { before: encoded_cursor(descending_nodes[2]) } } - - it 'returns all projects before the cursor' do - expect(subject.sliced_nodes).to eq(descending_nodes.first(2)) - end - end - - context 'when after cursor value is not NULL' do - let(:arguments) { { after: encoded_cursor(descending_nodes[1]) } } - - it 'returns all projects after the cursor' do - expect(subject.sliced_nodes).to eq(descending_nodes.last(3)) - end - end - - context 'when before and after cursor' do - let(:arguments) { { before: encoded_cursor(descending_nodes.last), after: encoded_cursor(descending_nodes.first) } } - - it 'returns all projects after the cursor' do - expect(subject.sliced_nodes).to eq(descending_nodes[1..3]) - end - end - end - - context 'when multiple orders with nil values are defined' do - let_it_be(:project1) { create(:project, last_repository_check_at: 10.days.ago) } # Asc: project5 Desc: project3 - let_it_be(:project2) { create(:project, last_repository_check_at: nil) } # Asc: project1 Desc: project1 - let_it_be(:project3) { create(:project, last_repository_check_at: 5.days.ago) } # Asc: project3 Desc: project5 - let_it_be(:project4) { create(:project, last_repository_check_at: nil) } # Asc: project2 Desc: project2 - let_it_be(:project5) { create(:project, last_repository_check_at: 20.days.ago) } # Asc: project4 Desc: project4 - - context 'when ascending' do - let_it_be(:order) { Gitlab::Pagination::Keyset::Order.build([column_order_last_repo, column_order_id]) } - let_it_be(:nodes) { Project.order(order) } - let_it_be(:ascending_nodes) { [project5, project1, project3, project2, project4] } - - it_behaves_like 'nodes are in ascending order' - - context 'when before cursor value is NULL' do - let(:arguments) { { before: encoded_cursor(project4) } } - - it 'returns all projects before the cursor' do - expect(subject.sliced_nodes).to eq([project5, project1, project3, project2]) - end - end - - context 'when after cursor value is NULL' do - let(:arguments) { { after: encoded_cursor(project2) } } - - it 'returns all projects after the cursor' do - expect(subject.sliced_nodes).to eq([project4]) - end - end - end - - context 'when descending' do - let_it_be(:order) { Gitlab::Pagination::Keyset::Order.build([column_order_last_repo_desc, column_order_id]) } - let_it_be(:nodes) { Project.order(order) } - let_it_be(:descending_nodes) { [project3, project1, project5, project2, project4] } - - it_behaves_like 'nodes are in descending order' - - context 'when before cursor value is NULL' do - let(:arguments) { { before: encoded_cursor(project4) } } - - it 'returns all projects before the cursor' do - expect(subject.sliced_nodes).to eq([project3, project1, project5, project2]) - end - end - - context 'when after cursor value is NULL' do - let(:arguments) { { after: encoded_cursor(project2) } } - - it 'returns all projects after the cursor' do - expect(subject.sliced_nodes).to eq([project4]) - end - end - end - end - - context 'when ordering by similarity' do - let_it_be(:project1) { create(:project, name: 'test') } - let_it_be(:project2) { create(:project, name: 'testing') } - let_it_be(:project3) { create(:project, name: 'tests') } - let_it_be(:project4) { create(:project, name: 'testing stuff') } - let_it_be(:project5) { create(:project, name: 'test') } - - let_it_be(:nodes) do - # Note: sorted_by_similarity_desc scope internally supports the generic keyset order. - Project.sorted_by_similarity_desc('test', include_in_select: true) - end - - let_it_be(:descending_nodes) { nodes.to_a } - - it_behaves_like 'nodes are in descending order' - end - - context 'when an invalid cursor is provided' do - let(:arguments) { { before: Base64Bp.urlsafe_encode64('invalidcursor', padding: false) } } - - it 'raises an error' do - expect { subject.sliced_nodes }.to raise_error(Gitlab::Graphql::Errors::ArgumentError) - end - end - end - - describe '#nodes' do - let_it_be(:all_nodes) { create_list(:project, 5) } - - let(:paged_nodes) { subject.nodes } - - it_behaves_like 'connection with paged nodes' do - let(:paged_nodes_size) { 3 } - end - - context 'when both are passed' do - let(:arguments) { { first: 2, last: 2 } } - - it 'raises an error' do - expect { paged_nodes }.to raise_error(Gitlab::Graphql::Errors::ArgumentError) - end - end - - context 'when primary key is not in original order' do - let(:nodes) { Project.order(last_repository_check_at: :desc) } - - it 'is added to end' do - sliced = subject.sliced_nodes - - order_sql = sliced.order_values.last.to_sql - - expect(order_sql).to end_with(Project.arel_table[:id].desc.to_sql) - end - end - - context 'when there is no primary key' do - before do - stub_const('NoPrimaryKey', Class.new(ActiveRecord::Base)) - NoPrimaryKey.class_eval do - self.table_name = 'no_primary_key' - self.primary_key = nil - end - end - - let(:nodes) { NoPrimaryKey.all } - - it 'raises an error' do - expect(NoPrimaryKey.primary_key).to be_nil - expect { subject.sliced_nodes }.to raise_error(ArgumentError, 'Relation must have a primary key') - end - end - end - - describe '#has_previous_page and #has_next_page' do - # using a list of 5 items with a max_page of 3 - let_it_be(:project_list) { create_list(:project, 5) } - let_it_be(:nodes) { Project.order(Gitlab::Pagination::Keyset::Order.build([column_order_id])) } - - context 'when default query' do - let(:arguments) { {} } - - it 'has no previous, but a next' do - expect(subject.has_previous_page).to be_falsey - expect(subject.has_next_page).to be_truthy - end - end - - context 'when before is first item' do - let(:arguments) { { before: encoded_cursor(project_list.first) } } - - it 'has no previous, but a next' do - expect(subject.has_previous_page).to be_falsey - expect(subject.has_next_page).to be_truthy - end - end - - describe 'using `before`' do - context 'when before is the last item' do - let(:arguments) { { before: encoded_cursor(project_list.last) } } - - it 'has no previous, but a next' do - expect(subject.has_previous_page).to be_falsey - expect(subject.has_next_page).to be_truthy - end - end - - context 'when before and last specified' do - let(:arguments) { { before: encoded_cursor(project_list.last), last: 2 } } - - it 'has a previous and a next' do - expect(subject.has_previous_page).to be_truthy - expect(subject.has_next_page).to be_truthy - end - end - - context 'when before and last does request all remaining nodes' do - let(:arguments) { { before: encoded_cursor(project_list[1]), last: 3 } } - - it 'has a previous and a next' do - expect(subject.has_previous_page).to be_falsey - expect(subject.has_next_page).to be_truthy - expect(subject.nodes).to eq [project_list[0]] - end - end - end - - describe 'using `after`' do - context 'when after is the first item' do - let(:arguments) { { after: encoded_cursor(project_list.first) } } - - it 'has a previous, and a next' do - expect(subject.has_previous_page).to be_truthy - expect(subject.has_next_page).to be_truthy - end - end - - context 'when after and first specified' do - let(:arguments) { { after: encoded_cursor(project_list.first), first: 2 } } - - it 'has a previous and a next' do - expect(subject.has_previous_page).to be_truthy - expect(subject.has_next_page).to be_truthy - end - end - - context 'when before and last does request all remaining nodes' do - let(:arguments) { { after: encoded_cursor(project_list[2]), last: 3 } } - - it 'has a previous but no next' do - expect(subject.has_previous_page).to be_truthy - expect(subject.has_next_page).to be_falsey - end - end - end - end - end end diff --git a/spec/lib/gitlab/http_connection_adapter_spec.rb b/spec/lib/gitlab/http_connection_adapter_spec.rb index a241a4b6490..5e2c6be8993 100644 --- a/spec/lib/gitlab/http_connection_adapter_spec.rb +++ b/spec/lib/gitlab/http_connection_adapter_spec.rb @@ -124,5 +124,16 @@ RSpec.describe Gitlab::HTTPConnectionAdapter do expect(connection.port).to eq(443) end end + + context 'when URL scheme is not HTTP/HTTPS' do + let(:uri) { URI('ssh://example.org') } + + it 'raises error' do + expect { subject }.to raise_error( + Gitlab::HTTP::BlockedUrlError, + "URL 'ssh://example.org' is blocked: Only allowed schemes are http, https" + ) + end + end end end diff --git a/spec/lib/gitlab/i18n_spec.rb b/spec/lib/gitlab/i18n_spec.rb index aae4a13bd73..b752d89bf0d 100644 --- a/spec/lib/gitlab/i18n_spec.rb +++ b/spec/lib/gitlab/i18n_spec.rb @@ -8,7 +8,7 @@ RSpec.describe Gitlab::I18n do describe '.selectable_locales' do include StubLanguagesTranslationPercentage - it 'does not return languages with low translation levels' do + it 'does not return languages with default translation levels 60%' do stub_languages_translation_percentage(pt_BR: 0, en: 100, es: 65) expect(described_class.selectable_locales).to eq({ @@ -16,6 +16,12 @@ RSpec.describe Gitlab::I18n do 'es' => 'Spanish - español' }) end + + it 'does not return languages with less than 100% translation levels' do + stub_languages_translation_percentage(pt_BR: 0, en: 100, es: 65) + + expect(described_class.selectable_locales(100)).to eq({ 'en' => 'English' }) + end end describe '.locale=' do diff --git a/spec/lib/gitlab/import/merge_request_helpers_spec.rb b/spec/lib/gitlab/import/merge_request_helpers_spec.rb index f858ab934bb..9626b7b893f 100644 --- a/spec/lib/gitlab/import/merge_request_helpers_spec.rb +++ b/spec/lib/gitlab/import/merge_request_helpers_spec.rb @@ -37,11 +37,8 @@ RSpec.describe Gitlab::Import::MergeRequestHelpers, type: :helper do attributes.merge(iid: iid, source_branch: iid.to_s)) end - # does ensure that we only load object twice - # 1. by #insert_and_return_id - # 2. by project.merge_requests.find - expect_any_instance_of(MergeRequest).to receive(:attributes) - .twice.times.and_call_original + # ensures that we only load object once by project.merge_requests.find + expect(MergeRequest).to receive(:allocate).once.and_call_original expect(subject.first).not_to be_nil expect(subject.second).to eq(false) diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index e9dde1c6180..b34399d20f1 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -157,6 +157,7 @@ merge_requests: - author - assignee - reviewers +- reviewed_by_users - updated_by - milestone - iteration @@ -408,6 +409,20 @@ project: - boards - last_event - integrations +- push_hooks_integrations +- tag_push_hooks_integrations +- issue_hooks_integrations +- confidential_issue_hooks_integrations +- merge_request_hooks_integrations +- note_hooks_integrations +- confidential_note_hooks_integrations +- job_hooks_integrations +- archive_trace_hooks_integrations +- pipeline_hooks_integrations +- wiki_page_hooks_integrations +- deployment_hooks_integrations +- alert_hooks_integrations +- vulnerability_hooks_integrations - campfire_integration - confluence_integration - datadog_integration @@ -423,7 +438,6 @@ project: - packagist_integration - pivotaltracker_integration - prometheus_integration -- flowdock_integration - assembla_integration - asana_integration - slack_integration @@ -649,6 +663,7 @@ project: - project_callouts - pipeline_metadata - disable_download_button +- dependency_list_exports award_emoji: - awardable - user diff --git a/spec/lib/gitlab/import_export/group/legacy_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/legacy_tree_restorer_spec.rb index dbd6cb243f6..a5b03974bc0 100644 --- a/spec/lib/gitlab/import_export/group/legacy_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/group/legacy_tree_restorer_spec.rb @@ -77,7 +77,7 @@ RSpec.describe Gitlab::ImportExport::Group::LegacyTreeRestorer do let(:group) { create(:group) } let(:shared) { Gitlab::ImportExport::Shared.new(group) } let(:group_tree_restorer) { described_class.new(user: importer_user, shared: shared, group: group, group_hash: nil) } - let(:group_json) { Gitlab::Json.parse(IO.read(File.join(shared.export_path, 'group.json'))) } + let(:group_json) { Gitlab::Json.parse(File.read(File.join(shared.export_path, 'group.json'))) } shared_examples 'excluded attributes' do excluded_attributes = %w[ diff --git a/spec/lib/gitlab/import_export/group/legacy_tree_saver_spec.rb b/spec/lib/gitlab/import_export/group/legacy_tree_saver_spec.rb index 31d647f883a..f5a4fc79c90 100644 --- a/spec/lib/gitlab/import_export/group/legacy_tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/group/legacy_tree_saver_spec.rb @@ -154,6 +154,6 @@ RSpec.describe Gitlab::ImportExport::Group::LegacyTreeSaver do end def group_json(filename) - ::JSON.parse(IO.read(filename)) + ::JSON.parse(File.read(filename)) end end diff --git a/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb index 1444897e136..aa30e24296e 100644 --- a/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::Group::TreeRestorer do +RSpec.describe Gitlab::ImportExport::Group::TreeRestorer, feature: :subgroups do include ImportExport::CommonUtil shared_examples 'group restoration' do @@ -112,7 +112,7 @@ RSpec.describe Gitlab::ImportExport::Group::TreeRestorer do let(:shared) { Gitlab::ImportExport::Shared.new(group) } let(:group_tree_restorer) { described_class.new(user: importer_user, shared: shared, group: group) } let(:exported_file) { File.join(shared.export_path, 'tree/groups/4352.json') } - let(:group_json) { Gitlab::Json.parse(IO.read(exported_file)) } + let(:group_json) { Gitlab::Json.parse(File.read(exported_file)) } shared_examples 'excluded attributes' do excluded_attributes = %w[ diff --git a/spec/lib/gitlab/import_export/import_test_coverage_spec.rb b/spec/lib/gitlab/import_export/import_test_coverage_spec.rb index b1f5574fba1..d8a4230e5da 100644 --- a/spec/lib/gitlab/import_export/import_test_coverage_spec.rb +++ b/spec/lib/gitlab/import_export/import_test_coverage_spec.rb @@ -86,7 +86,7 @@ RSpec.describe 'Test coverage of the Project Import' do end def relations_from_json(json_file) - json = Gitlab::Json.parse(IO.read(json_file)) + json = Gitlab::Json.parse(File.read(json_file)) [].tap { |res| gather_relations({ project: json }, res, []) } .map { |relation_names| relation_names.join('.') } diff --git a/spec/lib/gitlab/import_export/json/legacy_writer_spec.rb b/spec/lib/gitlab/import_export/json/legacy_writer_spec.rb index ed4368ba802..e8ecd98b1e1 100644 --- a/spec/lib/gitlab/import_export/json/legacy_writer_spec.rb +++ b/spec/lib/gitlab/import_export/json/legacy_writer_spec.rb @@ -96,6 +96,6 @@ RSpec.describe Gitlab::ImportExport::Json::LegacyWriter do def subject_json subject.close - ::JSON.parse(IO.read(subject.path)) + ::JSON.parse(File.read(subject.path)) end end diff --git a/spec/lib/gitlab/import_export/lfs_saver_spec.rb b/spec/lib/gitlab/import_export/lfs_saver_spec.rb index aa456736f78..5b6f50025ff 100644 --- a/spec/lib/gitlab/import_export/lfs_saver_spec.rb +++ b/spec/lib/gitlab/import_export/lfs_saver_spec.rb @@ -26,7 +26,7 @@ RSpec.describe Gitlab::ImportExport::LfsSaver do let(:lfs_json_file) { File.join(shared.export_path, Gitlab::ImportExport.lfs_objects_filename) } def lfs_json - Gitlab::Json.parse(IO.read(lfs_json_file)) + Gitlab::Json.parse(File.read(lfs_json_file)) end before do diff --git a/spec/lib/gitlab/import_export/model_configuration_spec.rb b/spec/lib/gitlab/import_export/model_configuration_spec.rb index 34591122a97..4f01f470ce7 100644 --- a/spec/lib/gitlab/import_export/model_configuration_spec.rb +++ b/spec/lib/gitlab/import_export/model_configuration_spec.rb @@ -23,8 +23,8 @@ RSpec.describe 'Import/Export model configuration' do # List of current models between models, in the format of # {model: [model_2, model3], ...} def setup_models - model_names.each_with_object({}) do |model_name, hash| - hash[model_name] = associations_for(relation_class_for_name(model_name)) - ['project'] + model_names.index_with do |model_name| + associations_for(relation_class_for_name(model_name)) - ['project'] end end diff --git a/spec/lib/gitlab/import_export/project/exported_relations_merger_spec.rb b/spec/lib/gitlab/import_export/project/exported_relations_merger_spec.rb index a781139acab..d70e89c6856 100644 --- a/spec/lib/gitlab/import_export/project/exported_relations_merger_spec.rb +++ b/spec/lib/gitlab/import_export/project/exported_relations_merger_spec.rb @@ -8,17 +8,17 @@ RSpec.describe Gitlab::ImportExport::Project::ExportedRelationsMerger do let(:shared) { Gitlab::ImportExport::Shared.new(export_job.project) } before do - create(:project_relation_export_upload, + create(:relation_export_upload, relation_export: create(:project_relation_export, relation: 'project', project_export_job: export_job), export_file: fixture_file_upload("spec/fixtures/gitlab/import_export/project.tar.gz") ) - create(:project_relation_export_upload, + create(:relation_export_upload, relation_export: create(:project_relation_export, relation: 'labels', project_export_job: export_job), export_file: fixture_file_upload("spec/fixtures/gitlab/import_export/labels.tar.gz") ) - create(:project_relation_export_upload, + create(:relation_export_upload, relation_export: create(:project_relation_export, relation: 'uploads', project_export_job: export_job), export_file: fixture_file_upload("spec/fixtures/gitlab/import_export/uploads.tar.gz") ) diff --git a/spec/lib/gitlab/import_export/project/relation_saver_spec.rb b/spec/lib/gitlab/import_export/project/relation_saver_spec.rb index 0467b63e918..5032dd864bb 100644 --- a/spec/lib/gitlab/import_export/project/relation_saver_spec.rb +++ b/spec/lib/gitlab/import_export/project/relation_saver_spec.rb @@ -111,7 +111,7 @@ RSpec.describe Gitlab::ImportExport::Project::RelationSaver do end def read_json(path) - Gitlab::Json.parse(IO.read(path)) + Gitlab::Json.parse(File.read(path)) end def read_ndjson(path) diff --git a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb index b753746cd8c..2699dc10b18 100644 --- a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb @@ -768,7 +768,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer do it 'overrides project feature access levels' do access_level_keys = ProjectFeature.available_features.map { |feature| ProjectFeature.access_level_attribute(feature) } - disabled_access_levels = access_level_keys.to_h { |item| [item, 'disabled'] } + disabled_access_levels = access_level_keys.index_with { 'disabled' } project.create_import_data(data: { override_params: disabled_access_levels }) diff --git a/spec/lib/gitlab/import_export/repo_restorer_spec.rb b/spec/lib/gitlab/import_export/repo_restorer_spec.rb index 727ca4f630b..3da7af7509e 100644 --- a/spec/lib/gitlab/import_export/repo_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/repo_restorer_spec.rb @@ -34,7 +34,7 @@ RSpec.describe Gitlab::ImportExport::RepoRestorer do Gitlab::Shell.new.remove_repository(project.repository_storage, project.disk_path) end - it 'restores the repo successfully' do + it 'restores the repo successfully', :aggregate_failures do expect(project.repository.exists?).to be false expect(subject.restore).to be_truthy @@ -42,7 +42,7 @@ RSpec.describe Gitlab::ImportExport::RepoRestorer do end context 'when the repository already exists' do - it 'deletes the existing repository before importing' do + it 'deletes the existing repository before importing', :aggregate_failures do allow(project.repository).to receive(:exists?).and_return(true) allow(project.repository).to receive(:disk_path).and_return('repository_path') @@ -69,7 +69,7 @@ RSpec.describe Gitlab::ImportExport::RepoRestorer do Gitlab::Shell.new.remove_repository(project.wiki.repository_storage, project.wiki.disk_path) end - it 'restores the wiki repo successfully' do + it 'restores the wiki repo successfully', :aggregate_failures do expect(project.wiki_repository_exists?).to be false subject.restore @@ -83,10 +83,21 @@ RSpec.describe Gitlab::ImportExport::RepoRestorer do let(:bundler) { Gitlab::ImportExport::WikiRepoSaver.new(exportable: project_without_wiki, shared: shared) } - it 'does not creates an empty wiki' do + it 'does not creates an empty wiki', :aggregate_failures do expect(subject.restore).to be true expect(project.wiki_repository_exists?).to be false end end + + context 'when wiki already exists' do + subject do + described_class.new(path_to_bundle: bundle_path, shared: shared, importable: ProjectWiki.new(project_with_repo)) + end + + it 'does not cause an error when restoring', :aggregate_failures do + expect(subject.restore).to be true + expect(shared.errors).to be_empty + end + end end end diff --git a/spec/lib/gitlab/import_export/saver_spec.rb b/spec/lib/gitlab/import_export/saver_spec.rb index f5eed81f73c..a34e68ecd19 100644 --- a/spec/lib/gitlab/import_export/saver_spec.rb +++ b/spec/lib/gitlab/import_export/saver_spec.rb @@ -33,7 +33,7 @@ RSpec.describe Gitlab::ImportExport::Saver do subject.save # rubocop:disable Rails/SaveBang expect(ImportExportUpload.find_by(project: project).export_file.url) - .to match(%r[\/uploads\/-\/system\/import_export_upload\/export_file.*]) + .to match(%r[/uploads/-/system/import_export_upload/export_file.*]) end it 'logs metrics after saving' do diff --git a/spec/lib/gitlab/import_sources_spec.rb b/spec/lib/gitlab/import_sources_spec.rb index 41ffcece221..393e0a9be10 100644 --- a/spec/lib/gitlab/import_sources_spec.rb +++ b/spec/lib/gitlab/import_sources_spec.rb @@ -11,7 +11,6 @@ RSpec.describe Gitlab::ImportSources do 'Bitbucket Cloud' => 'bitbucket', 'Bitbucket Server' => 'bitbucket_server', 'GitLab.com' => 'gitlab', - 'Google Code' => 'google_code', 'FogBugz' => 'fogbugz', 'Repository by URL' => 'git', 'GitLab export' => 'gitlab_project', @@ -32,7 +31,6 @@ RSpec.describe Gitlab::ImportSources do bitbucket bitbucket_server gitlab - google_code fogbugz git gitlab_project @@ -69,7 +67,6 @@ RSpec.describe Gitlab::ImportSources do 'bitbucket' => Gitlab::BitbucketImport::Importer, 'bitbucket_server' => Gitlab::BitbucketServerImport::Importer, 'gitlab' => Gitlab::GitlabImport::Importer, - 'google_code' => nil, 'fogbugz' => Gitlab::FogbugzImport::Importer, 'git' => nil, 'gitlab_project' => Gitlab::ImportExport::Importer, @@ -91,7 +88,6 @@ RSpec.describe Gitlab::ImportSources do 'bitbucket' => 'Bitbucket Cloud', 'bitbucket_server' => 'Bitbucket Server', 'gitlab' => 'GitLab.com', - 'google_code' => 'Google Code', 'fogbugz' => 'FogBugz', 'git' => 'Repository by URL', 'gitlab_project' => 'GitLab export', diff --git a/spec/lib/gitlab/incident_management/pager_duty/incident_issue_description_spec.rb b/spec/lib/gitlab/incident_management/pager_duty/incident_issue_description_spec.rb index c5288b9afbc..e2c67c68eb7 100644 --- a/spec/lib/gitlab/incident_management/pager_duty/incident_issue_description_spec.rb +++ b/spec/lib/gitlab/incident_management/pager_duty/incident_issue_description_spec.rb @@ -10,8 +10,8 @@ RSpec.describe Gitlab::IncidentManagement::PagerDuty::IncidentIssueDescription d [{ 'summary' => 'Laura Haley', 'url' => 'https://webdemo.pagerduty.com/users/P553OPV' }] end - let(:impacted_services) do - [{ 'summary' => 'Production XDB Cluster', 'url' => 'https://webdemo.pagerduty.com/services/PN49J75' }] + let(:impacted_service) do + { 'summary' => 'Production XDB Cluster', 'url' => 'https://webdemo.pagerduty.com/services/PN49J75' } end let(:incident_payload) do @@ -24,7 +24,7 @@ RSpec.describe Gitlab::IncidentManagement::PagerDuty::IncidentIssueDescription d 'urgency' => 'high', 'incident_key' => 'SOME-KEY', 'assignees' => assignees, - 'impacted_services' => impacted_services + 'impacted_service' => impacted_service } end @@ -40,7 +40,7 @@ RSpec.describe Gitlab::IncidentManagement::PagerDuty::IncidentIssueDescription d **Incident key:** SOME-KEY#{markdown_line_break} **Created at:** 26 September 2017, 3:14PM (UTC)#{markdown_line_break} **Assignees:** [Laura Haley](https://webdemo.pagerduty.com/users/P553OPV)#{markdown_line_break} - **Impacted services:** [Production XDB Cluster](https://webdemo.pagerduty.com/services/PN49J75) + **Impacted service:** [Production XDB Cluster](https://webdemo.pagerduty.com/services/PN49J75) MARKDOWN ) end @@ -78,18 +78,15 @@ RSpec.describe Gitlab::IncidentManagement::PagerDuty::IncidentIssueDescription d end end - context 'when there are several impacted services' do - let(:impacted_services) do - [ - { 'summary' => 'XDB Cluster', 'url' => 'https://xdb.pagerduty.com' }, - { 'summary' => 'BRB Cluster', 'url' => 'https://brb.pagerduty.com' } - ] + context 'when there is an impacted service' do + let(:impacted_service) do + { 'summary' => 'XDB Cluster', 'url' => 'https://xdb.pagerduty.com' } end - it 'impacted services is a list of links' do + it 'impacted service is a single link' do expect(to_s).to include( <<~MARKDOWN.chomp - **Impacted services:** [XDB Cluster](https://xdb.pagerduty.com), [BRB Cluster](https://brb.pagerduty.com) + **Impacted service:** [XDB Cluster](https://xdb.pagerduty.com) MARKDOWN ) end diff --git a/spec/lib/gitlab/instrumentation/redis_base_spec.rb b/spec/lib/gitlab/instrumentation/redis_base_spec.rb index f9dd0c94c97..656e6ffba05 100644 --- a/spec/lib/gitlab/instrumentation/redis_base_spec.rb +++ b/spec/lib/gitlab/instrumentation/redis_base_spec.rb @@ -1,8 +1,10 @@ # frozen_string_literal: true require 'spec_helper' +require 'rspec-parameterized' RSpec.describe Gitlab::Instrumentation::RedisBase, :request_store do + using RSpec::Parameterized::TableSyntax let(:instrumentation_class_a) do stub_const('InstanceA', Class.new(described_class)) end @@ -88,6 +90,44 @@ RSpec.describe Gitlab::Instrumentation::RedisBase, :request_store do end end + describe '.increment_cross_slot_request_count' do + context 'storage key overlapping' do + it 'keys do not overlap across storages' do + 3.times { instrumentation_class_a.increment_cross_slot_request_count } + 2.times { instrumentation_class_b.increment_cross_slot_request_count } + + expect(instrumentation_class_a.get_cross_slot_request_count).to eq(3) + expect(instrumentation_class_b.get_cross_slot_request_count).to eq(2) + end + + it 'increments by the given amount' do + instrumentation_class_a.increment_cross_slot_request_count(2) + instrumentation_class_a.increment_cross_slot_request_count(3) + + expect(instrumentation_class_a.get_cross_slot_request_count).to eq(5) + end + end + end + + describe '.increment_allowed_cross_slot_request_count' do + context 'storage key overlapping' do + it 'keys do not overlap across storages' do + 3.times { instrumentation_class_a.increment_allowed_cross_slot_request_count } + 2.times { instrumentation_class_b.increment_allowed_cross_slot_request_count } + + expect(instrumentation_class_a.get_allowed_cross_slot_request_count).to eq(3) + expect(instrumentation_class_b.get_allowed_cross_slot_request_count).to eq(2) + end + + it 'increments by the given amount' do + instrumentation_class_a.increment_allowed_cross_slot_request_count(2) + instrumentation_class_a.increment_allowed_cross_slot_request_count(3) + + expect(instrumentation_class_a.get_allowed_cross_slot_request_count).to eq(5) + end + end + end + describe '.increment_read_bytes' do context 'storage key overlapping' do it 'keys do not overlap across storages' do @@ -130,4 +170,44 @@ RSpec.describe Gitlab::Instrumentation::RedisBase, :request_store do end end end + + describe '.redis_cluster_validate!' do + let(:args) { [[:mget, 'foo', 'bar']] } + + before do + instrumentation_class_a.enable_redis_cluster_validation + end + + context 'Rails environments' do + where(:env, :allowed, :should_raise) do + 'production' | false | false + 'production' | true | false + 'staging' | false | false + 'staging' | true | false + 'development' | true | false + 'development' | false | true + 'test' | true | false + 'test' | false | true + end + + with_them do + it do + stub_rails_env(env) + + validation = -> { instrumentation_class_a.redis_cluster_validate!(args) } + under_test = if allowed + -> { Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands(&validation) } + else + validation + end + + if should_raise + expect(&under_test).to raise_error(::Gitlab::Instrumentation::RedisClusterValidator::CrossSlotError) + else + expect(&under_test).not_to raise_error + end + end + end + end + end end diff --git a/spec/lib/gitlab/instrumentation/redis_cluster_validator_spec.rb b/spec/lib/gitlab/instrumentation/redis_cluster_validator_spec.rb index 58c75bff9dd..892b8e69124 100644 --- a/spec/lib/gitlab/instrumentation/redis_cluster_validator_spec.rb +++ b/spec/lib/gitlab/instrumentation/redis_cluster_validator_spec.rb @@ -7,107 +7,102 @@ require 'rspec-parameterized' RSpec.describe Gitlab::Instrumentation::RedisClusterValidator do include RailsHelpers - describe '.validate!' do + describe '.validate' do using RSpec::Parameterized::TableSyntax - context 'Rails environments' do - where(:env, :should_raise) do - 'production' | false - 'staging' | false - 'development' | true - 'test' | true - end - - with_them do - it do - stub_rails_env(env) - - args = [[:mget, 'foo', 'bar']] - - if should_raise - expect { described_class.validate!(args) } - .to raise_error(described_class::CrossSlotError) - else - expect { described_class.validate!(args) }.not_to raise_error - end - end - end - end - - where(:command, :arguments, :should_raise) do - :rename | %w(foo bar) | true - :RENAME | %w(foo bar) | true - 'rename' | %w(foo bar) | true - 'RENAME' | %w(foo bar) | true - :rename | %w(iaa ahy) | false # 'iaa' and 'ahy' hash to the same slot - :rename | %w({foo}:1 {foo}:2) | false - :rename | %w(foo foo bar) | false # This is not a valid command but should not raise here - :mget | %w(foo bar) | true - :mget | %w(foo foo bar) | true - :mget | %w(foo foo) | false - :blpop | %w(foo bar 1) | true - :blpop | %w(foo foo 1) | false - :mset | %w(foo a bar a) | true - :mset | %w(foo a foo a) | false - :del | %w(foo bar) | true - :del | [%w(foo bar)] | true # Arguments can be a nested array - :del | %w(foo foo) | false - :hset | %w(foo bar) | false # Not a multi-key command - :mget | [] | false # This is invalid, but not because it's a cross-slot command + where(:command, :arguments, :keys, :is_valid) do + :rename | %w(foo bar) | 2 | false + :RENAME | %w(foo bar) | 2 | false + 'rename' | %w(foo bar) | 2 | false + 'RENAME' | %w(foo bar) | 2 | false + :rename | %w(iaa ahy) | 2 | true # 'iaa' and 'ahy' hash to the same slot + :rename | %w({foo}:1 {foo}:2) | 2 | true + :rename | %w(foo foo bar) | 2 | true # This is not a valid command but should not raise here + :mget | %w(foo bar) | 2 | false + :mget | %w(foo foo bar) | 3 | false + :mget | %w(foo foo) | 2 | true + :blpop | %w(foo bar 1) | 2 | false + :blpop | %w(foo foo 1) | 2 | true + :mset | %w(foo a bar a) | 2 | false + :mset | %w(foo a foo a) | 2 | true + :del | %w(foo bar) | 2 | false + :del | [%w(foo bar)] | 2 | false # Arguments can be a nested array + :del | %w(foo foo) | 2 | true + :hset | %w(foo bar) | 1 | nil # Single key write + :get | %w(foo) | 1 | nil # Single key read + :mget | [] | 0 | true # This is invalid, but not because it's a cross-slot command end with_them do it do args = [[command] + arguments] - - if should_raise - expect { described_class.validate!(args) } - .to raise_error(described_class::CrossSlotError) + if is_valid.nil? + expect(described_class.validate(args)).to eq(nil) else - expect { described_class.validate!(args) }.not_to raise_error + expect(described_class.validate(args)[:valid]).to eq(is_valid) + expect(described_class.validate(args)[:allowed]).to eq(false) + expect(described_class.validate(args)[:command_name]).to eq(command.to_s.upcase) + expect(described_class.validate(args)[:key_count]).to eq(keys) end end end - where(:arguments, :should_raise) do - [[:get, "foo"], [:get, "bar"]] | true - [[:get, "foo"], [:mget, "foo", "bar"]] | true # mix of single-key and multi-key cmds - [[:get, "{foo}:name"], [:get, "{foo}:profile"]] | false - [[:del, "foo"], [:del, "bar"]] | true - [] | false # pipeline or transaction opened and closed without ops + where(:arguments, :should_raise, :output) do + [ + [ + [[:get, "foo"], [:get, "bar"]], + true, + { valid: false, key_count: 2, command_name: 'PIPELINE/MULTI', allowed: false } + ], + [ + [[:get, "foo"], [:mget, "foo", "bar"]], + true, + { valid: false, key_count: 3, command_name: 'PIPELINE/MULTI', allowed: false } + ], + [ + [[:get, "{foo}:name"], [:get, "{foo}:profile"]], + false, + { valid: true, key_count: 2, command_name: 'PIPELINE/MULTI', allowed: false } + ], + [ + [[:del, "foo"], [:del, "bar"]], + true, + { valid: false, key_count: 2, command_name: 'PIPELINE/MULTI', allowed: false } + ], + [ + [], + false, + nil # pipeline or transaction opened and closed without ops + ] + ] end with_them do it do - if should_raise - expect { described_class.validate!(arguments) } - .to raise_error(described_class::CrossSlotError) - else - expect { described_class.validate!(arguments) }.not_to raise_error - end + expect(described_class.validate(arguments)).to eq(output) end end end describe '.allow_cross_slot_commands' do - it 'does not raise for invalid arguments' do - expect do + it 'skips validation for allowed commands' do + expect( described_class.allow_cross_slot_commands do - described_class.validate!([[:mget, 'foo', 'bar']]) + described_class.validate([[:mget, 'foo', 'bar']]) end - end.not_to raise_error + ).to eq({ valid: true, key_count: 2, command_name: 'MGET', allowed: true }) end it 'allows nested invocation' do - expect do + expect( described_class.allow_cross_slot_commands do described_class.allow_cross_slot_commands do - described_class.validate!([[:mget, 'foo', 'bar']]) + described_class.validate([[:mget, 'foo', 'bar']]) end - described_class.validate!([[:mget, 'foo', 'bar']]) + described_class.validate([[:mget, 'foo', 'bar']]) end - end.not_to raise_error + ).to eq({ valid: true, key_count: 2, command_name: 'MGET', allowed: true }) end end end diff --git a/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb b/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb index 02c5dfb7521..187a6ff1739 100644 --- a/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb +++ b/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb @@ -2,6 +2,7 @@ require 'spec_helper' require 'rspec-parameterized' +require 'support/helpers/rails_helpers' RSpec.describe Gitlab::Instrumentation::RedisInterceptor, :clean_gitlab_redis_shared_state, :request_store do using RSpec::Parameterized::TableSyntax @@ -74,6 +75,47 @@ RSpec.describe Gitlab::Instrumentation::RedisInterceptor, :clean_gitlab_redis_sh end end.to raise_exception(Redis::CommandError) end + + context 'in production environment' do + before do + stub_rails_env('production') # to avoid raising CrossSlotError + end + + it 'counts disallowed cross-slot requests' do + expect(instrumentation_class).to receive(:increment_cross_slot_request_count).and_call_original + expect(instrumentation_class).not_to receive(:increment_allowed_cross_slot_request_count).and_call_original + + Gitlab::Redis::SharedState.with { |redis| redis.call(:mget, 'foo', 'bar') } + end + + it 'does not count allowed cross-slot requests' do + expect(instrumentation_class).not_to receive(:increment_cross_slot_request_count).and_call_original + expect(instrumentation_class).to receive(:increment_allowed_cross_slot_request_count).and_call_original + + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + Gitlab::Redis::SharedState.with { |redis| redis.call(:mget, 'foo', 'bar') } + end + end + + it 'skips count for non-cross-slot requests' do + expect(instrumentation_class).not_to receive(:increment_cross_slot_request_count).and_call_original + expect(instrumentation_class).not_to receive(:increment_allowed_cross_slot_request_count).and_call_original + + Gitlab::Redis::SharedState.with { |redis| redis.call(:mget, '{foo}bar', '{foo}baz') } + end + end + + context 'without active RequestStore' do + before do + ::RequestStore.end! + end + + it 'still runs cross-slot validation' do + expect do + Gitlab::Redis::SharedState.with { |redis| redis.mget('foo', 'bar') } + end.to raise_error(instance_of(Gitlab::Instrumentation::RedisClusterValidator::CrossSlotError)) + end + end end describe 'latency' do @@ -103,7 +145,7 @@ RSpec.describe Gitlab::Instrumentation::RedisInterceptor, :clean_gitlab_redis_sh Gitlab::Redis::SharedState.with do |redis| redis.pipelined do |pipeline| - pipeline.call(:get, '{foobar}:buz') + pipeline.call(:get, '{foobar}buz') pipeline.call(:get, '{foobar}baz') end end @@ -123,37 +165,36 @@ RSpec.describe Gitlab::Instrumentation::RedisInterceptor, :clean_gitlab_redis_sh end describe 'commands not in the apdex' do - where(:command) do - [ - [%w[brpop foobar 0.01]], - [%w[blpop foobar 0.01]], - [%w[brpoplpush foobar bazqux 0.01]], - [%w[bzpopmin foobar 0.01]], - [%w[bzpopmax foobar 0.01]], - [%w[xread block 1 streams mystream 0-0]], - [%w[xreadgroup group mygroup myconsumer block 1 streams foobar 0-0]] - ] + where(:setup, :command) do + [['rpush', 'foobar', 1]] | ['brpop', 'foobar', 0] + [['rpush', 'foobar', 1]] | ['blpop', 'foobar', 0] + [['rpush', '{abc}foobar', 1]] | ['brpoplpush', '{abc}foobar', '{abc}bazqux', 0] + [['rpush', '{abc}foobar', 1]] | ['brpoplpush', '{abc}foobar', '{abc}bazqux', 0] + [['zadd', 'foobar', 1, 'a']] | ['bzpopmin', 'foobar', 0] + [['zadd', 'foobar', 1, 'a']] | ['bzpopmax', 'foobar', 0] + [['xadd', 'mystream', 1, 'myfield', 'mydata']] | ['xread', 'block', 1, 'streams', 'mystream', '0-0'] + [['xadd', 'foobar', 1, 'myfield', 'mydata'], ['xgroup', 'create', 'foobar', 'mygroup', 0]] | ['xreadgroup', 'group', 'mygroup', 'myconsumer', 'block', 1, 'streams', 'foobar', '0-0'] end with_them do it 'skips requests we do not want in the apdex' do + Gitlab::Redis::SharedState.with { |redis| setup.each { |cmd| redis.call(*cmd) } } + expect(instrumentation_class).not_to receive(:instance_observe_duration) - begin - Gitlab::Redis::SharedState.with { |redis| redis.call(*command) } - rescue Gitlab::Instrumentation::RedisClusterValidator::CrossSlotError, ::Redis::CommandError - end + Gitlab::Redis::SharedState.with { |redis| redis.call(*command) } end end context 'with pipelined commands' do - it 'skips requests that have blocking commands', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/373026' do + it 'skips requests that have blocking commands' do expect(instrumentation_class).not_to receive(:instance_observe_duration) Gitlab::Redis::SharedState.with do |redis| redis.pipelined do |pipeline| - pipeline.call(:get, 'foo') - pipeline.call(:brpop, 'foobar', '0.01') + pipeline.call(:get, '{foobar}buz') + pipeline.call(:rpush, '{foobar}baz', 1) + pipeline.call(:brpop, '{foobar}baz', 0) end end end diff --git a/spec/lib/gitlab/instrumentation/redis_spec.rb b/spec/lib/gitlab/instrumentation/redis_spec.rb index c01d06c97b0..3e02eadba4b 100644 --- a/spec/lib/gitlab/instrumentation/redis_spec.rb +++ b/spec/lib/gitlab/instrumentation/redis_spec.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'spec_helper' +require 'support/helpers/rails_helpers' RSpec.describe Gitlab::Instrumentation::Redis do def stub_storages(method, value) @@ -22,6 +23,8 @@ RSpec.describe Gitlab::Instrumentation::Redis do end it_behaves_like 'aggregation of redis storage data', :get_request_count + it_behaves_like 'aggregation of redis storage data', :get_cross_slot_request_count + it_behaves_like 'aggregation of redis storage data', :get_allowed_cross_slot_request_count it_behaves_like 'aggregation of redis storage data', :query_time it_behaves_like 'aggregation of redis storage data', :read_bytes it_behaves_like 'aggregation of redis storage data', :write_bytes @@ -35,20 +38,28 @@ RSpec.describe Gitlab::Instrumentation::Redis do Gitlab::Redis::Cache.with { |redis| redis.info } RequestStore.clear! - Gitlab::Redis::Cache.with { |redis| redis.set('cache-test', 321) } + stub_rails_env('staging') # to avoid raising CrossSlotError + Gitlab::Redis::Cache.with { |redis| redis.mset('cache-test', 321, 'cache-test-2', 321) } + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + Gitlab::Redis::Cache.with { |redis| redis.mget('cache-test', 'cache-test-2') } + end Gitlab::Redis::SharedState.with { |redis| redis.set('shared-state-test', 123) } end it 'returns payload filtering out zeroed values' do expected_payload = { # Aggregated results - redis_calls: 2, + redis_calls: 3, + redis_cross_slot_calls: 1, + redis_allowed_cross_slot_calls: 1, redis_duration_s: be >= 0, redis_read_bytes: be >= 0, redis_write_bytes: be >= 0, # Cache results - redis_cache_calls: 1, + redis_cache_calls: 2, + redis_cache_cross_slot_calls: 1, + redis_cache_allowed_cross_slot_calls: 1, redis_cache_duration_s: be >= 0, redis_cache_read_bytes: be >= 0, redis_cache_write_bytes: be >= 0, diff --git a/spec/lib/gitlab/instrumentation_helper_spec.rb b/spec/lib/gitlab/instrumentation_helper_spec.rb index d5ff39767c4..7d78d25f18e 100644 --- a/spec/lib/gitlab/instrumentation_helper_spec.rb +++ b/spec/lib/gitlab/instrumentation_helper_spec.rb @@ -2,6 +2,7 @@ require 'spec_helper' require 'rspec-parameterized' +require 'support/helpers/rails_helpers' RSpec.describe Gitlab::InstrumentationHelper do using RSpec::Parameterized::TableSyntax @@ -38,25 +39,33 @@ RSpec.describe Gitlab::InstrumentationHelper do context 'when Redis calls are made' do it 'adds Redis data and omits Gitaly data' do - Gitlab::Redis::Cache.with { |redis| redis.set('test-cache', 123) } + stub_rails_env('staging') # to avoid raising CrossSlotError + Gitlab::Redis::Cache.with { |redis| redis.mset('test-cache', 123, 'test-cache2', 123) } + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + Gitlab::Redis::Cache.with { |redis| redis.mget('cache-test', 'cache-test-2') } + end Gitlab::Redis::Queues.with { |redis| redis.set('test-queues', 321) } subject # Aggregated payload - expect(payload[:redis_calls]).to eq(2) + expect(payload[:redis_calls]).to eq(3) + expect(payload[:redis_cross_slot_calls]).to eq(1) + expect(payload[:redis_allowed_cross_slot_calls]).to eq(1) expect(payload[:redis_duration_s]).to be >= 0 expect(payload[:redis_read_bytes]).to be >= 0 expect(payload[:redis_write_bytes]).to be >= 0 - # Shared state payload + # Queue payload expect(payload[:redis_queues_calls]).to eq(1) expect(payload[:redis_queues_duration_s]).to be >= 0 expect(payload[:redis_queues_read_bytes]).to be >= 0 expect(payload[:redis_queues_write_bytes]).to be >= 0 # Cache payload - expect(payload[:redis_cache_calls]).to eq(1) + expect(payload[:redis_cache_calls]).to eq(2) + expect(payload[:redis_cache_cross_slot_calls]).to eq(1) + expect(payload[:redis_cache_allowed_cross_slot_calls]).to eq(1) expect(payload[:redis_cache_duration_s]).to be >= 0 expect(payload[:redis_cache_read_bytes]).to be >= 0 expect(payload[:redis_cache_write_bytes]).to be >= 0 @@ -67,6 +76,26 @@ RSpec.describe Gitlab::InstrumentationHelper do end end + context 'when LDAP requests are made' do + let(:provider) { 'ldapmain' } + let(:adapter) { Gitlab::Auth::Ldap::Adapter.new(provider) } + let(:conn) { instance_double(Net::LDAP::Connection, search: search) } + let(:search) { double(:search, result_code: 200) } # rubocop: disable RSpec/VerifiedDoubles + + it 'adds LDAP data' do + allow_next_instance_of(Net::LDAP) do |net_ldap| + allow(net_ldap).to receive(:use_connection).and_yield(conn) + end + + adapter.users('uid', 'foo') + subject + + # Query count should be 2, as it will call `open` then `search` + expect(payload[:net_ldap_count]).to eq(2) + expect(payload[:net_ldap_duration_s]).to be >= 0 + end + end + context 'when the request matched a Rack::Attack safelist' do it 'logs the safelist name' do Gitlab::Instrumentation::Throttle.safelist = 'foobar' @@ -122,7 +151,7 @@ RSpec.describe Gitlab::InstrumentationHelper do include MemoryInstrumentationHelper before do - skip_memory_instrumentation! + verify_memory_instrumentation_available! end it 'logs memory usage metrics' do diff --git a/spec/lib/gitlab/kubernetes/kube_client_spec.rb b/spec/lib/gitlab/kubernetes/kube_client_spec.rb index 8abd041fd4e..2ead188dc93 100644 --- a/spec/lib/gitlab/kubernetes/kube_client_spec.rb +++ b/spec/lib/gitlab/kubernetes/kube_client_spec.rb @@ -132,6 +132,14 @@ RSpec.describe Gitlab::Kubernetes::KubeClient do it_behaves_like 'local address' end + context 'when a non HTTP/HTTPS URL is provided' do + let(:api_url) { 'ssh://192.168.1.2' } + + it 'raises an error' do + expect { client }.to raise_error(Gitlab::UrlBlocker::BlockedUrlError) + end + end + it 'falls back to default options, but allows overriding' do client = described_class.new(api_url) defaults = Gitlab::Kubernetes::KubeClient::DEFAULT_KUBECLIENT_OPTIONS @@ -149,7 +157,7 @@ RSpec.describe Gitlab::Kubernetes::KubeClient do it_behaves_like 'a Kubeclient' it 'has the core API endpoint' do - expect(subject.api_endpoint.to_s).to match(%r{\/api\Z}) + expect(subject.api_endpoint.to_s).to match(%r{/api\Z}) end it 'has the api_version' do @@ -163,7 +171,7 @@ RSpec.describe Gitlab::Kubernetes::KubeClient do it_behaves_like 'a Kubeclient' it 'has the RBAC API group endpoint' do - expect(subject.api_endpoint.to_s).to match(%r{\/apis\/rbac.authorization.k8s.io\Z}) + expect(subject.api_endpoint.to_s).to match(%r{/apis/rbac.authorization.k8s.io\Z}) end it 'has the api_version' do @@ -177,7 +185,7 @@ RSpec.describe Gitlab::Kubernetes::KubeClient do it_behaves_like 'a Kubeclient' it 'has the Istio API group endpoint' do - expect(subject.api_endpoint.to_s).to match(%r{\/apis\/networking.istio.io\Z}) + expect(subject.api_endpoint.to_s).to match(%r{/apis/networking.istio.io\Z}) end it 'has the api_version' do @@ -191,7 +199,7 @@ RSpec.describe Gitlab::Kubernetes::KubeClient do it_behaves_like 'a Kubeclient' it 'has the extensions API group endpoint' do - expect(subject.api_endpoint.to_s).to match(%r{\/apis\/serving.knative.dev\Z}) + expect(subject.api_endpoint.to_s).to match(%r{/apis/serving.knative.dev\Z}) end it 'has the api_version' do @@ -205,7 +213,7 @@ RSpec.describe Gitlab::Kubernetes::KubeClient do it_behaves_like 'a Kubeclient' it 'has the networking API group endpoint' do - expect(subject.api_endpoint.to_s).to match(%r{\/apis\/networking.k8s.io\Z}) + expect(subject.api_endpoint.to_s).to match(%r{/apis/networking.k8s.io\Z}) end it 'has the api_version' do @@ -219,7 +227,7 @@ RSpec.describe Gitlab::Kubernetes::KubeClient do it_behaves_like 'a Kubeclient' it 'has the metrics API group endpoint' do - expect(subject.api_endpoint.to_s).to match(%r{\/apis\/metrics.k8s.io\Z}) + expect(subject.api_endpoint.to_s).to match(%r{/apis/metrics.k8s.io\Z}) end it 'has the api_version' do diff --git a/spec/lib/gitlab/markdown_cache/active_record/extension_spec.rb b/spec/lib/gitlab/markdown_cache/active_record/extension_spec.rb index 57f2b1cfd96..81d423598f2 100644 --- a/spec/lib/gitlab/markdown_cache/active_record/extension_spec.rb +++ b/spec/lib/gitlab/markdown_cache/active_record/extension_spec.rb @@ -2,6 +2,8 @@ require 'spec_helper' RSpec.describe Gitlab::MarkdownCache::ActiveRecord::Extension do + let_it_be(:project) { create(:project) } + let(:klass) do Class.new(ActiveRecord::Base) do self.table_name = 'issues' @@ -17,7 +19,12 @@ RSpec.describe Gitlab::MarkdownCache::ActiveRecord::Extension do end let(:cache_version) { Gitlab::MarkdownCache::CACHE_COMMONMARK_VERSION << 16 } - let(:thing) { klass.create!(title: markdown, title_html: html, cached_markdown_version: cache_version) } + let(:thing) do + klass.create!( + project_id: project.id, namespace_id: project.project_namespace_id, + title: markdown, title_html: html, cached_markdown_version: cache_version + ) + end let(:markdown) { '`Foo`' } let(:html) { '<p data-sourcepos="1:1-1:5" dir="auto"><code>Foo</code></p>' } @@ -26,7 +33,7 @@ RSpec.describe Gitlab::MarkdownCache::ActiveRecord::Extension do let(:updated_html) { '<p data-sourcepos="1:1-1:5" dir="auto"><code>Bar</code></p>' } context 'an unchanged markdown field' do - let(:thing) { klass.new(title: markdown) } + let(:thing) { klass.new(project_id: project.id, namespace_id: project.project_namespace_id, title: markdown) } before do thing.title = thing.title @@ -40,7 +47,12 @@ RSpec.describe Gitlab::MarkdownCache::ActiveRecord::Extension do end context 'a changed markdown field' do - let(:thing) { klass.create!(title: markdown, title_html: html, cached_markdown_version: cache_version) } + let(:thing) do + klass.create!( + project_id: project.id, namespace_id: project.project_namespace_id, + title: markdown, title_html: html, cached_markdown_version: cache_version + ) + end before do thing.title = updated_markdown @@ -72,7 +84,12 @@ RSpec.describe Gitlab::MarkdownCache::ActiveRecord::Extension do end context 'a non-markdown field changed' do - let(:thing) { klass.new(title: markdown, title_html: html, cached_markdown_version: cache_version) } + let(:thing) do + klass.new( + project_id: project.id, namespace_id: project.project_namespace_id, title: markdown, + title_html: html, cached_markdown_version: cache_version + ) + end before do thing.state_id = 2 @@ -86,7 +103,12 @@ RSpec.describe Gitlab::MarkdownCache::ActiveRecord::Extension do end context 'version is out of date' do - let(:thing) { klass.new(title: updated_markdown, title_html: html, cached_markdown_version: nil) } + let(:thing) do + klass.new( + project_id: project.id, namespace_id: project.project_namespace_id, + title: updated_markdown, title_html: html, cached_markdown_version: nil + ) + end before do thing.save! @@ -127,7 +149,12 @@ RSpec.describe Gitlab::MarkdownCache::ActiveRecord::Extension do end describe '#cached_html_up_to_date?' do - let(:thing) { klass.create!(title: updated_markdown, title_html: html, cached_markdown_version: nil) } + let(:thing) do + klass.create!( + project_id: project.id, namespace_id: project.project_namespace_id, + title: updated_markdown, title_html: html, cached_markdown_version: nil + ) + end subject { thing.cached_html_up_to_date?(:title) } diff --git a/spec/lib/gitlab/memory/instrumentation_spec.rb b/spec/lib/gitlab/memory/instrumentation_spec.rb index 069c45da18a..3d58f28ec1e 100644 --- a/spec/lib/gitlab/memory/instrumentation_spec.rb +++ b/spec/lib/gitlab/memory/instrumentation_spec.rb @@ -2,11 +2,11 @@ require 'spec_helper' -RSpec.describe Gitlab::Memory::Instrumentation do +RSpec.describe Gitlab::Memory::Instrumentation, feature_category: :application_performance do include MemoryInstrumentationHelper before do - skip_memory_instrumentation! + verify_memory_instrumentation_available! end describe '.available?' do diff --git a/spec/lib/gitlab/memory/jemalloc_spec.rb b/spec/lib/gitlab/memory/jemalloc_spec.rb index 414d6017534..8cce2278f8e 100644 --- a/spec/lib/gitlab/memory/jemalloc_spec.rb +++ b/spec/lib/gitlab/memory/jemalloc_spec.rb @@ -1,15 +1,14 @@ # frozen_string_literal: true require 'fast_spec_helper' -require 'tmpdir' +require 'tempfile' RSpec.describe Gitlab::Memory::Jemalloc do - let(:outdir) { Dir.mktmpdir } - let(:tmp_outdir) { Dir.mktmpdir } + let(:outfile) { Tempfile.new } after do - FileUtils.rm_f(outdir) - FileUtils.rm_f(tmp_outdir) + outfile.close + outfile.unlink end context 'when jemalloc is loaded' do @@ -31,12 +30,11 @@ RSpec.describe Gitlab::Memory::Jemalloc do describe '.dump_stats' do it 'writes stats JSON file' do - file_path = described_class.dump_stats(path: outdir, tmp_dir: tmp_outdir, format: format) + described_class.dump_stats(outfile, format: format) - file = Dir.entries(outdir).find { |e| e.match(/jemalloc_stats\.#{$$}\.\d+\.json$/) } - expect(file).not_to be_nil - expect(file_path).to eq(File.join(outdir, file)) - expect(File.read(file_path)).to eq(output) + outfile.rewind + + expect(outfile.read).to eq(output) end end end @@ -56,23 +54,12 @@ RSpec.describe Gitlab::Memory::Jemalloc do end describe '.dump_stats' do - shared_examples 'writes stats text file' do |filename_label, filename_pattern| - it do - described_class.dump_stats( - path: outdir, tmp_dir: tmp_outdir, format: format, filename_label: filename_label) - - file = Dir.entries(outdir).find { |e| e.match(filename_pattern) } - expect(file).not_to be_nil - expect(File.read(File.join(outdir, file))).to eq(output) - end - end + it 'writes stats text file' do + described_class.dump_stats(outfile, format: format) - context 'when custom filename label is passed' do - include_examples 'writes stats text file', 'puma_0', /jemalloc_stats\.#{$$}\.puma_0\.\d+\.txt$/ - end + outfile.rewind - context 'when custom filename label is not passed' do - include_examples 'writes stats text file', nil, /jemalloc_stats\.#{$$}\.\d+\.txt$/ + expect(outfile.read).to eq(output) end end end @@ -91,7 +78,7 @@ RSpec.describe Gitlab::Memory::Jemalloc do describe '.dump_stats' do it 'raises an error' do expect do - described_class.dump_stats(path: outdir, tmp_dir: tmp_outdir, format: format) + described_class.dump_stats(outfile, format: format) end.to raise_error(/format must be one of/) end end @@ -104,18 +91,18 @@ RSpec.describe Gitlab::Memory::Jemalloc do end describe '.stats' do - it 'returns nil' do - expect(described_class.stats).to be_nil + it 'returns empty string' do + expect(described_class.stats).to be_empty end end describe '.dump_stats' do it 'does nothing' do - stub_env('LD_PRELOAD', nil) + described_class.dump_stats(outfile) - described_class.dump_stats(path: outdir, tmp_dir: tmp_outdir) + outfile.rewind - expect(Dir.empty?(outdir)).to be(true) + expect(outfile.read).to be_empty end end end diff --git a/spec/lib/gitlab/memory/reporter_spec.rb b/spec/lib/gitlab/memory/reporter_spec.rb new file mode 100644 index 00000000000..924397ceb4f --- /dev/null +++ b/spec/lib/gitlab/memory/reporter_spec.rb @@ -0,0 +1,206 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Memory::Reporter, :aggregate_failures, feature_category: :application_performance do + let(:fake_report) do + Class.new do + def name + 'fake_report' + end + + def active? + true + end + + def run(writer) + writer << 'I ran' + end + end + end + + let(:logger) { instance_double(::Logger) } + let(:report) { fake_report.new } + + after do + FileUtils.rm_rf(reports_path) + end + + describe '#run_report', time_travel_to: '2020-02-02 10:30:45 0000' do + let(:report_duration_counter) { instance_double(::Prometheus::Client::Counter) } + let(:file_size) { 1_000_000 } + let(:report_file) { "#{reports_path}/fake_report.2020-02-02.10:30:45:000.worker_1.abc123.gz" } + + let(:input) { StringIO.new } + let(:output) { StringIO.new } + + before do + allow(SecureRandom).to receive(:uuid).and_return('abc123') + + allow(Gitlab::Metrics).to receive(:counter).and_return(report_duration_counter) + allow(report_duration_counter).to receive(:increment) + + allow(::Prometheus::PidProvider).to receive(:worker_id).and_return('worker_1') + allow(File).to receive(:size).with(report_file).and_return(file_size) + + allow(logger).to receive(:info) + + stub_gzip + end + + shared_examples 'runs and stores reports' do + it 'runs the given report and returns true' do + expect(reporter.run_report(report)).to be(true) + + expect(output.string).to eq('I ran') + end + + it 'closes read and write streams' do + expect(input).to receive(:close).ordered.at_least(:once) + expect(output).to receive(:close).ordered.at_least(:once) + + reporter.run_report(report) + end + + it 'logs start and finish event' do + expect(logger).to receive(:info).ordered.with( + hash_including( + message: 'started', + pid: Process.pid, + worker_id: 'worker_1', + perf_report_worker_uuid: 'abc123', + perf_report: 'fake_report' + )) + expect(logger).to receive(:info).ordered.with( + hash_including( + :duration_s, + :cpu_s, + perf_report_file: report_file, + perf_report_size_bytes: file_size, + message: 'finished', + pid: Process.pid, + worker_id: 'worker_1', + perf_report_worker_uuid: 'abc123', + perf_report: 'fake_report' + )) + + reporter.run_report(report) + end + + it 'increments Prometheus duration counter' do + expect(report_duration_counter).to receive(:increment).with({ report: 'fake_report' }, an_instance_of(Float)) + + reporter.run_report(report) + end + + context 'when the report returns invalid file path' do + before do + allow(File).to receive(:size).with(report_file).and_raise(Errno::ENOENT) + end + + it 'logs `0` as `perf_report_size_bytes`' do + expect(logger).to receive(:info).ordered.with( + hash_including(message: 'started') + ) + expect(logger).to receive(:info).ordered.with( + hash_including(message: 'finished', perf_report_size_bytes: 0) + ) + + reporter.run_report(report) + end + end + + context 'when an error occurs' do + before do + allow(report).to receive(:run).and_raise(RuntimeError.new('report failed')) + end + + it 'logs the error and returns false' do + expect(logger).to receive(:info).ordered.with(hash_including(message: 'started')) + expect(logger).to receive(:error).ordered.with( + hash_including( + message: 'failed', error: '#<RuntimeError: report failed>' + )) + + expect(reporter.run_report(report)).to be(false) + end + + it 'closes read and write streams' do + allow(logger).to receive(:info) + allow(logger).to receive(:error) + + expect(input).to receive(:close).ordered.at_least(:once) + expect(output).to receive(:close).ordered.at_least(:once) + + reporter.run_report(report) + end + + context 'when compression process is still running' do + it 'terminates the process' do + allow(logger).to receive(:info) + allow(logger).to receive(:error) + + expect(Gitlab::ProcessManagement).to receive(:signal).with(an_instance_of(Integer), :KILL) + + reporter.run_report(report) + end + end + end + + context 'when a report is disabled' do + it 'does nothing and returns false' do + expect(report).to receive(:active?).and_return(false) + expect(report).not_to receive(:run) + expect(logger).not_to receive(:info) + expect(report_duration_counter).not_to receive(:increment) + + reporter.run_report(report) + end + end + end + + context 'when reports path is specified directly' do + let(:reports_path) { Dir.mktmpdir } + + subject(:reporter) { described_class.new(reports_path: reports_path, logger: logger) } + + it_behaves_like 'runs and stores reports' + end + + context 'when reports path is specified via environment' do + let(:reports_path) { Dir.mktmpdir } + + subject(:reporter) { described_class.new(logger: logger) } + + before do + stub_env('GITLAB_DIAGNOSTIC_REPORTS_PATH', reports_path) + end + + it_behaves_like 'runs and stores reports' + end + + context 'when reports path is not specified' do + let(:reports_path) { reporter.reports_path } + + subject(:reporter) { described_class.new(logger: logger) } + + it 'defaults to a temporary location' do + expect(reports_path).not_to be_empty + end + + it_behaves_like 'runs and stores reports' + end + end + + # We need to stub out the call into gzip. We do this by intercepting the write + # end of the pipe and replacing it with a StringIO instead, which we can + # easily inspect for contents. + def stub_gzip + pid = 42 + allow(IO).to receive(:pipe).and_return([input, output]) + allow(Process).to receive(:spawn).with( + "gzip", "--fast", in: input, out: an_instance_of(File), err: an_instance_of(IO) + ).and_return(pid) + allow(Process).to receive(:waitpid).with(pid) + end +end diff --git a/spec/lib/gitlab/memory/reports/heap_dump_spec.rb b/spec/lib/gitlab/memory/reports/heap_dump_spec.rb new file mode 100644 index 00000000000..4e235a71bdb --- /dev/null +++ b/spec/lib/gitlab/memory/reports/heap_dump_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Memory::Reports::HeapDump, feature_category: :application_performance do + # Copy this class so we do not mess with its state. + let(:klass) { described_class.dup } + + subject(:report) { klass.new } + + describe '#name' do + # This is a bit silly, but it caused code coverage failures. + it 'is set' do + expect(report.name).to eq('heap_dump') + end + end + + describe '#active?' do + it 'is true when report_heap_dumps is enabled' do + expect(report).to be_active + end + + it 'is false when report_heap_dumps is disabled' do + stub_feature_flags(report_heap_dumps: false) + + expect(report).not_to be_active + end + end + + describe '#run' do + subject(:run) { report.run(writer) } + + let(:writer) { StringIO.new } + + context 'when no heap dump is enqueued' do + it 'does nothing and returns false' do + expect(ObjectSpace).not_to receive(:dump_all) + + expect(run).to be(false) + end + end + + context 'when a heap dump is enqueued', :aggregate_failures do + it 'dumps heap and returns true' do + expect(ObjectSpace).to receive(:dump_all).with(output: writer) do |output:| + output << 'heap contents' + end + + klass.enqueue! + + expect(run).to be(true) + expect(writer.string).to eq('heap contents') + end + end + end +end diff --git a/spec/lib/gitlab/memory/reports/jemalloc_stats_spec.rb b/spec/lib/gitlab/memory/reports/jemalloc_stats_spec.rb index b327a40bc2c..ce06c270a05 100644 --- a/spec/lib/gitlab/memory/reports/jemalloc_stats_spec.rb +++ b/spec/lib/gitlab/memory/reports/jemalloc_stats_spec.rb @@ -3,96 +3,16 @@ require 'spec_helper' RSpec.describe Gitlab::Memory::Reports::JemallocStats do - let_it_be(:outdir) { Dir.mktmpdir } + subject(:jemalloc_stats) { described_class.new } - let(:jemalloc_stats) { described_class.new(reports_path: outdir) } - - after do - FileUtils.rm_f(outdir) - end + let(:writer) { StringIO.new } describe '.run' do context 'when :report_jemalloc_stats ops FF is enabled' do - let(:worker_id) { 'puma_1' } - let(:report_name) { 'report.json' } - let(:report_path) { File.join(outdir, report_name) } - - before do - allow(Prometheus::PidProvider).to receive(:worker_id).and_return(worker_id) - end - - it 'invokes Jemalloc.dump_stats and returns file path' do - expect(Gitlab::Memory::Jemalloc) - .to receive(:dump_stats) - .with(path: outdir, - tmp_dir: File.join(outdir, '/tmp'), - filename_label: worker_id) - .and_return(report_path) - - expect(jemalloc_stats.run).to eq(report_path) - end - - describe 'reports cleanup' do - let(:jemalloc_stats) { described_class.new(reports_path: outdir) } - - before do - stub_env('GITLAB_DIAGNOSTIC_REPORTS_JEMALLOC_MAX_REPORTS_STORED', 3) - allow(Gitlab::Memory::Jemalloc).to receive(:dump_stats) - end - - context 'when number of reports exceeds `max_reports_stored`' do - let_it_be(:reports) do - now = Time.current - - (1..5).map do |i| - Tempfile.new("jemalloc_stats.#{i}.worker_#{i}.#{Time.current.to_i}.json", outdir).tap do |f| - FileUtils.touch(f, mtime: (now + i.second).to_i) - end - end - end - - after do - reports.each do |f| - f.close - f.unlink - rescue Errno::ENOENT - # Some of the files are already unlinked by the code we test; Ignore - end - end - - it 'keeps only `max_reports_stored` total newest files' do - expect { jemalloc_stats.run } - .to change { Dir.entries(outdir).count { |e| e.match(/jemalloc_stats.*/) } } - .from(5).to(3) - - # Keeps only the newest reports - expect(reports.last(3).all? { |r| File.exist?(r) }).to be true - end - end - - context 'when number of reports does not exceed `max_reports_stored`' do - let_it_be(:reports) do - now = Time.current - - (1..3).map do |i| - Tempfile.new("jemalloc_stats.#{i}.worker_#{i}.#{Time.current.to_i}.json", outdir).tap do |f| - FileUtils.touch(f, mtime: (now + i.second).to_i) - end - end - end - - after do - reports.each do |f| - f.close - f.unlink - end - end + it 'dumps jemalloc stats to the given writer' do + expect(Gitlab::Memory::Jemalloc).to receive(:dump_stats).with(writer) - it 'does not remove any reports' do - expect { jemalloc_stats.run } - .not_to change { Dir.entries(outdir).count { |e| e.match(/jemalloc_stats.*/) } } - end - end + jemalloc_stats.run(writer) end end @@ -101,10 +21,10 @@ RSpec.describe Gitlab::Memory::Reports::JemallocStats do stub_feature_flags(report_jemalloc_stats: false) end - it 'does not run the report and returns nil' do + it 'does not run the report' do expect(Gitlab::Memory::Jemalloc).not_to receive(:dump_stats) - expect(jemalloc_stats.run).to be_nil + jemalloc_stats.run(writer) end end end diff --git a/spec/lib/gitlab/memory/reports_daemon_spec.rb b/spec/lib/gitlab/memory/reports_daemon_spec.rb index 0473e170502..91c36c87253 100644 --- a/spec/lib/gitlab/memory/reports_daemon_spec.rb +++ b/spec/lib/gitlab/memory/reports_daemon_spec.rb @@ -3,94 +3,58 @@ require 'spec_helper' RSpec.describe Gitlab::Memory::ReportsDaemon, :aggregate_failures do - let(:daemon) { described_class.new } + let(:reporter) { instance_double(Gitlab::Memory::Reporter) } + let(:reports) { nil } - let_it_be(:tmp_dir) { Dir.mktmpdir } - - after(:all) do - FileUtils.remove_entry(tmp_dir) - end + subject(:daemon) { described_class.new(reporter: reporter, reports: reports) } describe '#run_thread' do - let(:report_duration_counter) { instance_double(::Prometheus::Client::Counter) } - let(:file_size) { 1_000_000 } - before do - allow(Gitlab::Metrics).to receive(:counter).and_return(report_duration_counter) - allow(report_duration_counter).to receive(:increment) - # make sleep no-op allow(daemon).to receive(:sleep) {} # let alive return 3 times: true, true, false allow(daemon).to receive(:alive).and_return(true, true, false) - - allow(File).to receive(:size).with(/#{daemon.reports_path}.*\.json/).and_return(file_size) - end - - it 'runs reports, logs and sets gauge' do - expect(daemon.send(:reports)) - .to all(receive(:run).twice { Tempfile.new("report.json", tmp_dir).path }) - - expect(::Prometheus::PidProvider).to receive(:worker_id).at_least(:once).and_return('worker_1') - - expect(Gitlab::AppLogger).to receive(:info).with( - hash_including( - :duration_s, - :cpu_s, - perf_report_size_bytes: file_size, - message: 'finished', - pid: Process.pid, - worker_id: 'worker_1', - perf_report: 'jemalloc_stats' - )).twice - - expect(report_duration_counter).to receive(:increment).with({ report: 'jemalloc_stats' }, an_instance_of(Float)) - - daemon.send(:run_thread) end - context 'when the report object returns invalid file path' do - before do - allow(File).to receive(:size).with(/#{daemon.reports_path}.*\.json/).and_raise(Errno::ENOENT) - end - - it 'logs `0` as `perf_report_size_bytes`' do - expect(daemon.send(:reports)) - .to all(receive(:run).twice { Tempfile.new("report.json", tmp_dir).path }) - - expect(Gitlab::AppLogger).to receive(:info).with(hash_including(perf_report_size_bytes: 0)).twice + context 'with default reports' do + it 'runs them using the given reporter' do + expect(reporter).to receive(:run_report).twice.with(an_instance_of(Gitlab::Memory::Reports::JemallocStats)) daemon.send(:run_thread) end end - it 'allows configure and run multiple reports' do + context 'with inactive reports' do # rubocop: disable RSpec/VerifiedDoubles # We test how ReportsDaemon could be extended in the future # We configure it with new reports classes which are not yet defined so we cannot make this an instance_double. - active_report_1 = double("Active Report 1", active?: true) - active_report_2 = double("Active Report 2", active?: true) - inactive_report = double("Inactive Report", active?: false) + let(:active_report_1) { double("Active Report 1", active?: true) } + let(:active_report_2) { double("Active Report 2", active?: true) } + let(:inactive_report) { double("Inactive Report", active?: false) } # rubocop: enable RSpec/VerifiedDoubles - allow(daemon).to receive(:reports).and_return([active_report_1, inactive_report, active_report_2]) + let(:reports) do + [active_report_1, active_report_2, inactive_report] + end - expect(active_report_1).to receive(:run).and_return(File.join(tmp_dir, 'report_1.json')).twice - expect(active_report_2).to receive(:run).and_return(File.join(tmp_dir, 'report_2.json')).twice - expect(inactive_report).not_to receive(:run) + it 'runs only active reports' do + expect(reporter).to receive(:run_report).ordered.with(active_report_1) + expect(reporter).to receive(:run_report).ordered.with(active_report_2) + expect(reporter).to receive(:run_report).ordered.with(active_report_1) + expect(reporter).to receive(:run_report).ordered.with(active_report_2) + expect(reporter).not_to receive(:run_report).with(inactive_report) - daemon.send(:run_thread) + daemon.send(:run_thread) + end end context 'sleep timers logic' do it 'wakes up every (fixed interval + defined delta), sleeps between reports each cycle' do stub_env('GITLAB_DIAGNOSTIC_REPORTS_SLEEP_MAX_DELTA_S', 1) # rand(1) == 0, so we will have fixed sleep interval - daemon = described_class.new + daemon = described_class.new(reporter: reporter, reports: reports) allow(daemon).to receive(:alive).and_return(true, true, false) - - expect(daemon.send(:reports)) - .to all(receive(:run).twice { Tempfile.new("report.json", tmp_dir).path }) + allow(reporter).to receive(:run_report) expect(daemon).to receive(:sleep).with(described_class::DEFAULT_SLEEP_S).ordered expect(daemon).to receive(:sleep).with(described_class::DEFAULT_SLEEP_BETWEEN_REPORTS_S).ordered @@ -116,7 +80,6 @@ RSpec.describe Gitlab::Memory::ReportsDaemon, :aggregate_failures do expect(daemon.sleep_s).to eq(described_class::DEFAULT_SLEEP_S) expect(daemon.sleep_max_delta_s).to eq(described_class::DEFAULT_SLEEP_MAX_DELTA_S) expect(daemon.sleep_between_reports_s).to eq(described_class::DEFAULT_SLEEP_BETWEEN_REPORTS_S) - expect(daemon.reports_path).to eq(described_class::DEFAULT_REPORTS_PATH) end end @@ -125,7 +88,6 @@ RSpec.describe Gitlab::Memory::ReportsDaemon, :aggregate_failures do stub_env('GITLAB_DIAGNOSTIC_REPORTS_SLEEP_S', 100) stub_env('GITLAB_DIAGNOSTIC_REPORTS_SLEEP_MAX_DELTA_S', 50) stub_env('GITLAB_DIAGNOSTIC_REPORTS_SLEEP_BETWEEN_REPORTS_S', 2) - stub_env('GITLAB_DIAGNOSTIC_REPORTS_PATH', tmp_dir) end it 'uses provided values' do @@ -134,7 +96,6 @@ RSpec.describe Gitlab::Memory::ReportsDaemon, :aggregate_failures do expect(daemon.sleep_s).to eq(100) expect(daemon.sleep_max_delta_s).to eq(50) expect(daemon.sleep_between_reports_s).to eq(2) - expect(daemon.reports_path).to eq(tmp_dir) end end end diff --git a/spec/lib/gitlab/memory/watchdog/configuration_spec.rb b/spec/lib/gitlab/memory/watchdog/configuration_spec.rb index 38a39f6a33a..9242344ead2 100644 --- a/spec/lib/gitlab/memory/watchdog/configuration_spec.rb +++ b/spec/lib/gitlab/memory/watchdog/configuration_spec.rb @@ -20,10 +20,14 @@ RSpec.describe Gitlab::Memory::Watchdog::Configuration do end end - describe '#logger' do - context 'when logger is not set, defaults to stdout logger' do - it 'defaults to Logger' do - expect(configuration.logger).to be_an_instance_of(::Gitlab::Logger) + describe '#event_reporter' do + context 'when event reporter is not set' do + before do + allow(Gitlab::Metrics).to receive(:counter) + end + + it 'defaults to EventReporter' do + expect(configuration.event_reporter).to be_an_instance_of(::Gitlab::Memory::Watchdog::EventReporter) end end end @@ -38,6 +42,8 @@ RSpec.describe Gitlab::Memory::Watchdog::Configuration do describe '#monitors' do context 'when monitors are configured to be used' do + let(:monitor_name1) { :monitor1 } + let(:monitor_name2) { :monitor2 } let(:payload1) do { message: 'monitor_1_text', @@ -96,7 +102,17 @@ RSpec.describe Gitlab::Memory::Watchdog::Configuration do expect(payloads).to eq([payload1, payload2]) expect(thresholds).to eq([false, true]) expect(strikes).to eq([false, true]) - expect(monitor_names).to eq([:monitor1, :monitor2]) + expect(monitor_names).to eq([monitor_name1, monitor_name2]) + end + + it 'monitors are not empty' do + expect(configuration.monitors).not_to be_empty + end + end + + context 'when monitors are not configured' do + it 'monitors are empty' do + expect(configuration.monitors).to be_empty end end @@ -119,18 +135,19 @@ RSpec.describe Gitlab::Memory::Watchdog::Configuration do include_examples 'executes monitors and returns correct results' end - end - context 'when same monitor class is configured twice' do - before do - configuration.monitors.push monitor_class_1, max_strikes: 1 - configuration.monitors.push monitor_class_1, max_strikes: 1 - end + context 'when monitors are configured with monitor name' do + let(:monitor_name1) { :mon_one } + let(:monitor_name2) { :mon_two } - it 'calls same monitor only once' do - expect do |b| - configuration.monitors.call_each(&b) - end.to yield_control.once + before do + configuration.monitors do |stack| + stack.push monitor_class_1, false, { message: 'monitor_1_text' }, max_strikes: 5, monitor_name: :mon_one + stack.push monitor_class_2, true, { message: 'monitor_2_text' }, max_strikes: 0, monitor_name: :mon_two + end + end + + include_examples 'executes monitors and returns correct results' end end end diff --git a/spec/lib/gitlab/memory/watchdog/configurator_spec.rb b/spec/lib/gitlab/memory/watchdog/configurator_spec.rb index 86cbb724cfd..a901be84a21 100644 --- a/spec/lib/gitlab/memory/watchdog/configurator_spec.rb +++ b/spec/lib/gitlab/memory/watchdog/configurator_spec.rb @@ -6,17 +6,23 @@ require 'sidekiq' require_dependency 'gitlab/cluster/lifecycle_events' RSpec.describe Gitlab::Memory::Watchdog::Configurator do - shared_examples 'as configurator' do |handler_class, sleep_time_env, sleep_time| + shared_examples 'as configurator' do |handler_class, event_reporter_class, sleep_time_env, sleep_time| it 'configures the correct handler' do configurator.call(configuration) expect(configuration.handler).to be_an_instance_of(handler_class) end + it 'configures the correct event reporter' do + configurator.call(configuration) + + expect(configuration.event_reporter).to be_an_instance_of(event_reporter_class) + end + it 'configures the correct logger' do configurator.call(configuration) - expect(configuration.logger).to eq(logger) + expect(configuration.event_reporter.logger).to eq(logger) end context 'when sleep_time_seconds is not passed through the environment' do @@ -87,12 +93,13 @@ RSpec.describe Gitlab::Memory::Watchdog::Configurator do it_behaves_like 'as configurator', Gitlab::Memory::Watchdog::PumaHandler, + Gitlab::Memory::Watchdog::EventReporter, 'GITLAB_MEMWD_SLEEP_TIME_SEC', - 60 + described_class::DEFAULT_SLEEP_INTERVAL_S context 'with DISABLE_PUMA_WORKER_KILLER set to true' do - let(:primary_memory) { 2048 } - let(:worker_memory) { max_mem_growth * primary_memory + 1 } + let(:primary_memory_bytes) { 2_097_152_000 } + let(:worker_memory_bytes) { max_mem_growth * primary_memory_bytes + 1 } let(:expected_payloads) do { heap_fragmentation: { @@ -105,9 +112,9 @@ RSpec.describe Gitlab::Memory::Watchdog::Configurator do }, unique_memory_growth: { message: 'memory limit exceeded', - memwd_uss_bytes: worker_memory, - memwd_ref_uss_bytes: primary_memory, - memwd_max_uss_bytes: max_mem_growth * primary_memory, + memwd_uss_bytes: worker_memory_bytes, + memwd_ref_uss_bytes: primary_memory_bytes, + memwd_max_uss_bytes: max_mem_growth * primary_memory_bytes, memwd_max_strikes: max_strikes, memwd_cur_strikes: 1 } @@ -117,10 +124,10 @@ RSpec.describe Gitlab::Memory::Watchdog::Configurator do before do stub_env('DISABLE_PUMA_WORKER_KILLER', true) allow(Gitlab::Metrics::Memory).to receive(:gc_heap_fragmentation).and_return(max_heap_fragmentation + 0.1) - allow(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).and_return({ uss: worker_memory }) + allow(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).and_return({ uss: worker_memory_bytes }) allow(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).with( pid: Gitlab::Cluster::PRIMARY_PID - ).and_return({ uss: primary_memory }) + ).and_return({ uss: primary_memory_bytes }) end context 'when settings are set via environment variables' do @@ -138,21 +145,22 @@ RSpec.describe Gitlab::Memory::Watchdog::Configurator do end context 'when settings are not set via environment variables' do - let(:max_heap_fragmentation) { 0.5 } - let(:max_mem_growth) { 3.0 } - let(:max_strikes) { 5 } + let(:max_heap_fragmentation) { described_class::DEFAULT_MAX_HEAP_FRAG } + let(:max_mem_growth) { described_class::DEFAULT_MAX_MEM_GROWTH } + let(:max_strikes) { described_class::DEFAULT_MAX_STRIKES } it_behaves_like 'as monitor configurator' end end context 'with DISABLE_PUMA_WORKER_KILLER set to false' do + let(:memory_limit_bytes) { memory_limit_mb.megabytes } let(:expected_payloads) do { rss_memory_limit: { message: 'rss memory limit exceeded', - memwd_rss_bytes: memory_limit + 1, - memwd_max_rss_bytes: memory_limit, + memwd_rss_bytes: memory_limit_bytes + 1, + memwd_max_rss_bytes: memory_limit_bytes, memwd_max_strikes: max_strikes, memwd_cur_strikes: 1 } @@ -161,15 +169,15 @@ RSpec.describe Gitlab::Memory::Watchdog::Configurator do before do stub_env('DISABLE_PUMA_WORKER_KILLER', false) - allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).and_return({ total: memory_limit + 1 }) + allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).and_return({ total: memory_limit_bytes + 1 }) end context 'when settings are set via environment variables' do - let(:memory_limit) { 1300.megabytes } + let(:memory_limit_mb) { 1300 } let(:max_strikes) { 4 } before do - stub_env('PUMA_WORKER_MAX_MEMORY', 1300) + stub_env('PUMA_WORKER_MAX_MEMORY', memory_limit_mb) stub_env('GITLAB_MEMWD_MAX_STRIKES', 4) end @@ -177,8 +185,8 @@ RSpec.describe Gitlab::Memory::Watchdog::Configurator do end context 'when settings are not set via environment variables' do - let(:memory_limit) { 1200.megabytes } - let(:max_strikes) { 5 } + let(:memory_limit_mb) { described_class::DEFAULT_PUMA_WORKER_RSS_LIMIT_MB } + let(:max_strikes) { described_class::DEFAULT_MAX_STRIKES } it_behaves_like 'as monitor configurator' end @@ -193,7 +201,115 @@ RSpec.describe Gitlab::Memory::Watchdog::Configurator do it_behaves_like 'as configurator', Gitlab::Memory::Watchdog::TermProcessHandler, + Gitlab::Memory::Watchdog::SidekiqEventReporter, 'SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', - 3 + described_class::DEFAULT_SIDEKIQ_SLEEP_INTERVAL_S + + context 'when sleep_time_seconds is less than MIN_SIDEKIQ_SLEEP_INTERVAL_S seconds' do + before do + stub_env('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', 0) + end + + it 'configures the correct sleep time' do + configurator.call(configuration) + + expect(configuration.sleep_time_seconds).to eq(described_class::MIN_SIDEKIQ_SLEEP_INTERVAL_S) + end + end + + context 'with monitors' do + let(:soft_limit_bytes) { soft_limit_kb.kilobytes } + let(:hard_limit_bytes) { hard_limit_kb.kilobytes } + + context 'when settings are set via environment variables' do + let(:soft_limit_kb) { 2000001 } + let(:hard_limit_kb) { 300000 } + let(:max_strikes) { 150 } + let(:grace_time) { 300 } + let(:expected_payloads) do + { + rss_memory_soft_limit: { + message: 'rss memory limit exceeded', + memwd_rss_bytes: soft_limit_bytes + 1, + memwd_max_rss_bytes: soft_limit_bytes, + memwd_max_strikes: max_strikes, + memwd_cur_strikes: 1 + }, + rss_memory_hard_limit: { + message: 'rss memory limit exceeded', + memwd_rss_bytes: hard_limit_bytes + 1, + memwd_max_rss_bytes: hard_limit_bytes, + memwd_max_strikes: 0, + memwd_cur_strikes: 1 + } + } + end + + before do + stub_env('SIDEKIQ_MEMORY_KILLER_MAX_RSS', soft_limit_kb) + stub_env('SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS', hard_limit_kb) + stub_env('SIDEKIQ_MEMORY_KILLER_GRACE_TIME', grace_time) + stub_env('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', 2) + allow(Gitlab::Metrics::System).to receive(:memory_usage_rss) + .and_return({ total: soft_limit_bytes + 1 }, { total: hard_limit_bytes + 1 }) + end + + it_behaves_like 'as monitor configurator' + end + + context 'when only SIDEKIQ_MEMORY_KILLER_MAX_RSS is set via environment variable' do + let(:soft_limit_kb) { 2000000 } + let(:max_strikes) do + described_class::DEFAULT_SIDEKIQ_GRACE_TIME_S / described_class::DEFAULT_SIDEKIQ_SLEEP_INTERVAL_S + end + + let(:expected_payloads) do + { + rss_memory_soft_limit: { + message: 'rss memory limit exceeded', + memwd_rss_bytes: soft_limit_bytes + 1, + memwd_max_rss_bytes: soft_limit_bytes, + memwd_max_strikes: max_strikes, + memwd_cur_strikes: 1 + } + } + end + + before do + allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).and_return({ total: soft_limit_bytes + 1 }) + stub_env('SIDEKIQ_MEMORY_KILLER_MAX_RSS', soft_limit_kb) + end + + it_behaves_like 'as monitor configurator' + end + + context 'when only SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS is set via environment variable' do + let(:hard_limit_kb) { 2000000 } + let(:expected_payloads) do + { + rss_memory_hard_limit: { + message: 'rss memory limit exceeded', + memwd_rss_bytes: hard_limit_bytes + 1, + memwd_max_rss_bytes: hard_limit_bytes, + memwd_max_strikes: 0, + memwd_cur_strikes: 1 + } + } + end + + before do + allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).and_return({ total: hard_limit_bytes + 1 }) + stub_env('SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS', hard_limit_kb) + end + + it_behaves_like 'as monitor configurator' + end + + context 'when both SIDEKIQ_MEMORY_KILLER_MAX_RSS and SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS are not set' do + let(:expected_payloads) { {} } + + it_behaves_like 'as monitor configurator' + end + end end end diff --git a/spec/lib/gitlab/memory/watchdog/event_reporter_spec.rb b/spec/lib/gitlab/memory/watchdog/event_reporter_spec.rb new file mode 100644 index 00000000000..f667bc724d2 --- /dev/null +++ b/spec/lib/gitlab/memory/watchdog/event_reporter_spec.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require 'prometheus/client' + +RSpec.describe Gitlab::Memory::Watchdog::EventReporter, feature_category: :application_performance do + let(:logger) { instance_double(::Logger) } + let(:violations_counter) { instance_double(::Prometheus::Client::Counter) } + let(:violations_handled_counter) { instance_double(::Prometheus::Client::Counter) } + let(:reporter) { described_class.new(logger: logger) } + + def stub_prometheus_metrics + allow(Gitlab::Metrics).to receive(:counter) + .with(:gitlab_memwd_violations_total, anything, anything) + .and_return(violations_counter) + allow(Gitlab::Metrics).to receive(:counter) + .with(:gitlab_memwd_violations_handled_total, anything, anything) + .and_return(violations_handled_counter) + + allow(violations_counter).to receive(:increment) + allow(violations_handled_counter).to receive(:increment) + end + + before do + stub_prometheus_metrics + allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).at_least(:once).and_return( + total: 1024 + ) + allow(::Prometheus::PidProvider).to receive(:worker_id).and_return('worker_1') + end + + describe '#logger' do + context 'when logger is not provided' do + let(:reporter) { described_class.new } + + it 'uses default Gitlab::AppLogger' do + expect(reporter.logger).to eq(Gitlab::AppLogger) + end + end + end + + describe '#started' do + it 'logs start message once' do + expect(logger).to receive(:info).once + .with( + pid: Process.pid, + worker_id: 'worker_1', + custom_label: 'dummy_label', + memwd_rss_bytes: 1024, + message: 'started') + + reporter.started(custom_label: 'dummy_label') + end + end + + describe '#stopped' do + subject { reporter.stopped(custom_label: 'dummy_label') } + + it 'logs stop message once' do + expect(logger).to receive(:info).once + .with( + pid: Process.pid, + worker_id: 'worker_1', + custom_label: 'dummy_label', + memwd_rss_bytes: 1024, + message: 'stopped') + + reporter.stopped(custom_label: 'dummy_label') + end + end + + describe '#threshold_violated' do + subject { reporter.threshold_violated(:monitor_name) } + + it 'increments violations counter' do + expect(violations_counter).to receive(:increment).with(reason: :monitor_name) + + subject + end + + it 'does not increment handled violations counter' do + expect(violations_handled_counter).not_to receive(:increment) + + subject + end + + it 'does not log violation' do + expect(logger).not_to receive(:warn) + + subject + end + end + + describe '#strikes_exceeded' do + subject { reporter.strikes_exceeded(:monitor_name, { message: 'dummy_text' }) } + + before do + allow(logger).to receive(:warn) + end + + it 'increments handled violations counter' do + expect(violations_handled_counter).to receive(:increment).with(reason: :monitor_name) + + subject + end + + it 'logs violation' do + expect(logger).to receive(:warn) + .with( + pid: Process.pid, + worker_id: 'worker_1', + memwd_rss_bytes: 1024, + message: 'dummy_text') + + subject + end + end +end diff --git a/spec/lib/gitlab/memory/watchdog/monitor/rss_memory_limit_spec.rb b/spec/lib/gitlab/memory/watchdog/monitor/rss_memory_limit_spec.rb index 9e25cfda782..4780b1eba53 100644 --- a/spec/lib/gitlab/memory/watchdog/monitor/rss_memory_limit_spec.rb +++ b/spec/lib/gitlab/memory/watchdog/monitor/rss_memory_limit_spec.rb @@ -4,25 +4,38 @@ require 'fast_spec_helper' require 'support/shared_examples/lib/gitlab/memory/watchdog/monitor_result_shared_examples' RSpec.describe Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit do - let(:memory_limit) { 2048 } - let(:worker_memory) { 1024 } + let(:max_rss_limit_gauge) { instance_double(::Prometheus::Client::Gauge) } + let(:memory_limit_bytes) { 2_097_152_000 } + let(:worker_memory_bytes) { 1_048_576_000 } subject(:monitor) do - described_class.new(memory_limit: memory_limit) + described_class.new(memory_limit_bytes: memory_limit_bytes) end before do - allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).and_return({ total: worker_memory }) + allow(Gitlab::Metrics).to receive(:gauge) + .with(:gitlab_memwd_max_memory_limit, anything) + .and_return(max_rss_limit_gauge) + allow(max_rss_limit_gauge).to receive(:set) + allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).and_return({ total: worker_memory_bytes }) + end + + describe '#initialize' do + it 'sets the max rss limit gauge' do + expect(max_rss_limit_gauge).to receive(:set).with({}, memory_limit_bytes) + + monitor + end end describe '#call' do context 'when process exceeds threshold' do - let(:worker_memory) { memory_limit + 1 } + let(:worker_memory_bytes) { memory_limit_bytes + 1 } let(:payload) do { message: 'rss memory limit exceeded', - memwd_rss_bytes: worker_memory, - memwd_max_rss_bytes: memory_limit + memwd_rss_bytes: worker_memory_bytes, + memwd_max_rss_bytes: memory_limit_bytes } end @@ -30,7 +43,7 @@ RSpec.describe Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit do end context 'when process does not exceed threshold' do - let(:worker_memory) { memory_limit - 1 } + let(:worker_memory_bytes) { memory_limit_bytes - 1 } let(:payload) { {} } include_examples 'returns Watchdog Monitor result', threshold_violated: false diff --git a/spec/lib/gitlab/memory/watchdog/monitor_state_spec.rb b/spec/lib/gitlab/memory/watchdog/monitor_state_spec.rb index ace1353c6e3..7802e274c53 100644 --- a/spec/lib/gitlab/memory/watchdog/monitor_state_spec.rb +++ b/spec/lib/gitlab/memory/watchdog/monitor_state_spec.rb @@ -7,6 +7,7 @@ RSpec.describe Gitlab::Memory::Watchdog::MonitorState do let(:payload) { { message: 'DummyMessage' } } let(:threshold_violated) { true } let(:monitor) { monitor_class.new(threshold_violated, payload) } + let(:monitor_name) { :dummy_monitor_name } let(:monitor_class) do Struct.new(:threshold_violated, :payload) do def call @@ -19,7 +20,7 @@ RSpec.describe Gitlab::Memory::Watchdog::MonitorState do end end - subject(:monitor_state) { described_class.new(monitor, max_strikes: max_strikes) } + subject(:monitor_state) { described_class.new(monitor, max_strikes: max_strikes, monitor_name: monitor_name) } shared_examples 'returns correct result' do it 'returns correct result', :aggregate_failures do @@ -29,7 +30,7 @@ RSpec.describe Gitlab::Memory::Watchdog::MonitorState do expect(result.strikes_exceeded?).to eq(strikes_exceeded) expect(result.threshold_violated?).to eq(threshold_violated) expect(result.payload).to eq(expected_payload) - expect(result.monitor_name).to eq(:monitor_name) + expect(result.monitor_name).to eq(monitor_name) end end @@ -63,10 +64,4 @@ RSpec.describe Gitlab::Memory::Watchdog::MonitorState do end end end - - describe '#monitor_class' do - subject { monitor_state.monitor_class } - - it { is_expected.to eq(monitor_class) } - end end diff --git a/spec/lib/gitlab/memory/watchdog/sidekiq_event_reporter_spec.rb b/spec/lib/gitlab/memory/watchdog/sidekiq_event_reporter_spec.rb new file mode 100644 index 00000000000..48595c3f172 --- /dev/null +++ b/spec/lib/gitlab/memory/watchdog/sidekiq_event_reporter_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Memory::Watchdog::SidekiqEventReporter, feature_category: :application_performance do + let(:counter) { instance_double(::Prometheus::Client::Counter) } + + before do + allow(Gitlab::Metrics).to receive(:counter).and_return(counter) + allow(counter).to receive(:increment) + end + + describe 'delegations' do + it { is_expected.to delegate_method(:started).to(:event_reporter) } + it { is_expected.to delegate_method(:stopped).to(:event_reporter) } + it { is_expected.to delegate_method(:threshold_violated).to(:event_reporter) } + it { is_expected.to delegate_method(:logger).to(:event_reporter) } + end + + describe '#strikes_exceeded' do + let(:sidekiq_event_reporter) { described_class.new(logger: logger) } + let(:sidekiq_watchdog_running_jobs_counter) { instance_double(::Prometheus::Client::Counter) } + let(:logger) { instance_double(::Logger) } + let(:queue) { 'default' } + let(:jid) { SecureRandom.hex } + let(:running_jobs) { { jid => { worker_class: DummyWorker } } } + let(:sidekiq_daemon_monitor) { instance_double(Gitlab::SidekiqDaemon::Monitor) } + let(:worker) do + Class.new do + def self.name + 'DummyWorker' + end + end + end + + before do + stub_const('DummyWorker', worker) + allow(Gitlab::SidekiqDaemon::Monitor).to receive(:instance).and_return(sidekiq_daemon_monitor) + allow(::Gitlab::Metrics).to receive(:counter) + .with(:sidekiq_watchdog_running_jobs_total, anything) + .and_return(sidekiq_watchdog_running_jobs_counter) + allow(sidekiq_watchdog_running_jobs_counter).to receive(:increment) + allow(logger).to receive(:warn) + + allow(sidekiq_daemon_monitor).to receive(:jobs).and_return(running_jobs) + end + + it 'delegates #strikes_exceeded with correct arguments' do + is_expected.to delegate_method(:strikes_exceeded).to(:event_reporter) + .with_arguments( + :monitor_name, + { + message: 'dummy_text', + running_jobs: [jid: jid, worker_class: 'DummyWorker'] + } + ) + end + + it 'increment running jobs counter' do + expect(sidekiq_watchdog_running_jobs_counter).to receive(:increment) + .with({ worker_class: "DummyWorker" }) + + sidekiq_event_reporter.strikes_exceeded(:monitor_name, { message: 'dummy_text' }) + end + end +end diff --git a/spec/lib/gitlab/memory/watchdog_spec.rb b/spec/lib/gitlab/memory/watchdog_spec.rb index 5d9599d6eab..668ea36d420 100644 --- a/spec/lib/gitlab/memory/watchdog_spec.rb +++ b/spec/lib/gitlab/memory/watchdog_spec.rb @@ -2,15 +2,13 @@ require 'spec_helper' -RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures do +RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures, feature_category: :application_performance do context 'watchdog' do let(:configuration) { instance_double(described_class::Configuration) } let(:handler) { instance_double(described_class::NullHandler) } - let(:logger) { instance_double(::Logger) } + let(:reporter) { instance_double(described_class::EventReporter) } let(:sleep_time_seconds) { 60 } let(:threshold_violated) { false } - let(:violations_counter) { instance_double(::Prometheus::Client::Counter) } - let(:violations_handled_counter) { instance_double(::Prometheus::Client::Counter) } let(:watchdog_iterations) { 1 } let(:name) { :monitor_name } let(:payload) { { message: 'dummy_text' } } @@ -37,18 +35,6 @@ RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures do end end - def stub_prometheus_metrics - allow(Gitlab::Metrics).to receive(:counter) - .with(:gitlab_memwd_violations_total, anything, anything) - .and_return(violations_counter) - allow(Gitlab::Metrics).to receive(:counter) - .with(:gitlab_memwd_violations_handled_total, anything, anything) - .and_return(violations_handled_counter) - - allow(violations_counter).to receive(:increment) - allow(violations_handled_counter).to receive(:increment) - end - describe '#initialize' do it 'initialize new configuration' do expect(described_class::Configuration).to receive(:new) @@ -59,33 +45,25 @@ RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures do describe '#call' do before do - stub_prometheus_metrics - allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).at_least(:once).and_return( - total: 1024 - ) - allow(::Prometheus::PidProvider).to receive(:worker_id).and_return('worker_1') - watchdog.configure do |config| config.handler = handler - config.logger = logger + config.event_reporter = reporter config.sleep_time_seconds = sleep_time_seconds - config.monitors.push monitor_class, threshold_violated, payload, max_strikes: max_strikes end allow(handler).to receive(:call).and_return(true) - allow(logger).to receive(:info) - allow(logger).to receive(:warn) + allow(reporter).to receive(:started) + allow(reporter).to receive(:stopped) + allow(reporter).to receive(:threshold_violated) + allow(reporter).to receive(:strikes_exceeded) end - it 'logs start message once' do - expect(logger).to receive(:info).once + it 'reports started event once' do + expect(reporter).to receive(:started).once .with( - pid: Process.pid, - worker_id: 'worker_1', memwd_handler_class: handler.class.name, - memwd_sleep_time_s: sleep_time_seconds, - memwd_rss_bytes: 1024, - message: 'started') + memwd_sleep_time_s: sleep_time_seconds + ) watchdog.call end @@ -96,55 +74,50 @@ RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures do watchdog.call end - context 'when gitlab_memory_watchdog ops toggle is off' do - before do - stub_feature_flags(gitlab_memory_watchdog: false) - end - - it 'does not trigger any monitor' do - expect(configuration).not_to receive(:monitors) - end - end - - context 'when process does not exceed threshold' do - it 'does not increment violations counters' do - expect(violations_counter).not_to receive(:increment) - expect(violations_handled_counter).not_to receive(:increment) - - watchdog.call - end - - it 'does not log violation' do - expect(logger).not_to receive(:warn) - - watchdog.call - end - - it 'does not execute handler' do - expect(handler).not_to receive(:call) + context 'when no monitors are configured' do + it 'reports stopped event once with correct reason' do + expect(reporter).to receive(:stopped).once + .with( + memwd_handler_class: handler.class.name, + memwd_sleep_time_s: sleep_time_seconds, + memwd_reason: 'monitors are not configured' + ) watchdog.call end end - context 'when process exceeds threshold' do - let(:threshold_violated) { true } + context 'when monitors are configured' do + before do + watchdog.configure do |config| + config.monitors.push monitor_class, threshold_violated, payload, max_strikes: max_strikes + end + end - it 'increments violations counter' do - expect(violations_counter).to receive(:increment).with(reason: name) + it 'reports stopped event once' do + expect(reporter).to receive(:stopped).once + .with( + memwd_handler_class: handler.class.name, + memwd_sleep_time_s: sleep_time_seconds + ) watchdog.call end - context 'when process does not exceed the allowed number of strikes' do - it 'does not increment handled violations counter' do - expect(violations_handled_counter).not_to receive(:increment) + context 'when gitlab_memory_watchdog ops toggle is off' do + before do + stub_feature_flags(gitlab_memory_watchdog: false) + end - watchdog.call + it 'does not trigger any monitor' do + expect(configuration).not_to receive(:monitors) end + end - it 'does not log violation' do - expect(logger).not_to receive(:warn) + context 'when process does not exceed threshold' do + it 'does not report violations event' do + expect(reporter).not_to receive(:threshold_violated) + expect(reporter).not_to receive(:strikes_exceeded) watchdog.call end @@ -156,81 +129,94 @@ RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures do end end - context 'when monitor exceeds the allowed number of strikes' do - let(:max_strikes) { 0 } + context 'when process exceeds threshold' do + let(:threshold_violated) { true } - it 'increments handled violations counter' do - expect(violations_handled_counter).to receive(:increment).with(reason: name) + it 'reports threshold violated event' do + expect(reporter).to receive(:threshold_violated).with(name) watchdog.call end - it 'logs violation' do - expect(logger).to receive(:warn) - .with( - pid: Process.pid, - worker_id: 'worker_1', - memwd_handler_class: handler.class.name, - memwd_sleep_time_s: sleep_time_seconds, - memwd_rss_bytes: 1024, - memwd_cur_strikes: 1, - memwd_max_strikes: max_strikes, - message: 'dummy_text') + context 'when process does not exceed the allowed number of strikes' do + it 'does not report strikes exceeded event' do + expect(reporter).not_to receive(:strikes_exceeded) - watchdog.call - end + watchdog.call + end - it 'executes handler' do - expect(handler).to receive(:call) + it 'does not execute handler' do + expect(handler).not_to receive(:call) - watchdog.call + watchdog.call + end end - context 'when enforce_memory_watchdog ops toggle is off' do - before do - stub_feature_flags(enforce_memory_watchdog: false) + context 'when monitor exceeds the allowed number of strikes' do + let(:max_strikes) { 0 } + + it 'reports strikes exceeded event' do + expect(reporter).to receive(:strikes_exceeded) + .with( + name, + memwd_handler_class: handler.class.name, + memwd_sleep_time_s: sleep_time_seconds, + memwd_cur_strikes: 1, + memwd_max_strikes: max_strikes, + message: "dummy_text" + ) + + watchdog.call end - it 'always uses the NullHandler' do - expect(handler).not_to receive(:call) - expect(described_class::NullHandler.instance).to receive(:call).and_return(true) + it 'executes handler and stops the watchdog' do + expect(handler).to receive(:call).and_return(true) + expect(reporter).to receive(:stopped).once + .with( + memwd_handler_class: handler.class.name, + memwd_sleep_time_s: sleep_time_seconds, + memwd_reason: 'successfully handled' + ) watchdog.call end - end - context 'when multiple monitors exceeds allowed number of strikes' do - before do - watchdog.configure do |config| - config.handler = handler - config.logger = logger - config.sleep_time_seconds = sleep_time_seconds - config.monitors.push monitor_class, threshold_violated, payload, max_strikes: max_strikes - config.monitors.push monitor_class, threshold_violated, payload, max_strikes: max_strikes + it 'schedules a heap dump' do + expect(Gitlab::Memory::Reports::HeapDump).to receive(:enqueue!) + + watchdog.call + end + + context 'when enforce_memory_watchdog ops toggle is off' do + before do + stub_feature_flags(enforce_memory_watchdog: false) + end + + it 'always uses the NullHandler' do + expect(handler).not_to receive(:call) + expect(described_class::NullHandler.instance).to receive(:call).and_return(true) + + watchdog.call end end - it 'only calls the handler once' do - expect(handler).to receive(:call).once.and_return(true) + context 'when multiple monitors exceeds allowed number of strikes' do + before do + watchdog.configure do |config| + config.monitors.push monitor_class, threshold_violated, payload, max_strikes: max_strikes + config.monitors.push monitor_class, threshold_violated, payload, max_strikes: max_strikes + end + end + + it 'only calls the handler once' do + expect(handler).to receive(:call).once.and_return(true) - watchdog.call + watchdog.call + end end end end end - - it 'logs stop message once' do - expect(logger).to receive(:info).once - .with( - pid: Process.pid, - worker_id: 'worker_1', - memwd_handler_class: handler.class.name, - memwd_sleep_time_s: sleep_time_seconds, - memwd_rss_bytes: 1024, - message: 'stopped') - - watchdog.call - end end describe '#configure' do @@ -255,6 +241,10 @@ RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures do subject(:handler) { described_class::TermProcessHandler.new(42) } describe '#call' do + before do + allow(Process).to receive(:kill) + end + it 'sends SIGTERM to the current process' do expect(Process).to receive(:kill).with(:TERM, 42) @@ -274,11 +264,12 @@ RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures do before do stub_const('::Puma::Cluster::WorkerHandle', puma_worker_handle_class) + allow(puma_worker_handle_class).to receive(:new).and_return(puma_worker_handle) + allow(puma_worker_handle).to receive(:term) end describe '#call' do it 'invokes orderly termination via Puma API' do - expect(puma_worker_handle_class).to receive(:new).and_return(puma_worker_handle) expect(puma_worker_handle).to receive(:term) expect(handler.call).to be(true) diff --git a/spec/lib/gitlab/merge_requests/commit_message_generator_spec.rb b/spec/lib/gitlab/merge_requests/message_generator_spec.rb index ad528dca81a..59aaffc4377 100644 --- a/spec/lib/gitlab/merge_requests/commit_message_generator_spec.rb +++ b/spec/lib/gitlab/merge_requests/message_generator_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::MergeRequests::CommitMessageGenerator do +RSpec.describe Gitlab::MergeRequests::MessageGenerator, feature_category: :code_review do let(:merge_commit_template) { nil } let(:squash_commit_template) { nil } let(:project) do @@ -59,7 +59,14 @@ RSpec.describe Gitlab::MergeRequests::CommitMessageGenerator do context 'when project has commit template with only the title' do let(:merge_request) do - double(:merge_request, title: 'Fixes', target_project: project, to_reference: '!123', metrics: nil, merge_user: nil) + double( + :merge_request, + title: 'Fixes', + target_project: project, + to_reference: '!123', + metrics: nil, + merge_user: nil + ) end let(message_template_name) { '%{title}' } @@ -214,7 +221,7 @@ RSpec.describe Gitlab::MergeRequests::CommitMessageGenerator do context 'when project has template with CRLF newlines' do let(message_template_name) do - "Merge branch '%{source_branch}' into '%{target_branch}'\r\n\r\n%{title}\r\n\r\n%{description}\r\n\r\nSee merge request %{reference}" + "Merge branch '%{source_branch}' into '%{target_branch}'\r\n\r\n%{title}\r\n\r\n%{description}\r\n\r\nSee merge request %{reference}" # rubocop: disable Layout/LineLength end it 'converts it to LF newlines' do @@ -289,6 +296,93 @@ RSpec.describe Gitlab::MergeRequests::CommitMessageGenerator do end end + context 'when project has merge commit template with reviewers' do + let(:user1) { create(:user) } + let(:user2) { create(:user) } + let(message_template_name) { <<~MSG.rstrip } + Merge branch '%{source_branch}' into '%{target_branch}' + + %{reviewed_by} + MSG + + context 'and mr has no reviewers' do + before do + merge_request.reviews = [] + end + + it 'removes variable and blank line' do + expect(result_message).to eq <<~MSG.rstrip + Merge branch 'feature' into 'master' + MSG + end + + context 'when there is blank line after reviewed_by' do + let(message_template_name) { <<~MSG.rstrip } + Merge branch '%{source_branch}' into '%{target_branch}' + + %{reviewed_by} + + Type: merge + MSG + + it 'removes blank line before it' do + expect(result_message).to eq <<~MSG.rstrip + Merge branch 'feature' into 'master' + + Type: merge + MSG + end + end + + context 'when there is no blank line after reviewed_by' do + let(message_template_name) { <<~MSG.rstrip } + Merge branch '%{source_branch}' into '%{target_branch}' + + %{reviewed_by} + Type: merge + MSG + + it 'does not remove blank line before it' do + expect(result_message).to eq <<~MSG.rstrip + Merge branch 'feature' into 'master' + + Type: merge + MSG + end + end + end + + context 'and mr has one reviewer' do + before do + merge_request.reviews.create!(project: merge_request.project, author: user1) + end + + it 'returns user name and email' do + expect(result_message).to eq <<~MSG.rstrip + Merge branch 'feature' into 'master' + + Reviewed-by: #{user1.name} <#{user1.email}> + MSG + end + end + + context 'and mr has multiple reviewers' do + before do + merge_request.reviews.create!(project: merge_request.project, author: user1) + merge_request.reviews.create!(project: merge_request.project, author: user2) + end + + it 'returns users names and emails' do + expect(result_message).to eq <<~MSG.rstrip + Merge branch 'feature' into 'master' + + Reviewed-by: #{user1.name} <#{user1.email}> + Reviewed-by: #{user2.name} <#{user2.email}> + MSG + end + end + end + context 'when project has merge commit template with approvers' do let(:user1) { create(:user) } let(:user2) { create(:user) } @@ -547,6 +641,7 @@ RSpec.describe Gitlab::MergeRequests::CommitMessageGenerator do first_commit:%{first_commit} first_multiline_commit:%{first_multiline_commit} url:%{url} + reviewed_by:%{reviewed_by} approved_by:%{approved_by} merged_by:%{merged_by} co_authored_by:%{co_authored_by} @@ -568,6 +663,7 @@ RSpec.describe Gitlab::MergeRequests::CommitMessageGenerator do Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> url:#{Gitlab::UrlBuilder.build(merge_request)} + reviewed_by: approved_by: merged_by:#{current_user.name} <#{current_user.commit_email_or_default}> co_authored_by:Co-authored-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> @@ -628,8 +724,8 @@ RSpec.describe Gitlab::MergeRequests::CommitMessageGenerator do end end - describe '#merge_message' do - let(:result_message) { subject.merge_message } + describe '#merge_commit_message' do + let(:result_message) { subject.merge_commit_message } it_behaves_like 'commit message with template', :merge_commit_template @@ -660,8 +756,8 @@ RSpec.describe Gitlab::MergeRequests::CommitMessageGenerator do end end - describe '#squash_message' do - let(:result_message) { subject.squash_message } + describe '#squash_commit_message' do + let(:result_message) { subject.squash_commit_message } it_behaves_like 'commit message with template', :squash_commit_template @@ -691,4 +787,95 @@ RSpec.describe Gitlab::MergeRequests::CommitMessageGenerator do end end end + + describe '#new_mr_description' do + let(:merge_request) do + build( + :merge_request, + source_project: project, + target_project: project, + target_branch: 'master', + source_branch: source_branch, + author: author, + description: merge_request_description, + title: merge_request_title + ) + end + + let(:result_message) { subject.new_mr_description } + + before do + compare = CompareService.new( + project, + merge_request.source_branch + ).execute( + project, + merge_request.target_branch + ) + + merge_request.compare_commits = compare.commits + merge_request.compare = compare + end + + context 'when project has template with all variables' do + let(:merge_request_description) { <<~MSG.rstrip } + source_branch:%{source_branch} + target_branch:%{target_branch} + title:%{title} + issues:%{issues} + description:%{description} + first_commit:%{first_commit} + first_multiline_commit:%{first_multiline_commit} + url:%{url} + approved_by:%{approved_by} + merged_by:%{merged_by} + co_authored_by:%{co_authored_by} + all_commits:%{all_commits} + MSG + + it 'renders only variables specific to a new non-persisted merge request' do + expect(result_message).to eq <<~MSG.rstrip + source_branch:feature + target_branch:master + title: + issues: + description: + first_commit:Feature added + + Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> + first_multiline_commit:Feature added + + Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> + url: + approved_by: + merged_by: + co_authored_by:Co-authored-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> + all_commits:* Feature added + + Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> + MSG + end + + context 'when no first commit exists' do + let(:source_branch) { 'master' } + + it 'does not populate any commit-related variables' do + expect(result_message).to eq <<~MSG.rstrip + source_branch:master + target_branch:master + title: + issues: + description: + first_commit: + first_multiline_commit:Bugfix + url: + approved_by: + merged_by: + co_authored_by: + all_commits: + MSG + end + end + end + end end diff --git a/spec/lib/gitlab/metrics/dashboard/validator_spec.rb b/spec/lib/gitlab/metrics/dashboard/validator_spec.rb index aaa9daf8fee..fb55b736354 100644 --- a/spec/lib/gitlab/metrics/dashboard/validator_spec.rb +++ b/spec/lib/gitlab/metrics/dashboard/validator_spec.rb @@ -143,56 +143,4 @@ RSpec.describe Gitlab::Metrics::Dashboard::Validator do end end end - - describe '#errors' do - context 'valid dashboard schema' do - it 'returns no errors' do - expect(described_class.errors(valid_dashboard)).to eq [] - end - - context 'with duplicate metric_ids' do - it 'returns errors' do - expect(described_class.errors(duplicate_id_dashboard)).to eq [Gitlab::Metrics::Dashboard::Validator::Errors::DuplicateMetricIds.new] - end - end - - context 'with dashboard_path and project' do - subject { described_class.errors(valid_dashboard, dashboard_path: 'test/path.yml', project: project) } - - context 'with no conflicting metric identifiers in db' do - it { is_expected.to eq [] } - end - - context 'with metric identifier present in current dashboard' do - before do - create(:prometheus_metric, - identifier: 'metric_a1', - dashboard_path: 'test/path.yml', - project: project - ) - end - - it { is_expected.to eq [] } - end - - context 'with metric identifier present in another dashboard' do - before do - create(:prometheus_metric, - identifier: 'metric_a1', - dashboard_path: 'some/other/dashboard/path.yml', - project: project - ) - end - - it { is_expected.to eq [Gitlab::Metrics::Dashboard::Validator::Errors::DuplicateMetricIds.new] } - end - end - end - - context 'invalid dashboard schema' do - it 'returns collection of validation errors' do - expect(described_class.errors(invalid_dashboard)).to all be_kind_of(Gitlab::Metrics::Dashboard::Validator::Errors::SchemaValidationError) - end - end - end end diff --git a/spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb b/spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb index fa50adb4e4f..6673cc50d67 100644 --- a/spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb +++ b/spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Metrics::Exporter::BaseExporter do +RSpec.describe Gitlab::Metrics::Exporter::BaseExporter, feature_category: :application_performance do let(:settings) { double('settings') } let(:log_enabled) { false } let(:exporter) { described_class.new(settings, log_enabled: log_enabled, log_file: 'test_exporter.log') } diff --git a/spec/lib/gitlab/metrics/global_search_slis_spec.rb b/spec/lib/gitlab/metrics/global_search_slis_spec.rb index c10d83664ea..1aa2c4398a7 100644 --- a/spec/lib/gitlab/metrics/global_search_slis_spec.rb +++ b/spec/lib/gitlab/metrics/global_search_slis_spec.rb @@ -5,12 +5,6 @@ require 'spec_helper' RSpec.describe Gitlab::Metrics::GlobalSearchSlis do using RSpec::Parameterized::TableSyntax - let(:error_rate_feature_flag_enabled) { true } - - before do - stub_feature_flags(global_search_error_rate_sli: error_rate_feature_flag_enabled) - end - describe '#initialize_slis!' do it 'initializes Apdex SLIs for global_search' do expect(Gitlab::Metrics::Sli::Apdex).to receive(:initialize_sli).with( @@ -21,27 +15,13 @@ RSpec.describe Gitlab::Metrics::GlobalSearchSlis do described_class.initialize_slis! end - context 'when global_search_error_rate_sli feature flag is enabled' do - let(:error_rate_feature_flag_enabled) { true } - - it 'initializes ErrorRate SLIs for global_search' do - expect(Gitlab::Metrics::Sli::ErrorRate).to receive(:initialize_sli).with( - :global_search, - a_kind_of(Array) - ) - - described_class.initialize_slis! - end - end - - context 'when global_search_error_rate_sli feature flag is disabled' do - let(:error_rate_feature_flag_enabled) { false } - - it 'does not initialize the ErrorRate SLIs for global_search' do - expect(Gitlab::Metrics::Sli::ErrorRate).not_to receive(:initialize_sli) + it 'initializes ErrorRate SLIs for global_search' do + expect(Gitlab::Metrics::Sli::ErrorRate).to receive(:initialize_sli).with( + :global_search, + a_kind_of(Array) + ) - described_class.initialize_slis! - end + described_class.initialize_slis! end end @@ -105,34 +85,15 @@ RSpec.describe Gitlab::Metrics::GlobalSearchSlis do end describe '#record_error_rate' do - context 'when global_search_error_rate_sli feature flag is enabled' do - let(:error_rate_feature_flag_enabled) { true } - - it 'calls increment on the error rate SLI' do - expect(Gitlab::Metrics::Sli::ErrorRate[:global_search]).to receive(:increment) - - described_class.record_error_rate( - error: true, - search_type: 'basic', - search_level: 'global', - search_scope: 'issues' - ) - end - end - - context 'when global_search_error_rate_sli feature flag is disabled' do - let(:error_rate_feature_flag_enabled) { false } - - it 'does not call increment on the error rate SLI' do - expect(Gitlab::Metrics::Sli::ErrorRate[:global_search]).not_to receive(:increment) - - described_class.record_error_rate( - error: true, - search_type: 'basic', - search_level: 'global', - search_scope: 'issues' - ) - end + it 'calls increment on the error rate SLI' do + expect(Gitlab::Metrics::Sli::ErrorRate[:global_search]).to receive(:increment) + + described_class.record_error_rate( + error: true, + search_type: 'basic', + search_level: 'global', + search_scope: 'issues' + ) end end end diff --git a/spec/lib/gitlab/metrics/rails_slis_spec.rb b/spec/lib/gitlab/metrics/rails_slis_spec.rb index b30eb57101f..9da102fb8b8 100644 --- a/spec/lib/gitlab/metrics/rails_slis_spec.rb +++ b/spec/lib/gitlab/metrics/rails_slis_spec.rb @@ -14,8 +14,8 @@ RSpec.describe Gitlab::Metrics::RailsSlis do end describe '.initialize_request_slis!' do - it "initializes the SLI for all possible endpoints if they weren't", :aggregate_failures do - possible_labels = [ + let(:possible_labels) do + [ { endpoint_id: "GET /api/:version/version", feature_category: :not_owned, @@ -27,17 +27,32 @@ RSpec.describe Gitlab::Metrics::RailsSlis do request_urgency: :default } ] + end - possible_graphql_labels = ['graphql:foo', 'graphql:bar', 'graphql:unknown'].map do |endpoint_id| + let(:possible_graphql_labels) do + ['graphql:foo', 'graphql:bar', 'graphql:unknown'].map do |endpoint_id| { endpoint_id: endpoint_id, feature_category: nil, query_urgency: ::Gitlab::EndpointAttributes::DEFAULT_URGENCY.name } end + end + + it "initializes the SLI for all possible endpoints if they weren't", :aggregate_failures do + expect(Gitlab::Metrics::Sli::Apdex).to receive(:initialize_sli).with(:rails_request, array_including(*possible_labels)).and_call_original + expect(Gitlab::Metrics::Sli::Apdex).to receive(:initialize_sli).with(:graphql_query, array_including(*possible_graphql_labels)).and_call_original + expect(Gitlab::Metrics::Sli::ErrorRate).to receive(:initialize_sli).with(:rails_request, array_including(*possible_labels)).and_call_original + + described_class.initialize_request_slis! + end + + it "initializes the SLI for all possible endpoints if they weren't given error rate feature flag is disabled", :aggregate_failures do + stub_feature_flags(gitlab_metrics_error_rate_sli: false) expect(Gitlab::Metrics::Sli::Apdex).to receive(:initialize_sli).with(:rails_request, array_including(*possible_labels)).and_call_original expect(Gitlab::Metrics::Sli::Apdex).to receive(:initialize_sli).with(:graphql_query, array_including(*possible_graphql_labels)).and_call_original + expect(Gitlab::Metrics::Sli::ErrorRate).not_to receive(:initialize_sli) described_class.initialize_request_slis! end @@ -51,6 +66,14 @@ RSpec.describe Gitlab::Metrics::RailsSlis do end end + describe '.request_error' do + it 'returns the initialized request error rate SLI object' do + described_class.initialize_request_slis! + + expect(described_class.request_error_rate).to be_initialized + end + end + describe '.graphql_query_apdex' do it 'returns the initialized request apdex SLI object' do described_class.initialize_request_slis! diff --git a/spec/lib/gitlab/metrics/requests_rack_middleware_spec.rb b/spec/lib/gitlab/metrics/requests_rack_middleware_spec.rb index ed78548ef62..61c690b85e9 100644 --- a/spec/lib/gitlab/metrics/requests_rack_middleware_spec.rb +++ b/spec/lib/gitlab/metrics/requests_rack_middleware_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do +RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures, feature_category: :error_budgets do let(:app) { double('app') } subject { described_class.new(app) } @@ -38,6 +38,29 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do expect(described_class).to receive_message_chain(:http_request_duration_seconds, :observe).with({ method: 'get' }, a_positive_execution_time) expect(Gitlab::Metrics::RailsSlis.request_apdex).to receive(:increment) .with(labels: { feature_category: 'unknown', endpoint_id: 'unknown', request_urgency: :default }, success: true) + expect(Gitlab::Metrics::RailsSlis.request_error_rate).to receive(:increment) + .with(labels: { feature_category: 'unknown', endpoint_id: 'unknown', request_urgency: :default }, error: false) + + subject.call(env) + end + + it 'guarantees SLI metrics are incremented with all the required labels' do + described_class.initialize_metrics + + expect(Gitlab::Metrics::RailsSlis.request_apdex).to receive(:increment).and_call_original + expect(Gitlab::Metrics::RailsSlis.request_error_rate).to receive(:increment).and_call_original + + subject.call(env) + end + + it 'does not track error rate when feature flag is disabled' do + stub_feature_flags(gitlab_metrics_error_rate_sli: false) + + expect(described_class).to receive_message_chain(:http_requests_total, :increment).with(method: 'get', status: '200', feature_category: 'unknown') + expect(described_class).to receive_message_chain(:http_request_duration_seconds, :observe).with({ method: 'get' }, a_positive_execution_time) + expect(Gitlab::Metrics::RailsSlis.request_apdex).to receive(:increment) + .with(labels: { feature_category: 'unknown', endpoint_id: 'unknown', request_urgency: :default }, success: true) + expect(Gitlab::Metrics::RailsSlis.request_error_rate).not_to receive(:increment) subject.call(env) end @@ -84,10 +107,23 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do context '@app.call returns an error code' do let(:status) { '500' } - it 'tracks count but not duration or apdex' do + it 'tracks count and error rate but not duration and apdex' do + expect(described_class).to receive_message_chain(:http_requests_total, :increment).with(method: 'get', status: '500', feature_category: 'unknown') + expect(described_class).not_to receive(:http_request_duration_seconds) + expect(Gitlab::Metrics::RailsSlis).not_to receive(:request_apdex) + expect(Gitlab::Metrics::RailsSlis.request_error_rate).to receive(:increment) + .with(labels: { feature_category: 'unknown', endpoint_id: 'unknown', request_urgency: :default }, error: true) + + subject.call(env) + end + + it 'does not track error rate when feature flag is disabled' do + stub_feature_flags(gitlab_metrics_error_rate_sli: false) + expect(described_class).to receive_message_chain(:http_requests_total, :increment).with(method: 'get', status: '500', feature_category: 'unknown') expect(described_class).not_to receive(:http_request_duration_seconds) expect(Gitlab::Metrics::RailsSlis).not_to receive(:request_apdex) + expect(Gitlab::Metrics::RailsSlis.request_error_rate).not_to receive(:increment) subject.call(env) end @@ -108,6 +144,7 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do expect(described_class).to receive_message_chain(:http_requests_total, :increment).with(method: 'get', status: 'undefined', feature_category: 'unknown') expect(described_class.http_request_duration_seconds).not_to receive(:observe) expect(Gitlab::Metrics::RailsSlis).not_to receive(:request_apdex) + expect(Gitlab::Metrics::RailsSlis.request_error_rate).not_to receive(:increment) expect { subject.call(env) }.to raise_error(StandardError) end @@ -124,6 +161,8 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do expect(described_class).not_to receive(:http_health_requests_total) expect(Gitlab::Metrics::RailsSlis.request_apdex) .to receive(:increment).with(labels: { feature_category: 'team_planning', endpoint_id: 'IssuesController#show', request_urgency: :default }, success: true) + expect(Gitlab::Metrics::RailsSlis.request_error_rate).to receive(:increment) + .with(labels: { feature_category: 'team_planning', endpoint_id: 'IssuesController#show', request_urgency: :default }, error: false) subject.call(env) end @@ -134,6 +173,7 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do expect(described_class).to receive_message_chain(:http_health_requests_total, :increment).with(method: 'get', status: '200') expect(described_class).not_to receive(:http_requests_total) expect(Gitlab::Metrics::RailsSlis).not_to receive(:request_apdex) + expect(Gitlab::Metrics::RailsSlis).not_to receive(:request_error_rate) subject.call(env) end @@ -147,8 +187,9 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do it 'adds the feature category to the labels for http_requests_total' do expect(described_class).to receive_message_chain(:http_requests_total, :increment).with(method: 'get', status: 'undefined', feature_category: 'team_planning') - expect(Gitlab::Metrics::RailsSlis).not_to receive(:request_apdex) + expect(Gitlab::Metrics::RailsSlis).not_to receive(:request_apdex) + expect(Gitlab::Metrics::RailsSlis).not_to receive(:request_error_rate) expect { subject.call(env) }.to raise_error(StandardError) end end @@ -159,6 +200,8 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do expect(described_class).not_to receive(:http_health_requests_total) expect(Gitlab::Metrics::RailsSlis.request_apdex).to receive(:increment) .with(labels: { feature_category: 'unknown', endpoint_id: 'unknown', request_urgency: :default }, success: true) + expect(Gitlab::Metrics::RailsSlis.request_error_rate).to receive(:increment) + .with(labels: { feature_category: 'unknown', endpoint_id: 'unknown', request_urgency: :default }, error: false) subject.call(env) end @@ -214,6 +257,15 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do }, success: success ) + expect(Gitlab::Metrics::RailsSlis.request_error_rate).to receive(:increment).with( + labels: { + feature_category: 'hello_world', + endpoint_id: 'GET /projects/:id/archive', + request_urgency: request_urgency_name + }, + error: false + ) + subject.call(env) end end @@ -247,6 +299,15 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do }, success: success ) + expect(Gitlab::Metrics::RailsSlis.request_error_rate).to receive(:increment).with( + labels: { + feature_category: 'hello_world', + endpoint_id: 'AnonymousController#index', + request_urgency: request_urgency_name + }, + error: false + ) + subject.call(env) end end @@ -273,6 +334,14 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do }, success: true ) + expect(Gitlab::Metrics::RailsSlis.request_error_rate).to receive(:increment).with( + labels: { + feature_category: 'unknown', + endpoint_id: 'unknown', + request_urgency: :default + }, + error: false + ) subject.call(env) allow(Gitlab::Metrics::System).to receive(:monotonic_time).and_return(100, 101) @@ -284,6 +353,14 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do }, success: false ) + expect(Gitlab::Metrics::RailsSlis.request_error_rate).to receive(:increment).with( + labels: { + feature_category: 'unknown', + endpoint_id: 'unknown', + request_urgency: :default + }, + error: false + ) subject.call(env) end end @@ -307,6 +384,14 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do }, success: true ) + expect(Gitlab::Metrics::RailsSlis.request_error_rate).to receive(:increment).with( + labels: { + feature_category: 'unknown', + endpoint_id: 'unknown', + request_urgency: :default + }, + error: false + ) subject.call(env) allow(Gitlab::Metrics::System).to receive(:monotonic_time).and_return(100, 101) @@ -318,6 +403,14 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do }, success: false ) + expect(Gitlab::Metrics::RailsSlis.request_error_rate).to receive(:increment).with( + labels: { + feature_category: 'unknown', + endpoint_id: 'unknown', + request_urgency: :default + }, + error: false + ) subject.call(env) end end @@ -337,6 +430,14 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do }, success: true ) + expect(Gitlab::Metrics::RailsSlis.request_error_rate).to receive(:increment).with( + labels: { + feature_category: 'unknown', + endpoint_id: 'unknown', + request_urgency: :default + }, + error: false + ) subject.call(env) allow(Gitlab::Metrics::System).to receive(:monotonic_time).and_return(100, 101) @@ -348,6 +449,14 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do }, success: false ) + expect(Gitlab::Metrics::RailsSlis.request_error_rate).to receive(:increment).with( + labels: { + feature_category: 'unknown', + endpoint_id: 'unknown', + request_urgency: :default + }, + error: false + ) subject.call(env) end end diff --git a/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb b/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb index 005c1ae2d0a..4569f3134ae 100644 --- a/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb @@ -7,7 +7,8 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do let(:env) { {} } let(:subscriber) { described_class.new } - let(:connection) { ActiveRecord::Base.retrieve_connection } + + let(:connection) { Gitlab::Database.database_base_models[:main].retrieve_connection } let(:db_config_name) { ::Gitlab::Database.db_config_name(connection) } describe '.load_balancing_metric_counter_keys' do @@ -155,7 +156,9 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do end it 'captures the metrics for web only' do - expect(web_transaction).to receive(:observe).with(:gitlab_database_transaction_seconds, 0.23, { db_config_name: db_config_name }) + expect(web_transaction).to receive(:observe).with( + :gitlab_database_transaction_seconds, 0.23, { db_config_name: db_config_name } + ) expect(background_transaction).not_to receive(:observe) expect(background_transaction).not_to receive(:increment) @@ -175,7 +178,9 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do end it 'captures the metrics for web only' do - expect(web_transaction).to receive(:observe).with(:gitlab_database_transaction_seconds, 0.23, { db_config_name: db_config_name }) + expect(web_transaction).to receive(:observe).with( + :gitlab_database_transaction_seconds, 0.23, { db_config_name: db_config_name } + ) expect(background_transaction).not_to receive(:observe) expect(background_transaction).not_to receive(:increment) @@ -195,7 +200,9 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do end it 'captures the metrics for web only' do - expect(background_transaction).to receive(:observe).with(:gitlab_database_transaction_seconds, 0.23, { db_config_name: db_config_name }) + expect(background_transaction).to receive(:observe).with( + :gitlab_database_transaction_seconds, 0.23, { db_config_name: db_config_name } + ) expect(web_transaction).not_to receive(:observe) expect(web_transaction).not_to receive(:increment) diff --git a/spec/lib/gitlab/metrics/subscribers/ldap_spec.rb b/spec/lib/gitlab/metrics/subscribers/ldap_spec.rb new file mode 100644 index 00000000000..b81000be62a --- /dev/null +++ b/spec/lib/gitlab/metrics/subscribers/ldap_spec.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe Gitlab::Metrics::Subscribers::Ldap, :request_store, feature_category: :logging do + let(:transaction) { Gitlab::Metrics::WebTransaction.new({}) } + let(:subscriber) { described_class.new } + + let(:attributes) do + [ + :altServer, :namingContexts, :supportedCapabilities, :supportedControl, + :supportedExtension, :supportedFeatures, :supportedLdapVersion, :supportedSASLMechanisms + ] + end + + let(:event_1) do + instance_double( + ActiveSupport::Notifications::Event, + name: "open.net_ldap", + payload: { + ignore_server_caps: true, + base: "", + scope: 0, + attributes: attributes, + result: nil + }, + time: Time.current, + duration: 0.321 + ) + end + + let(:event_2) do + instance_double( + ActiveSupport::Notifications::Event, + name: "search.net_ldap", + payload: { + ignore_server_caps: true, + base: "", + scope: 0, + attributes: attributes, + result: nil + }, + time: Time.current, + duration: 0.12 + ) + end + + let(:event_3) do + instance_double( + ActiveSupport::Notifications::Event, + name: "search.net_ldap", + payload: { + ignore_server_caps: true, + base: "", + scope: 0, + attributes: attributes, + result: nil + }, + time: Time.current, + duration: 5.3 + ) + end + + around do |example| + freeze_time { example.run } + end + + describe ".payload" do + context "when SafeRequestStore is empty" do + it "returns an empty array" do + expect(described_class.payload).to eql(net_ldap_count: 0, net_ldap_duration_s: 0.0) + end + end + + context "when LDAP recorded some values" do + before do + Gitlab::SafeRequestStore[:net_ldap_count] = 7 + Gitlab::SafeRequestStore[:net_ldap_duration_s] = 1.2 + end + + it "returns the populated payload" do + expect(described_class.payload).to eql(net_ldap_count: 7, net_ldap_duration_s: 1.2) + end + end + end + + describe "#observe_event" do + before do + allow(subscriber).to receive(:current_transaction).and_return(transaction) + end + + it "tracks LDAP request count" do + expect(transaction).to receive(:increment) + .with(:gitlab_net_ldap_total, 1, { name: "open" }) + expect(transaction).to receive(:increment) + .with(:gitlab_net_ldap_total, 1, { name: "search" }) + + subscriber.observe_event(event_1) + subscriber.observe_event(event_2) + end + + it "tracks LDAP request duration" do + expect(transaction).to receive(:observe) + .with(:gitlab_net_ldap_duration_seconds, 0.321, { name: "open" }) + expect(transaction).to receive(:observe) + .with(:gitlab_net_ldap_duration_seconds, 0.12, { name: "search" }) + expect(transaction).to receive(:observe) + .with(:gitlab_net_ldap_duration_seconds, 5.3, { name: "search" }) + + subscriber.observe_event(event_1) + subscriber.observe_event(event_2) + subscriber.observe_event(event_3) + end + + it "stores per-request counters" do + subscriber.observe_event(event_1) + subscriber.observe_event(event_2) + subscriber.observe_event(event_3) + + expect(Gitlab::SafeRequestStore[:net_ldap_count]).to eq(3) + expect(Gitlab::SafeRequestStore[:net_ldap_duration_s]).to eq(5.741) # 0.321 + 0.12 + 5.3 + end + end +end diff --git a/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb b/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb index 9aba6ac293c..59bfe2042fa 100644 --- a/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do let(:transaction) { Gitlab::Metrics::WebTransaction.new(env) } let(:subscriber) { described_class.new } - let(:event) { double(:event, duration: 15.2) } + let(:event) { double(:event, duration: 15.2, payload: { key: %w[a b c] }) } describe '#cache_read' do it 'increments the cache_read duration' do @@ -64,6 +64,40 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do end end + describe '#cache_read_multi' do + subject { subscriber.cache_read_multi(event) } + + context 'with a transaction' do + before do + allow(subscriber).to receive(:current_transaction) + .and_return(transaction) + end + + it 'observes multi-key count' do + expect(transaction).to receive(:observe) + .with(:gitlab_cache_read_multikey_count, event.payload[:key].size) + + subject + end + end + + context 'with no transaction' do + it 'does not observes multi-key count' do + expect(transaction).not_to receive(:observe) + .with(:gitlab_cache_read_multikey_count, event.payload[:key].size) + + subject + end + end + + it 'observes read_multi duration' do + expect(subscriber).to receive(:observe) + .with(:read_multi, event.duration) + + subject + end + end + describe '#cache_write' do it 'observes write duration' do expect(subscriber).to receive(:observe) diff --git a/spec/lib/gitlab/metrics_spec.rb b/spec/lib/gitlab/metrics_spec.rb index 366843a4c03..dbd6c07ef75 100644 --- a/spec/lib/gitlab/metrics_spec.rb +++ b/spec/lib/gitlab/metrics_spec.rb @@ -101,14 +101,32 @@ RSpec.describe Gitlab::Metrics do 401 | true nil | false 500 | false - 503 | false - '100' | false - '201' | true + 503 | false 'nothing' | false end with_them do specify { expect(described_class.record_duration_for_status?(status)).to be(should_record) } + specify { expect(described_class.record_duration_for_status?(status.to_s)).to be(should_record) } + end + end + + describe '.server_error?' do + using RSpec::Parameterized::TableSyntax + + where(:status, :should_record) do + 100 | false + 200 | false + 401 | false + 500 | true + 503 | true + nil | false + 'nothing' | false + end + + with_them do + specify { expect(described_class.server_error?(status)).to be(should_record) } + specify { expect(described_class.server_error?(status.to_s)).to be(should_record) } end end diff --git a/spec/lib/gitlab/middleware/compressed_json_spec.rb b/spec/lib/gitlab/middleware/compressed_json_spec.rb index 6d49ab58d5d..1444e6a9881 100644 --- a/spec/lib/gitlab/middleware/compressed_json_spec.rb +++ b/spec/lib/gitlab/middleware/compressed_json_spec.rb @@ -9,6 +9,7 @@ RSpec.describe Gitlab::Middleware::CompressedJson do let(:app) { double(:app) } let(:middleware) { described_class.new(app) } let(:content_type) { 'application/json' } + let(:relative_url_root) { '/gitlab' } let(:env) do { 'HTTP_CONTENT_ENCODING' => 'gzip', @@ -31,6 +32,43 @@ RSpec.describe Gitlab::Middleware::CompressedJson do end end + shared_examples 'passes input' do + it 'keeps the original input' do + expect(app).to receive(:call) + + middleware.call(env) + + expect(env['rack.input'].read).to eq(input) + expect(env['HTTP_CONTENT_ENCODING']).to eq('gzip') + end + end + + shared_context 'with relative url' do + before do + stub_config_setting(relative_url_root: relative_url_root) + end + end + + shared_examples 'handles non integer project ID' do + context 'with a URL-encoded project ID' do + let_it_be(:project_id) { 'gitlab-org%2fgitlab' } + + it_behaves_like 'decompress middleware' + end + + context 'with a non URL-encoded project ID' do + let_it_be(:project_id) { '1/repository/files/api/v4' } + + it_behaves_like 'passes input' + end + + context 'with a blank project ID' do + let_it_be(:project_id) { '' } + + it_behaves_like 'passes input' + end + end + describe '#call' do context 'with collector route' do let(:path) { '/api/v4/error_tracking/collector/1/store' } @@ -42,31 +80,80 @@ RSpec.describe Gitlab::Middleware::CompressedJson do it_behaves_like 'decompress middleware' end + + include_context 'with relative url' do + let(:path) { "#{relative_url_root}/api/v4/error_tracking/collector/1/store" } + + it_behaves_like 'decompress middleware' + end end - context 'with collector route under relative url' do - let(:path) { '/gitlab/api/v4/error_tracking/collector/1/store' } + context 'with packages route' do + context 'with instance level endpoint' do + context 'with npm advisory bulk url' do + let(:path) { '/api/v4/packages/npm/-/npm/v1/security/advisories/bulk' } + + it_behaves_like 'decompress middleware' + + include_context 'with relative url' do + let(:path) { "#{relative_url_root}/api/v4/packages/npm/-/npm/v1/security/advisories/bulk" } + + it_behaves_like 'decompress middleware' + end + end + + context 'with npm quick audit url' do + let(:path) { '/api/v4/packages/npm/-/npm/v1/security/audits/quick' } - before do - stub_config_setting(relative_url_root: '/gitlab') + it_behaves_like 'decompress middleware' + + include_context 'with relative url' do + let(:path) { "#{relative_url_root}/api/v4/packages/npm/-/npm/v1/security/audits/quick" } + + it_behaves_like 'decompress middleware' + end + end end - it_behaves_like 'decompress middleware' - end + context 'with project level endpoint' do + let_it_be(:project_id) { 1 } - context 'with some other route' do - let(:path) { '/api/projects/123' } + context 'with npm advisory bulk url' do + let(:path) { "/api/v4/projects/#{project_id}/packages/npm/-/npm/v1/security/advisories/bulk" } - it 'keeps the original input' do - expect(app).to receive(:call) + it_behaves_like 'decompress middleware' - middleware.call(env) + include_context 'with relative url' do + let(:path) { "#{relative_url_root}/api/v4/projects/#{project_id}/packages/npm/-/npm/v1/security/advisories/bulk" } # rubocop disable Layout/LineLength - expect(env['rack.input'].read).to eq(input) - expect(env['HTTP_CONTENT_ENCODING']).to eq('gzip') + it_behaves_like 'decompress middleware' + end + + it_behaves_like 'handles non integer project ID' + end + + context 'with npm quick audit url' do + let(:path) { "/api/v4/projects/#{project_id}/packages/npm/-/npm/v1/security/audits/quick" } + + it_behaves_like 'decompress middleware' + + include_context 'with relative url' do + let(:path) { "#{relative_url_root}/api/v4/projects/#{project_id}/packages/npm/-/npm/v1/security/audits/quick" } # rubocop disable Layout/LineLength + + it_behaves_like 'decompress middleware' + end + + it_behaves_like 'handles non integer project ID' + end end end + context 'with some other route' do + let(:path) { '/api/projects/123' } + + it_behaves_like 'passes input' + end + context 'payload is too large' do let(:body_limit) { Gitlab::Middleware::CompressedJson::MAXIMUM_BODY_SIZE } let(:decompressed_input) { 'a' * (body_limit + 100) } diff --git a/spec/lib/gitlab/middleware/go_spec.rb b/spec/lib/gitlab/middleware/go_spec.rb index bc1d53b2ccb..bed43c04460 100644 --- a/spec/lib/gitlab/middleware/go_spec.rb +++ b/spec/lib/gitlab/middleware/go_spec.rb @@ -278,7 +278,7 @@ RSpec.describe Gitlab::Middleware::Go do project_url = "http://#{Gitlab.config.gitlab.host}/#{path}" expect(response[0]).to eq(200) expect(response[1]['Content-Type']).to eq('text/html') - expected_body = %{<html><head><meta name="go-import" content="#{Gitlab.config.gitlab.host}/#{path} git #{repository_url}" /><meta name="go-source" content="#{Gitlab.config.gitlab.host}/#{path} #{project_url} #{project_url}/-/tree/#{branch}{/dir} #{project_url}/-/blob/#{branch}{/dir}/{file}#L{line}" /></head><body>go get #{Gitlab.config.gitlab.url}/#{path}</body></html>} + expected_body = %{<html><head><meta name="go-import" content="#{Gitlab.config.gitlab.host}/#{path} git #{repository_url}"><meta name="go-source" content="#{Gitlab.config.gitlab.host}/#{path} #{project_url} #{project_url}/-/tree/#{branch}{/dir} #{project_url}/-/blob/#{branch}{/dir}/{file}#L{line}"></head><body>go get #{Gitlab.config.gitlab.url}/#{path}</body></html>} expect(response[2]).to eq([expected_body]) end end diff --git a/spec/lib/gitlab/other_markup_spec.rb b/spec/lib/gitlab/other_markup_spec.rb index 26e60251abb..6b24c8a8710 100644 --- a/spec/lib/gitlab/other_markup_spec.rb +++ b/spec/lib/gitlab/other_markup_spec.rb @@ -5,7 +5,7 @@ require 'spec_helper' RSpec.describe Gitlab::OtherMarkup do let(:context) { {} } - context "XSS Checks" do + context 'XSS Checks' do links = { 'links' => { file: 'file.rdoc', @@ -20,6 +20,33 @@ RSpec.describe Gitlab::OtherMarkup do end end + context 'when rendering takes too long' do + let_it_be(:file_name) { 'foo.bar' } + let_it_be(:project) { create(:project, :repository) } + let_it_be(:context) { { project: project } } + let_it_be(:text) { +'Noël' } + + before do + stub_const('Gitlab::OtherMarkup::RENDER_TIMEOUT', 0.1) + allow(GitHub::Markup).to receive(:render) do + sleep(0.2) + text + end + end + + it 'times out' do + # expect twice because of timeout in SyntaxHighlightFilter + expect(Gitlab::RenderTimeout).to receive(:timeout).twice.and_call_original + expect(Gitlab::ErrorTracking).to receive(:track_exception).with( + instance_of(Timeout::Error), + project_id: context[:project].id, file_name: file_name, + class_name: described_class.name.demodulize + ) + + expect(render(file_name, text, context)).to eq("<p>#{text}</p>") + end + end + def render(*args) described_class.render(*args) end diff --git a/spec/lib/gitlab/pages/cache_control_spec.rb b/spec/lib/gitlab/pages/cache_control_spec.rb index 431c989e874..d46124e0e16 100644 --- a/spec/lib/gitlab/pages/cache_control_spec.rb +++ b/spec/lib/gitlab/pages/cache_control_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Pages::CacheControl do +RSpec.describe Gitlab::Pages::CacheControl, feature_category: :pages do describe '.for_namespace' do subject(:cache_control) { described_class.for_namespace(1) } @@ -11,8 +11,14 @@ RSpec.describe Gitlab::Pages::CacheControl do describe '#clear_cache' do it 'clears the cache' do expect(Rails.cache) - .to receive(:delete) - .with(/pages_domain_for_namespace_1_*/) + .to receive(:delete_multi) + .with( + array_including( + [ + "pages_domain_for_namespace_1", + /pages_domain_for_namespace_1_*/ + ] + )) subject.clear_cache end @@ -27,8 +33,14 @@ RSpec.describe Gitlab::Pages::CacheControl do describe '#clear_cache' do it 'clears the cache' do expect(Rails.cache) - .to receive(:delete) - .with(/pages_domain_for_project_1_*/) + .to receive(:delete_multi) + .with( + array_including( + [ + "pages_domain_for_project_1", + /pages_domain_for_project_1_*/ + ] + )) subject.clear_cache end @@ -58,6 +70,14 @@ RSpec.describe Gitlab::Pages::CacheControl do expect(described_class.new(type: :project, id: 1).cache_key).not_to eq(cache_key) end + + it 'caches the application settings hash' do + expect(Rails.cache) + .to receive(:write) + .with("pages_domain_for_project_1", kind_of(Set)) + + described_class.new(type: :project, id: 1).cache_key + end end it 'fails with invalid type' do diff --git a/spec/lib/gitlab/pagination/offset_pagination_spec.rb b/spec/lib/gitlab/pagination/offset_pagination_spec.rb index ebbd207cc11..b1c4ffd6c29 100644 --- a/spec/lib/gitlab/pagination/offset_pagination_spec.rb +++ b/spec/lib/gitlab/pagination/offset_pagination_spec.rb @@ -101,6 +101,28 @@ RSpec.describe Gitlab::Pagination::OffsetPagination do end end + context 'when without_count is true' do + it_behaves_like 'paginated response' + + it 'does not return the X-Total and X-Total-Pages headers' do + expect_no_header('X-Total') + expect_no_header('X-Total-Pages') + expect_header('X-Per-Page', '2') + expect_header('X-Page', '1') + expect_header('X-Next-Page', '2') + expect_header('X-Prev-Page', '') + + expect_header('Link', anything) do |_key, val| + expect(val).to include(%Q(<#{incoming_api_projects_url}?#{query.merge(page: 1).to_query}>; rel="first")) + expect(val).to include(%Q(<#{incoming_api_projects_url}?#{query.merge(page: 2).to_query}>; rel="next")) + expect(val).not_to include('rel="last"') + expect(val).not_to include('rel="prev"') + end + + expect { subject.paginate(resource, without_count: true) }.to make_queries_matching(/SELECT COUNT/, 0) + end + end + it 'does not return the total headers when excluding them' do expect_no_header('X-Total') expect_no_header('X-Total-Pages') diff --git a/spec/lib/gitlab/process_management_spec.rb b/spec/lib/gitlab/process_management_spec.rb index a71a476b540..fbd39702efb 100644 --- a/spec/lib/gitlab/process_management_spec.rb +++ b/spec/lib/gitlab/process_management_spec.rb @@ -41,15 +41,6 @@ RSpec.describe Gitlab::ProcessManagement do end end - describe '.wait_async' do - it 'waits for a process in a separate thread' do - thread = described_class.wait_async(Process.spawn('true')) - - # Upon success Process.wait just returns the PID. - expect(thread.value).to be_a_kind_of(Numeric) - end - end - # In the X_alive? checks, we check negative PIDs sometimes as a simple way # to be sure the pids are definitely for non-existent processes. # Note that -1 is special, and sends the signal to every process we have permission diff --git a/spec/lib/gitlab/process_supervisor_spec.rb b/spec/lib/gitlab/process_supervisor_spec.rb index 8356197805c..18de5053362 100644 --- a/spec/lib/gitlab/process_supervisor_spec.rb +++ b/spec/lib/gitlab/process_supervisor_spec.rb @@ -2,7 +2,7 @@ require_relative '../../../lib/gitlab/process_supervisor' -RSpec.describe Gitlab::ProcessSupervisor do +RSpec.describe Gitlab::ProcessSupervisor, feature_category: :application_performance do let(:health_check_interval_seconds) { 0.1 } let(:check_terminate_interval_seconds) { 1 } let(:forwarded_signals) { [] } diff --git a/spec/lib/gitlab/puma_logging/json_formatter_spec.rb b/spec/lib/gitlab/puma_logging/json_formatter_spec.rb index 64ace09e01b..d38f54bccf1 100644 --- a/spec/lib/gitlab/puma_logging/json_formatter_spec.rb +++ b/spec/lib/gitlab/puma_logging/json_formatter_spec.rb @@ -4,8 +4,8 @@ require 'spec_helper' RSpec.describe Gitlab::PumaLogging::JSONFormatter do it "generate json format with timestamp and pid" do - Timecop.freeze( Time.utc(2019, 12, 04, 9, 10, 11, 123456)) do - expect(subject.call('log message')).to eq "{\"timestamp\":\"2019-12-04T09:10:11.123Z\",\"pid\":#{Process.pid},\"message\":\"log message\"}" + travel_to(Time.utc(2019, 12, 04, 9, 10, 11)) do + expect(subject.call('log message')).to eq "{\"timestamp\":\"2019-12-04T09:10:11.000Z\",\"pid\":#{Process.pid},\"message\":\"log message\"}" end end end diff --git a/spec/lib/gitlab/quick_actions/dsl_spec.rb b/spec/lib/gitlab/quick_actions/dsl_spec.rb index 942d347424f..c0469537c68 100644 --- a/spec/lib/gitlab/quick_actions/dsl_spec.rb +++ b/spec/lib/gitlab/quick_actions/dsl_spec.rb @@ -3,9 +3,10 @@ require 'spec_helper' RSpec.describe Gitlab::QuickActions::Dsl do - before :all do - DummyClass = Struct.new(:project) do - include Gitlab::QuickActions::Dsl + before do + stub_const('DummyClass', Struct.new(:project)) + DummyClass.class_eval do + include Gitlab::QuickActions::Dsl # rubocop:disable RSpec/DescribedClass desc 'A command with no args' command :no_args, :none do diff --git a/spec/lib/gitlab/redis/multi_store_spec.rb b/spec/lib/gitlab/redis/multi_store_spec.rb index 207fe28e84e..0e7eedf66b1 100644 --- a/spec/lib/gitlab/redis/multi_store_spec.rb +++ b/spec/lib/gitlab/redis/multi_store_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Redis::MultiStore do +RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do using RSpec::Parameterized::TableSyntax let_it_be(:redis_store_class) do diff --git a/spec/lib/gitlab/reference_extractor_spec.rb b/spec/lib/gitlab/reference_extractor_spec.rb index 0ee8c35ae81..4d608c07736 100644 --- a/spec/lib/gitlab/reference_extractor_spec.rb +++ b/spec/lib/gitlab/reference_extractor_spec.rb @@ -262,8 +262,11 @@ RSpec.describe Gitlab::ReferenceExtractor do describe '#all' do let(:issue) { create(:issue, project: project) } + let(:issue2) { create(:issue, project: project) } + let(:issue2_url) { Rails.application.routes.url_helpers.project_issue_url(project, issue2) } let(:label) { create(:label, project: project) } - let(:text) { "Ref. #{issue.to_reference} and #{label.to_reference}" } + let(:alert) { create(:alert_management_alert, project: project) } + let(:text) { "Ref. #{issue.to_reference} and #{label.to_reference} and #{alert.to_reference} and #{issue2_url}" } before do project.add_developer(project.creator) @@ -271,7 +274,22 @@ RSpec.describe Gitlab::ReferenceExtractor do end it 'returns all referables' do - expect(subject.all).to match_array([issue, label]) + expect(subject.all).to match_array([issue, label, alert, issue2]) + end + end + + describe '#alerts' do + let(:alert1) { create(:alert_management_alert, project: project) } + let(:alert2) { create(:alert_management_alert, project: project) } + let(:text) { "Alert ref: #{alert1.to_reference} URL: #{alert2.details_url} Infalid ref: ^alert#0" } + + before do + project.add_developer(project.creator) + subject.analyze(text) + end + + it 'returns alert referables' do + expect(subject.alerts).to match_array([alert1, alert2]) end end diff --git a/spec/lib/gitlab/repository_archive_rate_limiter_spec.rb b/spec/lib/gitlab/repository_archive_rate_limiter_spec.rb index 49df70f3cb3..4599c647d5c 100644 --- a/spec/lib/gitlab/repository_archive_rate_limiter_spec.rb +++ b/spec/lib/gitlab/repository_archive_rate_limiter_spec.rb @@ -7,8 +7,7 @@ RSpec.describe ::Gitlab::RepositoryArchiveRateLimiter do Class.new do include ::Gitlab::RepositoryArchiveRateLimiter - def check_rate_limit!(**args) - end + def check_rate_limit!(**args); end end end diff --git a/spec/lib/gitlab/repository_cache_adapter_spec.rb b/spec/lib/gitlab/repository_cache_adapter_spec.rb index d14c3f44c6f..71a20cc58fd 100644 --- a/spec/lib/gitlab/repository_cache_adapter_spec.rb +++ b/spec/lib/gitlab/repository_cache_adapter_spec.rb @@ -27,8 +27,7 @@ RSpec.describe Gitlab::RepositoryCacheAdapter do 'foo/bar' end - def project - end + def project; end def cached_methods [:letters] diff --git a/spec/lib/gitlab/search/found_blob_spec.rb b/spec/lib/gitlab/search/found_blob_spec.rb index 8b1c91f689d..c41a051bc42 100644 --- a/spec/lib/gitlab/search/found_blob_spec.rb +++ b/spec/lib/gitlab/search/found_blob_spec.rb @@ -141,9 +141,8 @@ RSpec.describe Gitlab::Search::FoundBlob do subject { described_class.new(blob_path: path, project: project, ref: 'master') } before do - allow(Gitlab::Git::Blob).to receive(:batch).and_return([ - Gitlab::Git::Blob.new(path: path) - ]) + allow(Gitlab::Git::Blob) + .to receive(:batch).and_return([Gitlab::Git::Blob.new(path: path)]) end it { expect(subject.path).to eq('a/b/c.md') } diff --git a/spec/lib/gitlab/shell_spec.rb b/spec/lib/gitlab/shell_spec.rb index 785429aa3b0..049b8d4ed86 100644 --- a/spec/lib/gitlab/shell_spec.rb +++ b/spec/lib/gitlab/shell_spec.rb @@ -88,18 +88,11 @@ RSpec.describe Gitlab::Shell do let(:disk_path) { "#{project.disk_path}.git" } it 'returns true when the command succeeds' do - expect(TestEnv.storage_dir_exists?(project.repository_storage, disk_path)).to be(true) + expect(project.repository.raw).to exist expect(gitlab_shell.remove_repository(project.repository_storage, project.disk_path)).to be(true) - expect(TestEnv.storage_dir_exists?(project.repository_storage, disk_path)).to be(false) - end - - it 'keeps the namespace directory' do - gitlab_shell.remove_repository(project.repository_storage, project.disk_path) - - expect(TestEnv.storage_dir_exists?(project.repository_storage, disk_path)).to be(false) - expect(TestEnv.storage_dir_exists?(project.repository_storage, project.disk_path.gsub(project.name, ''))).to be(true) + expect(project.repository.raw).not_to exist end end @@ -107,21 +100,22 @@ RSpec.describe Gitlab::Shell do let!(:project2) { create(:project, :repository) } it 'returns true when the command succeeds' do - old_path = project2.disk_path + old_repo = project2.repository.raw new_path = "project/new_path" + new_repo = Gitlab::Git::Repository.new(project2.repository_storage, "#{new_path}.git", nil, nil) - expect(TestEnv.storage_dir_exists?(project2.repository_storage, "#{old_path}.git")).to be(true) - expect(TestEnv.storage_dir_exists?(project2.repository_storage, "#{new_path}.git")).to be(false) + expect(old_repo).to exist + expect(new_repo).not_to exist - expect(gitlab_shell.mv_repository(project2.repository_storage, old_path, new_path)).to be_truthy + expect(gitlab_shell.mv_repository(project2.repository_storage, project2.disk_path, new_path)).to be_truthy - expect(TestEnv.storage_dir_exists?(project2.repository_storage, "#{old_path}.git")).to be(false) - expect(TestEnv.storage_dir_exists?(project2.repository_storage, "#{new_path}.git")).to be(true) + expect(old_repo).not_to exist + expect(new_repo).to exist end it 'returns false when the command fails' do expect(gitlab_shell.mv_repository(project2.repository_storage, project2.disk_path, '')).to be_falsy - expect(TestEnv.storage_dir_exists?(project2.repository_storage, "#{project2.disk_path}.git")).to be(true) + expect(project2.repository.raw).to exist end end end @@ -133,9 +127,11 @@ RSpec.describe Gitlab::Shell do describe '#add_namespace' do it 'creates a namespace' do - Gitlab::GitalyClient::NamespaceService.allow { subject.add_namespace(storage, "mepmep") } + Gitlab::GitalyClient::NamespaceService.allow do + subject.add_namespace(storage, "mepmep") - expect(TestEnv.storage_dir_exists?(storage, "mepmep")).to be(true) + expect(Gitlab::GitalyClient::NamespaceService.new(storage).exists?("mepmep")).to be(true) + end end end @@ -160,9 +156,9 @@ RSpec.describe Gitlab::Shell do Gitlab::GitalyClient::NamespaceService.allow do subject.add_namespace(storage, "mepmep") subject.rm_namespace(storage, "mepmep") - end - expect(TestEnv.storage_dir_exists?(storage, "mepmep")).to be(false) + expect(Gitlab::GitalyClient::NamespaceService.new(storage).exists?("mepmep")).to be(false) + end end end @@ -171,10 +167,10 @@ RSpec.describe Gitlab::Shell do Gitlab::GitalyClient::NamespaceService.allow do subject.add_namespace(storage, "mepmep") subject.mv_namespace(storage, "mepmep", "2mep") - end - expect(TestEnv.storage_dir_exists?(storage, "mepmep")).to be(false) - expect(TestEnv.storage_dir_exists?(storage, "2mep")).to be(true) + expect(Gitlab::GitalyClient::NamespaceService.new(storage).exists?("mepmep")).to be(false) + expect(Gitlab::GitalyClient::NamespaceService.new(storage).exists?("2mep")).to be(true) + end end end end diff --git a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb index 8c9a1abba5a..5baeec93036 100644 --- a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb +++ b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb @@ -4,11 +4,23 @@ require 'spec_helper' RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do let(:memory_killer) { described_class.new } + let(:sidekiq_daemon_monitor) { instance_double(Gitlab::SidekiqDaemon::Monitor) } + let(:running_jobs) { {} } let(:pid) { 12345 } + let(:worker) do + Class.new do + def self.name + 'DummyWorker' + end + end + end before do + stub_const('DummyWorker', worker) allow(Sidekiq.logger).to receive(:info) allow(Sidekiq.logger).to receive(:warn) + allow(Gitlab::SidekiqDaemon::Monitor).to receive(:instance).and_return(sidekiq_daemon_monitor) + allow(sidekiq_daemon_monitor).to receive(:jobs).and_return(running_jobs) allow(memory_killer).to receive(:pid).and_return(pid) # make sleep no-op @@ -306,31 +318,37 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do stub_const("#{described_class}::CHECK_INTERVAL_SECONDS", check_interval_seconds) end - it 'send signal and return when all jobs finished' do - expect(Process).to receive(:kill).with(signal, pid).ordered - expect(Gitlab::Metrics::System).to receive(:monotonic_time).and_call_original + context 'when all jobs are finished' do + let(:running_jobs) { {} } - expect(memory_killer).to receive(:enabled?).and_return(true) - expect(memory_killer).to receive(:any_jobs?).and_return(false) + it 'send signal and return when all jobs finished' do + expect(Process).to receive(:kill).with(signal, pid).ordered + expect(Gitlab::Metrics::System).to receive(:monotonic_time).and_call_original - expect(memory_killer).not_to receive(:sleep) + expect(memory_killer).to receive(:enabled?).and_return(true) - subject + expect(memory_killer).not_to receive(:sleep) + + subject + end end - it 'send signal and wait till deadline if any job not finished' do - expect(Process).to receive(:kill) - .with(signal, pid) - .ordered + context 'when there are still running jobs' do + let(:running_jobs) { { 'jid1' => { worker_class: DummyWorker } } } - expect(Gitlab::Metrics::System).to receive(:monotonic_time) - .and_call_original - .at_least(:once) + it 'send signal and wait till deadline if any job not finished' do + expect(Process).to receive(:kill) + .with(signal, pid) + .ordered + + expect(Gitlab::Metrics::System).to receive(:monotonic_time) + .and_call_original + .at_least(:once) - expect(memory_killer).to receive(:enabled?).and_return(true).at_least(:once) - expect(memory_killer).to receive(:any_jobs?).and_return(true).at_least(:once) + expect(memory_killer).to receive(:enabled?).and_return(true).at_least(:once) - subject + subject + end end end @@ -377,21 +395,11 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do let(:jid) { 1 } let(:reason) { 'rss out of range reason description' } let(:queue) { 'default' } - let(:running_jobs) { [{ jid: jid, worker_class: 'DummyWorker' }] } - let(:metrics) { memory_killer.instance_variable_get(:@metrics) } - let(:worker) do - Class.new do - def self.name - 'DummyWorker' - end - include ApplicationWorker - end - end + let(:metrics) { memory_killer.instance_variable_get(:@metrics) } + let(:running_jobs) { { jid => { worker_class: DummyWorker } } } before do - stub_const("DummyWorker", worker) - allow(memory_killer).to receive(:get_rss_kb).and_return(*current_rss) allow(memory_killer).to receive(:get_soft_limit_rss_kb).and_return(soft_limit_rss) allow(memory_killer).to receive(:get_hard_limit_rss_kb).and_return(hard_limit_rss) @@ -413,15 +421,13 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do hard_limit_rss: hard_limit_rss, soft_limit_rss: soft_limit_rss, reason: reason, - running_jobs: running_jobs, + running_jobs: [jid: jid, worker_class: 'DummyWorker'], memory_total_kb: memory_total) expect(metrics[:sidekiq_memory_killer_running_jobs]).to receive(:increment) .with({ worker_class: "DummyWorker", deadline_exceeded: true }) - Gitlab::SidekiqDaemon::Monitor.instance.within_job(DummyWorker, jid, queue) do - subject - end + subject end end @@ -452,6 +458,7 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do expect(subject).to eq("current_rss(#{rss}) > soft_limit_rss(#{soft_limit}) longer than GRACE_BALLOON_SECONDS(#{grace_balloon_seconds})") end end + context 'deadline not exceeded' do let(:deadline_exceeded) { false } @@ -463,21 +470,24 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do end describe '#rss_increase_by_jobs' do - let(:running_jobs) { { id1: 'job1', id2: 'job2' } } + let(:running_jobs) { { 'job1' => { worker_class: "Job1" }, 'job2' => { worker_class: "Job2" } } } subject { memory_killer.send(:rss_increase_by_jobs) } + before do + allow(memory_killer).to receive(:rss_increase_by_job).and_return(11, 22) + end + it 'adds up individual rss_increase_by_job' do - allow(Gitlab::SidekiqDaemon::Monitor).to receive_message_chain(:instance, :jobs_mutex, :synchronize).and_yield - expect(Gitlab::SidekiqDaemon::Monitor).to receive_message_chain(:instance, :jobs).and_return(running_jobs) - expect(memory_killer).to receive(:rss_increase_by_job).and_return(11, 22) expect(subject).to eq(33) end - it 'return 0 if no job' do - allow(Gitlab::SidekiqDaemon::Monitor).to receive_message_chain(:instance, :jobs_mutex, :synchronize).and_yield - expect(Gitlab::SidekiqDaemon::Monitor).to receive_message_chain(:instance, :jobs).and_return({}) - expect(subject).to eq(0) + context 'when there is no running job' do + let(:running_jobs) { {} } + + it 'return 0 if no job' do + expect(subject).to eq(0) + end end end diff --git a/spec/lib/gitlab/sidekiq_daemon/monitor_spec.rb b/spec/lib/gitlab/sidekiq_daemon/monitor_spec.rb index f93c0e28fc0..479ef29bbf9 100644 --- a/spec/lib/gitlab/sidekiq_daemon/monitor_spec.rb +++ b/spec/lib/gitlab/sidekiq_daemon/monitor_spec.rb @@ -6,9 +6,15 @@ RSpec.describe Gitlab::SidekiqDaemon::Monitor do let(:monitor) { described_class.new } describe '#within_job' do - it 'tracks thread' do + it 'tracks thread, jid and worker_class' do blk = proc do - expect(monitor.jobs.dig('jid', :thread)).not_to be_nil + monitor.jobs do |jobs| + jobs.each do |jid, job| + expect(job[:thread]).not_to be_nil + expect(jid).to eq('jid') + expect(job[:worker_class]).to eq('worker_class') + end + end "OK" end @@ -37,6 +43,17 @@ RSpec.describe Gitlab::SidekiqDaemon::Monitor do end end + describe '#jobs' do + it 'returns running jobs hash' do + jid = SecureRandom.hex + running_jobs = { jid => hash_including(worker_class: 'worker_class') } + + monitor.within_job('worker_class', jid, 'queue') do + expect(monitor.jobs).to match(running_jobs) + end + end + end + describe '#run_thread when notification channel not enabled' do subject { monitor.send(:run_thread) } @@ -220,7 +237,7 @@ RSpec.describe Gitlab::SidekiqDaemon::Monitor do let(:thread) { Thread.new { sleep 1000 } } before do - monitor.jobs[jid] = { worker_class: 'worker_class', thread: thread, started_at: Time.now.to_i } + allow(monitor).to receive(:find_thread_unsafe).with(jid).and_return(thread) end after do diff --git a/spec/lib/gitlab/sidekiq_middleware/client_metrics_spec.rb b/spec/lib/gitlab/sidekiq_middleware/client_metrics_spec.rb index dca00c85e30..472591bde5e 100644 --- a/spec/lib/gitlab/sidekiq_middleware/client_metrics_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/client_metrics_spec.rb @@ -40,8 +40,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::ClientMetrics do TestWorker.class_eval do include Sidekiq::Worker - def perform(*args) - end + def perform(*args); end end allow(Gitlab::Metrics).to receive(:counter).and_return(Gitlab::Metrics::NullMetric.instance) diff --git a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/client_spec.rb b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/client_spec.rb index 44c8df73463..14eb568b974 100644 --- a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/client_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/client_spec.rb @@ -17,8 +17,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::Client, :clean_gitlab_r include ApplicationWorker - def perform(*args) - end + def perform(*args); end end end diff --git a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb index 09548d21106..1b01793d80d 100644 --- a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb @@ -18,8 +18,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::Server, :clean_gitlab_r self.class.work end - def self.work - end + def self.work; end end end diff --git a/spec/lib/gitlab/sidekiq_middleware/instrumentation_logger_spec.rb b/spec/lib/gitlab/sidekiq_middleware/instrumentation_logger_spec.rb index 8cf65e1be5b..cfb2c7ab5c3 100644 --- a/spec/lib/gitlab/sidekiq_middleware/instrumentation_logger_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/instrumentation_logger_spec.rb @@ -13,8 +13,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::InstrumentationLogger do include ApplicationWorker - def perform(*args) - end + def perform(*args); end end end diff --git a/spec/lib/gitlab/sidekiq_middleware/query_analyzer_spec.rb b/spec/lib/gitlab/sidekiq_middleware/query_analyzer_spec.rb index e58af1d60fe..c31f05f00e4 100644 --- a/spec/lib/gitlab/sidekiq_middleware/query_analyzer_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/query_analyzer_spec.rb @@ -10,8 +10,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::QueryAnalyzer, query_analyzers: false let(:queue) { 'some-queue' } let(:middleware) { described_class.new } - def do_queries - end + def do_queries; end subject { middleware.call(worker, job, queue) { do_queries } } diff --git a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb index 1a53a9b8701..f7cee6beb58 100644 --- a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb @@ -231,8 +231,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do include Sidekiq::Worker include WorkerAttributes - def perform(*args) - end + def perform(*args); end end allow(::Gitlab::Database::LoadBalancing).to receive_message_chain(:proxy, :load_balancer).and_return(load_balancer) @@ -306,8 +305,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do feature_category :not_owned end - def perform - end + def perform; end end end diff --git a/spec/lib/gitlab/sidekiq_middleware/worker_context/client_spec.rb b/spec/lib/gitlab/sidekiq_middleware/worker_context/client_spec.rb index 821d8b8fe7b..1b6cd7ac5fb 100644 --- a/spec/lib/gitlab/sidekiq_middleware/worker_context/client_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/worker_context/client_spec.rb @@ -17,8 +17,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Client do jobs.find { |job| job['args'] == args } end - def perform(*args) - end + def perform(*args); end end end @@ -38,8 +37,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Client do 'TestMailer' end - def test_mail - end + def test_mail; end end end diff --git a/spec/lib/gitlab/sidekiq_middleware/worker_context/server_spec.rb b/spec/lib/gitlab/sidekiq_middleware/worker_context/server_spec.rb index 05b328e55d3..2deab3064eb 100644 --- a/spec/lib/gitlab/sidekiq_middleware/worker_context/server_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/worker_context/server_spec.rb @@ -41,8 +41,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Server do include Sidekiq::Worker - def perform - end + def perform; end end end diff --git a/spec/lib/gitlab/slash_commands/application_help_spec.rb b/spec/lib/gitlab/slash_commands/application_help_spec.rb index b182c0e5cc6..d0cefdf4895 100644 --- a/spec/lib/gitlab/slash_commands/application_help_spec.rb +++ b/spec/lib/gitlab/slash_commands/application_help_spec.rb @@ -4,13 +4,11 @@ require 'spec_helper' RSpec.describe Gitlab::SlashCommands::ApplicationHelp do let(:params) { { command: '/gitlab', text: 'help' } } - let_it_be(:user) { create(:user) } - let_it_be(:chat_user) { create(:chat_name, user: user) } let(:project) { build(:project) } describe '#execute' do subject do - described_class.new(project, chat_user, params).execute + described_class.new(project, params).execute end it 'displays the help section' do diff --git a/spec/lib/gitlab/slash_commands/deploy_spec.rb b/spec/lib/gitlab/slash_commands/deploy_spec.rb index 5af234ff88e..94a95fb417f 100644 --- a/spec/lib/gitlab/slash_commands/deploy_spec.rb +++ b/spec/lib/gitlab/slash_commands/deploy_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::SlashCommands::Deploy do +RSpec.describe Gitlab::SlashCommands::Deploy, feature_category: :team_planning do describe '#execute' do let(:project) { create(:project, :repository) } let(:user) { create(:user) } diff --git a/spec/lib/gitlab/sql/pattern_spec.rb b/spec/lib/gitlab/sql/pattern_spec.rb index 60bb006673f..a34ddf8773c 100644 --- a/spec/lib/gitlab/sql/pattern_spec.rb +++ b/spec/lib/gitlab/sql/pattern_spec.rb @@ -210,7 +210,7 @@ RSpec.describe Gitlab::SQL::Pattern do let(:query) { 'foo' } it 'returns a single ILIKE condition' do - expect(fuzzy_arel_match.to_sql).to match(/title.*I?LIKE '\%foo\%'/) + expect(fuzzy_arel_match.to_sql).to match(/title.*I?LIKE '%foo%'/) end end @@ -232,7 +232,7 @@ RSpec.describe Gitlab::SQL::Pattern do let(:query) { 'foo baz' } it 'returns a joining LIKE condition using a AND' do - expect(fuzzy_arel_match.to_sql).to match(/title.+I?LIKE '\%foo\%' AND .*title.*I?LIKE '\%baz\%'/) + expect(fuzzy_arel_match.to_sql).to match(/title.+I?LIKE '%foo%' AND .*title.*I?LIKE '%baz%'/) end end @@ -248,7 +248,7 @@ RSpec.describe Gitlab::SQL::Pattern do let(:query) { 'foo ba' } it 'returns a single ILIKE condition using the longer word' do - expect(fuzzy_arel_match.to_sql).to match(/title.+I?LIKE '\%foo\%'/) + expect(fuzzy_arel_match.to_sql).to match(/title.+I?LIKE '%foo%'/) end end @@ -256,7 +256,7 @@ RSpec.describe Gitlab::SQL::Pattern do let(:query) { 'foo "really bar" baz' } it 'returns a joining LIKE condition using a AND' do - expect(fuzzy_arel_match.to_sql).to match(/title.+I?LIKE '\%foo\%' AND .*title.*I?LIKE '\%baz\%' AND .*title.*I?LIKE '\%really bar\%'/) + expect(fuzzy_arel_match.to_sql).to match(/title.+I?LIKE '%foo%' AND .*title.*I?LIKE '%baz%' AND .*title.*I?LIKE '%really bar%'/) end end @@ -266,7 +266,7 @@ RSpec.describe Gitlab::SQL::Pattern do subject(:fuzzy_arel_match) { Project.fuzzy_arel_match(Route.arel_table[:path], query) } it 'returns a condition with the table and column name' do - expect(fuzzy_arel_match.to_sql).to match(/"routes"."path".*ILIKE '\%foo\%'/) + expect(fuzzy_arel_match.to_sql).to match(/"routes"."path".*ILIKE '%foo%'/) end end end diff --git a/spec/lib/gitlab/ssh/signature_spec.rb b/spec/lib/gitlab/ssh/signature_spec.rb index e8d366f0762..5149972dbf9 100644 --- a/spec/lib/gitlab/ssh/signature_spec.rb +++ b/spec/lib/gitlab/ssh/signature_spec.rb @@ -5,20 +5,20 @@ require 'spec_helper' RSpec.describe Gitlab::Ssh::Signature do # ssh-keygen -t ed25519 let_it_be(:committer_email) { 'ssh-commit-test@example.com' } - let_it_be(:public_key_text) { 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJKOfqOH0fDde+Ua/1SObkXB1CEDF5M6UfARMpW3F87u' } + let_it_be(:public_key_text) { 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHZ8NHEnCIpC4mnot+BRxv6L+fq+TnN1CgsRrHWLmfwb' } let_it_be_with_reload(:user) { create(:user, email: committer_email) } - let_it_be_with_reload(:key) { create(:key, key: public_key_text, user: user) } + let_it_be_with_reload(:key) { create(:key, usage_type: :signing, key: public_key_text, user: user) } let(:signed_text) { 'This message was signed by an ssh key' } let(:signature_text) do - # ssh-keygen -Y sign -n file -f id_test message.txt + # ssh-keygen -Y sign -n git -f id_test message.txt <<~SIG -----BEGIN SSH SIGNATURE----- - U1NIU0lHAAAAAQAAADMAAAALc3NoLWVkMjU1MTkAAAAgko5+o4fR8N175Rr/VI5uRcHUIQ - MXkzpR8BEylbcXzu4AAAAEZmlsZQAAAAAAAAAGc2hhNTEyAAAAUwAAAAtzc2gtZWQyNTUx - OQAAAECQa95KgBkgbMwIPNwHRjHu0WYrKvAc5O/FaBXlTDcPWQHi8WRDhbPNN6MqSYLg/S - hsei6Y8VYPv85StrEHYdoF + U1NIU0lHAAAAAQAAADMAAAALc3NoLWVkMjU1MTkAAAAgdnw0cScIikLiaei34FHG/ov5+r + 5Oc3UKCxGsdYuZ/BsAAAADZ2l0AAAAAAAAAAZzaGE1MTIAAABTAAAAC3NzaC1lZDI1NTE5 + AAAAQDWOEauf0jXyA9caa5bOgK5QZD6c69pm+EbG3GMw5QBL3N/Gt+r413McCSJFohWWBk + Lxemg8NzZ0nB7lTFbaxQc= -----END SSH SIGNATURE----- SIG end @@ -51,37 +51,37 @@ RSpec.describe Gitlab::Ssh::Signature do context 'when using an RSA key' do let(:public_key_text) do <<~KEY.delete("\n") - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCr3ucg9tLf87S2TxgeDaO4Cs5Mzv7wwi5w - OnSG8hE/Zj7xzf0kXAYns/dHhPilkQMCulMQuGGprGzDJXZ9WrrVDHgBj2+kLB8cc+XYIb29 - HPsoz5a1T776wWrzs5cw3Vbb0ZEMPG27SfJ+HtIqnIAcgBoRxgP/+I9we7tVxrTuog/9jSzU - H1IscwfwgKdUrvN5cyhqqxWspwZVlf6s4jaVjC9sKlF7u9CBCxqM2G7GZRKH2sEV2Tw0mT4z - 39UQ5uz9+4hxWChosiQChrT9zSJDGWQm3WGn5ubYPeB/xINEKkFxuEupnSK7l8PQxeLAwlcN - YHKMkHdO16O6PlpxvcLR1XVy4F12NXCxFjTr8GmFvJTvevf9iuFRmYQpffqm+EMN0shuhPag - Z1poVK7ZMO49b4HD6csGwDjXEgNAnyi7oPV1WMHVy+xi2j+yaAgiVk50kgTwp9sGkHTiMTM8 - YWjCq+Hb+HXLINmqO5V1QChT7PAFYycmQ0Fe2x39eLLMHy0= + ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDkq6ko8LMxf2NwyJKh+77KSDc7/ynPgUJD + IopkhqftuHFYe2Y+V3MBJnpzfSRwR2xGfXQUUzLU9AGyfZIO/ZLK2yvfhlO3k//5PbAaZb3y + urlnF9T1d2nhtfi8wuzsEn7Boh6qdoWPFIsloAL/X0PXH1HWKmzyNer92HKGrnWFfaaEMo0n + T3ureAhRG4IONyUcOK+DyoH+YbxXSlHnLO2oHHlWaP9RrJCHbfAQbfDhaZCI0cNkXXOwUwA4 + yWGzDibfXZTvaYxpjbz1xoHmCAq8IrobCgkQaEg3PH3vPGnbP0TpViXjMnZyBZyT7tg9WHBV + kAsl0CizyUgZHPAPYuqKy5JNlnjVjeqYeIgdN4Tj7hpJ1n0hVpRk4zQNYRmAAj3GNqgPAsd0 + 3i4rW8cqmhO0fmhP5DgQ7Mt5S9AgcTcCr6niPacK34XrwKiRjxXmCLjr36q8wuRU3QdMt+MK + Zxk/qJdAUIltz+nuGiwct0w+sWefYzmiRXu6hljBBrRAvnU= KEY end let(:signature_text) do <<~SIG -----BEGIN SSH SIGNATURE----- - U1NIU0lHAAAAAQAAAZcAAAAHc3NoLXJzYQAAAAMBAAEAAAGBAKve5yD20t/ztLZPGB4No7 - gKzkzO/vDCLnA6dIbyET9mPvHN/SRcBiez90eE+KWRAwK6UxC4YamsbMMldn1autUMeAGP - b6QsHxxz5dghvb0c+yjPlrVPvvrBavOzlzDdVtvRkQw8bbtJ8n4e0iqcgByAGhHGA//4j3 - B7u1XGtO6iD/2NLNQfUixzB/CAp1Su83lzKGqrFaynBlWV/qziNpWML2wqUXu70IELGozY - bsZlEofawRXZPDSZPjPf1RDm7P37iHFYKGiyJAKGtP3NIkMZZCbdYafm5tg94H/Eg0QqQX - G4S6mdIruXw9DF4sDCVw1gcoyQd07Xo7o+WnG9wtHVdXLgXXY1cLEWNOvwaYW8lO969/2K - 4VGZhCl9+qb4Qw3SyG6E9qBnWmhUrtkw7j1vgcPpywbAONcSA0CfKLug9XVYwdXL7GLaP7 - JoCCJWTnSSBPCn2waQdOIxMzxhaMKr4dv4dcsg2ao7lXVAKFPs8AVjJyZDQV7bHf14sswf - LQAAAARmaWxlAAAAAAAAAAZzaGE1MTIAAAGUAAAADHJzYS1zaGEyLTUxMgAAAYAXgXpXWw - A1fYHTUON+e1yrTw8AKB4ymfqpR9Zr1OUmYUKJ9xXvvyNCfKHL6XD14CkMu1Tx8Z3TTPG9 - C6uAXBniKRwwaLVOKffZMshf5sbjcy65KkqBPC7n/cDiCAeoJ8Y05trEDV62+pOpB2lLdv - pwwg2o0JaoLbdRcKCD0pw1u0O7VDDngTKFZ4ghHrEslxwlFruht1h9hs3rmdITlT0RMNuU - PHGAIB56u4E4UeoMd3D5rga+4Boj0s6551VgP3vCmcz9ZojPHhTCQdUZU1yHdEBTadYTq6 - UWHhQwDCUDkSNKCRxWo6EyKZQeTakedAt4qkdSpSUCKOJGWKmPOfAm2/sDEmSxffRdxRRg - QUe8lklyFTZd6U/ZkJ/y7VR46fcSkEqLSLd9jAZT/3HJXbZfULpwsTcvcLcJLkCuzHEaU1 - LRyJBsanLCYHTv7ep5PvIuAngUWrXK2eb7oacVs94mWXfs1PG482Ym4+bZA5u0QliGTVaC - M2EMhRTf0cqFuA4= + U1NIU0lHAAAAAQAAAZcAAAAHc3NoLXJzYQAAAAMBAAEAAAGBAOSrqSjwszF/Y3DIkqH7vs + pINzv/Kc+BQkMiimSGp+24cVh7Zj5XcwEmenN9JHBHbEZ9dBRTMtT0AbJ9kg79ksrbK9+G + U7eT//k9sBplvfK6uWcX1PV3aeG1+LzC7OwSfsGiHqp2hY8UiyWgAv9fQ9cfUdYqbPI16v + 3YcoaudYV9poQyjSdPe6t4CFEbgg43JRw4r4PKgf5hvFdKUecs7agceVZo/1GskIdt8BBt + 8OFpkIjRw2Rdc7BTADjJYbMOJt9dlO9pjGmNvPXGgeYICrwiuhsKCRBoSDc8fe88ads/RO + lWJeMydnIFnJPu2D1YcFWQCyXQKLPJSBkc8A9i6orLkk2WeNWN6ph4iB03hOPuGknWfSFW + lGTjNA1hGYACPcY2qA8Cx3TeLitbxyqaE7R+aE/kOBDsy3lL0CBxNwKvqeI9pwrfhevAqJ + GPFeYIuOvfqrzC5FTdB0y34wpnGT+ol0BQiW3P6e4aLBy3TD6xZ59jOaJFe7qGWMEGtEC+ + dQAAAANnaXQAAAAAAAAABnNoYTUxMgAAAZQAAAAMcnNhLXNoYTItNTEyAAABgEnuYyYOlM + CSR+wvmBY7eKHzFor5ByM7N4F7VZAGKK/vbS3C38xDdiJZwsZUscpe5WspJVCWUTkFxXjn + GW7vseIfJBVkyqnu2uN8X1j/VDLFESEajcchPhPxtfAMK1/NL99O7rCrYX2pmpkm9tWsFk + NX5B93sRyDUnHAOkB+zdqU8P0xdzc8kmBl5OOqu1rSjZIgnQjcauEIRIUN+rFuiRRmIvJp + UvMhkKSsRCH93btGW7A6x5e4iPzP+Em0UFYJdOx2lvu9aVAktQzysGwDN+9c4IC+07UHKT + UIE5jSbR1QKfavcywNQnCltQ2bTxpnm4A6QHKcdr9Q57dV014FgtmtT/Pw03iyl5MwbEqW + 7YEHSkMyAcd1rjEpOCN2pJjjbrOKLePG0R2ffgvVJnTWGFklCxsJ1/7IASHst1wg1/gu1g + Kx/TEv+gOKpehAgs2Sz/4kZtFuHO2dbHYC3UrPR5HT8JnQWeCfiT0qwsVQ6xribw0jEYyd + ZBNWKkPdNocAbA== -----END SSH SIGNATURE----- SIG end @@ -98,10 +98,10 @@ RSpec.describe Gitlab::Ssh::Signature do let(:signature_text) do <<~SIG -----BEGIN SSH SIGNATURE----- - U1NIU0lHAAAAAQAAADMAAAALc3NoLWVkMjU1MTkAAAAgko5+o4fR8N175Rr/VI5uRcHUIQ - MXkzpR8BEylbcXzu4AAAAEZmlsZQAAAAAAAAAGc2hhNTEyAAAAUwAAAAtzc2gtZWQyNTUx - OQAAAEC1y2I7o3KqKFlnM+MLkhIo+uRX3YQOYCqycfibyfvmkZTcwqMxgNBInBM9pY3VvS - sbW2iEdgz34agHbi+1BHIM + U1NIU0lHAAAAAQAAADMAAAALc3NoLWVkMjU1MTkAAAAgdnw0cScIikLiaei34FHG/ov5+r + 5Oc3UKCxGsdYuZ/BsAAAADZ2l0AAAAAAAAAAZzaGE1MTIAAABTAAAAC3NzaC1lZDI1NTE5 + AAAAQP2liwaQ44PC9oXf5Xzjq20WLdWEK9nyonvDGtduGUXMOL4yP5A6WvKz7kSt7Vba/U + MNK0nmnNc7Aokfh/2eRQE= -----END SSH SIGNATURE----- SIG end @@ -151,16 +151,32 @@ RSpec.describe Gitlab::Ssh::Signature do context 'when user email is not verified' do before do + email = user.emails.find_by(email: committer_email) + email.update!(confirmed_at: nil) user.update!(confirmed_at: nil) end - it_behaves_like 'unverified signature' + it 'reports unverified status' do + expect(signature.verification_status).to eq(:unverified) + end + end + + context 'when no user exist with the committer email' do + before do + user.delete + end + + it 'reports other_user status' do + expect(signature.verification_status).to eq(:other_user) + end end context 'when no user exists with the committer email' do let(:committer_email) { 'different-email+ssh-commit-test@example.com' } - it_behaves_like 'unverified signature' + it 'reports other_user status' do + expect(signature.verification_status).to eq(:other_user) + end end context 'when signature is invalid' do @@ -178,6 +194,21 @@ RSpec.describe Gitlab::Ssh::Signature do it_behaves_like 'unverified signature' end + context 'when signature is for a different namespace' do + let(:signature_text) do + <<~SIG + -----BEGIN SSH SIGNATURE----- + U1NIU0lHAAAAAQAAADMAAAALc3NoLWVkMjU1MTkAAAAgdnw0cScIikLiaei34FHG/ov5+r + 5Oc3UKCxGsdYuZ/BsAAAAEZmlsZQAAAAAAAAAGc2hhNTEyAAAAUwAAAAtzc2gtZWQyNTUx + OQAAAEAd6Psg4D/5IdSVTy35D4t2iNX4udJnX8JrUCjQl0GoPl1vzPjgyvxdzdoQl6bh1w + 4rror3RuzUYBGzIioIc1MP + -----END SSH SIGNATURE----- + SIG + end + + it_behaves_like 'unverified signature' + end + context 'when signature is for a different message' do let(:signature_text) do <<~SIG @@ -204,13 +235,25 @@ RSpec.describe Gitlab::Ssh::Signature do it_behaves_like 'unverified signature' end - context 'when key does not exist in GitLab' do - before do - key.delete + context 'when the signing key does not exist in GitLab' do + context 'when the key is not a signing one' do + before do + key.auth! + end + + it 'reports unknown_key status' do + expect(signature.verification_status).to eq(:unknown_key) + end end - it 'reports unknown_key status' do - expect(signature.verification_status).to eq(:unknown_key) + context 'when the key is removed' do + before do + key.delete + end + + it 'reports unknown_key status' do + expect(signature.verification_status).to eq(:unknown_key) + end end end diff --git a/spec/lib/gitlab/ssh_public_key_spec.rb b/spec/lib/gitlab/ssh_public_key_spec.rb index a2524314458..d4b0b1ea53b 100644 --- a/spec/lib/gitlab/ssh_public_key_spec.rb +++ b/spec/lib/gitlab/ssh_public_key_spec.rb @@ -260,8 +260,7 @@ RSpec.describe Gitlab::SSHPublicKey, lib: true, fips_mode: false do context 'when the key is represented by a subclass of the class that is in the list of supported technologies' do it 'raises error' do rsa_subclass = Class.new(described_class.technology(:rsa).key_class) do - def initialize - end + def initialize; end end key = rsa_subclass.new diff --git a/spec/lib/gitlab/tracking/destinations/snowplow_spec.rb b/spec/lib/gitlab/tracking/destinations/snowplow_spec.rb index 1d4725cf405..e79535358f9 100644 --- a/spec/lib/gitlab/tracking/destinations/snowplow_spec.rb +++ b/spec/lib/gitlab/tracking/destinations/snowplow_spec.rb @@ -3,8 +3,11 @@ require 'spec_helper' RSpec.describe Gitlab::Tracking::Destinations::Snowplow, :do_not_stub_snowplow_by_default do - let(:emitter) { SnowplowTracker::Emitter.new('localhost', buffer_size: 1) } - let(:tracker) { SnowplowTracker::Tracker.new(emitter, SnowplowTracker::Subject.new, 'namespace', 'app_id') } + let(:emitter) { SnowplowTracker::Emitter.new(endpoint: 'localhost', options: { buffer_size: 1 }) } + let(:tracker) do + SnowplowTracker::Tracker.new(emitters: [emitter], subject: SnowplowTracker::Subject.new, namespace: 'namespace', + app_id: 'app_id') + end before do stub_application_setting(snowplow_collector_hostname: 'gitfoo.com') @@ -21,16 +24,19 @@ RSpec.describe Gitlab::Tracking::Destinations::Snowplow, :do_not_stub_snowplow_b expect(SnowplowTracker::AsyncEmitter) .to receive(:new) - .with('gitfoo.com', - { protocol: 'https', - on_success: subject.method(:increment_successful_events_emissions), - on_failure: subject.method(:failure_callback) }) + .with(endpoint: 'gitfoo.com', + options: { protocol: 'https', + on_success: subject.method(:increment_successful_events_emissions), + on_failure: subject.method(:failure_callback) }) .and_return(emitter) expect(SnowplowTracker::Tracker) .to receive(:new) - .with(emitter, an_instance_of(SnowplowTracker::Subject), described_class::SNOWPLOW_NAMESPACE, '_abc123_') - .and_return(tracker) + .with(emitters: [emitter], + subject: an_instance_of(SnowplowTracker::Subject), + namespace: described_class::SNOWPLOW_NAMESPACE, + app_id: '_abc123_') + .and_return(tracker) end describe '#event' do @@ -41,7 +47,8 @@ RSpec.describe Gitlab::Tracking::Destinations::Snowplow, :do_not_stub_snowplow_b expect(tracker) .to have_received(:track_struct_event) - .with('category', 'action', 'label', 'property', 1.5, nil, (Time.now.to_f * 1000).to_i) + .with(category: 'category', action: 'action', label: 'label', property: 'property', value: 1.5, context: nil, + tstamp: (Time.now.to_f * 1000).to_i) end it 'increase total snowplow events counter' do diff --git a/spec/lib/gitlab/tracking/event_definition_spec.rb b/spec/lib/gitlab/tracking/event_definition_spec.rb index 623009e9a30..c8e616b092b 100644 --- a/spec/lib/gitlab/tracking/event_definition_spec.rb +++ b/spec/lib/gitlab/tracking/event_definition_spec.rb @@ -83,6 +83,11 @@ RSpec.describe Gitlab::Tracking::EventDefinition do subject { described_class.definitions } + after do + FileUtils.rm_rf(metric1) + FileUtils.rm_rf(metric2) + end + it 'has empty list when there are no definition files' do is_expected.to be_empty end @@ -92,10 +97,5 @@ RSpec.describe Gitlab::Tracking::EventDefinition do is_expected.to be_one end - - after do - FileUtils.rm_rf(metric1) - FileUtils.rm_rf(metric2) - end end end diff --git a/spec/lib/gitlab/tracking/service_ping_context_spec.rb b/spec/lib/gitlab/tracking/service_ping_context_spec.rb index d70dfaa4e0b..7530650b902 100644 --- a/spec/lib/gitlab/tracking/service_ping_context_spec.rb +++ b/spec/lib/gitlab/tracking/service_ping_context_spec.rb @@ -4,16 +4,55 @@ require 'spec_helper' RSpec.describe Gitlab::Tracking::ServicePingContext do describe '#init' do - it 'does not accept unsupported data sources' do - expect { described_class.new(data_source: :random, event: 'event a') }.to raise_error(ArgumentError) + using RSpec::Parameterized::TableSyntax + + context 'with valid configuration' do + where(:data_source, :event, :key_path) do + :redis | nil | 'counts.some_metric' + :redis_hll | 'some_event' | nil + end + + with_them do + it 'does not raise errors' do + expect { described_class.new(data_source: data_source, event: event, key_path: key_path) }.not_to raise_error + end + end + end + + context 'with invalid configuration' do + where(:data_source, :event, :key_path) do + :redis | nil | nil + :redis | 'some_event' | nil + :redis_hll | nil | nil + :redis_hll | nil | 'some key_path' + :random | 'some_event' | nil + end + + with_them do + subject(:new_instance) { described_class.new(data_source: data_source, event: event, key_path: key_path) } + + it 'does not raise errors' do + expect { new_instance }.to raise_error(ArgumentError) + end + end end end describe '#to_context' do - let(:subject) { described_class.new(data_source: :redis_hll, event: 'sample_event') } + context 'for redis_hll data source' do + let(:context_instance) { described_class.new(data_source: :redis_hll, event: 'sample_event') } + + it 'contains event_name' do + expect(context_instance.to_context.to_json.dig(:data, :event_name)).to eq('sample_event') + end + end + + context 'for redis data source' do + let(:context_instance) { described_class.new(data_source: :redis, key_path: 'counts.sample_metric') } - it 'contains event_name' do - expect(subject.to_context.to_json.dig(:data, :event_name)).to eq('sample_event') + it 'contains event_name' do + expect(context_instance.to_context.to_json.dig(:data, :key_path)).to eq('counts.sample_metric') + end end end end diff --git a/spec/lib/gitlab/tracking_spec.rb b/spec/lib/gitlab/tracking_spec.rb index e11175c776d..99ca402616a 100644 --- a/spec/lib/gitlab/tracking_spec.rb +++ b/spec/lib/gitlab/tracking_spec.rb @@ -180,15 +180,6 @@ RSpec.describe Gitlab::Tracking do it_behaves_like 'delegates to destination', Gitlab::Tracking::Destinations::SnowplowMicro end - - it 'tracks errors' do - expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).with( - an_instance_of(ContractError), - snowplow_category: nil, snowplow_action: 'some_action' - ) - - described_class.event(nil, 'some_action') - end end describe '.definition' do diff --git a/spec/lib/gitlab/url_blocker_spec.rb b/spec/lib/gitlab/url_blocker_spec.rb index 8f505606e04..05f7af7606d 100644 --- a/spec/lib/gitlab/url_blocker_spec.rb +++ b/spec/lib/gitlab/url_blocker_spec.rb @@ -5,8 +5,10 @@ require 'spec_helper' RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do include StubRequests + let(:schemes) { %w[http https] } + describe '#validate!' do - subject { described_class.validate!(import_url) } + subject { described_class.validate!(import_url, schemes: schemes) } shared_examples 'validates URI and hostname' do it 'runs the url validations' do @@ -59,7 +61,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end context 'when allow_object_storage is true' do - subject { described_class.validate!(import_url, allow_object_storage: true) } + subject { described_class.validate!(import_url, allow_object_storage: true, schemes: schemes) } context 'with a local domain name' do let(:host) { 'http://review-minio-svc.svc:9000' } @@ -218,7 +220,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end context 'disabled DNS rebinding protection' do - subject { described_class.validate!(import_url, dns_rebind_protection: false) } + subject { described_class.validate!(import_url, dns_rebind_protection: false, schemes: schemes) } context 'when URI is internal' do let(:import_url) { 'http://localhost' } @@ -278,115 +280,114 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do it 'allows imports from configured web host and port' do import_url = "http://#{Gitlab.host_with_port}/t.git" - expect(described_class.blocked_url?(import_url)).to be false + expect(described_class.blocked_url?(import_url, schemes: schemes)).to be false end it 'allows mirroring from configured SSH host and port' do import_url = "ssh://#{Gitlab.config.gitlab_shell.ssh_host}:#{Gitlab.config.gitlab_shell.ssh_port}/t.git" - expect(described_class.blocked_url?(import_url)).to be false + expect(described_class.blocked_url?(import_url, schemes: schemes)).to be false end it 'returns true for bad localhost hostname' do - expect(described_class.blocked_url?('https://localhost:65535/foo/foo.git')).to be true + expect(described_class.blocked_url?('https://localhost:65535/foo/foo.git', schemes: schemes)).to be true end it 'returns true for bad port' do - expect(described_class.blocked_url?('https://gitlab.com:25/foo/foo.git', ports: ports)).to be true + expect(described_class.blocked_url?('https://gitlab.com:25/foo/foo.git', ports: ports, schemes: schemes)).to be true end it 'returns true for bad scheme' do expect(described_class.blocked_url?('https://gitlab.com/foo/foo.git', schemes: ['https'])).to be false - expect(described_class.blocked_url?('https://gitlab.com/foo/foo.git')).to be false expect(described_class.blocked_url?('https://gitlab.com/foo/foo.git', schemes: ['http'])).to be true end it 'returns true for bad protocol on configured web/SSH host and ports' do web_url = "javascript://#{Gitlab.host_with_port}/t.git%0aalert(1)" - expect(described_class.blocked_url?(web_url)).to be true + expect(described_class.blocked_url?(web_url, schemes: schemes)).to be true ssh_url = "javascript://#{Gitlab.config.gitlab_shell.ssh_host}:#{Gitlab.config.gitlab_shell.ssh_port}/t.git%0aalert(1)" - expect(described_class.blocked_url?(ssh_url)).to be true + expect(described_class.blocked_url?(ssh_url, schemes: schemes)).to be true end it 'returns true for localhost IPs' do - expect(described_class.blocked_url?('https://[0:0:0:0:0:0:0:0]/foo/foo.git')).to be true - expect(described_class.blocked_url?('https://0.0.0.0/foo/foo.git')).to be true - expect(described_class.blocked_url?('https://[::]/foo/foo.git')).to be true + expect(described_class.blocked_url?('https://[0:0:0:0:0:0:0:0]/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://0.0.0.0/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://[::]/foo/foo.git', schemes: schemes)).to be true end it 'returns true for loopback IP' do - expect(described_class.blocked_url?('https://127.0.0.2/foo/foo.git')).to be true - expect(described_class.blocked_url?('https://127.0.0.1/foo/foo.git')).to be true - expect(described_class.blocked_url?('https://[::1]/foo/foo.git')).to be true + expect(described_class.blocked_url?('https://127.0.0.2/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://127.0.0.1/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://[::1]/foo/foo.git', schemes: schemes)).to be true end it 'returns true for alternative version of 127.0.0.1 (0177.1)' do - expect(described_class.blocked_url?('https://0177.1:65535/foo/foo.git')).to be true + expect(described_class.blocked_url?('https://0177.1:65535/foo/foo.git', schemes: schemes)).to be true end it 'returns true for alternative version of 127.0.0.1 (017700000001)' do - expect(described_class.blocked_url?('https://017700000001:65535/foo/foo.git')).to be true + expect(described_class.blocked_url?('https://017700000001:65535/foo/foo.git', schemes: schemes)).to be true end it 'returns true for alternative version of 127.0.0.1 (0x7f.1)' do - expect(described_class.blocked_url?('https://0x7f.1:65535/foo/foo.git')).to be true + expect(described_class.blocked_url?('https://0x7f.1:65535/foo/foo.git', schemes: schemes)).to be true end it 'returns true for alternative version of 127.0.0.1 (0x7f.0.0.1)' do - expect(described_class.blocked_url?('https://0x7f.0.0.1:65535/foo/foo.git')).to be true + expect(described_class.blocked_url?('https://0x7f.0.0.1:65535/foo/foo.git', schemes: schemes)).to be true end it 'returns true for alternative version of 127.0.0.1 (0x7f000001)' do - expect(described_class.blocked_url?('https://0x7f000001:65535/foo/foo.git')).to be true + expect(described_class.blocked_url?('https://0x7f000001:65535/foo/foo.git', schemes: schemes)).to be true end it 'returns true for alternative version of 127.0.0.1 (2130706433)' do - expect(described_class.blocked_url?('https://2130706433:65535/foo/foo.git')).to be true + expect(described_class.blocked_url?('https://2130706433:65535/foo/foo.git', schemes: schemes)).to be true end it 'returns true for alternative version of 127.0.0.1 (127.000.000.001)' do - expect(described_class.blocked_url?('https://127.000.000.001:65535/foo/foo.git')).to be true + expect(described_class.blocked_url?('https://127.000.000.001:65535/foo/foo.git', schemes: schemes)).to be true end it 'returns true for alternative version of 127.0.0.1 (127.0.1)' do - expect(described_class.blocked_url?('https://127.0.1:65535/foo/foo.git')).to be true + expect(described_class.blocked_url?('https://127.0.1:65535/foo/foo.git', schemes: schemes)).to be true end context 'with ipv6 mapped address' do it 'returns true for localhost IPs' do - expect(described_class.blocked_url?('https://[0:0:0:0:0:ffff:0.0.0.0]/foo/foo.git')).to be true - expect(described_class.blocked_url?('https://[::ffff:0.0.0.0]/foo/foo.git')).to be true - expect(described_class.blocked_url?('https://[::ffff:0:0]/foo/foo.git')).to be true + expect(described_class.blocked_url?('https://[0:0:0:0:0:ffff:0.0.0.0]/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://[::ffff:0.0.0.0]/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://[::ffff:0:0]/foo/foo.git', schemes: schemes)).to be true end it 'returns true for loopback IPs' do - expect(described_class.blocked_url?('https://[0:0:0:0:0:ffff:127.0.0.1]/foo/foo.git')).to be true - expect(described_class.blocked_url?('https://[::ffff:127.0.0.1]/foo/foo.git')).to be true - expect(described_class.blocked_url?('https://[::ffff:7f00:1]/foo/foo.git')).to be true - expect(described_class.blocked_url?('https://[0:0:0:0:0:ffff:127.0.0.2]/foo/foo.git')).to be true - expect(described_class.blocked_url?('https://[::ffff:127.0.0.2]/foo/foo.git')).to be true - expect(described_class.blocked_url?('https://[::ffff:7f00:2]/foo/foo.git')).to be true + expect(described_class.blocked_url?('https://[0:0:0:0:0:ffff:127.0.0.1]/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://[::ffff:127.0.0.1]/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://[::ffff:7f00:1]/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://[0:0:0:0:0:ffff:127.0.0.2]/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://[::ffff:127.0.0.2]/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://[::ffff:7f00:2]/foo/foo.git', schemes: schemes)).to be true end end it 'returns true for a non-alphanumeric hostname' do aggregate_failures do - expect(described_class).to be_blocked_url('ssh://-oProxyCommand=whoami/a') + expect(described_class).to be_blocked_url('ssh://-oProxyCommand=whoami/a', schemes: ['ssh']) # The leading character here is a Unicode "soft hyphen" - expect(described_class).to be_blocked_url('ssh://oProxyCommand=whoami/a') + expect(described_class).to be_blocked_url('ssh://oProxyCommand=whoami/a', schemes: ['ssh']) # Unicode alphanumerics are allowed - expect(described_class).not_to be_blocked_url('ssh://ğitlab.com/a') + expect(described_class).not_to be_blocked_url('ssh://ğitlab.com/a', schemes: ['ssh']) end end it 'returns true for invalid URL' do - expect(described_class.blocked_url?('http://:8080')).to be true + expect(described_class.blocked_url?('http://:8080', schemes: schemes)).to be true end it 'returns false for legitimate URL' do - expect(described_class.blocked_url?('https://gitlab.com/foo/foo.git')).to be false + expect(described_class.blocked_url?('https://gitlab.com/foo/foo.git', schemes: schemes)).to be false end context 'when allow_local_network is' do @@ -471,33 +472,33 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end context 'true (default)' do - it_behaves_like 'allows local requests', { allow_localhost: true, allow_local_network: true } + it_behaves_like 'allows local requests', { allow_localhost: true, allow_local_network: true, schemes: %w[http https] } end context 'false' do it 'blocks urls from private networks' do local_ips.each do |ip| stub_domain_resolv(fake_domain, ip) do - expect(described_class).to be_blocked_url("http://#{fake_domain}", allow_local_network: false) + expect(described_class).to be_blocked_url("http://#{fake_domain}", allow_local_network: false, schemes: schemes) end - expect(described_class).to be_blocked_url("http://#{ip}", allow_local_network: false) + expect(described_class).to be_blocked_url("http://#{ip}", allow_local_network: false, schemes: schemes) end end it 'blocks IPv4 link-local endpoints' do - expect(described_class).to be_blocked_url('http://169.254.169.254', allow_local_network: false) - expect(described_class).to be_blocked_url('http://169.254.168.100', allow_local_network: false) + expect(described_class).to be_blocked_url('http://169.254.169.254', allow_local_network: false, schemes: schemes) + expect(described_class).to be_blocked_url('http://169.254.168.100', allow_local_network: false, schemes: schemes) end it 'blocks IPv6 link-local endpoints' do - expect(described_class).to be_blocked_url('http://[0:0:0:0:0:ffff:169.254.169.254]', allow_local_network: false) - expect(described_class).to be_blocked_url('http://[::ffff:169.254.169.254]', allow_local_network: false) - expect(described_class).to be_blocked_url('http://[::ffff:a9fe:a9fe]', allow_local_network: false) - expect(described_class).to be_blocked_url('http://[0:0:0:0:0:ffff:169.254.168.100]', allow_local_network: false) - expect(described_class).to be_blocked_url('http://[::ffff:169.254.168.100]', allow_local_network: false) - expect(described_class).to be_blocked_url('http://[::ffff:a9fe:a864]', allow_local_network: false) - expect(described_class).to be_blocked_url('http://[fe80::c800:eff:fe74:8]', allow_local_network: false) + expect(described_class).to be_blocked_url('http://[0:0:0:0:0:ffff:169.254.169.254]', allow_local_network: false, schemes: schemes) + expect(described_class).to be_blocked_url('http://[::ffff:169.254.169.254]', allow_local_network: false, schemes: schemes) + expect(described_class).to be_blocked_url('http://[::ffff:a9fe:a9fe]', allow_local_network: false, schemes: schemes) + expect(described_class).to be_blocked_url('http://[0:0:0:0:0:ffff:169.254.168.100]', allow_local_network: false, schemes: schemes) + expect(described_class).to be_blocked_url('http://[::ffff:169.254.168.100]', allow_local_network: false, schemes: schemes) + expect(described_class).to be_blocked_url('http://[::ffff:a9fe:a864]', allow_local_network: false, schemes: schemes) + expect(described_class).to be_blocked_url('http://[fe80::c800:eff:fe74:8]', allow_local_network: false, schemes: schemes) end it 'blocks limited broadcast address 255.255.255.255 and variants' do @@ -507,7 +508,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') limited_broadcast_address_variants.each do |variant| - expect(described_class).to be_blocked_url("https://#{variant}", allow_local_network: false), "Expected #{variant} to be blocked" + expect(described_class).to be_blocked_url("https://#{variant}", allow_local_network: false, schemes: schemes), "Expected #{variant} to be blocked" end end @@ -515,7 +516,8 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do let(:url_blocker_attributes) do { allow_localhost: false, - allow_local_network: false + allow_local_network: false, + schemes: schemes } end @@ -545,7 +547,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do ] end - it_behaves_like 'allows local requests', { allow_localhost: false, allow_local_network: false } + it_behaves_like 'allows local requests', { allow_localhost: false, allow_local_network: false, schemes: %w[http https] } it 'allows IP when dns_rebind_protection is disabled' do url = "http://example.com" @@ -622,7 +624,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end it do - expect(described_class).not_to be_blocked_url(url, dns_rebind_protection: dns_rebind_value) + expect(described_class).not_to be_blocked_url(url, dns_rebind_protection: dns_rebind_value, schemes: schemes) end end @@ -676,26 +678,26 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do context 'when enforce_user is' do context 'false (default)' do it 'does not block urls with a non-alphanumeric username' do - expect(described_class).not_to be_blocked_url('ssh://-oProxyCommand=whoami@example.com/a') + expect(described_class).not_to be_blocked_url('ssh://-oProxyCommand=whoami@example.com/a', schemes: ['ssh']) # The leading character here is a Unicode "soft hyphen" - expect(described_class).not_to be_blocked_url('ssh://oProxyCommand=whoami@example.com/a') + expect(described_class).not_to be_blocked_url('ssh://oProxyCommand=whoami@example.com/a', schemes: ['ssh']) # Unicode alphanumerics are allowed - expect(described_class).not_to be_blocked_url('ssh://ğitlab@example.com/a') + expect(described_class).not_to be_blocked_url('ssh://ğitlab@example.com/a', schemes: ['ssh']) end end context 'true' do it 'blocks urls with a non-alphanumeric username' do aggregate_failures do - expect(described_class).to be_blocked_url('ssh://-oProxyCommand=whoami@example.com/a', enforce_user: true) + expect(described_class).to be_blocked_url('ssh://-oProxyCommand=whoami@example.com/a', enforce_user: true, schemes: ['ssh']) # The leading character here is a Unicode "soft hyphen" - expect(described_class).to be_blocked_url('ssh://oProxyCommand=whoami@example.com/a', enforce_user: true) + expect(described_class).to be_blocked_url('ssh://oProxyCommand=whoami@example.com/a', enforce_user: true, schemes: ['ssh']) # Unicode alphanumerics are allowed - expect(described_class).not_to be_blocked_url('ssh://ğitlab@example.com/a', enforce_user: true) + expect(described_class).not_to be_blocked_url('ssh://ğitlab@example.com/a', enforce_user: true, schemes: ['ssh']) end end end @@ -703,35 +705,35 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do context 'when ascii_only is true' do it 'returns true for unicode domain' do - expect(described_class.blocked_url?('https://𝕘itⅼαƄ.com/foo/foo.bar', ascii_only: true)).to be true + expect(described_class.blocked_url?('https://𝕘itⅼαƄ.com/foo/foo.bar', ascii_only: true, schemes: schemes)).to be true end it 'returns true for unicode tld' do - expect(described_class.blocked_url?('https://gitlab.ᴄοｍ/foo/foo.bar', ascii_only: true)).to be true + expect(described_class.blocked_url?('https://gitlab.ᴄοｍ/foo/foo.bar', ascii_only: true, schemes: schemes)).to be true end it 'returns true for unicode path' do - expect(described_class.blocked_url?('https://gitlab.com/𝒇οο/𝒇οο.Ƅαꮁ', ascii_only: true)).to be true + expect(described_class.blocked_url?('https://gitlab.com/𝒇οο/𝒇οο.Ƅαꮁ', ascii_only: true, schemes: schemes)).to be true end it 'returns true for IDNA deviations' do - expect(described_class.blocked_url?('https://mißile.com/foo/foo.bar', ascii_only: true)).to be true - expect(described_class.blocked_url?('https://miςςile.com/foo/foo.bar', ascii_only: true)).to be true - expect(described_class.blocked_url?('https://gitlab.com/foo/foo.bar', ascii_only: true)).to be true - expect(described_class.blocked_url?('https://gitlab.com/foo/foo.bar', ascii_only: true)).to be true + expect(described_class.blocked_url?('https://mißile.com/foo/foo.bar', ascii_only: true, schemes: schemes)).to be true + expect(described_class.blocked_url?('https://miςςile.com/foo/foo.bar', ascii_only: true, schemes: schemes)).to be true + expect(described_class.blocked_url?('https://gitlab.com/foo/foo.bar', ascii_only: true, schemes: schemes)).to be true + expect(described_class.blocked_url?('https://gitlab.com/foo/foo.bar', ascii_only: true, schemes: schemes)).to be true end end it 'blocks urls with invalid ip address' do stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') - expect(described_class).to be_blocked_url('http://8.8.8.8.8') + expect(described_class).to be_blocked_url('http://8.8.8.8.8', schemes: schemes) end it 'blocks urls whose hostname cannot be resolved' do stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') - expect(described_class).to be_blocked_url('http://foobar.x') + expect(described_class).to be_blocked_url('http://foobar.x', schemes: schemes) end context 'when gitlab is running on a non-default port' do @@ -743,13 +745,13 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do it 'returns true for url targeting the wrong port' do stub_domain_resolv('gitlab.local', '127.0.0.1') do - expect(described_class).to be_blocked_url("http://gitlab.local/foo") + expect(described_class).to be_blocked_url("http://gitlab.local/foo", schemes: schemes) end end it 'does not block url on gitlab port' do stub_domain_resolv('gitlab.local', '127.0.0.1') do - expect(described_class).not_to be_blocked_url("http://gitlab.local:#{gitlab_port}/foo") + expect(described_class).not_to be_blocked_url("http://gitlab.local:#{gitlab_port}/foo", schemes: schemes) end end end diff --git a/spec/lib/gitlab/usage/metric_definition_spec.rb b/spec/lib/gitlab/usage/metric_definition_spec.rb index 931340947a2..4b835d11975 100644 --- a/spec/lib/gitlab/usage/metric_definition_spec.rb +++ b/spec/lib/gitlab/usage/metric_definition_spec.rb @@ -233,6 +233,11 @@ RSpec.describe Gitlab::Usage::MetricDefinition do subject { described_class.send(:load_all!) } + after do + FileUtils.rm_rf(metric1) + FileUtils.rm_rf(metric2) + end + it 'has empty list when there are no definition files' do is_expected.to be_empty end @@ -251,11 +256,6 @@ RSpec.describe Gitlab::Usage::MetricDefinition do subject end - - after do - FileUtils.rm_rf(metric1) - FileUtils.rm_rf(metric2) - end end describe 'dump_metrics_yaml' do diff --git a/spec/lib/gitlab/usage/metrics/aggregates/aggregate_spec.rb b/spec/lib/gitlab/usage/metrics/aggregates/aggregate_spec.rb index 1f00f7bbec3..10e336e9235 100644 --- a/spec/lib/gitlab/usage/metrics/aggregates/aggregate_spec.rb +++ b/spec/lib/gitlab/usage/metrics/aggregates/aggregate_spec.rb @@ -12,6 +12,12 @@ RSpec.describe Gitlab::Usage::Metrics::Aggregates::Aggregate, :clean_gitlab_redi describe '.calculate_count_for_aggregation' do using RSpec::Parameterized::TableSyntax + before do + %w[event1 event2].each do |event_name| + allow(Gitlab::UsageDataCounters::HLLRedisCounter).to receive(:known_event?).with(event_name).and_return(true) + end + end + context 'with valid configuration' do where(:number_of_days, :operator, :datasource, :expected_method) do 28 | 'AND' | 'redis_hll' | :calculate_metrics_intersections diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/count_merge_request_authors_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/count_merge_request_authors_metric_spec.rb deleted file mode 100644 index 92459e92eac..00000000000 --- a/spec/lib/gitlab/usage/metrics/instrumentations/count_merge_request_authors_metric_spec.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountMergeRequestAuthorsMetric do - let(:expected_value) { 1 } - let(:start) { 30.days.ago.to_s(:db) } - let(:finish) { 2.days.ago.to_s(:db) } - - let(:expected_query) do - "SELECT COUNT(DISTINCT \"merge_requests\".\"author_id\") FROM \"merge_requests\"" \ - " WHERE \"merge_requests\".\"created_at\" BETWEEN '#{start}' AND '#{finish}'" - end - - before do - user = create(:user) - user2 = create(:user) - - create(:merge_request, created_at: 1.year.ago, author: user) - create(:merge_request, created_at: 1.week.ago, author: user2) - create(:merge_request, created_at: 1.week.ago, author: user2) - end - - it_behaves_like 'a correct instrumented metric value and query', { time_frame: '28d' } -end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/database_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/database_metric_spec.rb index f1ecc8c8ab5..8ca42a6f007 100644 --- a/spec/lib/gitlab/usage/metrics/instrumentations/database_metric_spec.rb +++ b/spec/lib/gitlab/usage/metrics/instrumentations/database_metric_spec.rb @@ -196,6 +196,22 @@ RSpec.describe Gitlab::Usage::Metrics::Instrumentations::DatabaseMetric do end end + context 'with 7 days time frame' do + subject do + database_metric_class.tap do |metric_class| + metric_class.relation { Issue } + metric_class.operation :count + end.new(time_frame: '7d') + end + + it 'calculates a correct result' do + create(:issue, created_at: 10.days.ago) + create(:issue, created_at: 5.days.ago) + + expect(subject.value).to eq(1) + end + end + context 'with additional parameters passed via options' do subject do database_metric_class.tap do |metric_class| diff --git a/spec/lib/gitlab/usage/metrics/name_suggestion_spec.rb b/spec/lib/gitlab/usage/metrics/name_suggestion_spec.rb index 24107727a8e..9dba64ff59f 100644 --- a/spec/lib/gitlab/usage/metrics/name_suggestion_spec.rb +++ b/spec/lib/gitlab/usage/metrics/name_suggestion_spec.rb @@ -44,7 +44,7 @@ RSpec.describe Gitlab::Usage::Metrics::NameSuggestion do let(:operation) { :count } let(:relation) { Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot) } let(:column) { nil } - let(:name_suggestion) { /count_<adjective describing\: '\(issues\.author_id != \d+\)'>_issues_<with>_alert_management_alerts/ } + let(:name_suggestion) { /count_<adjective describing: '\(issues\.author_id != \d+\)'>_issues_<with>_alert_management_alerts/ } end end end @@ -55,7 +55,7 @@ RSpec.describe Gitlab::Usage::Metrics::NameSuggestion do let(:operation) { :distinct_count } let(:relation) { ::Clusters::Cluster.aws_installed.enabled.where(created_at: 30.days.ago..2.days.ago ) } let(:column) { :user_id } - let(:constraints) { /<adjective describing\: '\(clusters.provider_type = \d+ AND \(cluster_providers_aws\.status IN \(\d+\)\) AND clusters\.enabled = TRUE\)'>/ } + let(:constraints) { /<adjective describing: '\(clusters.provider_type = \d+ AND \(cluster_providers_aws\.status IN \(\d+\)\) AND clusters\.enabled = TRUE\)'>/ } let(:name_suggestion) { /count_distinct_user_id_from_#{constraints}_clusters_<with>_#{constraints}_cluster_providers_aws/ } end end @@ -66,7 +66,7 @@ RSpec.describe Gitlab::Usage::Metrics::NameSuggestion do let(:operation) { :sum } let(:relation) { JiraImportState.finished } let(:column) { :imported_issues_count } - let(:name_suggestion) { /sum_imported_issues_count_from_<adjective describing\: '\(jira_imports\.status = \d+\)'>_jira_imports/ } + let(:name_suggestion) { /sum_imported_issues_count_from_<adjective describing: '\(jira_imports\.status = \d+\)'>_jira_imports/ } end end diff --git a/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb b/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb index 7e8b15d23db..83a4ea8e948 100644 --- a/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb +++ b/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb @@ -45,7 +45,7 @@ RSpec.describe Gitlab::Usage::Metrics::NamesSuggestions::Generator do it_behaves_like 'name suggestion' do # corresponding metric is collected with count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id) let(:key_path) { 'counts.issues_created_manually_from_alerts' } - let(:name_suggestion) { /count_<adjective describing\: '\(issues\.author_id != \d+\)'>_issues_<with>_alert_management_alerts/ } + let(:name_suggestion) { /count_<adjective describing: '\(issues\.author_id != \d+\)'>_issues_<with>_alert_management_alerts/ } end end end @@ -54,7 +54,7 @@ RSpec.describe Gitlab::Usage::Metrics::NamesSuggestions::Generator do it_behaves_like 'name suggestion' do # corresponding metric is collected with distinct_count(::Clusters::Cluster.aws_installed.enabled.where(time_period), :user_id) let(:key_path) { 'usage_activity_by_stage_monthly.configure.clusters_platforms_eks' } - let(:constraints) { /<adjective describing\: '\(clusters.provider_type = \d+ AND \(cluster_providers_aws\.status IN \(\d+\)\) AND clusters\.enabled = TRUE\)'>/ } + let(:constraints) { /<adjective describing: '\(clusters.provider_type = \d+ AND \(cluster_providers_aws\.status IN \(\d+\)\) AND clusters\.enabled = TRUE\)'>/ } let(:name_suggestion) { /count_distinct_user_id_from_#{constraints}_clusters_<with>_#{constraints}_cluster_providers_aws/ } end end @@ -63,7 +63,7 @@ RSpec.describe Gitlab::Usage::Metrics::NamesSuggestions::Generator do it_behaves_like 'name suggestion' do # corresponding metric is collected with sum(JiraImportState.finished, :imported_issues_count) let(:key_path) { 'counts.jira_imports_total_imported_issues_count' } - let(:name_suggestion) { /sum_imported_issues_count_from_<adjective describing\: '\(jira_imports\.status = \d+\)'>_jira_imports/ } + let(:name_suggestion) { /sum_imported_issues_count_from_<adjective describing: '\(jira_imports\.status = \d+\)'>_jira_imports/ } end end @@ -71,7 +71,7 @@ RSpec.describe Gitlab::Usage::Metrics::NamesSuggestions::Generator do it_behaves_like 'name suggestion' do # corresponding metric is collected with add(data[:personal_snippets], data[:project_snippets]) let(:key_path) { 'counts.snippets' } - let(:name_suggestion) { /add_count_<adjective describing\: '\(snippets\.type = 'PersonalSnippet'\)'>_snippets_and_count_<adjective describing\: '\(snippets\.type = 'ProjectSnippet'\)'>_snippets/ } + let(:name_suggestion) { /add_count_<adjective describing: '\(snippets\.type = 'PersonalSnippet'\)'>_snippets_and_count_<adjective describing: '\(snippets\.type = 'ProjectSnippet'\)'>_snippets/ } end end diff --git a/spec/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins_spec.rb b/spec/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins_spec.rb index fb3bd564e34..3e72d118ac6 100644 --- a/spec/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins_spec.rb +++ b/spec/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins_spec.rb @@ -4,7 +4,9 @@ require 'spec_helper' RSpec.describe Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Joins do describe '#accept' do - let(:collector) { Arel::Collectors::SubstituteBinds.new(ActiveRecord::Base.connection, Arel::Collectors::SQLString.new) } + let(:collector) do + Arel::Collectors::SubstituteBinds.new(ApplicationRecord.connection, Arel::Collectors::SQLString.new) + end context 'with join added via string' do it 'collects join parts' do @@ -33,7 +35,10 @@ RSpec.describe Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Joins result = described_class.new(ApplicationRecord.connection).accept(arel) - expect(result).to match_array [{ source: "joins", constraints: "records.id = joins.records_id" }, { source: "second_level_joins", constraints: "joins.id = second_level_joins.joins_id" }] + expect(result).to match_array [ + { source: "joins", constraints: "records.id = joins.records_id" }, + { source: "second_level_joins", constraints: "joins.id = second_level_joins.joins_id" } + ] end end end diff --git a/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb b/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb index e122d9a3026..63a1da490ed 100644 --- a/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb @@ -13,7 +13,7 @@ RSpec.describe 'Code review events' do definition.attributes.dig(:options, :events) end.uniq - exceptions = %w[i_code_review_mr_diffs i_code_review_mr_with_invalid_approvers i_code_review_mr_single_file_diffs i_code_review_total_suggestions_applied i_code_review_total_suggestions_added i_code_review_create_note_in_ipynb_diff i_code_review_create_note_in_ipynb_diff_mr i_code_review_create_note_in_ipynb_diff_commit] + exceptions = %w[i_code_review_create_mr i_code_review_mr_diffs i_code_review_mr_with_invalid_approvers i_code_review_mr_single_file_diffs i_code_review_total_suggestions_applied i_code_review_total_suggestions_added i_code_review_create_note_in_ipynb_diff i_code_review_create_note_in_ipynb_diff_mr i_code_review_create_note_in_ipynb_diff_commit] code_review_aggregated_events += exceptions expect(code_review_events - code_review_aggregated_events).to be_empty diff --git a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb index 0bea06f602f..1d980c48c72 100644 --- a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb @@ -23,70 +23,6 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s described_class.clear_memoization(:known_events) end - context 'migration to instrumentation classes data collection' do - let_it_be(:instrumented_events) do - instrumentation_classes = %w[AggregatedMetric RedisHLLMetric] - ::Gitlab::Usage::MetricDefinition.all.map do |definition| - next unless definition.available? - next unless instrumentation_classes.include?(definition.attributes[:instrumentation_class]) - - definition.attributes.dig(:options, :events)&.sort - end.compact.to_set - end - - def not_instrumented_events(category) - described_class - .events_for_category(category) - .sort - .reject do |event| - instrumented_events.include?([event]) - end - end - - def not_instrumented_aggregate(category) - events = described_class.events_for_category(category).sort - - return unless described_class::CATEGORIES_FOR_TOTALS.include?(category) - return unless described_class.send(:eligible_for_totals?, events) - return if instrumented_events.include?(events) - - events - end - - describe 'Gitlab::UsageDataCounters::HLLRedisCounter::CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS' do - it 'includes only fully migrated categories' do - wrong_skipped_events = described_class::CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS.map do |category| - next if not_instrumented_events(category).empty? && not_instrumented_aggregate(category).nil? - - [category, [not_instrumented_events(category), not_instrumented_aggregate(category)].compact] - end.compact.to_h - - expect(wrong_skipped_events).to be_empty - end - - context 'with not instrumented category' do - let(:instrumented_events) { [] } - - it 'can detect not migrated category' do - wrong_skipped_events = described_class::CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS.map do |category| - next if not_instrumented_events(category).empty? && not_instrumented_aggregate(category).nil? - - [category, [not_instrumented_events(category), not_instrumented_aggregate(category)].compact] - end.compact.to_h - - expect(wrong_skipped_events).not_to be_empty - end - end - end - - describe '.unique_events_data' do - it 'does not include instrumented categories' do - expect(described_class.unique_events_data.keys) - .not_to include(*described_class.categories_collected_from_metrics_definitions) - end - end - end - describe '.categories' do it 'gets CE unique category names' do expect(described_class.categories).to include( @@ -138,14 +74,14 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s File.open(ce_temp_file.path, "w+b") { |f| f.write [ce_event].to_yaml } end - it 'returns ce events' do - expect(described_class.known_events).to include(ce_event) - end - after do ce_temp_file.unlink FileUtils.remove_entry(ce_temp_dir) if Dir.exist?(ce_temp_dir) end + + it 'returns ce events' do + expect(described_class.known_events).to include(ce_event) + end end describe 'known_events' do @@ -273,6 +209,22 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s expect { described_class.track_event('unknown', values: entity1, time: Date.current) }.to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::UnknownEvent) end + context 'when Rails environment is production' do + before do + allow(Rails.env).to receive(:development?).and_return(false) + allow(Rails.env).to receive(:test?).and_return(false) + end + + it 'reports only UnknownEvent exception' do + expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception) + .with(Gitlab::UsageDataCounters::HLLRedisCounter::UnknownEvent) + .once + .and_call_original + + expect { described_class.track_event('unknown', values: entity1, time: Date.current) }.not_to raise_error + end + end + it 'reports an error if Feature.enabled raise an error' do expect(Feature).to receive(:enabled?).and_raise(StandardError.new) expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception) @@ -342,7 +294,7 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s context 'with valid contex' do it 'increments context event counter' do expect(Gitlab::Redis::HLL).to receive(:add) do |kwargs| - expect(kwargs[:key]).to match(/^#{default_context}\_.*/) + expect(kwargs[:key]).to match(/^#{default_context}_.*/) end described_class.track_event_in_context(context_event, values: entity1, context: default_context) @@ -544,53 +496,6 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s end end - describe 'unique_events_data' do - let(:known_events) do - [ - { name: 'event1_slot', redis_slot: "slot", category: 'category1', aggregation: "weekly" }, - { name: 'event2_slot', redis_slot: "slot", category: 'category1', aggregation: "weekly" }, - { name: 'event3', category: 'category2', aggregation: "weekly" }, - { name: 'event4', category: 'category2', aggregation: "weekly" } - ].map(&:with_indifferent_access) - end - - before do - allow(described_class).to receive(:known_events).and_return(known_events) - allow(described_class).to receive(:categories).and_return(%w(category1 category2)) - - stub_const('Gitlab::UsageDataCounters::HLLRedisCounter::CATEGORIES_FOR_TOTALS', %w(category1 category2)) - - described_class.track_event('event1_slot', values: entity1, time: 2.days.ago) - described_class.track_event('event2_slot', values: entity2, time: 2.days.ago) - described_class.track_event('event2_slot', values: entity3, time: 2.weeks.ago) - - # events in different slots - described_class.track_event('event3', values: entity2, time: 2.days.ago) - described_class.track_event('event4', values: entity2, time: 2.days.ago) - end - - it 'returns the number of unique events for all known events' do - results = { - "category1" => { - "event1_slot_weekly" => 1, - "event1_slot_monthly" => 1, - "event2_slot_weekly" => 1, - "event2_slot_monthly" => 2, - "category1_total_unique_counts_weekly" => 2, - "category1_total_unique_counts_monthly" => 3 - }, - "category2" => { - "event3_weekly" => 1, - "event3_monthly" => 1, - "event4_weekly" => 1, - "event4_monthly" => 1 - } - } - - expect(subject.unique_events_data).to eq(results) - end - end - describe '.calculate_events_union' do let(:time_range) { { start_date: 7.days.ago, end_date: DateTime.current } } let(:known_events) do diff --git a/spec/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter_spec.rb index 74e63d219bd..9a1ffd8d01d 100644 --- a/spec/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter_spec.rb @@ -50,11 +50,29 @@ RSpec.describe Gitlab::UsageDataCounters::MergeRequestActivityUniqueCounter, :cl end describe '.track_create_mr_action' do - subject { described_class.track_create_mr_action(user: user) } + subject { described_class.track_create_mr_action(user: user, merge_request: merge_request) } + + let(:merge_request) { create(:merge_request) } + let(:target_project) { merge_request.target_project } + + it_behaves_like 'a tracked merge request unique event' do + let(:action) { described_class::MR_USER_CREATE_ACTION } + end it_behaves_like 'a tracked merge request unique event' do let(:action) { described_class::MR_CREATE_ACTION } end + + it_behaves_like 'Snowplow event tracking with RedisHLL context' do + let(:action) { :create } + let(:category) { described_class.name } + let(:project) { target_project } + let(:namespace) { project.namespace.reload } + let(:user) { project.creator } + let(:feature_flag_name) { :route_hll_to_snowplow_phase2 } + let(:label) { 'redis_hll_counters.code_review.i_code_review_create_mr_monthly' } + let(:property) { described_class::MR_CREATE_ACTION } + end end describe '.track_close_mr_action' do @@ -94,30 +112,15 @@ RSpec.describe Gitlab::UsageDataCounters::MergeRequestActivityUniqueCounter, :cl let(:action) { described_class::MR_APPROVE_ACTION } end - it 'records correct payload with Snowplow event', :snowplow do - stub_feature_flags(route_hll_to_snowplow_phase2: true) - - subject - - expect_snowplow_event( - category: 'merge_requests', - action: 'i_code_review_user_approve_mr', - namespace: target_project.namespace, - user: user, - project: target_project - ) - end - - context 'when FF is disabled' do - before do - stub_feature_flags(route_hll_to_snowplow_phase2: false) - end - - it 'doesnt emit snowplow events', :snowplow do - subject - - expect_no_snowplow_event - end + it_behaves_like 'Snowplow event tracking with RedisHLL context' do + let(:action) { :approve } + let(:category) { described_class.name } + let(:project) { target_project } + let(:namespace) { project.namespace.reload } + let(:user) { project.creator } + let(:feature_flag_name) { :route_hll_to_snowplow_phase2 } + let(:label) { 'redis_hll_counters.code_review.i_code_review_user_approve_mr_monthly' } + let(:property) { described_class::MR_APPROVE_ACTION } end end diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index d8f50fa27bb..214331e15e8 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -598,35 +598,10 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do external_diffs: { enabled: false }, lfs: { enabled: true, object_store: { enabled: false, direct_upload: true, background_upload: false, provider: "AWS" } }, uploads: { enabled: nil, object_store: { enabled: false, direct_upload: true, background_upload: false, provider: "AWS" } }, - packages: { enabled: true, object_store: { enabled: false, direct_upload: false, background_upload: true, provider: "AWS" } } } + packages: { enabled: true, object_store: { enabled: false, direct_upload: false, background_upload: false, provider: "AWS" } } } ) end - context 'with existing container expiration policies' do - let_it_be(:disabled) { create(:container_expiration_policy, enabled: false) } - let_it_be(:enabled) { create(:container_expiration_policy, enabled: true) } - - ::ContainerExpirationPolicy.older_than_options.keys.each do |value| - let_it_be("container_expiration_policy_with_older_than_set_to_#{value}") { create(:container_expiration_policy, older_than: value) } - end - - let_it_be('container_expiration_policy_with_older_than_set_to_null') { create(:container_expiration_policy, older_than: nil) } - - let(:inactive_policies) { ::ContainerExpirationPolicy.where(enabled: false) } - let(:active_policies) { ::ContainerExpirationPolicy.active } - - subject { described_class.data[:counts] } - - it 'gathers usage data' do - expect(subject[:projects_with_expiration_policy_enabled_with_older_than_unset]).to eq 1 - expect(subject[:projects_with_expiration_policy_enabled_with_older_than_set_to_7d]).to eq 1 - expect(subject[:projects_with_expiration_policy_enabled_with_older_than_set_to_14d]).to eq 1 - expect(subject[:projects_with_expiration_policy_enabled_with_older_than_set_to_30d]).to eq 1 - expect(subject[:projects_with_expiration_policy_enabled_with_older_than_set_to_60d]).to eq 1 - expect(subject[:projects_with_expiration_policy_enabled_with_older_than_set_to_90d]).to eq 2 - end - end - context 'when queries time out' do let(:metric_method) { :count } @@ -860,7 +835,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do 'direct_upload' => true, 'connection' => { 'provider' => 'AWS', 'aws_access_key_id' => 'minio', 'aws_secret_access_key' => 'gdk-minio', 'region' => 'gdk', 'endpoint' => 'http://127.0.0.1:9000', 'path_style' => true }, - 'background_upload' => false, 'proxy_download' => false } }) expect(subject).to eq( @@ -1135,36 +1109,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do end end - describe 'redis_hll_counters' do - subject { described_class.redis_hll_counters } - - let(:migrated_categories) do - ::Gitlab::UsageDataCounters::HLLRedisCounter.categories_collected_from_metrics_definitions - end - - let(:categories) { ::Gitlab::UsageDataCounters::HLLRedisCounter.categories - migrated_categories } - let(:ignored_metrics) { ["i_package_composer_deploy_token_weekly"] } - - it 'has all known_events' do - expect(subject).to have_key(:redis_hll_counters) - - expect(subject[:redis_hll_counters].keys).to match_array(categories) - - categories.each do |category| - keys = ::Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category(category) - - metrics = keys.map { |key| "#{key}_weekly" } + keys.map { |key| "#{key}_monthly" } - metrics -= ignored_metrics - - if ::Gitlab::UsageDataCounters::HLLRedisCounter::CATEGORIES_FOR_TOTALS.include?(category) - metrics.append("#{category}_total_unique_counts_weekly", "#{category}_total_unique_counts_monthly") - end - - expect(subject[:redis_hll_counters][category].keys).to match_array(metrics) - end - end - end - describe '.service_desk_counts' do subject { described_class.send(:service_desk_counts) } diff --git a/spec/lib/gitlab/utils/delegator_override/validator_spec.rb b/spec/lib/gitlab/utils/delegator_override/validator_spec.rb index a58bc65c708..4fcf01ea256 100644 --- a/spec/lib/gitlab/utils/delegator_override/validator_spec.rb +++ b/spec/lib/gitlab/utils/delegator_override/validator_spec.rb @@ -7,8 +7,7 @@ RSpec.describe Gitlab::Utils::DelegatorOverride::Validator do Class.new(::SimpleDelegator) do extend(::Gitlab::Utils::DelegatorOverride) - def foo - end + def foo; end end.prepend(ee_delegator_extension) end @@ -16,18 +15,15 @@ RSpec.describe Gitlab::Utils::DelegatorOverride::Validator do Module.new do extend(::Gitlab::Utils::DelegatorOverride) - def bar - end + def bar; end end end let(:target_class) do Class.new do - def foo - end + def foo; end - def bar - end + def bar; end end end diff --git a/spec/lib/gitlab/utils/delegator_override_spec.rb b/spec/lib/gitlab/utils/delegator_override_spec.rb index 2dafa75e344..b566b7a2cad 100644 --- a/spec/lib/gitlab/utils/delegator_override_spec.rb +++ b/spec/lib/gitlab/utils/delegator_override_spec.rb @@ -7,25 +7,21 @@ RSpec.describe Gitlab::Utils::DelegatorOverride do Class.new(::SimpleDelegator) do extend(::Gitlab::Utils::DelegatorOverride) - def foo - end + def foo; end end end let(:target_class) do Class.new do - def foo - end + def foo; end - def bar - end + def bar; end end end let(:dummy_module) do Module.new do - def foobar - end + def foobar; end end end diff --git a/spec/lib/gitlab/utils/override_spec.rb b/spec/lib/gitlab/utils/override_spec.rb index a5e53c1dfc1..63f7b1623d8 100644 --- a/spec/lib/gitlab/utils/override_spec.rb +++ b/spec/lib/gitlab/utils/override_spec.rb @@ -35,8 +35,7 @@ RSpec.describe Gitlab::Utils::Override do override :good if bad_arity - def good(num) - end + def good(num); end elsif negative_arity def good(*args) super.succ diff --git a/spec/lib/gitlab/utils/strong_memoize_spec.rb b/spec/lib/gitlab/utils/strong_memoize_spec.rb index 236b6d29ba7..287858579d6 100644 --- a/spec/lib/gitlab/utils/strong_memoize_spec.rb +++ b/spec/lib/gitlab/utils/strong_memoize_spec.rb @@ -23,7 +23,7 @@ RSpec.describe Gitlab::Utils::StrongMemoize do end def method_name - strong_memoize(:method_name) do + strong_memoize(:method_name) do # rubocop: disable Gitlab/StrongMemoizeAttr trace << value value end @@ -59,22 +59,19 @@ RSpec.describe Gitlab::Utils::StrongMemoize do protected - def private_method - end + def private_method; end private :private_method strong_memoize_attr :private_method public - def protected_method - end + def protected_method; end protected :protected_method strong_memoize_attr :protected_method private - def public_method - end + def public_method; end public :public_method strong_memoize_attr :public_method end @@ -219,6 +216,10 @@ RSpec.describe Gitlab::Utils::StrongMemoize do it 'calls the existing .method_added' do expect(klass.method_added_list).to include(:method_name_attr) end + + it 'retains method arity' do + expect(klass.instance_method(member_name).arity).to eq(0) + end end context "memoized before method definition with different member name and value #{value}" do @@ -280,5 +281,22 @@ RSpec.describe Gitlab::Utils::StrongMemoize do expect { subject }.to raise_error(NameError, %r{undefined method `nonexistent_method' for class}) end end + + context 'when memoized method has parameters' do + it 'raises an error' do + expected_message = /Using `strong_memoize_attr` on methods with parameters is not supported/ + + expect do + strong_memoize_class = described_class + + Class.new do + include strong_memoize_class + + def method_with_parameters(params); end + strong_memoize_attr :method_with_parameters + end + end.to raise_error(RuntimeError, expected_message) + end + end end end diff --git a/spec/lib/gitlab/work_items/work_item_hierarchy_spec.rb b/spec/lib/gitlab/work_items/work_item_hierarchy_spec.rb new file mode 100644 index 00000000000..b2f298a7d05 --- /dev/null +++ b/spec/lib/gitlab/work_items/work_item_hierarchy_spec.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::WorkItems::WorkItemHierarchy, feature_category: :portfolio_management do + let_it_be(:project) { create(:project) } + let_it_be(:type1) { create(:work_item_type, namespace: project.namespace) } + let_it_be(:type2) { create(:work_item_type, namespace: project.namespace) } + let_it_be(:hierarchy_restriction1) { create(:hierarchy_restriction, parent_type: type1, child_type: type2) } + let_it_be(:hierarchy_restriction2) { create(:hierarchy_restriction, parent_type: type2, child_type: type2) } + let_it_be(:hierarchy_restriction3) { create(:hierarchy_restriction, parent_type: type2, child_type: type1) } + let_it_be(:item1) { create(:work_item, work_item_type: type1, project: project) } + let_it_be(:item2) { create(:work_item, work_item_type: type2, project: project) } + let_it_be(:item3) { create(:work_item, work_item_type: type2, project: project) } + let_it_be(:item4) { create(:work_item, work_item_type: type1, project: project) } + let_it_be(:ignored1) { create(:work_item, work_item_type: type1, project: project) } + let_it_be(:ignored2) { create(:work_item, work_item_type: type2, project: project) } + let_it_be(:link1) { create(:parent_link, work_item_parent: item1, work_item: item2) } + let_it_be(:link2) { create(:parent_link, work_item_parent: item2, work_item: item3) } + let_it_be(:link3) { create(:parent_link, work_item_parent: item3, work_item: item4) } + + let(:options) { {} } + + describe '#base_and_ancestors' do + subject { described_class.new(::WorkItem.where(id: item3.id), options: options) } + + it 'includes the base and its ancestors' do + relation = subject.base_and_ancestors + + expect(relation).to eq([item3, item2, item1]) + end + + context 'when same_type option is used' do + let(:options) { { same_type: true } } + + it 'includes the base and its ancestors' do + relation = subject.base_and_ancestors + + expect(relation).to eq([item3, item2]) + end + end + + it 'can find ancestors upto a certain level' do + relation = subject.base_and_ancestors(upto: item1) + + expect(relation).to eq([item3, item2]) + end + + describe 'hierarchy_order option' do + let(:relation) do + subject.base_and_ancestors(hierarchy_order: hierarchy_order) + end + + context 'for :asc' do + let(:hierarchy_order) { :asc } + + it 'orders by child to ancestor' do + expect(relation).to eq([item3, item2, item1]) + end + end + + context 'for :desc' do + let(:hierarchy_order) { :desc } + + it 'orders by ancestor to child' do + expect(relation).to eq([item1, item2, item3]) + end + end + end + end + + describe '#base_and_descendants' do + subject { described_class.new(::WorkItem.where(id: item2.id), options: options) } + + it 'includes the base and its descendants' do + relation = subject.base_and_descendants + + expect(relation).to eq([item2, item3, item4]) + end + + context 'when same_type option is used' do + let(:options) { { same_type: true } } + + it 'includes the base and its ancestors' do + relation = subject.base_and_descendants + + expect(relation).to eq([item2, item3]) + end + end + + context 'when with_depth is true' do + let(:relation) do + subject.base_and_descendants(with_depth: true) + end + + it 'includes depth in the results' do + object_depths = { + item2.id => 1, + item3.id => 2, + item4.id => 3 + } + + relation.each do |object| + expect(object.depth).to eq(object_depths[object.id]) + end + end + end + end +end diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb index 5c9a3cc0a24..3c7542ea5f9 100644 --- a/spec/lib/gitlab/workhorse_spec.rb +++ b/spec/lib/gitlab/workhorse_spec.rb @@ -4,6 +4,7 @@ require 'spec_helper' RSpec.describe Gitlab::Workhorse do let_it_be(:project) { create(:project, :repository) } + let(:features) { { 'gitaly-feature-enforce-requests-limits' => 'true' } } let(:repository) { project.repository } @@ -42,7 +43,7 @@ RSpec.describe Gitlab::Workhorse do expect(command).to eq('git-archive') expect(params).to eq({ 'GitalyServer' => { - features: { 'gitaly-feature-enforce-requests-limits' => 'true' }, + 'call_metadata' => features, address: Gitlab::GitalyClient.address(project.repository_storage), token: Gitlab::GitalyClient.token(project.repository_storage) }, @@ -92,7 +93,7 @@ RSpec.describe Gitlab::Workhorse do expect(command).to eq("git-format-patch") expect(params).to eq({ 'GitalyServer' => { - features: { 'gitaly-feature-enforce-requests-limits' => 'true' }, + 'call_metadata' => features, address: Gitlab::GitalyClient.address(project.repository_storage), token: Gitlab::GitalyClient.token(project.repository_storage) }, @@ -155,7 +156,7 @@ RSpec.describe Gitlab::Workhorse do expect(command).to eq("git-diff") expect(params).to eq({ 'GitalyServer' => { - features: { 'gitaly-feature-enforce-requests-limits' => 'true' }, + 'call_metadata' => features, address: Gitlab::GitalyClient.address(project.repository_storage), token: Gitlab::GitalyClient.token(project.repository_storage) }, @@ -208,6 +209,16 @@ RSpec.describe Gitlab::Workhorse do describe '.git_http_ok' do let(:user) { create(:user) } + let(:gitaly_params) do + { + GitalyServer: { + call_metadata: call_metadata, + address: Gitlab::GitalyClient.address('default'), + token: Gitlab::GitalyClient.token('default') + } + } + end + let(:repo_path) { 'ignored but not allowed to be empty in gitlab-workhorse' } let(:action) { 'info_refs' } let(:params) do @@ -219,6 +230,13 @@ RSpec.describe Gitlab::Workhorse do } end + let(:call_metadata) do + features.merge({ + 'user_id' => params[:GL_ID], + 'username' => params[:GL_USERNAME] + }) + end + subject { described_class.git_http_ok(repository, Gitlab::GlRepository::PROJECT, user, action) } it { expect(subject).to include(params) } @@ -238,100 +256,79 @@ RSpec.describe Gitlab::Workhorse do it { expect(subject).to include(params) } end - context 'when Gitaly is enabled' do - let(:gitaly_params) do - { - GitalyServer: { - features: { 'gitaly-feature-enforce-requests-limits' => 'true' }, - address: Gitlab::GitalyClient.address('default'), - token: Gitlab::GitalyClient.token('default') - } - } - end + it 'includes a Repository param' do + repo_param = { + storage_name: 'default', + relative_path: project.disk_path + '.git', + gl_repository: "project-#{project.id}" + } - before do - allow(Gitlab.config.gitaly).to receive(:enabled).and_return(true) - end + expect(subject[:Repository]).to include(repo_param) + end - it 'includes a Repository param' do - repo_param = { - storage_name: 'default', - relative_path: project.disk_path + '.git', - gl_repository: "project-#{project.id}" - } + context "when git_upload_pack action is passed" do + let(:action) { 'git_upload_pack' } - expect(subject[:Repository]).to include(repo_param) - end + it { expect(subject).to include(gitaly_params) } - context "when git_upload_pack action is passed" do - let(:action) { 'git_upload_pack' } - let(:feature_flag) { :post_upload_pack } + context 'show_all_refs enabled' do + subject { described_class.git_http_ok(repository, Gitlab::GlRepository::PROJECT, user, action, show_all_refs: true) } - it 'includes Gitaly params in the returned value' do - allow(Gitlab::GitalyClient).to receive(:feature_enabled?).with(feature_flag).and_return(true) + it { is_expected.to include(ShowAllRefs: true) } + end - expect(subject).to include(gitaly_params) + context 'when a feature flag is set for a single project' do + before do + stub_feature_flags(gitaly_mep_mep: project) end - context 'show_all_refs enabled' do - subject { described_class.git_http_ok(repository, Gitlab::GlRepository::PROJECT, user, action, show_all_refs: true) } + it 'sets the flag to true for that project' do + response = described_class.git_http_ok(repository, Gitlab::GlRepository::PROJECT, user, action) - it { is_expected.to include(ShowAllRefs: true) } + expect(response.dig(:GitalyServer, :call_metadata)).to include('gitaly-feature-enforce-requests-limits' => 'true', + 'gitaly-feature-mep-mep' => 'true') end - context 'when a feature flag is set for a single project' do - before do - stub_feature_flags(gitaly_mep_mep: project) - end + it 'sets the flag to false for other projects' do + other_project = create(:project, :public, :repository) + response = described_class.git_http_ok(other_project.repository, Gitlab::GlRepository::PROJECT, user, action) - it 'sets the flag to true for that project' do - response = described_class.git_http_ok(repository, Gitlab::GlRepository::PROJECT, user, action) - - expect(response.dig(:GitalyServer, :features)).to eq('gitaly-feature-enforce-requests-limits' => 'true', - 'gitaly-feature-mep-mep' => 'true') - end - - it 'sets the flag to false for other projects' do - other_project = create(:project, :public, :repository) - response = described_class.git_http_ok(other_project.repository, Gitlab::GlRepository::PROJECT, user, action) - - expect(response.dig(:GitalyServer, :features)).to eq('gitaly-feature-enforce-requests-limits' => 'true', - 'gitaly-feature-mep-mep' => 'false') - end + expect(response.dig(:GitalyServer, :call_metadata)).to include('gitaly-feature-enforce-requests-limits' => 'true', + 'gitaly-feature-mep-mep' => 'false') + end - it 'sets the flag to false when there is no project' do - snippet = create(:personal_snippet, :repository) - response = described_class.git_http_ok(snippet.repository, Gitlab::GlRepository::SNIPPET, user, action) + it 'sets the flag to false when there is no project' do + snippet = create(:personal_snippet, :repository) + response = described_class.git_http_ok(snippet.repository, Gitlab::GlRepository::SNIPPET, user, action) - expect(response.dig(:GitalyServer, :features)).to eq('gitaly-feature-enforce-requests-limits' => 'true', - 'gitaly-feature-mep-mep' => 'false') - end + expect(response.dig(:GitalyServer, :call_metadata)).to include('gitaly-feature-enforce-requests-limits' => 'true', + 'gitaly-feature-mep-mep' => 'false') end end + end - context "when git_receive_pack action is passed" do - let(:action) { 'git_receive_pack' } + context "when git_receive_pack action is passed" do + let(:action) { 'git_receive_pack' } - it { expect(subject).to include(gitaly_params) } - end + it { expect(subject).to include(gitaly_params) } + end - context "when info_refs action is passed" do - let(:action) { 'info_refs' } + context "when info_refs action is passed" do + let(:action) { 'info_refs' } - it { expect(subject).to include(gitaly_params) } + it { expect(subject).to include(gitaly_params) } - context 'show_all_refs enabled' do - subject { described_class.git_http_ok(repository, Gitlab::GlRepository::PROJECT, user, action, show_all_refs: true) } + context 'show_all_refs enabled' do + subject { described_class.git_http_ok(repository, Gitlab::GlRepository::PROJECT, user, action, show_all_refs: true) } - it { is_expected.to include(ShowAllRefs: true) } - end + it { is_expected.to include(ShowAllRefs: true) } end + end - context 'when action passed is not supported by Gitaly' do - let(:action) { 'download' } + context 'when action passed is not supported by Gitaly' do + let(:action) { 'download' } - it { expect { subject }.to raise_exception('Unsupported action: download') } - end + it { expect { subject }.to raise_exception('Unsupported action: download') } end context 'when receive_max_input_size has been updated' do @@ -349,6 +346,23 @@ RSpec.describe Gitlab::Workhorse do expect(subject[:GitConfigOptions]).to be_empty end end + + context 'when remote_ip is available in the application context' do + it 'includes a RemoteIP params' do + result = {} + Gitlab::ApplicationContext.with_context(remote_ip: "1.2.3.4") do + result = described_class.git_http_ok(repository, Gitlab::GlRepository::PROJECT, user, action) + end + expect(result[:GitalyServer][:call_metadata]['remote_ip']).to eql("1.2.3.4") + end + end + + context 'when remote_ip is not available in the application context' do + it 'does not include RemoteIP params' do + result = described_class.git_http_ok(repository, Gitlab::GlRepository::PROJECT, user, action) + expect(result[:GitalyServer][:call_metadata]).not_to have_key('remote_ip') + end + end end describe '.set_key_and_notify' do @@ -428,7 +442,7 @@ RSpec.describe Gitlab::Workhorse do expect(command).to eq('git-blob') expect(params).to eq({ 'GitalyServer' => { - features: { 'gitaly-feature-enforce-requests-limits' => 'true' }, + 'call_metadata' => features, address: Gitlab::GitalyClient.address(project.repository_storage), token: Gitlab::GitalyClient.token(project.repository_storage) }, @@ -508,7 +522,7 @@ RSpec.describe Gitlab::Workhorse do expect(command).to eq('git-snapshot') expect(params).to eq( 'GitalyServer' => { - 'features' => { 'gitaly-feature-enforce-requests-limits' => 'true' }, + 'call_metadata' => features, 'address' => Gitlab::GitalyClient.address(project.repository_storage), 'token' => Gitlab::GitalyClient.token(project.repository_storage) }, diff --git a/spec/lib/gitlab/x509/signature_spec.rb b/spec/lib/gitlab/x509/signature_spec.rb index 31f66232f38..eb8c0bd0aff 100644 --- a/spec/lib/gitlab/x509/signature_spec.rb +++ b/spec/lib/gitlab/x509/signature_spec.rb @@ -11,6 +11,17 @@ RSpec.describe Gitlab::X509::Signature do } end + it_behaves_like 'signature with type checking', :x509 do + subject(:signature) do + described_class.new( + X509Helpers::User1.signed_commit_signature, + X509Helpers::User1.signed_commit_base_data, + X509Helpers::User1.certificate_email, + X509Helpers::User1.signed_commit_time + ) + end + end + shared_examples "a verified signature" do let!(:user) { create(:user, email: X509Helpers::User1.certificate_email) } @@ -271,21 +282,21 @@ RSpec.describe Gitlab::X509::Signature do end end - describe '#user' do + describe '#signed_by_user' do subject do described_class.new( X509Helpers::User1.signed_tag_signature, X509Helpers::User1.signed_tag_base_data, X509Helpers::User1.certificate_email, X509Helpers::User1.signed_commit_time - ).user + ).signed_by_user end context 'if email is assigned to a user' do - let!(:user) { create(:user, email: X509Helpers::User1.certificate_email) } + let!(:signed_by_user) { create(:user, email: X509Helpers::User1.certificate_email) } it 'returns user' do - is_expected.to eq(user) + is_expected.to eq(signed_by_user) end end diff --git a/spec/lib/google_api/cloud_platform/client_spec.rb b/spec/lib/google_api/cloud_platform/client_spec.rb index 0c207161927..82ab6c089da 100644 --- a/spec/lib/google_api/cloud_platform/client_spec.rb +++ b/spec/lib/google_api/cloud_platform/client_spec.rb @@ -106,8 +106,8 @@ RSpec.describe GoogleApi::CloudPlatform::Client do let(:enable_addons) { [] } let(:addons_config) do - enable_addons.each_with_object({}) do |addon, hash| - hash[addon] = { disabled: false } + enable_addons.index_with do + { disabled: false } end end diff --git a/spec/lib/json_web_token/hmac_token_spec.rb b/spec/lib/json_web_token/hmac_token_spec.rb index cf7e5c54f45..016084eaf69 100644 --- a/spec/lib/json_web_token/hmac_token_spec.rb +++ b/spec/lib/json_web_token/hmac_token_spec.rb @@ -1,9 +1,11 @@ # frozen_string_literal: true require 'json' -require 'timecop' +require 'active_support/testing/time_helpers' RSpec.describe JSONWebToken::HMACToken do + include ActiveSupport::Testing::TimeHelpers + let(:secret) { 'shh secret squirrel' } shared_examples 'a valid, non-expired token' do @@ -54,13 +56,13 @@ RSpec.describe JSONWebToken::HMACToken do end context 'that is expired' do - # Needs the ! so Timecop.freeze() is effective + # Needs the ! so freeze_time() is effective let!(:encoded_token) { described_class.new(secret).encoded } it "raises exception saying 'Signature has expired'" do # Needs to be 120 seconds, because the default expiry is 60 seconds # with an additional 60 second leeway. - Timecop.freeze(Time.now + 120) do + travel_to(Time.now + 120) do expect { decoded_token }.to raise_error(JWT::ExpiredSignature, 'Signature has expired') end end @@ -77,19 +79,19 @@ RSpec.describe JSONWebToken::HMACToken do context 'that has expired' do let(:expire_time) { 0 } + around do |example| + travel_to(Time.now + 1) { example.run } + end + context 'with the default leeway' do - Timecop.freeze(Time.now + 1) do - it_behaves_like 'a valid, non-expired token' - end + it_behaves_like 'a valid, non-expired token' end context 'with a leeway of 0 seconds' do let(:leeway) { 0 } it "raises exception saying 'Signature has expired'" do - Timecop.freeze(Time.now + 1) do - expect { decoded_token }.to raise_error(JWT::ExpiredSignature, 'Signature has expired') - end + expect { decoded_token }.to raise_error(JWT::ExpiredSignature, 'Signature has expired') end end end diff --git a/spec/lib/mattermost/session_spec.rb b/spec/lib/mattermost/session_spec.rb index d208ef93224..87e2e341777 100644 --- a/spec/lib/mattermost/session_spec.rb +++ b/spec/lib/mattermost/session_spec.rb @@ -14,15 +14,15 @@ RSpec.describe Mattermost::Session, type: :request do subject { described_class.new(user) } # Needed for doorkeeper to function + before do + subject.base_uri = mattermost_url + end + it { is_expected.to respond_to(:current_resource_owner) } it { is_expected.to respond_to(:request) } it { is_expected.to respond_to(:authorization) } it { is_expected.to respond_to(:strategy) } - before do - subject.base_uri = mattermost_url - end - describe '#with session' do let(:location) { 'http://location.tld' } let(:cookie_header) { 'MMOAUTH=taskik8az7rq8k6rkpuas7htia; Path=/;' } diff --git a/spec/lib/pager_duty/webhook_payload_parser_spec.rb b/spec/lib/pager_duty/webhook_payload_parser_spec.rb index 647f19e3d3a..1606d03c746 100644 --- a/spec/lib/pager_duty/webhook_payload_parser_spec.rb +++ b/spec/lib/pager_duty/webhook_payload_parser_spec.rb @@ -10,23 +10,27 @@ RSpec.describe PagerDuty::WebhookPayloadParser do let(:triggered_event) do { - 'event' => 'incident.trigger', + 'event' => 'incident.triggered', 'incident' => { - 'url' => 'https://webdemo.pagerduty.com/incidents/PRORDTY', - 'incident_number' => 33, - 'title' => 'My new incident', + 'url' => 'https://gitlab-1.pagerduty.com/incidents/Q1XZUF87W1HB5A', + 'incident_number' => 2, + 'title' => '[FILTERED]', 'status' => 'triggered', - 'created_at' => '2017-09-26T15:14:36Z', + 'created_at' => '2022-11-30T08:46:19Z', 'urgency' => 'high', - 'incident_key' => nil, - 'assignees' => [{ - 'summary' => 'Laura Haley', - 'url' => 'https://webdemo.pagerduty.com/users/P553OPV' - }], - 'impacted_services' => [{ - 'summary' => 'Production XDB Cluster', - 'url' => 'https://webdemo.pagerduty.com/services/PN49J75' - }] + 'incident_key' => '[FILTERED]', + 'assignees' => + [ + { + 'summary' => 'Rajendra Kadam', + 'url' => 'https://gitlab-1.pagerduty.com/users/PIN0B5C' + } + ], + 'impacted_service' => + { + 'summary' => 'Test service', + 'url' => 'https://gitlab-1.pagerduty.com/services/PK6IKMT' + } } } end @@ -37,74 +41,50 @@ RSpec.describe PagerDuty::WebhookPayloadParser do let(:payload) { Gitlab::Json.parse(fixture_file) } it 'returns parsed payload' do - is_expected.to eq([triggered_event]) + is_expected.to eq(triggered_event) end context 'when assignments summary and html_url are blank' do before do - payload['messages'].each do |m| - m['incident']['assignments'] = [{ 'assignee' => { 'summary' => '', 'html_url' => '' } }] - end + payload['event']['data']['assignees'] = [{ 'summary' => '', 'html_url' => '' }] end it 'returns parsed payload with blank assignees' do - assignees = parse.map { |events| events['incident'].slice('assignees') } + assignees = parse['incident'].slice('assignees') - expect(assignees).to eq([{ 'assignees' => [] }]) + expect(assignees).to eq({ 'assignees' => [] }) end end context 'when impacted_services summary and html_url are blank' do before do - payload['messages'].each do |m| - m['incident']['impacted_services'] = [{ 'summary' => '', 'html_url' => '' }] - end + payload['event']['data']['service'] = { 'summary' => '', 'html_url' => '' } end - it 'returns parsed payload with blank assignees' do - assignees = parse.map { |events| events['incident'].slice('impacted_services') } + it 'returns parsed payload with blank impacted service' do + assignees = parse['incident'].slice('impacted_service') - expect(assignees).to eq([{ 'impacted_services' => [] }]) + expect(assignees).to eq({ 'impacted_service' => {} }) end end end context 'when payload schema is invalid' do - let(:payload) { { 'messages' => [{ 'event' => 'incident.trigger' }] } } + let(:payload) { { 'event' => 'incident.triggered' } } - it 'returns payload with blank incidents' do - is_expected.to eq([]) + it 'returns payload with blank incident' do + is_expected.to eq({}) end end - context 'when payload consists of two messages' do - context 'when one of the messages has no incident data' do - let(:payload) do - valid_payload = Gitlab::Json.parse(fixture_file) - event = { 'event' => 'incident.trigger' } - valid_payload['messages'] = valid_payload['messages'].append(event) - valid_payload - end - - it 'returns parsed payload with valid events only' do - is_expected.to eq([triggered_event]) - end + context 'when event is unknown' do + let(:payload) do + valid_payload = Gitlab::Json.parse(fixture_file) + valid_payload['event'] = 'incident.unknown' end - context 'when one of the messages has unknown event' do - let(:payload) do - valid_payload = Gitlab::Json.parse(fixture_file) - event = { 'event' => 'incident.unknown', 'incident' => valid_payload['messages'].first['incident'] } - valid_payload['messages'] = valid_payload['messages'].append(event) - valid_payload - end - - it 'returns parsed payload' do - unknown_event = triggered_event.dup - unknown_event['event'] = 'incident.unknown' - - is_expected.to contain_exactly(triggered_event, unknown_event) - end + it 'returns empty payload' do + is_expected.to be_empty end end end diff --git a/spec/lib/peek/views/active_record_spec.rb b/spec/lib/peek/views/active_record_spec.rb index 7bc15f40065..fc768bdcb82 100644 --- a/spec/lib/peek/views/active_record_spec.rb +++ b/spec/lib/peek/views/active_record_spec.rb @@ -61,7 +61,7 @@ RSpec.describe Peek::Views::ActiveRecord, :request_store do end it 'includes db role data and db_config_name name' do - Timecop.freeze(2021, 2, 23, 10, 0) do + travel_to(Time.utc(2021, 2, 23, 10, 0)) do ActiveSupport::Notifications.publish('sql.active_record', Time.current, Time.current + 1.second, '1', event_1) ActiveSupport::Notifications.publish('sql.active_record', Time.current, Time.current + 2.seconds, '2', event_2) ActiveSupport::Notifications.publish('sql.active_record', Time.current, Time.current + 3.seconds, '3', event_3) diff --git a/spec/lib/sbom/package_url/argument_validator_spec.rb b/spec/lib/sbom/package_url/argument_validator_spec.rb index 246da1c0bda..56dc1d54ba9 100644 --- a/spec/lib/sbom/package_url/argument_validator_spec.rb +++ b/spec/lib/sbom/package_url/argument_validator_spec.rb @@ -5,7 +5,7 @@ require 'rspec-parameterized' require_relative '../../../support/shared_contexts/lib/sbom/package_url_shared_contexts' -RSpec.describe Sbom::PackageUrl::ArgumentValidator do +RSpec.describe Sbom::PackageUrl::ArgumentValidator, feature_category: :dependency_management do let(:mock_package_url) { Struct.new(:type, :namespace, :name, :version, :qualifiers, keyword_init: true) } let(:package) do mock_package_url.new( diff --git a/spec/lib/sbom/package_url/decoder_spec.rb b/spec/lib/sbom/package_url/decoder_spec.rb index 5b480475b7c..3c092b35ea2 100644 --- a/spec/lib/sbom/package_url/decoder_spec.rb +++ b/spec/lib/sbom/package_url/decoder_spec.rb @@ -5,7 +5,7 @@ require 'rspec-parameterized' require_relative '../../../support/shared_contexts/lib/sbom/package_url_shared_contexts' -RSpec.describe Sbom::PackageUrl::Decoder do +RSpec.describe Sbom::PackageUrl::Decoder, feature_category: :dependency_management do describe '#decode' do subject(:decode) { described_class.new(purl).decode! } diff --git a/spec/lib/sbom/package_url/encoder_spec.rb b/spec/lib/sbom/package_url/encoder_spec.rb index bdbd61636b5..a0b51007008 100644 --- a/spec/lib/sbom/package_url/encoder_spec.rb +++ b/spec/lib/sbom/package_url/encoder_spec.rb @@ -5,7 +5,7 @@ require 'rspec-parameterized' require_relative '../../../support/shared_contexts/lib/sbom/package_url_shared_contexts' -RSpec.describe Sbom::PackageUrl::Encoder do +RSpec.describe Sbom::PackageUrl::Encoder, feature_category: :dependency_management do describe '#encode' do let(:package) do ::Sbom::PackageUrl.new( diff --git a/spec/lib/sbom/package_url/normalizer_spec.rb b/spec/lib/sbom/package_url/normalizer_spec.rb index bbc2bd3ca13..3ad548a5c84 100644 --- a/spec/lib/sbom/package_url/normalizer_spec.rb +++ b/spec/lib/sbom/package_url/normalizer_spec.rb @@ -5,7 +5,7 @@ require 'rspec-parameterized' require_relative '../../../support/shared_contexts/lib/sbom/package_url_shared_contexts' -RSpec.describe Sbom::PackageUrl::Normalizer do +RSpec.describe Sbom::PackageUrl::Normalizer, feature_category: :dependency_management do shared_examples 'name normalization' do context 'with bitbucket url' do let(:type) { 'bitbucket' } diff --git a/spec/lib/sbom/package_url_spec.rb b/spec/lib/sbom/package_url_spec.rb index 6760b0a68e5..92490b184df 100644 --- a/spec/lib/sbom/package_url_spec.rb +++ b/spec/lib/sbom/package_url_spec.rb @@ -29,7 +29,7 @@ require 'rspec-parameterized' require_relative '../../support/helpers/next_instance_of' require_relative '../../support/shared_contexts/lib/sbom/package_url_shared_contexts' -RSpec.describe Sbom::PackageUrl do +RSpec.describe Sbom::PackageUrl, feature_category: :dependency_management do include NextInstanceOf describe '#initialize' do diff --git a/spec/lib/security/ci_configuration/container_scanning_build_action_spec.rb b/spec/lib/security/ci_configuration/container_scanning_build_action_spec.rb index 38066e41c53..5b1db66beb0 100644 --- a/spec/lib/security/ci_configuration/container_scanning_build_action_spec.rb +++ b/spec/lib/security/ci_configuration/container_scanning_build_action_spec.rb @@ -33,7 +33,7 @@ RSpec.describe Security::CiConfiguration::ContainerScanningBuildAction do RANDOM: make sure this persists include: - template: existing.yml - - template: Security/Container-Scanning.gitlab-ci.yml + - template: Jobs/Container-Scanning.gitlab-ci.yml CI_YML end @@ -85,7 +85,7 @@ RSpec.describe Security::CiConfiguration::ContainerScanningBuildAction do variables: RANDOM: make sure this persists include: - - template: Security/Container-Scanning.gitlab-ci.yml + - template: Jobs/Container-Scanning.gitlab-ci.yml CI_YML end @@ -93,7 +93,7 @@ RSpec.describe Security::CiConfiguration::ContainerScanningBuildAction do let(:gitlab_ci_content) do { "stages" => %w(test), "variables" => { "RANDOM" => "make sure this persists" }, - "include" => [{ "template" => "Security/Container-Scanning.gitlab-ci.yml" }] } + "include" => [{ "template" => "Jobs/Container-Scanning.gitlab-ci.yml" }] } end it 'generates the correct YML' do @@ -106,7 +106,7 @@ RSpec.describe Security::CiConfiguration::ContainerScanningBuildAction do let(:gitlab_ci_content) do { "stages" => %w(test), "variables" => { "RANDOM" => "make sure this persists" }, - "include" => { "template" => "Security/Container-Scanning.gitlab-ci.yml" } } + "include" => { "template" => "Jobs/Container-Scanning.gitlab-ci.yml" } } end it 'generates the correct YML' do @@ -138,7 +138,7 @@ RSpec.describe Security::CiConfiguration::ContainerScanningBuildAction do # DOCKER_USER: ... # DOCKER_PASSWORD: ... include: - - template: Security/Container-Scanning.gitlab-ci.yml + - template: Jobs/Container-Scanning.gitlab-ci.yml CI_YML end diff --git a/spec/lib/security/weak_passwords_spec.rb b/spec/lib/security/weak_passwords_spec.rb index 9d12c352abf..afa9448e746 100644 --- a/spec/lib/security/weak_passwords_spec.rb +++ b/spec/lib/security/weak_passwords_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Security::WeakPasswords do +RSpec.describe Security::WeakPasswords, feature_category: :authentication_and_authorization do describe "#weak_for_user?" do using RSpec::Parameterized::TableSyntax @@ -34,6 +34,7 @@ RSpec.describe Security::WeakPasswords do "!@mCwEaKy" | true "A1B2pass" | true "A1B2C3jr" | false # jr is too short + "3e18a7f60a908e329958396d68131d39e1b66a03ea420725e2a0fce7cb17pass" | false # Password is >= 64 chars # Predictable username substrings "56d4ab689a" | true diff --git a/spec/lib/serializers/json_spec.rb b/spec/lib/serializers/json_spec.rb deleted file mode 100644 index 96a57cde056..00000000000 --- a/spec/lib/serializers/json_spec.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -require 'fast_spec_helper' -require 'oj' - -RSpec.describe Serializers::Json do - describe '.dump' do - let(:obj) { { key: "value" } } - - subject { described_class.dump(obj) } - - it 'returns a hash' do - is_expected.to eq(obj) - end - end - - describe '.load' do - let(:data_string) { '{"key":"value","variables":[{"key":"VAR1","value":"VALUE1"}]}' } - let(:data_hash) { Gitlab::Json.parse(data_string) } - - context 'when loading a hash' do - subject { described_class.load(data_hash) } - - it 'decodes a string' do - is_expected.to be_a(Hash) - end - - it 'allows to access with symbols' do - expect(subject[:key]).to eq('value') - expect(subject[:variables].first[:key]).to eq('VAR1') - end - - it 'allows to access with strings' do - expect(subject["key"]).to eq('value') - expect(subject["variables"].first["key"]).to eq('VAR1') - end - end - - context 'when loading a nil' do - subject { described_class.load(nil) } - - it 'returns nil' do - is_expected.to be_nil - end - end - end -end diff --git a/spec/lib/sidebars/projects/menus/deployments_menu_spec.rb b/spec/lib/sidebars/projects/menus/deployments_menu_spec.rb index 685ba0c31c7..ce971915174 100644 --- a/spec/lib/sidebars/projects/menus/deployments_menu_spec.rb +++ b/spec/lib/sidebars/projects/menus/deployments_menu_spec.rb @@ -47,44 +47,16 @@ RSpec.describe Sidebars::Projects::Menus::DeploymentsMenu do end end - shared_examples 'split_operations_visibility_permissions FF disabled' do - before do - stub_feature_flags(split_operations_visibility_permissions: false) - end - - it { is_expected.not_to be_nil } - - context 'and the feature is disabled' do - before do - project.update_attribute("#{item_id}_access_level", 'disabled') - end - - it { is_expected.not_to be_nil } - end - - context 'and operations is disabled' do - before do - project.update_attribute(:operations_access_level, 'disabled') - end - - it do - is_expected.to be_nil if [:environments, :feature_flags].include?(item_id) - end - end - end - describe 'Feature Flags' do let(:item_id) { :feature_flags } it_behaves_like 'access rights checks' - it_behaves_like 'split_operations_visibility_permissions FF disabled' end describe 'Environments' do let(:item_id) { :environments } it_behaves_like 'access rights checks' - it_behaves_like 'split_operations_visibility_permissions FF disabled' end describe 'Releases' do diff --git a/spec/lib/sidebars/projects/menus/infrastructure_menu_spec.rb b/spec/lib/sidebars/projects/menus/infrastructure_menu_spec.rb index 64408ac3b88..116948b7cb0 100644 --- a/spec/lib/sidebars/projects/menus/infrastructure_menu_spec.rb +++ b/spec/lib/sidebars/projects/menus/infrastructure_menu_spec.rb @@ -31,43 +31,18 @@ RSpec.describe Sidebars::Projects::Menus::InfrastructureMenu do let(:enabled) { Featurable::PRIVATE } let(:disabled) { Featurable::DISABLED } - where(:operations_access_level, :infrastructure_access_level, :render) do - ref(:disabled) | ref(:enabled) | true - ref(:disabled) | ref(:disabled) | false - ref(:enabled) | ref(:enabled) | true - ref(:enabled) | ref(:disabled) | false + where(:infrastructure_access_level, :render) do + ref(:enabled) | true + ref(:disabled) | false end with_them do it 'renders based on the infrastructure access level' do - project.project_feature.update!(operations_access_level: operations_access_level) project.project_feature.update!(infrastructure_access_level: infrastructure_access_level) expect(subject.render?).to be render end end - - context 'when `split_operations_visibility_permissions` feature flag is disabled' do - before do - stub_feature_flags(split_operations_visibility_permissions: false) - end - - where(:operations_access_level, :infrastructure_access_level, :render) do - ref(:disabled) | ref(:enabled) | false - ref(:disabled) | ref(:disabled) | false - ref(:enabled) | ref(:enabled) | true - ref(:enabled) | ref(:disabled) | true - end - - with_them do - it 'renders based on the operations access level' do - project.project_feature.update!(operations_access_level: operations_access_level) - project.project_feature.update!(infrastructure_access_level: infrastructure_access_level) - - expect(subject.render?).to be render - end - end - end end end diff --git a/spec/lib/sidebars/projects/menus/monitor_menu_spec.rb b/spec/lib/sidebars/projects/menus/monitor_menu_spec.rb index f6a8dd7367d..a1e6ae13e68 100644 --- a/spec/lib/sidebars/projects/menus/monitor_menu_spec.rb +++ b/spec/lib/sidebars/projects/menus/monitor_menu_spec.rb @@ -16,40 +16,30 @@ RSpec.describe Sidebars::Projects::Menus::MonitorMenu do let(:enabled) { Featurable::PRIVATE } let(:disabled) { Featurable::DISABLED } - where(:flag_enabled, :operations_access_level, :monitor_level, :render) do - true | ref(:disabled) | ref(:enabled) | true - true | ref(:disabled) | ref(:disabled) | false - true | ref(:enabled) | ref(:enabled) | true - true | ref(:enabled) | ref(:disabled) | false - false | ref(:disabled) | ref(:enabled) | false - false | ref(:disabled) | ref(:disabled) | false - false | ref(:enabled) | ref(:enabled) | true - false | ref(:enabled) | ref(:disabled) | true + where(:monitor_level, :render) do + ref(:enabled) | true + ref(:disabled) | false end with_them do it 'renders when expected to' do - stub_feature_flags(split_operations_visibility_permissions: flag_enabled) - project.project_feature.update!(operations_access_level: operations_access_level) project.project_feature.update!(monitor_access_level: monitor_level) expect(subject.render?).to be render end end - context 'when operation feature is enabled' do - context 'when menu does not have any renderable menu items' do - it 'returns false' do - allow(subject).to receive(:has_renderable_items?).and_return(false) + context 'when menu does not have any renderable menu items' do + it 'returns false' do + allow(subject).to receive(:has_renderable_items?).and_return(false) - expect(subject.render?).to be false - end + expect(subject.render?).to be false end + end - context 'when menu has menu items' do - it 'returns true' do - expect(subject.render?).to be true - end + context 'when menu has menu items' do + it 'returns true' do + expect(subject.render?).to be true end end end diff --git a/spec/lib/sidebars/projects/menus/repository_menu_spec.rb b/spec/lib/sidebars/projects/menus/repository_menu_spec.rb index f26433306b6..e7aa2b7edca 100644 --- a/spec/lib/sidebars/projects/menus/repository_menu_spec.rb +++ b/spec/lib/sidebars/projects/menus/repository_menu_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Sidebars::Projects::Menus::RepositoryMenu do +RSpec.describe Sidebars::Projects::Menus::RepositoryMenu, feature_category: :source_code_management do let_it_be(:project) { create(:project, :repository) } let(:user) { project.first_owner } @@ -36,12 +36,68 @@ RSpec.describe Sidebars::Projects::Menus::RepositoryMenu do end context 'for menu items' do - subject { described_class.new(context).renderable_items.index { |e| e.item_id == item_id } } + shared_examples_for 'repository menu item link for' do |item_id| + let(:ref) { 'master' } + let(:item_id) { item_id } + subject { described_class.new(context).renderable_items.find { |e| e.item_id == item_id }.link } + + using RSpec::Parameterized::TableSyntax + + let(:context) do + Sidebars::Projects::Context.new(current_user: user, container: project, current_ref: ref, + ref_type: ref_type) + end + + where(:feature_flag_enabled, :ref_type, :link) do + true | nil | lazy { "#{route}?ref_type=heads" } + true | 'heads' | lazy { "#{route}?ref_type=heads" } + true | 'tags' | lazy { "#{route}?ref_type=tags" } + false | nil | lazy { route } + false | 'heads' | lazy { route } + false | 'tags' | lazy { route } + end + + with_them do + before do + stub_feature_flags(use_ref_type_parameter: feature_flag_enabled) + end + + it 'has a link with the fully qualifed ref route' do + expect(subject).to eq(link) + end + end + + context 'when ref is not the default' do + let(:ref) { 'nonmain' } + + context 'and ref_type is not provided' do + let(:ref_type) { nil } + + it { is_expected.to eq(route) } + end + + context 'and ref_type is provided' do + let(:ref_type) { 'heads' } + + it { is_expected.to eq("#{route}?ref_type=heads") } + end + end + end + + describe 'Commits' do + let_it_be(:item_id) { :commits } + + it_behaves_like 'repository menu item link for', :commits do + let(:route) { "/#{project.full_path}/-/commits/#{ref}" } + end + end describe 'Contributors' do let_it_be(:item_id) { :contributors } context 'when analytics is disabled' do + subject { described_class.new(context).renderable_items.find { |e| e.item_id == item_id } } + before do project.project_feature.update!(analytics_access_level: ProjectFeature::DISABLED) end @@ -54,7 +110,15 @@ RSpec.describe Sidebars::Projects::Menus::RepositoryMenu do project.project_feature.update!(analytics_access_level: ProjectFeature::ENABLED) end - it { is_expected.not_to be_nil } + it_behaves_like 'repository menu item link for', :contributors do + let(:route) { "/#{project.full_path}/-/graphs/#{ref}" } + end + end + end + + describe 'Network' do + it_behaves_like 'repository menu item link for', :graphs do + let(:route) { "/#{project.full_path}/-/network/#{ref}" } end end end diff --git a/spec/lib/system_check/app/gitlab_cable_config_exists_check_spec.rb b/spec/lib/system_check/app/gitlab_cable_config_exists_check_spec.rb new file mode 100644 index 00000000000..8e127bb715c --- /dev/null +++ b/spec/lib/system_check/app/gitlab_cable_config_exists_check_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe SystemCheck::App::GitlabCableConfigExistsCheck, feature_category: :redis do + subject(:system_check) { described_class.new } + + describe '#check?' do + subject { system_check.check? } + + context 'when config/cable.yml exists' do + before do + allow(File).to receive(:exist?).and_return(true) + end + + it { is_expected.to eq(true) } + end + + context 'when config/cable.yml does not exist' do + before do + allow(File).to receive(:exist?).and_return(false) + end + + it { is_expected.to eq(false) } + end + end +end diff --git a/spec/lib/system_check/app/gitlab_resque_config_exists_check_spec.rb b/spec/lib/system_check/app/gitlab_resque_config_exists_check_spec.rb new file mode 100644 index 00000000000..d2e5dec7460 --- /dev/null +++ b/spec/lib/system_check/app/gitlab_resque_config_exists_check_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe SystemCheck::App::GitlabResqueConfigExistsCheck, feature_category: :redis do + subject(:system_check) { described_class.new } + + describe '#check?' do + subject { system_check.check? } + + context 'when config/resque.yml exists' do + before do + allow(File).to receive(:exist?).and_return(true) + end + + it { is_expected.to eq(true) } + end + + context 'when config/resque.yml does not exist' do + before do + allow(File).to receive(:exist?).and_return(false) + end + + it { is_expected.to eq(false) } + end + end +end diff --git a/spec/lib/system_check/base_check_spec.rb b/spec/lib/system_check/base_check_spec.rb index 241c3b33777..168bda07791 100644 --- a/spec/lib/system_check/base_check_spec.rb +++ b/spec/lib/system_check/base_check_spec.rb @@ -12,7 +12,7 @@ RSpec.describe SystemCheck::BaseCheck do it 'responds to Gitlab::TaskHelpers methods' do expect(subject).to respond_to :ask_to_continue, :os_name, :prompt, :run_and_match, :run_command, - :run_command!, :uid_for, :gid_for, :gitlab_user, :gitlab_user?, :warn_user_is_not_gitlab, :all_repos, + :run_command!, :uid_for, :gid_for, :gitlab_user, :gitlab_user?, :warn_user_is_not_gitlab, :repository_storage_paths_args, :user_home, :checkout_or_clone_version, :clone_repo, :checkout_version end end diff --git a/spec/lib/system_check/sidekiq_check_spec.rb b/spec/lib/system_check/sidekiq_check_spec.rb index c2f61e0e4b7..ff4eece8f7c 100644 --- a/spec/lib/system_check/sidekiq_check_spec.rb +++ b/spec/lib/system_check/sidekiq_check_spec.rb @@ -37,45 +37,53 @@ RSpec.describe SystemCheck::SidekiqCheck do ) end - it 'succeeds when one cluster process and one or more worker processes are running' do - stub_ps_output <<~PS - root 2193947 0.9 0.1 146564 18104 ? Ssl 17:34 0:00 ruby bin/sidekiq-cluster * -P ... - root 2193955 92.2 3.1 4675972 515516 ? Sl 17:34 0:13 sidekiq 5.2.9 ... - root 2193956 92.2 3.1 4675972 515516 ? Sl 17:34 0:13 sidekiq 5.2.9 ... - PS - - expect_check_output <<~OUTPUT - Running? ... yes - Number of Sidekiq processes (cluster/worker) ... 1/2 - OUTPUT - end - - # TODO: Running without a cluster is deprecated and will be removed in GitLab 14.0 - # https://gitlab.com/gitlab-org/gitlab/-/issues/323225 - context 'when running without a cluster' do - it 'fails when more than one worker process is running' do + context 'when only a worker process is running' do + before do stub_ps_output <<~PS root 2193955 92.2 3.1 4675972 515516 ? Sl 17:34 0:13 sidekiq 5.2.9 ... - root 2193956 92.2 3.1 4675972 515516 ? Sl 17:34 0:13 sidekiq 5.2.9 ... PS + end - expect_check_output include( - 'Running? ... yes', - 'Number of Sidekiq processes (cluster/worker) ... 0/2', - 'Please fix the error above and rerun the checks.' - ) + it 'fails with the right message for systemd' do + allow(File).to receive(:symlink?).with(described_class::SYSTEMD_UNIT_PATH).and_return(true) + + expect_check_output <<~OUTPUT + Running? ... yes + Number of Sidekiq processes (cluster/worker) ... 0/1 + Try fixing it: + sudo systemctl restart gitlab-sidekiq.service + Please fix the error above and rerun the checks. + OUTPUT end - it 'succeeds when one worker process is running' do - stub_ps_output <<~PS - root 2193955 92.2 3.1 4675972 515516 ? Sl 17:34 0:13 sidekiq 5.2.9 ... - PS + it 'fails with the right message for sysvinit' do + allow(File).to receive(:symlink?).with(described_class::SYSTEMD_UNIT_PATH).and_return(false) + allow(subject).to receive(:gitlab_user).and_return('git') expect_check_output <<~OUTPUT Running? ... yes Number of Sidekiq processes (cluster/worker) ... 0/1 + Try fixing it: + sudo service gitlab stop + sudo pkill -u git -f sidekiq + sleep 10 && sudo pkill -9 -u git -f sidekiq + sudo service gitlab start + Please fix the error above and rerun the checks. OUTPUT end end + + it 'succeeds when one cluster process and one or more worker processes are running' do + stub_ps_output <<~PS + root 2193947 0.9 0.1 146564 18104 ? Ssl 17:34 0:00 ruby bin/sidekiq-cluster * -P ... + root 2193955 92.2 3.1 4675972 515516 ? Sl 17:34 0:13 sidekiq 5.2.9 ... + root 2193956 92.2 3.1 4675972 515516 ? Sl 17:34 0:13 sidekiq 5.2.9 ... + PS + + expect_check_output <<~OUTPUT + Running? ... yes + Number of Sidekiq processes (cluster/worker) ... 1/2 + OUTPUT + end end end diff --git a/spec/lib/version_check_spec.rb b/spec/lib/version_check_spec.rb index 1803dd66ba7..4aa8975b7cf 100644 --- a/spec/lib/version_check_spec.rb +++ b/spec/lib/version_check_spec.rb @@ -2,7 +2,9 @@ require 'spec_helper' -RSpec.describe VersionCheck do +RSpec.describe VersionCheck, :use_clean_rails_memory_store_caching do + include ReactiveCachingHelpers + describe '.url' do it 'returns the correct URL' do expect(described_class.url).to match(%r{\A#{Regexp.escape(described_class.host)}/check\.json\?gitlab_info=\w+}) @@ -24,13 +26,25 @@ RSpec.describe VersionCheck do end describe '#calculate_reactive_cache' do - context 'response code is 200' do + context 'response code is 200 with valid body' do before do stub_request(:get, described_class.url).to_return(status: 200, body: '{ "status": "success" }', headers: {}) end it 'returns the response object' do - expect(described_class.new.calculate_reactive_cache).to eq("{ \"status\": \"success\" }") + expect(described_class.new.calculate_reactive_cache).to eq({ "status" => "success" }) + end + end + + context 'response code is 200 with invalid body' do + before do + stub_request(:get, described_class.url).to_return(status: 200, body: '{ "invalid: json" }', headers: {}) + end + + it 'returns an error hash' do + expect(described_class.new.calculate_reactive_cache).to eq( + { error: 'parsing version check response failed', status: 200 } + ) end end @@ -39,38 +53,61 @@ RSpec.describe VersionCheck do stub_request(:get, described_class.url).to_return(status: 500, body: nil, headers: {}) end - it 'returns nil' do - expect(described_class.new.calculate_reactive_cache).to be(nil) + it 'returns an error hash' do + expect(described_class.new.calculate_reactive_cache).to eq({ error: 'version check failed', status: 500 }) end end end describe '#response' do - context 'cache returns value' do - let(:response) { { "severity" => "success" }.to_json } - + # see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106254 + context "with old string value in cache" do before do - allow_next_instance_of(described_class) do |instance| - allow(instance).to receive(:with_reactive_cache).and_return(response) - end + old_version_check = described_class.new + allow(old_version_check).to receive(:id).and_return(Gitlab::VERSION) + write_reactive_cache(old_version_check, + "{\"latest_stable_versions\":[],\"latest_version\":\"15.6.2\",\"severity\":\"success\",\"details\":\"\"}" + ) end - it 'returns the response object' do - expect(described_class.new.response).to be(response) + it 'returns nil' do + version_check = described_class.new + expect(version_check.response).to be_nil end end - context 'cache returns nil' do - let(:response) { nil } + # see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106254 + context "with non-hash value in cache" do + it 'returns nil and invalidates the reactive cache' do + version_check = described_class.new + stub_reactive_cache(version_check, + "{\"latest_stable_versions\":[],\"latest_version\":\"15.6.2\",\"severity\":\"success\",\"details\":\"\"}" + ) - before do - allow_next_instance_of(described_class) do |instance| - allow(instance).to receive(:with_reactive_cache).and_return(response) - end + expect(version_check).to receive(:refresh_reactive_cache!).and_call_original + expect(version_check.response).to be_nil + expect(read_reactive_cache(version_check)).to be_nil end + end - it 'returns nil' do - expect(described_class.new.response).to be(nil) + context 'cache returns value' do + it 'returns the response object' do + version_check = described_class.new + data = { status: 'success' } + stub_reactive_cache(version_check, data) + + expect(version_check.response).to eq(data) + end + end + + context 'cache returns error' do + it 'returns nil and invalidates the reactive cache' do + version_check = described_class.new + stub_reactive_cache(version_check, error: 'version check failed') + + expect(version_check).to receive(:refresh_reactive_cache!).and_call_original + expect(version_check.response).to be_nil + expect(read_reactive_cache(version_check)).to be_nil end end end |