diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-08-30 12:13:15 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-08-30 12:13:15 +0300 |
commit | 0c918eb567a29de7ecfc890e8e2dd90e900e7e58 (patch) | |
tree | b4ad20b2eacec3fbb2fb85b1ca141afd87b8d3ee /spec | |
parent | 55cd0a88bb0c7b899ca7e292fd3e30c5174063d2 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'spec')
8 files changed, 272 insertions, 9 deletions
diff --git a/spec/factories/protected_branches.rb b/spec/factories/protected_branches.rb index bac1cf21596..cee7b577236 100644 --- a/spec/factories/protected_branches.rb +++ b/spec/factories/protected_branches.rb @@ -11,6 +11,12 @@ FactoryBot.define do default_access_level { true } end + after(:create) do |protected_branch, evaluator| + break unless protected_branch.project&.persisted? + + ProtectedBranches::CacheService.new(protected_branch.project).refresh + end + trait :create_branch_on_repository do association :project, factory: [:project, :repository] diff --git a/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb b/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb new file mode 100644 index 00000000000..f6c877c03ed --- /dev/null +++ b/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::RemoveBackfilledJobArtifactsExpireAtBatchingStrategy, '#next_batch' do # rubocop:disable Layout/LineLength + let_it_be(:namespace) { table(:namespaces).create!(id: 1, name: 'user', path: 'user') } + let_it_be(:project) do + table(:projects).create!( + id: 1, + name: 'gitlab1', + path: 'gitlab1', + project_namespace_id: 1, + namespace_id: namespace.id + ) + end + + let(:batching_strategy) { described_class.new(connection: Ci::ApplicationRecord.connection) } + let(:job_artifact) { table(:ci_job_artifacts, database: :ci) } + + # job artifacts expiring at midnight in various timezones + let!(:ci_job_artifact_1) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) } + let!(:ci_job_artifact_2) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-21 01:30:00.000')) } + let!(:ci_job_artifact_3) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:00:00.000')) } + let!(:ci_job_artifact_4) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:30:00.000')) } + let!(:ci_job_artifact_5) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:00:00.000')) } + let!(:ci_job_artifact_6) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:30:00.000')) } + let!(:ci_job_artifact_7) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-23 06:45:00.000')) } + # out ot scope job artifacts + let!(:ci_job_artifact_8) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.001')) } + let!(:ci_job_artifact_9) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-19 12:00:00.000')) } + # job artifacts of trace type (file_type: 3) + let!(:ci_job_artifact_10) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) } + let!(:ci_job_artifact_11) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) } + # out ot scope job artifacts + let!(:ci_job_artifact_12) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-24 23:30:00.000')) } + let!(:ci_job_artifact_13) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:30:00.000')) } + # job artifacts of trace type (file_type: 3) + let!(:ci_job_artifact_14) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) } + let!(:ci_job_artifact_15) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) } + + it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchingStrategy } + + context 'when starting on the first batch' do + it 'returns the bounds of the next batch' do + batch_bounds = batching_strategy.next_batch( + :ci_job_artifacts, + :id, + batch_min_value: ci_job_artifact_1.id, + batch_size: 5, + job_arguments: [] + ) + expect(batch_bounds).to eq([ci_job_artifact_1.id, ci_job_artifact_5.id]) + end + end + + context 'when the range includes out of scope records' do + it 'returns the bounds of the next batch, skipping records outside the scope' do + batch_bounds = batching_strategy.next_batch( + :ci_job_artifacts, + :id, + batch_min_value: ci_job_artifact_1.id, + batch_size: 10, + job_arguments: [] + ) + expect(batch_bounds).to eq([ci_job_artifact_1.id, ci_job_artifact_14.id]) + end + end + + context 'when the range begins on out of scope records' do + it 'returns the bounds of the next batch, skipping records outside the scope' do + batch_bounds = batching_strategy.next_batch( + :ci_job_artifacts, + :id, + batch_min_value: ci_job_artifact_8.id, + batch_size: 3, + job_arguments: [] + ) + expect(batch_bounds).to eq([ci_job_artifact_10.id, ci_job_artifact_14.id]) + end + end + + context 'when no additional batch remain' do + it 'returns nil' do + batch_bounds = batching_strategy.next_batch( + :ci_job_artifacts, + :id, + batch_min_value: ci_job_artifact_15.id + 1, + batch_size: 10, + job_arguments: [] + ) + expect(batch_bounds).to be_nil + end + end + + private + + def create_job_artifact(file_type:, expire_at:) + job = table(:ci_builds, database: :ci).create! + job_artifact.create!(job_id: job.id, expire_at: expire_at, project_id: project.id, file_type: file_type) + end +end diff --git a/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb new file mode 100644 index 00000000000..41266cb24da --- /dev/null +++ b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::RemoveBackfilledJobArtifactsExpireAt do + it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchedMigrationJob } + + describe '#perform' do + let(:job_artifact) { table(:ci_job_artifacts, database: :ci) } + + let(:test_worker) do + described_class.new( + start_id: 1, + end_id: 100, + batch_table: :ci_job_artifacts, + batch_column: :id, + sub_batch_size: 10, + pause_ms: 0, + connection: Ci::ApplicationRecord.connection + ) + end + + let_it_be(:namespace) { table(:namespaces).create!(id: 1, name: 'user', path: 'user') } + let_it_be(:project) do + table(:projects).create!( + id: 1, + name: 'gitlab1', + path: 'gitlab1', + project_namespace_id: 1, + namespace_id: namespace.id + ) + end + + subject { test_worker.perform } + + context 'with artifacts that has backfilled expire_at' do + let!(:created_on_00_30_45_minutes_on_21_22_23) do + create_job_artifact(id: 1, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) + create_job_artifact(id: 2, file_type: 1, expire_at: Time.zone.parse('2022-01-21 01:30:00.000')) + create_job_artifact(id: 3, file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:00:00.000')) + create_job_artifact(id: 4, file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:30:00.000')) + create_job_artifact(id: 5, file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:00:00.000')) + create_job_artifact(id: 6, file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:30:00.000')) + create_job_artifact(id: 7, file_type: 1, expire_at: Time.zone.parse('2022-01-23 06:45:00.000')) + end + + let!(:created_close_to_00_or_30_minutes) do + create_job_artifact(id: 8, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.001')) + create_job_artifact(id: 9, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:30:00.999')) + end + + let!(:created_on_00_or_30_minutes_on_other_dates) do + create_job_artifact(id: 10, file_type: 1, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) + create_job_artifact(id: 11, file_type: 1, expire_at: Time.zone.parse('2022-01-19 12:00:00.000')) + create_job_artifact(id: 12, file_type: 1, expire_at: Time.zone.parse('2022-01-24 23:30:00.000')) + end + + let!(:created_at_other_times) do + create_job_artifact(id: 13, file_type: 1, expire_at: Time.zone.parse('2022-01-19 00:00:00.000')) + create_job_artifact(id: 14, file_type: 1, expire_at: Time.zone.parse('2022-01-19 00:30:00.000')) + create_job_artifact(id: 15, file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:00:00.000')) + create_job_artifact(id: 16, file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:30:00.000')) + end + + it 'removes expire_at on job artifacts that have expire_at on 00, 30 or 45 minute of 21, 22, 23 of the month' do + expect { subject }.to change { job_artifact.where(expire_at: nil).count }.from(0).to(7) + end + + it 'keeps expire_at on other job artifacts' do + expect { subject }.to change { job_artifact.where.not(expire_at: nil).count }.from(16).to(9) + end + end + + context 'with trace artifacts that has backfilled expire_at' do + let!(:trace_artifacts) do + create_job_artifact(id: 1, file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) + create_job_artifact(id: 2, file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) + end + + it 'removes expire_at on trace job artifacts' do + expect { subject }.to change { job_artifact.where(expire_at: nil).count }.from(0).to(2) + end + end + + private + + def create_job_artifact(id:, file_type:, expire_at:) + job = table(:ci_builds, database: :ci).create!(id: id) + job_artifact.create!(id: id, job_id: job.id, expire_at: expire_at, project_id: project.id, file_type: file_type) + end + end +end diff --git a/spec/lib/gitlab/metrics/system_spec.rb b/spec/lib/gitlab/metrics/system_spec.rb index 26026a9540b..7739501dd95 100644 --- a/spec/lib/gitlab/metrics/system_spec.rb +++ b/spec/lib/gitlab/metrics/system_spec.rb @@ -72,10 +72,20 @@ RSpec.describe Gitlab::Metrics::System do end describe '.memory_usage_rss' do - it "returns the process' resident set size (RSS) in bytes" do - mock_existing_proc_file('/proc/self/status', proc_status) + context 'without PID' do + it "returns the current process' resident set size (RSS) in bytes" do + mock_existing_proc_file('/proc/self/status', proc_status) + + expect(described_class.memory_usage_rss).to eq(2527232) + end + end + + context 'with PID' do + it "returns the given process' resident set size (RSS) in bytes" do + mock_existing_proc_file('/proc/7/status', proc_status) - expect(described_class.memory_usage_rss).to eq(2527232) + expect(described_class.memory_usage_rss(pid: 7)).to eq(2527232) + end end end @@ -96,11 +106,22 @@ RSpec.describe Gitlab::Metrics::System do end describe '.memory_usage_uss_pss' do - it "returns the process' unique and porportional set size (USS/PSS) in bytes" do - mock_existing_proc_file('/proc/self/smaps_rollup', proc_smaps_rollup) + context 'without PID' do + it "returns the current process' unique and porportional set size (USS/PSS) in bytes" do + mock_existing_proc_file('/proc/self/smaps_rollup', proc_smaps_rollup) + + # (Private_Clean (152 kB) + Private_Dirty (312 kB) + Private_Hugetlb (0 kB)) * 1024 + expect(described_class.memory_usage_uss_pss).to eq(uss: 475136, pss: 515072) + end + end + + context 'with PID' do + it "returns the given process' unique and porportional set size (USS/PSS) in bytes" do + mock_existing_proc_file('/proc/7/smaps_rollup', proc_smaps_rollup) - # (Private_Clean (152 kB) + Private_Dirty (312 kB) + Private_Hugetlb (0 kB)) * 1024 - expect(described_class.memory_usage_uss_pss).to eq(uss: 475136, pss: 515072) + # (Private_Clean (152 kB) + Private_Dirty (312 kB) + Private_Hugetlb (0 kB)) * 1024 + expect(described_class.memory_usage_uss_pss(pid: 7)).to eq(uss: 475136, pss: 515072) + end end end diff --git a/spec/migrations/20220606054503_fix_incorrect_job_artifacts_expire_at_spec.rb b/spec/migrations/20220606054503_fix_incorrect_job_artifacts_expire_at_spec.rb new file mode 100644 index 00000000000..5921dd64c0e --- /dev/null +++ b/spec/migrations/20220606054503_fix_incorrect_job_artifacts_expire_at_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe FixIncorrectJobArtifactsExpireAt, migration: :gitlab_ci do + let_it_be(:batched_migration) { described_class::MIGRATION } + + it 'does not schedule background jobs when Gitlab.com is true' do + allow(Gitlab).to receive(:com?).and_return(true) + + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + end + end + + it 'schedules background job on non Gitlab.com' do + allow(Gitlab).to receive(:com?).and_return(false) + + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + gitlab_schema: :gitlab_ci, + table_name: :ci_job_artifacts, + column_name: :id, + interval: described_class::INTERVAL, + batch_size: described_class::BATCH_SIZE + ) + } + end + end +end diff --git a/spec/rubocop/cop/gitlab/mark_used_feature_flags_spec.rb b/spec/rubocop/cop/gitlab/mark_used_feature_flags_spec.rb index c61376d7760..515620bd820 100644 --- a/spec/rubocop/cop/gitlab/mark_used_feature_flags_spec.rb +++ b/spec/rubocop/cop/gitlab/mark_used_feature_flags_spec.rb @@ -48,6 +48,7 @@ RSpec.describe RuboCop::Cop::Gitlab::MarkUsedFeatureFlags do Feature.enabled? Feature.disabled? push_frontend_feature_flag + YamlProcessor::FeatureFlags.enabled? ].each do |feature_flag_method| context "#{feature_flag_method} method" do context 'a string feature flag' do diff --git a/spec/services/protected_branches/destroy_service_spec.rb b/spec/services/protected_branches/destroy_service_spec.rb index 9fa07820148..123deeea005 100644 --- a/spec/services/protected_branches/destroy_service_spec.rb +++ b/spec/services/protected_branches/destroy_service_spec.rb @@ -5,7 +5,7 @@ require 'spec_helper' RSpec.describe ProtectedBranches::DestroyService do let_it_be_with_reload(:project) { create(:project) } - let(:protected_branch) { create(:protected_branch, project: project) } + let!(:protected_branch) { create(:protected_branch, project: project) } let(:user) { project.first_owner } subject(:service) { described_class.new(project, user) } diff --git a/spec/services/protected_branches/update_service_spec.rb b/spec/services/protected_branches/update_service_spec.rb index c4fe4d78070..2ff6c3c489a 100644 --- a/spec/services/protected_branches/update_service_spec.rb +++ b/spec/services/protected_branches/update_service_spec.rb @@ -5,7 +5,7 @@ require 'spec_helper' RSpec.describe ProtectedBranches::UpdateService do let_it_be_with_reload(:project) { create(:project) } - let(:protected_branch) { create(:protected_branch, project: project) } + let!(:protected_branch) { create(:protected_branch, project: project) } let(:user) { project.first_owner } let(:params) { { name: new_name } } |