Add latest changes from gitlab-org/gitlab@master

8 files changed, 272 insertions, 9 deletions

diff --git a/spec/factories/protected_branches.rb b/spec/factories/protected_branches.rb
index bac1cf21596..cee7b577236 100644
--- a/spec/factories/protected_branches.rb
+++ b/spec/factories/protected_branches.rb
@@ -11,6 +11,12 @@ FactoryBot.define do
       default_access_level { true }
     end
 
+    after(:create) do |protected_branch, evaluator|
+      break unless protected_branch.project&.persisted?
+
+      ProtectedBranches::CacheService.new(protected_branch.project).refresh
+    end
+
     trait :create_branch_on_repository do
       association :project, factory: [:project, :repository]
 
diff --git a/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb b/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb
new file mode 100644
index 00000000000..f6c877c03ed
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb
@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::RemoveBackfilledJobArtifactsExpireAtBatchingStrategy, '#next_batch' do # rubocop:disable Layout/LineLength
+  let_it_be(:namespace) { table(:namespaces).create!(id: 1, name: 'user', path: 'user') }
+  let_it_be(:project) do
+    table(:projects).create!(
+      id: 1,
+      name: 'gitlab1',
+      path: 'gitlab1',
+      project_namespace_id: 1,
+      namespace_id: namespace.id
+    )
+  end
+
+  let(:batching_strategy) { described_class.new(connection: Ci::ApplicationRecord.connection) }
+  let(:job_artifact) { table(:ci_job_artifacts, database: :ci) }
+
+  # job artifacts expiring at midnight in various timezones
+  let!(:ci_job_artifact_1) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) }
+  let!(:ci_job_artifact_2) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-21 01:30:00.000')) }
+  let!(:ci_job_artifact_3) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:00:00.000')) }
+  let!(:ci_job_artifact_4) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:30:00.000')) }
+  let!(:ci_job_artifact_5) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:00:00.000')) }
+  let!(:ci_job_artifact_6) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:30:00.000')) }
+  let!(:ci_job_artifact_7) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-23 06:45:00.000')) }
+  # out ot scope job artifacts
+  let!(:ci_job_artifact_8) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.001')) }
+  let!(:ci_job_artifact_9) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-19 12:00:00.000')) }
+  # job artifacts of trace type (file_type: 3)
+  let!(:ci_job_artifact_10) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) }
+  let!(:ci_job_artifact_11) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) }
+  # out ot scope job artifacts
+  let!(:ci_job_artifact_12) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-24 23:30:00.000')) }
+  let!(:ci_job_artifact_13) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:30:00.000')) }
+  # job artifacts of trace type (file_type: 3)
+  let!(:ci_job_artifact_14) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) }
+  let!(:ci_job_artifact_15) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) }
+
+  it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchingStrategy }
+
+  context 'when starting on the first batch' do
+    it 'returns the bounds of the next batch' do
+      batch_bounds = batching_strategy.next_batch(
+        :ci_job_artifacts,
+        :id,
+        batch_min_value: ci_job_artifact_1.id,
+        batch_size: 5,
+        job_arguments: []
+      )
+      expect(batch_bounds).to eq([ci_job_artifact_1.id, ci_job_artifact_5.id])
+    end
+  end
+
+  context 'when the range includes out of scope records' do
+    it 'returns the bounds of the next batch, skipping records outside the scope' do
+      batch_bounds = batching_strategy.next_batch(
+        :ci_job_artifacts,
+        :id,
+        batch_min_value: ci_job_artifact_1.id,
+        batch_size: 10,
+        job_arguments: []
+      )
+      expect(batch_bounds).to eq([ci_job_artifact_1.id, ci_job_artifact_14.id])
+    end
+  end
+
+  context 'when the range begins on out of scope records' do
+    it 'returns the bounds of the next batch, skipping records outside the scope' do
+      batch_bounds = batching_strategy.next_batch(
+        :ci_job_artifacts,
+        :id,
+        batch_min_value: ci_job_artifact_8.id,
+        batch_size: 3,
+        job_arguments: []
+      )
+      expect(batch_bounds).to eq([ci_job_artifact_10.id, ci_job_artifact_14.id])
+    end
+  end
+
+  context 'when no additional batch remain' do
+    it 'returns nil' do
+      batch_bounds = batching_strategy.next_batch(
+        :ci_job_artifacts,
+        :id,
+        batch_min_value: ci_job_artifact_15.id + 1,
+        batch_size: 10,
+        job_arguments: []
+      )
+      expect(batch_bounds).to be_nil
+    end
+  end
+
+  private
+
+  def create_job_artifact(file_type:, expire_at:)
+    job = table(:ci_builds, database: :ci).create!
+    job_artifact.create!(job_id: job.id, expire_at: expire_at, project_id: project.id, file_type: file_type)
+  end
+end
diff --git a/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb
new file mode 100644
index 00000000000..41266cb24da
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb
@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::RemoveBackfilledJobArtifactsExpireAt do
+  it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchedMigrationJob }
+
+  describe '#perform' do
+    let(:job_artifact) { table(:ci_job_artifacts, database: :ci) }
+
+    let(:test_worker) do
+      described_class.new(
+        start_id: 1,
+        end_id: 100,
+        batch_table: :ci_job_artifacts,
+        batch_column: :id,
+        sub_batch_size: 10,
+        pause_ms: 0,
+        connection: Ci::ApplicationRecord.connection
+      )
+    end
+
+    let_it_be(:namespace) { table(:namespaces).create!(id: 1, name: 'user', path: 'user') }
+    let_it_be(:project) do
+      table(:projects).create!(
+        id: 1,
+        name: 'gitlab1',
+        path: 'gitlab1',
+        project_namespace_id: 1,
+        namespace_id: namespace.id
+      )
+    end
+
+    subject { test_worker.perform }
+
+    context 'with artifacts that has backfilled expire_at' do
+      let!(:created_on_00_30_45_minutes_on_21_22_23) do
+        create_job_artifact(id: 1, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.000'))
+        create_job_artifact(id: 2, file_type: 1, expire_at: Time.zone.parse('2022-01-21 01:30:00.000'))
+        create_job_artifact(id: 3, file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:00:00.000'))
+        create_job_artifact(id: 4, file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:30:00.000'))
+        create_job_artifact(id: 5, file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:00:00.000'))
+        create_job_artifact(id: 6, file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:30:00.000'))
+        create_job_artifact(id: 7, file_type: 1, expire_at: Time.zone.parse('2022-01-23 06:45:00.000'))
+      end
+
+      let!(:created_close_to_00_or_30_minutes) do
+        create_job_artifact(id: 8, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.001'))
+        create_job_artifact(id: 9, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:30:00.999'))
+      end
+
+      let!(:created_on_00_or_30_minutes_on_other_dates) do
+        create_job_artifact(id: 10, file_type: 1, expire_at: Time.zone.parse('2022-01-01 00:00:00.000'))
+        create_job_artifact(id: 11, file_type: 1, expire_at: Time.zone.parse('2022-01-19 12:00:00.000'))
+        create_job_artifact(id: 12, file_type: 1, expire_at: Time.zone.parse('2022-01-24 23:30:00.000'))
+      end
+
+      let!(:created_at_other_times) do
+        create_job_artifact(id: 13, file_type: 1, expire_at: Time.zone.parse('2022-01-19 00:00:00.000'))
+        create_job_artifact(id: 14, file_type: 1, expire_at: Time.zone.parse('2022-01-19 00:30:00.000'))
+        create_job_artifact(id: 15, file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:00:00.000'))
+        create_job_artifact(id: 16, file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:30:00.000'))
+      end
+
+      it 'removes expire_at on job artifacts that have expire_at on 00, 30 or 45 minute of 21, 22, 23 of the month' do
+        expect { subject }.to change { job_artifact.where(expire_at: nil).count }.from(0).to(7)
+      end
+
+      it 'keeps expire_at on other job artifacts' do
+        expect { subject }.to change { job_artifact.where.not(expire_at: nil).count }.from(16).to(9)
+      end
+    end
+
+    context 'with trace artifacts that has backfilled expire_at' do
+      let!(:trace_artifacts) do
+        create_job_artifact(id: 1, file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000'))
+        create_job_artifact(id: 2, file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000'))
+      end
+
+      it 'removes expire_at on trace job artifacts' do
+        expect { subject }.to change { job_artifact.where(expire_at: nil).count }.from(0).to(2)
+      end
+    end
+
+    private
+
+    def create_job_artifact(id:, file_type:, expire_at:)
+      job = table(:ci_builds, database: :ci).create!(id: id)
+      job_artifact.create!(id: id, job_id: job.id, expire_at: expire_at, project_id: project.id, file_type: file_type)
+    end
+  end
+end
diff --git a/spec/lib/gitlab/metrics/system_spec.rb b/spec/lib/gitlab/metrics/system_spec.rb
index 26026a9540b..7739501dd95 100644
--- a/spec/lib/gitlab/metrics/system_spec.rb
+++ b/spec/lib/gitlab/metrics/system_spec.rb
@@ -72,10 +72,20 @@ RSpec.describe Gitlab::Metrics::System do
     end
 
     describe '.memory_usage_rss' do
-      it "returns the process' resident set size (RSS) in bytes" do
-        mock_existing_proc_file('/proc/self/status', proc_status)
+      context 'without PID' do
+        it "returns the current process' resident set size (RSS) in bytes" do
+          mock_existing_proc_file('/proc/self/status', proc_status)
+
+          expect(described_class.memory_usage_rss).to eq(2527232)
+        end
+      end
+
+      context 'with PID' do
+        it "returns the given process' resident set size (RSS) in bytes" do
+          mock_existing_proc_file('/proc/7/status', proc_status)
 
-        expect(described_class.memory_usage_rss).to eq(2527232)
+          expect(described_class.memory_usage_rss(pid: 7)).to eq(2527232)
+        end
       end
     end
 
@@ -96,11 +106,22 @@ RSpec.describe Gitlab::Metrics::System do
     end
 
     describe '.memory_usage_uss_pss' do
-      it "returns the process' unique and porportional set size (USS/PSS) in bytes" do
-        mock_existing_proc_file('/proc/self/smaps_rollup', proc_smaps_rollup)
+      context 'without PID' do
+        it "returns the current process' unique and porportional set size (USS/PSS) in bytes" do
+          mock_existing_proc_file('/proc/self/smaps_rollup', proc_smaps_rollup)
+
+          # (Private_Clean (152 kB) + Private_Dirty (312 kB) + Private_Hugetlb (0 kB)) * 1024
+          expect(described_class.memory_usage_uss_pss).to eq(uss: 475136, pss: 515072)
+        end
+      end
+
+      context 'with PID' do
+        it "returns the given process' unique and porportional set size (USS/PSS) in bytes" do
+          mock_existing_proc_file('/proc/7/smaps_rollup', proc_smaps_rollup)
 
-        # (Private_Clean (152 kB) + Private_Dirty (312 kB) + Private_Hugetlb (0 kB)) * 1024
-        expect(described_class.memory_usage_uss_pss).to eq(uss: 475136, pss: 515072)
+          # (Private_Clean (152 kB) + Private_Dirty (312 kB) + Private_Hugetlb (0 kB)) * 1024
+          expect(described_class.memory_usage_uss_pss(pid: 7)).to eq(uss: 475136, pss: 515072)
+        end
       end
     end
 
diff --git a/spec/migrations/20220606054503_fix_incorrect_job_artifacts_expire_at_spec.rb b/spec/migrations/20220606054503_fix_incorrect_job_artifacts_expire_at_spec.rb
new file mode 100644
index 00000000000..5921dd64c0e
--- /dev/null
+++ b/spec/migrations/20220606054503_fix_incorrect_job_artifacts_expire_at_spec.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe FixIncorrectJobArtifactsExpireAt, migration: :gitlab_ci do
+  let_it_be(:batched_migration) { described_class::MIGRATION }
+
+  it 'does not schedule background jobs when Gitlab.com is true' do
+    allow(Gitlab).to receive(:com?).and_return(true)
+
+    reversible_migration do |migration|
+      migration.before -> {
+        expect(batched_migration).not_to have_scheduled_batched_migration
+      }
+
+      migration.after -> {
+        expect(batched_migration).not_to have_scheduled_batched_migration
+      }
+    end
+  end
+
+  it 'schedules background job on non Gitlab.com' do
+    allow(Gitlab).to receive(:com?).and_return(false)
+
+    reversible_migration do |migration|
+      migration.before -> {
+        expect(batched_migration).not_to have_scheduled_batched_migration
+      }
+
+      migration.after -> {
+        expect(batched_migration).to have_scheduled_batched_migration(
+          gitlab_schema: :gitlab_ci,
+          table_name: :ci_job_artifacts,
+          column_name: :id,
+          interval: described_class::INTERVAL,
+          batch_size: described_class::BATCH_SIZE
+        )
+      }
+    end
+  end
+end
diff --git a/spec/rubocop/cop/gitlab/mark_used_feature_flags_spec.rb b/spec/rubocop/cop/gitlab/mark_used_feature_flags_spec.rb
index c61376d7760..515620bd820 100644
--- a/spec/rubocop/cop/gitlab/mark_used_feature_flags_spec.rb
+++ b/spec/rubocop/cop/gitlab/mark_used_feature_flags_spec.rb
@@ -48,6 +48,7 @@ RSpec.describe RuboCop::Cop::Gitlab::MarkUsedFeatureFlags do
     Feature.enabled?
     Feature.disabled?
     push_frontend_feature_flag
+    YamlProcessor::FeatureFlags.enabled?
   ].each do |feature_flag_method|
     context "#{feature_flag_method} method" do
       context 'a string feature flag' do
diff --git a/spec/services/protected_branches/destroy_service_spec.rb b/spec/services/protected_branches/destroy_service_spec.rb
index 9fa07820148..123deeea005 100644
--- a/spec/services/protected_branches/destroy_service_spec.rb
+++ b/spec/services/protected_branches/destroy_service_spec.rb
@@ -5,7 +5,7 @@ require 'spec_helper'
 RSpec.describe ProtectedBranches::DestroyService do
   let_it_be_with_reload(:project) { create(:project) }
 
-  let(:protected_branch) { create(:protected_branch, project: project) }
+  let!(:protected_branch) { create(:protected_branch, project: project) }
   let(:user) { project.first_owner }
 
   subject(:service) { described_class.new(project, user) }
diff --git a/spec/services/protected_branches/update_service_spec.rb b/spec/services/protected_branches/update_service_spec.rb
index c4fe4d78070..2ff6c3c489a 100644
--- a/spec/services/protected_branches/update_service_spec.rb
+++ b/spec/services/protected_branches/update_service_spec.rb
@@ -5,7 +5,7 @@ require 'spec_helper'
 RSpec.describe ProtectedBranches::UpdateService do
   let_it_be_with_reload(:project) { create(:project) }
 
-  let(:protected_branch) { create(:protected_branch, project: project) }
+  let!(:protected_branch) { create(:protected_branch, project: project) }
   let(:user) { project.first_owner }
   let(:params) { { name: new_name } }