Add latest changes from gitlab-org/gitlab@master

26 files changed, 309 insertions, 170 deletions

diff --git a/spec/controllers/projects/incidents_controller_spec.rb b/spec/controllers/projects/incidents_controller_spec.rb
index 20cf0dcfd3a..460821634b0 100644
--- a/spec/controllers/projects/incidents_controller_spec.rb
+++ b/spec/controllers/projects/incidents_controller_spec.rb
@@ -43,7 +43,6 @@ RSpec.describe Projects::IncidentsController do
 
       expect(response).to have_gitlab_http_status(:ok)
       expect(response).to render_template(:index)
-      expect(Gon.features).to include('incidentEscalations' => true)
     end
 
     context 'when user is unauthorized' do
diff --git a/spec/factories/terraform/state.rb b/spec/factories/terraform/state.rb
index fb63c845073..c40fa14b4f8 100644
--- a/spec/factories/terraform/state.rb
+++ b/spec/factories/terraform/state.rb
@@ -12,6 +12,10 @@ FactoryBot.define do
       locked_by_user { association(:user) }
     end
 
+    trait :deletion_in_progress do
+      deleted_at { Time.current }
+    end
+
     trait :with_version do
       after(:create) do |state|
         create(:terraform_state_version, terraform_state: state)
diff --git a/spec/features/incidents/incidents_list_spec.rb b/spec/features/incidents/incidents_list_spec.rb
index 789cc89e083..3241e71f537 100644
--- a/spec/features/incidents/incidents_list_spec.rb
+++ b/spec/features/incidents/incidents_list_spec.rb
@@ -44,18 +44,5 @@ RSpec.describe 'Incident Management index', :js do
       expect(table).to have_content('Date created')
       expect(table).to have_content('Assignees')
     end
-
-    context 'when :incident_escalations feature is disabled' do
-      before do
-        stub_feature_flags(incident_escalations: false)
-      end
-
-      it 'does not include the Status columns' do
-        visit project_incidents_path(project)
-        wait_for_requests
-
-        expect(page.find('.gl-table')).not_to have_content('Status')
-      end
-    end
   end
 end
diff --git a/spec/frontend/incidents/components/incidents_list_spec.js b/spec/frontend/incidents/components/incidents_list_spec.js
index a556f3c17f3..356480f931e 100644
--- a/spec/frontend/incidents/components/incidents_list_spec.js
+++ b/spec/frontend/incidents/components/incidents_list_spec.js
@@ -85,7 +85,6 @@ describe('Incidents List', () => {
           assigneeUsernameQuery: '',
           slaFeatureAvailable: true,
           canCreateIncident: true,
-          incidentEscalationsAvailable: true,
           ...provide,
         },
         stubs: {
@@ -211,20 +210,6 @@ describe('Incidents List', () => {
           expect(status.classes('gl-text-truncate')).toBe(true);
         });
       });
-
-      describe('when feature is disabled', () => {
-        beforeEach(() => {
-          mountComponent({
-            data: { incidents: { list: mockIncidents }, incidentsCount },
-            provide: { incidentEscalationsAvailable: false },
-            loading: false,
-          });
-        });
-
-        it('is absent if feature flag is disabled', () => {
-          expect(findEscalationStatus().length).toBe(0);
-        });
-      });
     });
 
     it('contains a link to the incident details page', async () => {
diff --git a/spec/graphql/mutations/issues/set_escalation_status_spec.rb b/spec/graphql/mutations/issues/set_escalation_status_spec.rb
index d41118b1812..f04d396efb8 100644
--- a/spec/graphql/mutations/issues/set_escalation_status_spec.rb
+++ b/spec/graphql/mutations/issues/set_escalation_status_spec.rb
@@ -50,16 +50,6 @@ RSpec.describe Mutations::Issues::SetEscalationStatus do
             expect { result }.to raise_error(Gitlab::Graphql::Errors::ResourceNotAvailable, 'Feature unavailable for provided issue')
           end
         end
-
-        context 'with feature disabled' do
-          before do
-            stub_feature_flags(incident_escalations: false)
-          end
-
-          it 'raises an error' do
-            expect { result }.to raise_error(Gitlab::Graphql::Errors::ResourceNotAvailable, 'Feature unavailable for provided issue')
-          end
-        end
       end
     end
   end
diff --git a/spec/graphql/mutations/terraform/state/delete_spec.rb b/spec/graphql/mutations/terraform/state/delete_spec.rb
index 313a85a4bac..66d4b50741f 100644
--- a/spec/graphql/mutations/terraform/state/delete_spec.rb
+++ b/spec/graphql/mutations/terraform/state/delete_spec.rb
@@ -34,12 +34,12 @@ RSpec.describe Mutations::Terraform::State::Delete do
         state.project.add_maintainer(user)
       end
 
-      it 'deletes the state', :aggregate_failures do
-        expect do
-          expect(subject).to eq(errors: [])
-        end.to change { ::Terraform::State.count }.by(-1)
+      it 'schedules the state for deletion', :aggregate_failures do
+        expect_next_instance_of(Terraform::States::TriggerDestroyService, state, current_user: user) do |service|
+          expect(service).to receive(:execute).once.and_return(ServiceResponse.success)
+        end
 
-        expect { state.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        subject
       end
     end
 
diff --git a/spec/graphql/resolvers/issues_resolver_spec.rb b/spec/graphql/resolvers/issues_resolver_spec.rb
index e6ec9d8c895..1f2e2aec023 100644
--- a/spec/graphql/resolvers/issues_resolver_spec.rb
+++ b/spec/graphql/resolvers/issues_resolver_spec.rb
@@ -573,22 +573,6 @@ RSpec.describe Resolvers::IssuesResolver do
             issues = resolve_issues(sort: :created_desc).to_a
             expect(issues).to eq([resolved_incident, issue_no_status, triggered_incident])
           end
-
-          context 'when incident_escalations feature flag is disabled' do
-            before do
-              stub_feature_flags(incident_escalations: false)
-            end
-
-            it 'defaults ascending status sort to created_desc' do
-              issues = resolve_issues(sort: :escalation_status_asc).to_a
-              expect(issues).to eq([resolved_incident, issue_no_status, triggered_incident])
-            end
-
-            it 'defaults descending status sort to created_desc' do
-              issues = resolve_issues(sort: :escalation_status_desc).to_a
-              expect(issues).to eq([resolved_incident, issue_no_status, triggered_incident])
-            end
-          end
         end
 
         context 'when sorting with non-stable cursors' do
diff --git a/spec/graphql/types/issue_type_spec.rb b/spec/graphql/types/issue_type_spec.rb
index 920aadd2faf..e7454b85357 100644
--- a/spec/graphql/types/issue_type_spec.rb
+++ b/spec/graphql/types/issue_type_spec.rb
@@ -291,14 +291,6 @@ RSpec.describe GitlabSchema.types['Issue'] do
         let!(:escalation_status) { create(:incident_management_issuable_escalation_status, issue: issue) }
 
         it { is_expected.to eq(escalation_status.status_name.to_s.upcase) }
-
-        context 'with feature disabled' do
-          before do
-            stub_feature_flags(incident_escalations: false)
-          end
-
-          it { is_expected.to be_nil }
-        end
       end
     end
   end
diff --git a/spec/lib/gitlab/database/load_balancing/transaction_leaking_spec.rb b/spec/lib/gitlab/database/load_balancing/transaction_leaking_spec.rb
new file mode 100644
index 00000000000..30e5fbbd803
--- /dev/null
+++ b/spec/lib/gitlab/database/load_balancing/transaction_leaking_spec.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'Load balancer behavior with errors inside a transaction', :redis, :delete do
+  let(:model) { ApplicationRecord }
+  let(:db_host) { model.connection_pool.db_config.host }
+
+  let(:test_table_name) { '_test_foo' }
+
+  before do
+    # Patch in our load balancer config, simply pointing at the test database twice
+    allow(Gitlab::Database::LoadBalancing::Configuration).to receive(:for_model) do |base_model|
+      Gitlab::Database::LoadBalancing::Configuration.new(base_model, [db_host, db_host])
+    end
+
+    Gitlab::Database::LoadBalancing::Setup.new(ApplicationRecord).setup
+
+    model.connection.execute(<<~SQL)
+      CREATE TABLE IF NOT EXISTS #{test_table_name} (id SERIAL PRIMARY KEY, value INTEGER)
+    SQL
+  end
+
+  after do
+    model.connection.execute(<<~SQL)
+      DROP TABLE IF EXISTS #{test_table_name}
+    SQL
+  end
+
+  def execute(conn)
+    conn.execute("INSERT INTO #{test_table_name} (value) VALUES (1)")
+    backend_pid = conn.execute("SELECT pg_backend_pid() AS pid").to_a.first['pid']
+
+    # This will result in a PG error, which is not raised.
+    # Instead, we retry the statement on a fresh connection (where the pid is different and it does nothing)
+    # and the load balancer continues with a fresh connection and no transaction if a transaction was open previously
+    conn.execute(<<~SQL)
+      SELECT CASE
+      WHEN pg_backend_pid() = #{backend_pid} THEN
+        pg_terminate_backend(#{backend_pid})
+      END
+    SQL
+
+    # This statement will execute on a new connection, and violate transaction semantics
+    # if we were in a transaction before
+    conn.execute("INSERT INTO #{test_table_name} (value) VALUES (2)")
+  end
+
+  it 'logs a warning when violating transaction semantics with writes' do
+    conn = model.connection
+
+    expect(::Gitlab::Database::LoadBalancing::Logger).to receive(:warn).with(hash_including(event: :transaction_leak))
+
+    conn.transaction do
+      expect(conn).to be_transaction_open
+
+      execute(conn)
+
+      expect(conn).not_to be_transaction_open
+    end
+
+    values = conn.execute("SELECT value FROM #{test_table_name}").to_a.map { |row| row['value'] }
+    expect(values).to contain_exactly(2) # Does not include 1 because the transaction was aborted and leaked
+  end
+
+  it 'does not log a warning when no transaction is open to be leaked' do
+    conn = model.connection
+
+    expect(::Gitlab::Database::LoadBalancing::Logger)
+      .not_to receive(:warn).with(hash_including(event: :transaction_leak))
+
+    expect(conn).not_to be_transaction_open
+
+    execute(conn)
+
+    expect(conn).not_to be_transaction_open
+
+    values = conn.execute("SELECT value FROM #{test_table_name}").to_a.map { |row| row['value'] }
+    expect(values).to contain_exactly(1, 2) # Includes both rows because there was no transaction to roll back
+  end
+end
diff --git a/spec/models/concerns/issuable_spec.rb b/spec/models/concerns/issuable_spec.rb
index e8e9c263d23..87821de3cf5 100644
--- a/spec/models/concerns/issuable_spec.rb
+++ b/spec/models/concerns/issuable_spec.rb
@@ -982,14 +982,6 @@ RSpec.describe Issuable do
       subject { issuable.supports_escalation? }
 
       it { is_expected.to eq(supports_escalation) }
-
-      context 'with feature disabled' do
-        before do
-          stub_feature_flags(incident_escalations: false)
-        end
-
-        it { is_expected.to eq(false) }
-      end
     end
   end
 
diff --git a/spec/models/merge_request/cleanup_schedule_spec.rb b/spec/models/merge_request/cleanup_schedule_spec.rb
index 85208f901fd..9c50b64f2bd 100644
--- a/spec/models/merge_request/cleanup_schedule_spec.rb
+++ b/spec/models/merge_request/cleanup_schedule_spec.rb
@@ -120,6 +120,41 @@ RSpec.describe MergeRequest::CleanupSchedule do
     end
   end
 
+  describe '.stuck' do
+    let!(:cleanup_schedule_1) { create(:merge_request_cleanup_schedule, updated_at: 1.day.ago) }
+    let!(:cleanup_schedule_2) { create(:merge_request_cleanup_schedule, :running, updated_at: 5.hours.ago) }
+    let!(:cleanup_schedule_3) { create(:merge_request_cleanup_schedule, :running, updated_at: 7.hours.ago) }
+    let!(:cleanup_schedule_4) { create(:merge_request_cleanup_schedule, :completed, updated_at: 1.day.ago) }
+    let!(:cleanup_schedule_5) { create(:merge_request_cleanup_schedule, :failed, updated_at: 1.day.ago) }
+
+    it 'returns records that has been in running state for more than 6 hours' do
+      expect(described_class.stuck).to match_array([cleanup_schedule_3])
+    end
+  end
+
+  describe '.stuck_retry!' do
+    let!(:cleanup_schedule_1) { create(:merge_request_cleanup_schedule, :running, updated_at: 5.hours.ago) }
+    let!(:cleanup_schedule_2) { create(:merge_request_cleanup_schedule, :running, updated_at: 7.hours.ago) }
+
+    it 'sets stuck records to unstarted' do
+      expect { described_class.stuck_retry! }.to change { cleanup_schedule_2.reload.unstarted? }.from(false).to(true)
+    end
+
+    context 'when there are more than 5 stuck schedules' do
+      before do
+        create_list(:merge_request_cleanup_schedule, 5, :running, updated_at: 1.day.ago)
+      end
+
+      it 'only retries 5 stuck schedules at once' do
+        expect(described_class.stuck.count).to eq 6
+
+        described_class.stuck_retry!
+
+        expect(described_class.stuck.count).to eq 1
+      end
+    end
+  end
+
   describe '.start_next' do
     let!(:cleanup_schedule_1) { create(:merge_request_cleanup_schedule, :completed, scheduled_at: 1.day.ago) }
     let!(:cleanup_schedule_2) { create(:merge_request_cleanup_schedule, scheduled_at: 2.days.ago) }
diff --git a/spec/models/terraform/state_spec.rb b/spec/models/terraform/state_spec.rb
index a113ae37203..a484952bfe9 100644
--- a/spec/models/terraform/state_spec.rb
+++ b/spec/models/terraform/state_spec.rb
@@ -11,8 +11,6 @@ RSpec.describe Terraform::State do
   it { is_expected.to validate_presence_of(:name) }
   it { is_expected.to validate_presence_of(:project_id) }
 
-  it { is_expected.to validate_uniqueness_of(:name).scoped_to(:project_id) }
-
   describe 'scopes' do
     describe '.ordered_by_name' do
       let_it_be(:project) { create(:project) }
@@ -40,22 +38,6 @@ RSpec.describe Terraform::State do
     end
   end
 
-  describe '#destroy' do
-    let(:terraform_state) { create(:terraform_state) }
-    let(:user) { terraform_state.project.creator }
-
-    it 'deletes when the state is unlocked' do
-      expect(terraform_state.destroy).to be_truthy
-    end
-
-    it 'fails to delete when the state is locked', :aggregate_failures do
-      terraform_state.update!(lock_xid: SecureRandom.uuid, locked_by_user: user, locked_at: Time.current)
-
-      expect(terraform_state.destroy).to be_falsey
-      expect(terraform_state.errors.full_messages).to eq(["You cannot remove the State file because it's locked. Unlock the State file first before removing it."])
-    end
-  end
-
   describe '#latest_file' do
     let(:terraform_state) { create(:terraform_state, :with_version) }
     let(:latest_version) { terraform_state.latest_version }
diff --git a/spec/models/terraform/state_version_spec.rb b/spec/models/terraform/state_version_spec.rb
index 7af9b7897ff..22b1397f30a 100644
--- a/spec/models/terraform/state_version_spec.rb
+++ b/spec/models/terraform/state_version_spec.rb
@@ -4,6 +4,7 @@ require 'spec_helper'
 
 RSpec.describe Terraform::StateVersion do
   it { is_expected.to be_a FileStoreMounter }
+  it { is_expected.to be_a EachBatch }
 
   it { is_expected.to belong_to(:terraform_state).required }
   it { is_expected.to belong_to(:created_by_user).class_name('User').optional }
diff --git a/spec/requests/api/graphql/mutations/issues/set_escalation_status_spec.rb b/spec/requests/api/graphql/mutations/issues/set_escalation_status_spec.rb
index 0166871502b..a81364d37b2 100644
--- a/spec/requests/api/graphql/mutations/issues/set_escalation_status_spec.rb
+++ b/spec/requests/api/graphql/mutations/issues/set_escalation_status_spec.rb
@@ -49,14 +49,6 @@ RSpec.describe 'Setting the escalation status of an incident' do
     it_behaves_like 'a mutation that returns top-level errors', errors: ['Feature unavailable for provided issue']
   end
 
-  context 'with feature disabled' do
-    before do
-      stub_feature_flags(incident_escalations: false)
-    end
-
-    it_behaves_like 'a mutation that returns top-level errors', errors: ['Feature unavailable for provided issue']
-  end
-
   it 'sets given escalation_policy to the escalation status for the issue' do
     post_graphql_mutation(mutation, current_user: current_user)
 
diff --git a/spec/requests/api/graphql/mutations/user_preferences/update_spec.rb b/spec/requests/api/graphql/mutations/user_preferences/update_spec.rb
index 85194e6eb20..e1c7fd9d60d 100644
--- a/spec/requests/api/graphql/mutations/user_preferences/update_spec.rb
+++ b/spec/requests/api/graphql/mutations/user_preferences/update_spec.rb
@@ -28,17 +28,6 @@ RSpec.describe Mutations::UserPreferences::Update do
       expect(current_user.user_preference.persisted?).to eq(true)
       expect(current_user.user_preference.issues_sort).to eq(Types::IssueSortEnum.values[sort_value].value.to_s)
     end
-
-    context 'when incident_escalations feature flag is disabled' do
-      let(:sort_value) { 'ESCALATION_STATUS_ASC' }
-
-      before do
-        stub_feature_flags(incident_escalations: false)
-      end
-
-      it_behaves_like 'a mutation that returns top-level errors',
-        errors: ['Feature flag `incident_escalations` must be enabled to use this sort order.']
-    end
   end
 
   context 'when user has existing preference' do
@@ -56,16 +45,5 @@ RSpec.describe Mutations::UserPreferences::Update do
 
       expect(current_user.user_preference.issues_sort).to eq(Types::IssueSortEnum.values[sort_value].value.to_s)
     end
-
-    context 'when incident_escalations feature flag is disabled' do
-      let(:sort_value) { 'ESCALATION_STATUS_DESC' }
-
-      before do
-        stub_feature_flags(incident_escalations: false)
-      end
-
-      it_behaves_like 'a mutation that returns top-level errors',
-        errors: ['Feature flag `incident_escalations` must be enabled to use this sort order.']
-    end
   end
 end
diff --git a/spec/requests/api/graphql/terraform/state/delete_spec.rb b/spec/requests/api/graphql/terraform/state/delete_spec.rb
index 35927d03b49..ba0619ea611 100644
--- a/spec/requests/api/graphql/terraform/state/delete_spec.rb
+++ b/spec/requests/api/graphql/terraform/state/delete_spec.rb
@@ -12,12 +12,12 @@ RSpec.describe 'delete a terraform state' do
   let(:mutation) { graphql_mutation(:terraform_state_delete, id: state.to_global_id.to_s) }
 
   before do
+    expect_next_instance_of(Terraform::States::TriggerDestroyService, state, current_user: user) do |service|
+      expect(service).to receive(:execute).once.and_return(ServiceResponse.success)
+    end
+
     post_graphql_mutation(mutation, current_user: user)
   end
 
   include_examples 'a working graphql query'
-
-  it 'deletes the state' do
-    expect { state.reload }.to raise_error(ActiveRecord::RecordNotFound)
-  end
 end
diff --git a/spec/requests/api/terraform/state_spec.rb b/spec/requests/api/terraform/state_spec.rb
index f49e799cf5d..e8458db4a4d 100644
--- a/spec/requests/api/terraform/state_spec.rb
+++ b/spec/requests/api/terraform/state_spec.rb
@@ -71,6 +71,18 @@ RSpec.describe API::Terraform::State, :snowplow do
     end
   end
 
+  shared_context 'cannot access a state that is scheduled for deletion' do
+    before do
+      state.update!(deleted_at: Time.current)
+    end
+
+    it 'returns unprocessable entity' do
+      request
+
+      expect(response).to have_gitlab_http_status(:unprocessable_entity)
+    end
+  end
+
   describe 'GET /projects/:id/terraform/state/:name' do
     subject(:request) { get api(state_path), headers: auth_header }
 
@@ -106,6 +118,8 @@ RSpec.describe API::Terraform::State, :snowplow do
             expect(response).to have_gitlab_http_status(:not_found)
           end
         end
+
+        it_behaves_like 'cannot access a state that is scheduled for deletion'
       end
 
       context 'with developer permissions' do
@@ -191,6 +205,8 @@ RSpec.describe API::Terraform::State, :snowplow do
             expect(response).to have_gitlab_http_status(:unprocessable_entity)
           end
         end
+
+        it_behaves_like 'cannot access a state that is scheduled for deletion'
       end
 
       context 'without body' do
@@ -269,13 +285,19 @@ RSpec.describe API::Terraform::State, :snowplow do
 
     context 'with maintainer permissions' do
       let(:current_user) { maintainer }
+      let(:deletion_service) { instance_double(Terraform::States::TriggerDestroyService) }
 
-      it 'deletes the state and returns empty body' do
-        expect { request }.to change { Terraform::State.count }.by(-1)
+      it 'schedules the state for deletion and returns empty body' do
+        expect(Terraform::States::TriggerDestroyService).to receive(:new).and_return(deletion_service)
+        expect(deletion_service).to receive(:execute).once
+
+        request
 
         expect(response).to have_gitlab_http_status(:ok)
         expect(Gitlab::Json.parse(response.body)).to be_empty
       end
+
+      it_behaves_like 'cannot access a state that is scheduled for deletion'
     end
 
     context 'with developer permissions' do
@@ -305,6 +327,7 @@ RSpec.describe API::Terraform::State, :snowplow do
     subject(:request) { post api("#{state_path}/lock"), headers: auth_header, params: params }
 
     it_behaves_like 'endpoint with unique user tracking'
+    it_behaves_like 'cannot access a state that is scheduled for deletion'
 
     it 'locks the terraform state' do
       request
@@ -359,6 +382,10 @@ RSpec.describe API::Terraform::State, :snowplow do
       let(:lock_id) { 'irrelevant to this test, just needs to be present' }
     end
 
+    it_behaves_like 'cannot access a state that is scheduled for deletion' do
+      let(:lock_id) { 'irrelevant to this test, just needs to be present' }
+    end
+
     context 'with the correct lock id' do
       let(:lock_id) { '123-456' }
 
diff --git a/spec/serializers/issue_sidebar_basic_entity_spec.rb b/spec/serializers/issue_sidebar_basic_entity_spec.rb
index 564ffb1aea9..64a271e359a 100644
--- a/spec/serializers/issue_sidebar_basic_entity_spec.rb
+++ b/spec/serializers/issue_sidebar_basic_entity_spec.rb
@@ -59,16 +59,6 @@ RSpec.describe IssueSidebarBasicEntity do
             expect(entity[:current_user][:can_update_escalation_status]).to be(false)
           end
         end
-
-        context 'with :incident_escalations feature flag disabled' do
-          before do
-            stub_feature_flags(incident_escalations: false)
-          end
-
-          it 'is not present' do
-            expect(entity[:current_user]).not_to include(:can_update_escalation_status)
-          end
-        end
       end
     end
   end
diff --git a/spec/services/alert_management/alerts/update_service_spec.rb b/spec/services/alert_management/alerts/update_service_spec.rb
index f02607b8174..9bdc9970807 100644
--- a/spec/services/alert_management/alerts/update_service_spec.rb
+++ b/spec/services/alert_management/alerts/update_service_spec.rb
@@ -291,14 +291,6 @@ RSpec.describe AlertManagement::Alerts::UpdateService do
 
               it_behaves_like 'does not sync with the incident status'
             end
-
-            context 'when feature flag is disabled' do
-              before do
-                stub_feature_flags(incident_escalations: false)
-              end
-
-              it_behaves_like 'does not sync with the incident status'
-            end
           end
         end
       end
diff --git a/spec/services/incident_management/issuable_escalation_statuses/prepare_update_service_spec.rb b/spec/services/incident_management/issuable_escalation_statuses/prepare_update_service_spec.rb
index 25164df40ca..6c99631fcb0 100644
--- a/spec/services/incident_management/issuable_escalation_statuses/prepare_update_service_spec.rb
+++ b/spec/services/incident_management/issuable_escalation_statuses/prepare_update_service_spec.rb
@@ -42,14 +42,6 @@ RSpec.describe IncidentManagement::IssuableEscalationStatuses::PrepareUpdateServ
 
   it_behaves_like 'successful response', { status_event: :acknowledge }
 
-  context 'when feature flag is disabled' do
-    before do
-      stub_feature_flags(incident_escalations: false)
-    end
-
-    it_behaves_like 'availability error response'
-  end
-
   context 'when user is anonymous' do
     let(:current_user) { nil }
 
diff --git a/spec/services/issues/update_service_spec.rb b/spec/services/issues/update_service_spec.rb
index d496857bb25..d11fe772023 100644
--- a/spec/services/issues/update_service_spec.rb
+++ b/spec/services/issues/update_service_spec.rb
@@ -1230,14 +1230,6 @@ RSpec.describe Issues::UpdateService, :mailer do
 
           it_behaves_like 'updates the escalation status record', :acknowledged
         end
-
-        context 'with :incident_escalations feature flag disabled' do
-          before do
-            stub_feature_flags(incident_escalations: false)
-          end
-
-          it_behaves_like 'does not change the status record'
-        end
       end
 
       context 'when issue type is not incident' do
diff --git a/spec/services/terraform/remote_state_handler_spec.rb b/spec/services/terraform/remote_state_handler_spec.rb
index ca392849d49..19c1d4109e9 100644
--- a/spec/services/terraform/remote_state_handler_spec.rb
+++ b/spec/services/terraform/remote_state_handler_spec.rb
@@ -33,6 +33,14 @@ RSpec.describe Terraform::RemoteStateHandler do
         it 'returns the state' do
           expect(subject.find_with_lock).to eq(state)
         end
+
+        context 'with a state scheduled for deletion' do
+          let!(:state) { create(:terraform_state, :deletion_in_progress, project: project, name: 'state') }
+
+          it 'raises an exception' do
+            expect { subject.find_with_lock }.to raise_error(described_class::StateDeletedError)
+          end
+        end
       end
     end
   end
@@ -84,6 +92,13 @@ RSpec.describe Terraform::RemoteStateHandler do
           .to raise_error(described_class::StateLockedError)
       end
 
+      it 'raises an exception if the state is scheduled for deletion' do
+        create(:terraform_state, :deletion_in_progress, project: project, name: 'new-state')
+
+        expect { handler.handle_with_lock }
+          .to raise_error(described_class::StateDeletedError)
+      end
+
       context 'user does not have permission to modify state' do
         let(:user) { developer }
 
@@ -127,24 +142,28 @@ RSpec.describe Terraform::RemoteStateHandler do
 
         expect { handler.lock! }.to raise_error(described_class::StateLockedError)
       end
+
+      it 'raises an exception when the state exists and is scheduled for deletion' do
+        create(:terraform_state, :deletion_in_progress, project: project, name: 'new-state')
+
+        expect { handler.lock! }.to raise_error(described_class::StateDeletedError)
+      end
     end
 
     describe '#unlock!' do
-      let(:lock_id) { 'abc-abc' }
+      let_it_be(:state) { create(:terraform_state, :locked, project: project, name: 'new-state', lock_xid: 'abc-abc') }
+
+      let(:lock_id) { state.lock_xid }
 
       subject(:handler) do
         described_class.new(
           project,
           user,
-          name: 'new-state',
+          name: state.name,
           lock_id: lock_id
         )
       end
 
-      before do
-        create(:terraform_state, :locked, project: project, name: 'new-state', lock_xid: 'abc-abc')
-      end
-
       it 'unlocks the state' do
         state = handler.unlock!
 
@@ -169,6 +188,15 @@ RSpec.describe Terraform::RemoteStateHandler do
             .to raise_error(described_class::StateLockedError)
         end
       end
+
+      context 'with a state scheduled for deletion' do
+        it 'raises an exception' do
+          state.update!(deleted_at: Time.current)
+
+          expect { handler.unlock! }
+            .to raise_error(described_class::StateDeletedError)
+        end
+      end
     end
   end
 end
diff --git a/spec/services/terraform/states/destroy_service_spec.rb b/spec/services/terraform/states/destroy_service_spec.rb
new file mode 100644
index 00000000000..16fcc8f93db
--- /dev/null
+++ b/spec/services/terraform/states/destroy_service_spec.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Terraform::States::DestroyService do
+  let_it_be(:state) { create(:terraform_state, :with_version, :deletion_in_progress) }
+  let_it_be(:file) { double }
+
+  before do
+    allow_next_found_instance_of(Terraform::StateVersion) do |version|
+      allow(version).to receive(:file).and_return(file)
+    end
+  end
+
+  describe '#execute' do
+    subject { described_class.new(state).execute }
+
+    it 'removes version files from object storage, followed by the state record' do
+      expect(file).to receive(:remove!).once
+      expect(state).to receive(:destroy!)
+
+      subject
+    end
+
+    context 'state is not marked for deletion' do
+      let(:state) { create(:terraform_state) }
+
+      it 'does not delete the state' do
+        expect(state).not_to receive(:destroy!)
+
+        subject
+      end
+    end
+  end
+end
diff --git a/spec/services/terraform/states/trigger_destroy_service_spec.rb b/spec/services/terraform/states/trigger_destroy_service_spec.rb
new file mode 100644
index 00000000000..2e96331779c
--- /dev/null
+++ b/spec/services/terraform/states/trigger_destroy_service_spec.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Terraform::States::TriggerDestroyService do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:user) { create(:user, maintainer_projects: [project]) }
+
+  describe '#execute', :aggregate_failures do
+    let_it_be(:state) { create(:terraform_state, project: project) }
+
+    subject { described_class.new(state, current_user: user).execute }
+
+    it 'marks the state as deleted and schedules a cleanup worker' do
+      expect(Terraform::States::DestroyWorker).to receive(:perform_async).with(state.id).once
+
+      expect(subject).to be_success
+      expect(state.deleted_at).to be_like_time(Time.current)
+    end
+
+    shared_examples 'unable to delete state' do
+      it 'does not modify the state' do
+        expect(Terraform::States::DestroyWorker).not_to receive(:perform_async)
+
+        expect { subject }.not_to change(state, :deleted_at)
+        expect(subject).to be_error
+        expect(subject.message).to eq(message)
+      end
+    end
+
+    context 'user does not have permission' do
+      let(:user) { create(:user, developer_projects: [project]) }
+      let(:message) { 'You have insufficient permissions to delete this state' }
+
+      include_examples 'unable to delete state'
+    end
+
+    context 'state is locked' do
+      let(:state) { create(:terraform_state, :locked, project: project) }
+      let(:message) { 'Cannot remove a locked state' }
+
+      include_examples 'unable to delete state'
+    end
+  end
+end
diff --git a/spec/workers/schedule_merge_request_cleanup_refs_worker_spec.rb b/spec/workers/schedule_merge_request_cleanup_refs_worker_spec.rb
index ef515e43474..49730d9ab8c 100644
--- a/spec/workers/schedule_merge_request_cleanup_refs_worker_spec.rb
+++ b/spec/workers/schedule_merge_request_cleanup_refs_worker_spec.rb
@@ -25,6 +25,12 @@ RSpec.describe ScheduleMergeRequestCleanupRefsWorker do
       end
     end
 
+    it 'retries stuck cleanup schedules' do
+      expect(MergeRequest::CleanupSchedule).to receive(:stuck_retry!)
+
+      worker.perform
+    end
+
     include_examples 'an idempotent worker' do
       it 'schedules MergeRequestCleanupRefsWorker to be performed with capacity' do
         expect(MergeRequestCleanupRefsWorker).to receive(:perform_with_capacity).twice
diff --git a/spec/workers/terraform/states/destroy_worker_spec.rb b/spec/workers/terraform/states/destroy_worker_spec.rb
new file mode 100644
index 00000000000..02e79373279
--- /dev/null
+++ b/spec/workers/terraform/states/destroy_worker_spec.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Terraform::States::DestroyWorker do
+  let(:state) { create(:terraform_state) }
+
+  describe '#perform' do
+    let(:state_id) { state.id }
+    let(:deletion_service) { instance_double(Terraform::States::DestroyService, execute: true) }
+
+    subject { described_class.new.perform(state_id) }
+
+    it 'calls the deletion service' do
+      expect(deletion_service).to receive(:execute).once
+      expect(Terraform::States::DestroyService).to receive(:new)
+        .with(state).and_return(deletion_service)
+
+      subject
+    end
+
+    context 'state no longer exists' do
+      let(:state_id) { -1 }
+
+      it 'completes without error' do
+        expect { subject }.not_to raise_error
+      end
+    end
+  end
+end