Add latest changes from gitlab-org/gitlab@master

24 files changed, 260 insertions, 207 deletions

diff --git a/.gitlab/ci/qa-common/main.gitlab-ci.yml b/.gitlab/ci/qa-common/main.gitlab-ci.yml
index 883624a58ff..c9b60b078bb 100644
--- a/.gitlab/ci/qa-common/main.gitlab-ci.yml
+++ b/.gitlab/ci/qa-common/main.gitlab-ci.yml
@@ -6,7 +6,7 @@ workflow:
 
 include:
   - project: gitlab-org/quality/pipeline-common
-    ref: 5.3.0
+    ref: 5.4.0
     file:
       - /ci/base.gitlab-ci.yml
       - /ci/allure-report.yml
diff --git a/Gemfile b/Gemfile
index 927a4e4958b..256121cda0a 100644
--- a/Gemfile
+++ b/Gemfile
@@ -111,7 +111,7 @@ gem 'net-ldap', '~> 0.17.1'
 gem 'grape', '~> 1.5.2'
 gem 'grape-entity', '~> 0.10.0'
 gem 'rack-cors', '~> 1.1.1', require: 'rack/cors'
-gem 'grape-swagger', '~>1.5.0', group: [:development, :test]
+gem 'grape-swagger', '~> 1.6.1', group: [:development, :test]
 gem 'grape-swagger-entity', '~> 0.5.1', group: [:development, :test]
 
 # GraphQL API
@@ -335,11 +335,11 @@ gem 'sentry-sidekiq', '~> 5.8.0'
 
 # PostgreSQL query parsing
 #
-gem 'pg_query', '~> 2.2', '>= 2.2.1'
+gem 'pg_query', '~> 4.2.1'
 
 gem 'premailer-rails', '~> 1.10.3'
 
-gem 'gitlab-labkit', '~> 0.32.0'
+gem 'gitlab-labkit', '~> 0.33.0'
 gem 'thrift', '>= 0.16.0'
 
 # I18n
diff --git a/Gemfile.checksum b/Gemfile.checksum
index cc3ed31f5af..71ed6d7124d 100644
--- a/Gemfile.checksum
+++ b/Gemfile.checksum
@@ -213,7 +213,7 @@
 {"name":"gitlab-dangerfiles","version":"3.10.0","platform":"ruby","checksum":"df4cfe051f52529c0256346d89d06d5ef2bb630928754eb620b5233eb9b14041"},
 {"name":"gitlab-experiment","version":"0.7.1","platform":"ruby","checksum":"166dddb3aa83428bcaa93c35684ed01dc4d61f321fd2ae40b020806dc54a7824"},
 {"name":"gitlab-fog-azure-rm","version":"1.7.0","platform":"ruby","checksum":"969c67943c54ad4c259a6acd040493f13922fbdf2211bb4eca00e71505263dc2"},
-{"name":"gitlab-labkit","version":"0.32.0","platform":"ruby","checksum":"f30a33edc53586c059fd0b5d748acd2a12be75f6fc72a87669a0a08fe922866e"},
+{"name":"gitlab-labkit","version":"0.33.0","platform":"ruby","checksum":"d1fba8d30fde314a3f5dee1921ac31860bed4fecd8aa98ac6671f2627479e05b"},
 {"name":"gitlab-license","version":"2.2.2","platform":"ruby","checksum":"2ccbc763828d013524b0b3b9ee671e58d5277693e5ffb2e5463cbac87e8aed1e"},
 {"name":"gitlab-mail_room","version":"0.0.23","platform":"ruby","checksum":"23564fa4dab24ec5011d4c64a801fc0228301d5b0f046a26a1d8e96e36c19997"},
 {"name":"gitlab-markup","version":"1.9.0","platform":"ruby","checksum":"7eda045a08ec2d110084252fa13a8c9eac8bdac0e302035ca7db4b82bcbd7ed4"},
@@ -260,7 +260,7 @@
 {"name":"grape","version":"1.5.2","platform":"ruby","checksum":"1df3b734c3862e235174232bc629587eddda9ef3df648230827575186700ae29"},
 {"name":"grape-entity","version":"0.10.0","platform":"ruby","checksum":"9aed1e7cbbc96d9e73f72e5f32c776d4ba8a5baf54c3acda2682008dba2b2cfe"},
 {"name":"grape-path-helpers","version":"1.7.1","platform":"ruby","checksum":"2e27271a20d4073e3a3b2b955425c7f803e198be3ba8f6e59e3d59643c5381e2"},
-{"name":"grape-swagger","version":"1.5.0","platform":"ruby","checksum":"9c885b326ab0644abecf7df4ce866abc2411f359cfd59cbcca545b9b3b25c8ff"},
+{"name":"grape-swagger","version":"1.6.1","platform":"ruby","checksum":"0fd2d38476524b66e8d06de71e6c481d34286d895b12161f5df4427d66d5c69f"},
 {"name":"grape-swagger-entity","version":"0.5.1","platform":"ruby","checksum":"f51e372d00ac96cf90d948f87b3f4eb287ab053976ca57ad503d442ad8605523"},
 {"name":"grape_logging","version":"1.8.4","platform":"ruby","checksum":"efcc3e322dbd5d620a68f078733b7db043cf12680144cd03c982f14115c792d1"},
 {"name":"graphiql-rails","version":"1.8.0","platform":"ruby","checksum":"02e2c5098be2c6c29219a0e9b2910a2cd3c494301587a3199a7c4484d8038ed1"},
@@ -445,7 +445,7 @@
 {"name":"pg","version":"1.5.3","platform":"x64-mingw-ucrt","checksum":"1f2a6b2afaf0ccb8afe8b6a00131bce8151fbd6e8826b2d944288f6f2b615389"},
 {"name":"pg","version":"1.5.3","platform":"x64-mingw32","checksum":"ab7f5f3020323094a2b16f9638166b04c103e152a9079a1b8e795f4bf79765e0"},
 {"name":"pg","version":"1.5.3","platform":"x86-mingw32","checksum":"aa6ddda9887462d30a6d49d875eb9d27fca8cdb7185103b650e7351b38f15ddf"},
-{"name":"pg_query","version":"2.2.1","platform":"ruby","checksum":"6086972bbf4eab86d8425b35f14ca8b6fe41e4341423582801c1ec86ff5f8cea"},
+{"name":"pg_query","version":"4.2.1","platform":"ruby","checksum":"b04820a9d1c0c1608e3240b7d84baabbee1b95a7302f29fdd0f00e901c604833"},
 {"name":"plist","version":"3.6.0","platform":"ruby","checksum":"f468bcf6b72ec6d1585ed6744eb4817c1932a5bf91895ed056e69b7f12ca10f2"},
 {"name":"png_quantizator","version":"0.2.1","platform":"ruby","checksum":"6023d4d064125c3a7e02929c95b7320ed6ac0d7341f9e8de0c9ea6576ef3106b"},
 {"name":"po_to_json","version":"1.0.1","platform":"ruby","checksum":"6a7188aa6c42a22c9718f9b39062862ef7f3d8f6a7b4177cae058c3308b56af7"},
diff --git a/Gemfile.lock b/Gemfile.lock
index 3380ec995f6..9fa33aa29e8 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -602,13 +602,13 @@ GEM
       fog-json (~> 1.2.0)
       mime-types
       ms_rest_azure (~> 0.12.0)
-    gitlab-labkit (0.32.0)
+    gitlab-labkit (0.33.0)
       actionpack (>= 5.0.0, < 8.0.0)
       activesupport (>= 5.0.0, < 8.0.0)
       grpc (>= 1.37)
       jaeger-client (~> 1.1.0)
       opentracing (~> 0.4)
-      pg_query (~> 2.1)
+      pg_query (~> 4.2.1)
       redis (> 3.0.0, < 6.0.0)
     gitlab-license (2.2.2)
     gitlab-mail_room (0.0.23)
@@ -722,7 +722,7 @@ GEM
       grape (~> 1.3)
       rake (> 12)
       ruby2_keywords (~> 0.0.2)
-    grape-swagger (1.5.0)
+    grape-swagger (1.6.1)
       grape (~> 1.3)
     grape-swagger-entity (0.5.1)
       grape-entity (>= 0.6.0)
@@ -1128,8 +1128,8 @@ GEM
     peek (1.1.0)
       railties (>= 4.0.0)
     pg (1.5.3)
-    pg_query (2.2.1)
-      google-protobuf (>= 3.19.2)
+    pg_query (4.2.1)
+      google-protobuf (>= 3.22.3)
     plist (3.6.0)
     png_quantizator (0.2.1)
     po_to_json (1.0.1)
@@ -1743,7 +1743,7 @@ DEPENDENCIES
   gitlab-dangerfiles (~> 3.10.0)
   gitlab-experiment (~> 0.7.1)
   gitlab-fog-azure-rm (~> 1.7.0)
-  gitlab-labkit (~> 0.32.0)
+  gitlab-labkit (~> 0.33.0)
   gitlab-license (~> 2.2.1)
   gitlab-mail_room (~> 0.0.23)
   gitlab-markup (~> 1.9.0)
@@ -1769,7 +1769,7 @@ DEPENDENCIES
   grape (~> 1.5.2)
   grape-entity (~> 0.10.0)
   grape-path-helpers (~> 1.7.1)
-  grape-swagger (~> 1.5.0)
+  grape-swagger (~> 1.6.1)
   grape-swagger-entity (~> 0.5.1)
   grape_logging (~> 1.8)
   graphiql-rails (~> 1.8)
@@ -1856,7 +1856,7 @@ DEPENDENCIES
   parslet (~> 1.8)
   peek (~> 1.1)
   pg (~> 1.5.3)
-  pg_query (~> 2.2, >= 2.2.1)
+  pg_query (~> 4.2.1)
   png_quantizator (~> 0.2.1)
   premailer-rails (~> 1.10.3)
   prometheus-client-mmap (~> 0.24)
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 51c2cea1983..056c76f6326 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -417,6 +417,15 @@
   :weight: 1
   :idempotent: true
   :tags: []
+- :name: cronjob:database_monitor_locked_tables
+  :worker_name: Database::MonitorLockedTablesWorker
+  :feature_category: :cell
+  :has_external_dependencies: false
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: cronjob:database_partition_management
   :worker_name: Database::PartitionManagementWorker
   :feature_category: :database
diff --git a/app/workers/database/monitor_locked_tables_worker.rb b/app/workers/database/monitor_locked_tables_worker.rb
new file mode 100644
index 00000000000..66296ea1c0d
--- /dev/null
+++ b/app/workers/database/monitor_locked_tables_worker.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Database
+  class MonitorLockedTablesWorker
+    include ApplicationWorker
+    include CronjobQueue # rubocop: disable Scalability/CronWorkerContext
+
+    sidekiq_options retry: false
+    feature_category :cell
+    data_consistency :sticky
+    idempotent!
+
+    version 1
+
+    INITIAL_DATABASE_RESULT = {
+      tables_need_lock: [],
+      tables_need_lock_count: 0,
+      tables_need_unlock: [],
+      tables_need_unlock_count: 0
+    }.freeze
+
+    def perform
+      return unless Gitlab::Database.database_mode == Gitlab::Database::MODE_MULTIPLE_DATABASES
+      return if Feature.disabled?(:monitor_database_locked_tables, type: :ops)
+
+      lock_writes_results = ::Gitlab::Database::TablesLocker.new(dry_run: true, include_partitions: false).lock_writes
+
+      tables_lock_info_per_db = ::Gitlab::Database.database_base_models_with_gitlab_shared.keys.to_h do |db_name, _|
+        [db_name, INITIAL_DATABASE_RESULT.deep_dup]
+      end
+
+      lock_writes_results.each do |result|
+        handle_lock_writes_result(tables_lock_info_per_db, result)
+      end
+
+      log_extra_metadata_on_done(:results, tables_lock_info_per_db)
+    end
+
+    private
+
+    def handle_lock_writes_result(results, result)
+      case result[:action]
+      when "needs_lock"
+        results[result[:database]][:tables_need_lock] << result[:table]
+        results[result[:database]][:tables_need_lock_count] += 1
+      when "needs_unlock"
+        results[result[:database]][:tables_need_unlock] << result[:table]
+        results[result[:database]][:tables_need_unlock_count] += 1
+      end
+    end
+  end
+end
diff --git a/config/feature_flags/development/ci_batch_project_includes_context.yml b/config/feature_flags/development/ci_batch_project_includes_context.yml
deleted file mode 100644
index 634ed19bf34..00000000000
--- a/config/feature_flags/development/ci_batch_project_includes_context.yml
+++ /dev/null
@@ -1,8 +0,0 @@
----
-name: ci_batch_project_includes_context
-introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/112570
-rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/392746
-milestone: '15.10'
-type: development
-group: group::pipeline authoring
-default_enabled: false
diff --git a/config/feature_flags/ops/monitor_database_locked_tables.yml b/config/feature_flags/ops/monitor_database_locked_tables.yml
new file mode 100644
index 00000000000..5aee694a674
--- /dev/null
+++ b/config/feature_flags/ops/monitor_database_locked_tables.yml
@@ -0,0 +1,8 @@
+---
+name: monitor_database_locked_tables
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/120926
+rollout_issue_url:
+milestone: '16.1'
+type: ops
+group: group::tenant scale
+default_enabled: false
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 20b7bd6d399..76935b5903c 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -711,6 +711,9 @@ Gitlab.ee do
   Settings.cron_jobs['adjourned_projects_deletion_cron_worker'] ||= {}
   Settings.cron_jobs['adjourned_projects_deletion_cron_worker']['cron'] ||= '0 7 * * *'
   Settings.cron_jobs['adjourned_projects_deletion_cron_worker']['job_class'] = 'AdjournedProjectsDeletionCronWorker'
+  Settings.cron_jobs['database_monitor_locked_tables_cron_worker'] ||= {}
+  Settings.cron_jobs['database_monitor_locked_tables_cron_worker']['cron'] ||= '30 7 */3 * *'
+  Settings.cron_jobs['database_monitor_locked_tables_cron_worker']['job_class'] = 'Database::MonitorLockedTablesWorker'
   Settings.cron_jobs['geo_verification_cron_worker'] ||= {}
   Settings.cron_jobs['geo_verification_cron_worker']['cron'] ||= '* * * * *'
   Settings.cron_jobs['geo_verification_cron_worker']['job_class'] ||= 'Geo::VerificationCronWorker'
diff --git a/doc/administration/geo/setup/index.md b/doc/administration/geo/setup/index.md
index 6674ea7bd0d..eef915bb5d8 100644
--- a/doc/administration/geo/setup/index.md
+++ b/doc/administration/geo/setup/index.md
@@ -18,6 +18,7 @@ type: howto
 
 - Ensure the **primary** site has a [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) subscription to unlock Geo. You only need one license for all the sites.
 - Confirm the [requirements for running Geo](../index.md#requirements-for-running-geo) are met by all sites. For example, sites must use the same GitLab version, and sites must be able to communicate with each other over certain ports.
+- Confirm the **primary** and **secondary** site storage configurations match. If the primary Geo site uses object storage, the secondary Geo site must use it too. See [Geo with Object storage] (../object_storage.md) for more details.
 
 ## Using Omnibus GitLab
 
diff --git a/doc/ci/testing/code_quality.md b/doc/ci/testing/code_quality.md
index 62058e4f1cb..6eab50a1f61 100644
--- a/doc/ci/testing/code_quality.md
+++ b/doc/ci/testing/code_quality.md
@@ -227,7 +227,7 @@ Code Quality can be customized by defining available CI/CD variables:
 | CI/CD variable              | Description |
 | --------------------------- | ----------- |
 | `SOURCE_CODE`               | Path to the source code to scan. |
-| `TIMEOUT_SECONDS`           | Custom timeout for the `codeclimate analyze` command. |
+| `TIMEOUT_SECONDS`           | Custom timeout per engine container for the `codeclimate analyze` command, default is 900 seconds (15 minutes). |
 | `CODECLIMATE_DEBUG`         | Set to enable [Code Climate debug mode](https://github.com/codeclimate/codeclimate#environment-variables) |
 | `CODECLIMATE_DEV`           | Set to enable `--dev` mode which lets you run engines not known to the CLI. |
 | `REPORT_STDOUT`             | Set to print the report to `STDOUT` instead of generating the usual report file. |
diff --git a/doc/raketasks/backup_gitlab.md b/doc/raketasks/backup_gitlab.md
index e04d63da2ad..890cbaf8b3b 100644
--- a/doc/raketasks/backup_gitlab.md
+++ b/doc/raketasks/backup_gitlab.md
@@ -146,7 +146,7 @@ the GitLab container according to the documentation, it should be in the
 
 For [GitLab Helm chart installations](https://gitlab.com/gitlab-org/charts/gitlab)
 on a Kubernetes cluster, you must follow the
-[Back up the secrets](https://docs.gitlab.com/charts/backup-restore/backup.html#backup-the-secrets)
+[Back up the secrets](https://docs.gitlab.com/charts/backup-restore/backup.html#back-up-the-secrets)
 instructions.
 
 You may also want to back up any TLS keys and certificates (`/etc/gitlab/ssl`, `/etc/gitlab/trusted-certs`), and your
diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb
index 16a6bc8a692..de726b57053 100644
--- a/lib/gitlab/ci/config/external/file/project.rb
+++ b/lib/gitlab/ci/config/external/file/project.rb
@@ -68,8 +68,6 @@ module Gitlab
             private
 
             def project
-              return legacy_project if ::Feature.disabled?(:ci_batch_project_includes_context, context.project)
-
               # Although we use `where_full_path_in`, this BatchLoader does not reduce the number of queries to 1.
               # That's because we use it in the `can_access_local_content?` and `sha` BatchLoaders
               # as the `for` parameter. And this loads the project immediately.
@@ -83,10 +81,6 @@ module Gitlab
             end
 
             def can_access_local_content?
-              if ::Feature.disabled?(:ci_batch_project_includes_context, context.project)
-                return legacy_can_access_local_content?
-              end
-
               return if project.nil?
 
               # We are force-loading the project with the `itself` method
@@ -103,7 +97,6 @@ module Gitlab
             end
 
             def sha
-              return legacy_sha if ::Feature.disabled?(:ci_batch_project_includes_context, context.project)
               return if project.nil?
 
               # with `itself`, we are force-loading the project
@@ -128,26 +121,6 @@ module Gitlab
               end
             end
 
-            def legacy_project
-              strong_memoize(:legacy_project) do
-                ::Project.find_by_full_path(project_name)
-              end
-            end
-
-            def legacy_can_access_local_content?
-              strong_memoize(:legacy_can_access_local_content) do
-                context.logger.instrument(:config_file_project_validate_access) do
-                  Ability.allowed?(context.user, :download_code, project)
-                end
-              end
-            end
-
-            def legacy_sha
-              strong_memoize(:legacy_sha) do
-                project.commit(ref_name).try(:sha)
-              end
-            end
-
             override :expand_context_attrs
             def expand_context_attrs
               {
diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb
index 3472f2c581a..95975e4661b 100644
--- a/lib/gitlab/ci/config/external/mapper/verifier.rb
+++ b/lib/gitlab/ci/config/external/mapper/verifier.rb
@@ -11,10 +11,6 @@ module Gitlab
 
             # rubocop: disable Metrics/CyclomaticComplexity
             def process_without_instrumentation(files)
-              if ::Feature.disabled?(:ci_batch_project_includes_context, context.project)
-                return legacy_process_without_instrumentation(files)
-              end
-
               files.each do |file|
                 # When running a pipeline, some Ci::ProjectConfig sources prepend the config content with an
                 # "internal" `include`. We use this condition to exclude that `include` from the included file set.
@@ -45,30 +41,6 @@ module Gitlab
             end
             # rubocop: enable Metrics/CyclomaticComplexity
 
-            def legacy_process_without_instrumentation(files)
-              files.each do |file|
-                # When running a pipeline, some Ci::ProjectConfig sources prepend the config content with an
-                # "internal" `include`. We use this condition to exclude that `include` from the included file set.
-                context.expandset << file unless context.internal_include?
-                verify_max_includes!
-
-                verify_execution_time!
-
-                file.validate_location!
-                file.validate_context! if file.valid?
-                file.content if file.valid?
-              end
-
-              # We do not combine the loops because we need to load the content of all files before continuing
-              # to call `BatchLoader` for all locations.
-              files.each do |file| # rubocop:disable Style/CombinableLoops
-                verify_execution_time!
-
-                file.validate_content! if file.valid?
-                file.load_and_validate_expanded_hash! if file.valid?
-              end
-            end
-
             def verify_max_includes!
               return if context.expandset.count <= context.max_includes
 
diff --git a/lib/gitlab/database/lock_writes_manager.rb b/lib/gitlab/database/lock_writes_manager.rb
index 43e71e6bda2..8ddd871f93c 100644
--- a/lib/gitlab/database/lock_writes_manager.rb
+++ b/lib/gitlab/database/lock_writes_manager.rb
@@ -51,10 +51,15 @@ module Gitlab
 
         execute_sql_statement(sql_statement)
 
-        result_hash(action: 'locked')
+        result_hash(action: dry_run ? 'needs_lock' : 'locked')
       end
 
       def unlock_writes
+        unless table_locked_for_writes?
+          logger&.info "Skipping unlock_writes, because #{table_name} is already unlocked for writes"
+          return result_hash(action: 'skipped')
+        end
+
         logger&.info "Database: '#{database_name}', Table: '#{table_name}': Allow Writes".color(:green)
         sql_statement = <<~SQL
           DROP TRIGGER IF EXISTS #{write_trigger_name} ON #{table_name};
@@ -62,7 +67,7 @@ module Gitlab
 
         execute_sql_statement(sql_statement)
 
-        result_hash(action: 'unlocked')
+        result_hash(action: dry_run ? 'needs_unlock' : 'unlocked')
       end
 
       private
diff --git a/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb b/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb
index 420195d89dd..20814b098c1 100644
--- a/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb
+++ b/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb
@@ -104,7 +104,7 @@ module Gitlab
             when :func_call
               "#{parse_node(node.func_call.funcname.first)}()"
             when :a_const
-              parse_node(node.a_const.val)
+              parse_a_const(node.a_const)
             when :type_cast
               value = parse_node(node.type_cast.arg)
               type = type(node.type_cast.type_name)
@@ -112,10 +112,21 @@ module Gitlab
 
               [MAPPINGS.fetch(value, "'#{value}'"), separator].compact.join('')
             else
-              node.to_h[node.node].values.last
+              get_value_from_key(node, key: node.node)
             end
           end
 
+          def parse_a_const(a_const)
+            return unless a_const
+
+            type = a_const.val
+            get_value_from_key(a_const, key: type)
+          end
+
+          def get_value_from_key(node, key:)
+            node.to_h[key].values.last
+          end
+
           def partition_keys
             return [] unless partitioning_stmt
 
diff --git a/lib/gitlab/database/tables_locker.rb b/lib/gitlab/database/tables_locker.rb
index 0b0d46f4b0e..02e0da022f9 100644
--- a/lib/gitlab/database/tables_locker.rb
+++ b/lib/gitlab/database/tables_locker.rb
@@ -5,10 +5,11 @@ module Gitlab
     class TablesLocker
       GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_embedding gitlab_geo].freeze
 
-      def initialize(logger: nil, dry_run: false)
+      def initialize(logger: nil, dry_run: false, include_partitions: true)
         @logger = logger
         @dry_run = dry_run
         @result = []
+        @include_partitions = include_partitions
       end
 
       def unlock_writes
@@ -50,6 +51,7 @@ module Gitlab
       # Unlocks the writes on the table and its partitions
       def unlock_writes_on_table(table_name, connection, database_name)
         @result << lock_writes_manager(table_name, connection, database_name).unlock_writes
+        return unless @include_partitions
 
         table_attached_partitions(table_name, connection) do |postgres_partition|
           @result << lock_writes_manager(postgres_partition.identifier, connection, database_name).unlock_writes
@@ -59,6 +61,7 @@ module Gitlab
       # It locks the writes on the table and its partitions
       def lock_writes_on_table(table_name, connection, database_name)
         @result << lock_writes_manager(table_name, connection, database_name).lock_writes
+        return unless @include_partitions
 
         table_attached_partitions(table_name, connection) do |postgres_partition|
           @result << lock_writes_manager(postgres_partition.identifier, connection, database_name).lock_writes
@@ -67,6 +70,7 @@ module Gitlab
 
       def tables_to_lock(connection, &block)
         Gitlab::Database::GitlabSchema.tables_to_schema.each(&block)
+        return unless @include_partitions
 
         Gitlab::Database::SharedModel.using_connection(connection) do
           Postgresql::DetachedPartition.find_each do |detached_partition|
diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_multiple_projects_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_multiple_projects_spec.rb
index dc8db7ec387..59591fbe1cd 100644
--- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_multiple_projects_spec.rb
+++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_multiple_projects_spec.rb
@@ -1,10 +1,7 @@
 # frozen_string_literal: true
 
 module QA
-  RSpec.describe 'Verify', :runner, product_group: :pipeline_authoring, feature_flag: {
-    name: 'ci_batch_project_includes_context',
-    scope: :global
-  } do
+  RSpec.describe 'Verify', :runner, product_group: :pipeline_authoring do
     describe 'Include multiple files from multiple projects' do
       let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(number: 8)}" }
 
@@ -34,7 +31,7 @@ module QA
         end
       end
 
-      def before_do
+      before do
         Flow::Login.sign_in
 
         add_included_files_for(main_project)
@@ -50,44 +47,17 @@ module QA
         runner.remove_via_api!
       end
 
-      context 'when FF is on', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/396374' do
-        before do
-          Runtime::Feature.enable(:ci_batch_project_includes_context, project: main_project)
-          sleep 60
-
-          before_do
-        end
-
-        it 'runs the pipeline with composed config' do
-          Page::Project::Pipeline::Show.perform do |pipeline|
-            aggregate_failures 'pipeline has all expected jobs' do
-              expect(pipeline).to have_job('test_for_main')
-              expect(pipeline).to have_job("test1_for_#{project1.full_path}")
-              expect(pipeline).to have_job("test1_for_#{project2.full_path}")
-              expect(pipeline).to have_job("test2_for_#{project1.full_path}")
-              expect(pipeline).to have_job("test2_for_#{main_project.full_path}")
-            end
-          end
-        end
-      end
-
-      context 'when FF is off', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/396375' do
-        before do
-          Runtime::Feature.disable(:ci_batch_project_includes_context, project: main_project)
-          sleep 60
-
-          before_do
-        end
-
-        it 'runs the pipeline with composed config' do
-          Page::Project::Pipeline::Show.perform do |pipeline|
-            aggregate_failures 'pipeline has all expected jobs' do
-              expect(pipeline).to have_job('test_for_main')
-              expect(pipeline).to have_job("test1_for_#{project1.full_path}")
-              expect(pipeline).to have_job("test1_for_#{project2.full_path}")
-              expect(pipeline).to have_job("test2_for_#{project1.full_path}")
-              expect(pipeline).to have_job("test2_for_#{main_project.full_path}")
-            end
+      it(
+        'runs the pipeline with composed config',
+        testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/396374'
+      ) do
+        Page::Project::Pipeline::Show.perform do |pipeline|
+          aggregate_failures 'pipeline has all expected jobs' do
+            expect(pipeline).to have_job('test_for_main')
+            expect(pipeline).to have_job("test1_for_#{project1.full_path}")
+            expect(pipeline).to have_job("test1_for_#{project2.full_path}")
+            expect(pipeline).to have_job("test2_for_#{project1.full_path}")
+            expect(pipeline).to have_job("test2_for_#{main_project.full_path}")
           end
         end
       end
diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/pipeline_editor_tabs_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/pipeline_editor_tabs_spec.rb
index 1a8f07ba621..a9f46e394f7 100644
--- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/pipeline_editor_tabs_spec.rb
+++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/pipeline_editor_tabs_spec.rb
@@ -84,11 +84,7 @@ module QA
       context 'when CI has invalid syntax' do
         it(
           'shows invalid validations',
-          testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/368333',
-          quarantine: {
-            issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/412769',
-            type: :broken
-          }
+          testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/368333'
         ) do
           invalid_msg = 'syntax is invalid'
 
@@ -107,10 +103,10 @@ module QA
 
               show.go_to_full_configuration_tab
 
-              # TODO: remove this retry when
+              # TODO: remove page reload when
               # https://gitlab.com/gitlab-org/gitlab/-/issues/378536 is resolved
-              show.retry_until(max_attempts: 2, reload: true, sleep_interval: 1) { show.has_no_alert? }
-              expect(show).to have_source_editor
+              expect { show.has_source_editor? }
+                .to eventually_be_truthy.within(max_attempts: 2, reload_page: show, sleep_interval: 1)
 
               expect(show.ci_syntax_validate_message).to have_content('CI configuration is invalid')
             end
diff --git a/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb
index 1ee46daa196..e7dd5bd5079 100644
--- a/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb
+++ b/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb
@@ -147,43 +147,6 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category:
         expect(access_check_queries.values.sum).to eq(2)
       end
 
-      context 'when the FF ci_batch_project_includes_context is disabled' do
-        before do
-          stub_feature_flags(ci_batch_project_includes_context: false)
-        end
-
-        it 'returns an array of file objects' do
-          expect(process.map(&:location)).to contain_exactly(
-            'myfolder/file1.yml', 'myfolder/file2.yml', 'myfolder/file3.yml',
-            'myfolder/file1.yml', 'myfolder/file2.yml'
-          )
-        end
-
-        it 'adds files to the expandset' do
-          expect { process }.to change { context.expandset.count }.by(5)
-        end
-
-        it 'calls Gitaly for all files', :request_store do
-          files # calling this to load project creations and the `project.commit.id` call
-
-          # 5 for the sha check, 2 for the files in batch
-          expect { process }.to change { Gitlab::GitalyClient.get_request_count }.by(7)
-        end
-
-        it 'queries without batch', :use_sql_query_cache do
-          files # calling this to load project creations and the `project.commit.id` call
-
-          queries = ActiveRecord::QueryRecorder.new(skip_cached: false) { process }
-          projects_queries = queries.occurrences_starting_with('SELECT "projects"')
-          access_check_queries = queries.occurrences_starting_with(
-            'SELECT MAX("project_authorizations"."access_level")'
-          )
-
-          expect(projects_queries.values.sum).to eq(5)
-          expect(access_check_queries.values.sum).to eq(5)
-        end
-      end
-
       context 'when a project is missing' do
         let(:files) do
           [
@@ -203,20 +166,6 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category:
 
           expect(process.all?(&:valid?)).to be_falsey
         end
-
-        context 'when the FF ci_batch_project_includes_context is disabled' do
-          before do
-            stub_feature_flags(ci_batch_project_includes_context: false)
-          end
-
-          it 'returns an array of file objects' do
-            expect(process.map(&:location)).to contain_exactly(
-              'myfolder/file1.yml', 'myfolder/file2.yml'
-            )
-
-            expect(process.all?(&:valid?)).to be_falsey
-          end
-        end
       end
     end
 
diff --git a/spec/lib/gitlab/database/lock_writes_manager_spec.rb b/spec/lib/gitlab/database/lock_writes_manager_spec.rb
index 2aa95372338..899f3760132 100644
--- a/spec/lib/gitlab/database/lock_writes_manager_spec.rb
+++ b/spec/lib/gitlab/database/lock_writes_manager_spec.rb
@@ -55,7 +55,9 @@ RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: :
 
   describe '#lock_writes' do
     it 'prevents any writes on the table' do
-      subject.lock_writes
+      expect(subject.lock_writes).to eq(
+        { action: "locked", database: "main", dry_run: dry_run, table: test_table }
+      )
 
       expect do
         connection.execute("delete from #{test_table}")
@@ -116,19 +118,13 @@ RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: :
       expect(connection).not_to receive(:execute).with(/CREATE TRIGGER/)
 
       expect do
-        subject.lock_writes
+        result = subject.lock_writes
+        expect(result).to eq({ action: "skipped", database: "main", dry_run: false, table: test_table })
       end.not_to change {
         number_of_triggers_on(connection, test_table)
       }
     end
 
-    it 'returns result hash with action skipped' do
-      subject.lock_writes
-
-      expect(subject.lock_writes).to eq({ action: "skipped", database: "main", dry_run: false,
-table: test_table })
-    end
-
     context 'when running in dry_run mode' do
       let(:dry_run) { true }
 
@@ -154,9 +150,10 @@ table: test_table })
         end.not_to raise_error
       end
 
-      it 'returns result hash with action locked' do
-        expect(subject.lock_writes).to eq({ action: "locked", database: "main", dry_run: dry_run,
-table: test_table })
+      it 'returns result hash with action needs_lock' do
+        expect(subject.lock_writes).to eq(
+          { action: "needs_lock", database: "main", dry_run: true, table: test_table }
+        )
       end
     end
   end
@@ -175,13 +172,24 @@ table: test_table })
     end
 
     it 'allows writing on the table again' do
-      subject.unlock_writes
+      expect(subject.unlock_writes).to eq(
+        { action: "unlocked", database: "main", dry_run: dry_run, table: test_table }
+      )
 
       expect do
         connection.execute("delete from #{test_table}")
       end.not_to raise_error
     end
 
+    it 'skips unlocking the table if the table was already unlocked for writes' do
+      subject.unlock_writes
+
+      expect(subject).not_to receive(:execute_sql_statement)
+      expect(subject.unlock_writes).to eq(
+        { action: "skipped", database: "main", dry_run: dry_run, table: test_table }
+      )
+    end
+
     it 'removes the write protection triggers from the gitlab_main tables on the ci database' do
       expect do
         subject.unlock_writes
@@ -198,11 +206,6 @@ table: test_table })
       subject.unlock_writes
     end
 
-    it 'returns result hash with action unlocked' do
-      expect(subject.unlock_writes).to eq({ action: "unlocked", database: "main", dry_run: dry_run,
-table: test_table })
-    end
-
     context 'when running in dry_run mode' do
       let(:dry_run) { true }
 
@@ -225,8 +228,9 @@ table: test_table })
       end
 
       it 'returns result hash with dry_run true' do
-        expect(subject.unlock_writes).to eq({ action: "unlocked", database: "main", dry_run: dry_run,
-table: test_table })
+        expect(subject.unlock_writes).to eq(
+          { action: "needs_unlock", database: "main", dry_run: true, table: test_table }
+        )
       end
     end
   end
diff --git a/spec/lib/gitlab/database/tables_locker_spec.rb b/spec/lib/gitlab/database/tables_locker_spec.rb
index aaafe27f7ca..0e7e929d54b 100644
--- a/spec/lib/gitlab/database/tables_locker_spec.rb
+++ b/spec/lib/gitlab/database/tables_locker_spec.rb
@@ -251,6 +251,31 @@ RSpec.describe Gitlab::Database::TablesLocker, :suppress_gitlab_schemas_validate
       it_behaves_like 'unlock partitions', gitlab_main_detached_partition, 'ci'
     end
 
+    context 'when not including partitions' do
+      subject { described_class.new(include_partitions: false).lock_writes }
+
+      it 'does not include any table partitions' do
+        gitlab_main_partition = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.security_findings_test_partition"
+
+        expect(Gitlab::Database::LockWritesManager).not_to receive(:new).with(
+          hash_including(table_name: gitlab_main_partition)
+        )
+
+        subject
+      end
+
+      it 'does not include any detached partitions' do
+        detached_partition_name = "_test_gitlab_main_part_20220101"
+        gitlab_main_detached_partition = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{detached_partition_name}"
+
+        expect(Gitlab::Database::LockWritesManager).not_to receive(:new).with(
+          hash_including(table_name: gitlab_main_detached_partition)
+        )
+
+        subject
+      end
+    end
+
     context 'when running in dry_run mode' do
       subject { described_class.new(dry_run: true).lock_writes }
 
diff --git a/spec/workers/database/monitor_locked_tables_worker_spec.rb b/spec/workers/database/monitor_locked_tables_worker_spec.rb
new file mode 100644
index 00000000000..47475a0ad4a
--- /dev/null
+++ b/spec/workers/database/monitor_locked_tables_worker_spec.rb
@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Database::MonitorLockedTablesWorker, feature_category: :cell do
+  let(:worker) { described_class.new }
+  let(:tables_locker) { instance_double(Gitlab::Database::TablesLocker, lock_writes: nil) }
+
+  describe '#perform' do
+    context 'when running with single database' do
+      before do
+        skip_if_database_exists(:ci)
+      end
+
+      it 'skips executing the job' do
+        expect(Gitlab::Database::TablesLocker).not_to receive(:new)
+        worker.perform
+      end
+    end
+
+    context 'when running in decomposed database' do
+      context 'when the feature flag is disabled' do
+        before do
+          stub_feature_flags(monitor_database_locked_tables: false)
+        end
+
+        it 'skips executing the job' do
+          expect(Gitlab::Database::TablesLocker).not_to receive(:new)
+          worker.perform
+        end
+      end
+
+      context 'when the feature flag is enabled' do
+        before do
+          skip_if_shared_database(:ci)
+          stub_feature_flags(monitor_database_locked_tables: true)
+          allow(Gitlab::Database::TablesLocker).to receive(:new).and_return(tables_locker)
+        end
+
+        it 'calls TablesLocker with dry_run enabled' do
+          expect(tables_locker).to receive(:lock_writes).and_return([])
+          expect(worker).to receive(:log_extra_metadata_on_done)
+
+          worker.perform
+        end
+
+        it 'reports the tables that need to be locked on both databases main and ci' do
+          lock_writes_results = [
+            { table: 'users', database: 'ci', action: 'needs_lock' },
+            { table: 'projects', database: 'ci', action: 'needs_lock' },
+            { table: 'ci_builds', database: 'ci', action: 'skipped' },
+            { table: 'ci_builds', database: 'main', action: 'needs_lock' },
+            { table: 'users', database: 'main', action: 'skipped' },
+            { table: 'projects', database: 'main', action: 'skipped' },
+            { table: 'issues', database: 'main', action: 'needs_unlock' } # if a table was locked by mistake
+          ]
+          expected_log_results = {
+            'ci' => {
+              tables_need_lock: %w[users projects],
+              tables_need_lock_count: 2,
+              tables_need_unlock: [],
+              tables_need_unlock_count: 0
+            },
+            'main' => {
+              tables_need_lock: ['ci_builds'],
+              tables_need_lock_count: 1,
+              tables_need_unlock: ['issues'],
+              tables_need_unlock_count: 1
+            }
+          }
+          expect(tables_locker).to receive(:lock_writes).and_return(lock_writes_results)
+          expect(worker).to receive(:log_extra_metadata_on_done).with(:results, expected_log_results)
+
+          worker.perform
+        end
+      end
+    end
+  end
+end
diff --git a/vendor/project_templates/serverless_framework.tar.gz b/vendor/project_templates/serverless_framework.tar.gz
index 279d0f2eb5c..6b5def03ed3 100644
--- a/vendor/project_templates/serverless_framework.tar.gz
+++ b/vendor/project_templates/serverless_framework.tar.gz