Add latest changes from gitlab-org/gitlab@master

24 files changed, 264 insertions, 9 deletions

diff --git a/db/click_house/migrate/main/20240115122100_drop_audit_events.rb b/db/click_house/migrate/main/20240115122100_drop_audit_events.rb
new file mode 100644
index 00000000000..198e83ff7ee
--- /dev/null
+++ b/db/click_house/migrate/main/20240115122100_drop_audit_events.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class DropAuditEvents < ClickHouse::Migration
+  def up
+    execute <<~SQL
+      DROP TABLE IF EXISTS audit_events;
+    SQL
+  end
+
+  def down
+    execute <<~SQL
+      CREATE TABLE IF NOT EXISTS audit_events
+      (
+          id UInt64 DEFAULT 0,
+          author_id UInt64 DEFAULT 0,
+          author_name String DEFAULT '',
+          created_at DateTime64(6, 'UTC') DEFAULT now(),
+          details String DEFAULT '',
+          entity_id UInt64 DEFAULT 0,
+          entity_path String DEFAULT '',
+          entity_type LowCardinality(String) DEFAULT '',
+          ip_address String DEFAULT '',
+          target_details String DEFAULT '',
+          target_id UInt64 DEFAULT 0,
+          target_type LowCardinality(String) DEFAULT '',
+          is_deleted UInt8 DEFAULT 0,
+      ) ENGINE = ReplacingMergeTree(created_at, is_deleted)
+      PARTITION BY toYear(created_at)
+      ORDER BY (entity_type, entity_id, author_id, created_at, id);
+    SQL
+
+    execute <<~SQL
+      ALTER TABLE audit_events
+      ADD PROJECTION IF NOT EXISTS by_id (SELECT * ORDER BY id);
+    SQL
+  end
+end
diff --git a/db/click_house/migrate/main/20240115162101_recreate_audit_events.rb b/db/click_house/migrate/main/20240115162101_recreate_audit_events.rb
new file mode 100644
index 00000000000..0fcef508c4f
--- /dev/null
+++ b/db/click_house/migrate/main/20240115162101_recreate_audit_events.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class RecreateAuditEvents < ClickHouse::Migration
+  def up
+    execute <<~SQL
+      CREATE TABLE IF NOT EXISTS audit_events
+      (
+          id UInt64 DEFAULT 0,
+          author_id Int64 DEFAULT 0,
+          author_name String DEFAULT '',
+          created_at DateTime64(6, 'UTC') DEFAULT now(),
+          details String DEFAULT '',
+          entity_id Int64 DEFAULT 0,
+          entity_path String DEFAULT '',
+          entity_type LowCardinality(String) DEFAULT '',
+          ip_address String DEFAULT '',
+          target_details String DEFAULT '',
+          target_id Int64 DEFAULT 0,
+          target_type LowCardinality(String) DEFAULT '',
+          is_deleted UInt8 DEFAULT 0,
+      ) ENGINE = ReplacingMergeTree(created_at, is_deleted)
+      PARTITION BY toYear(created_at)
+      ORDER BY (entity_type, entity_id, author_id, created_at, id);
+    SQL
+
+    execute <<~SQL
+      ALTER TABLE audit_events
+      ADD PROJECTION IF NOT EXISTS by_id (SELECT * ORDER BY id);
+    SQL
+  end
+
+  def down
+    execute <<~SQL
+      DROP TABLE audit_events
+    SQL
+  end
+end
diff --git a/db/docs/batched_background_migrations/delete_invalid_protected_branch_merge_access_levels.yml b/db/docs/batched_background_migrations/delete_invalid_protected_branch_merge_access_levels.yml
index 6dccaeeeb50..a09e0ee8d05 100644
--- a/db/docs/batched_background_migrations/delete_invalid_protected_branch_merge_access_levels.yml
+++ b/db/docs/batched_background_migrations/delete_invalid_protected_branch_merge_access_levels.yml
@@ -5,3 +5,4 @@ feature_category: source_code_management
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/134337
 milestone: '16.6'
 queued_migration_version: 20231016173129
+finalized_by: 20240111195101
diff --git a/db/docs/batched_background_migrations/delete_invalid_protected_branch_push_access_levels.yml b/db/docs/batched_background_migrations/delete_invalid_protected_branch_push_access_levels.yml
index 72e59e6e00f..c884d7cfbf8 100644
--- a/db/docs/batched_background_migrations/delete_invalid_protected_branch_push_access_levels.yml
+++ b/db/docs/batched_background_migrations/delete_invalid_protected_branch_push_access_levels.yml
@@ -5,3 +5,4 @@ feature_category: source_code_management
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/134337
 milestone: '16.6'
 queued_migration_version: 20231016194927
+finalized_by: 20240111194808
diff --git a/db/docs/batched_background_migrations/delete_invalid_protected_tag_create_access_levels.yml b/db/docs/batched_background_migrations/delete_invalid_protected_tag_create_access_levels.yml
index 35c76b78894..41bf2ab9e03 100644
--- a/db/docs/batched_background_migrations/delete_invalid_protected_tag_create_access_levels.yml
+++ b/db/docs/batched_background_migrations/delete_invalid_protected_tag_create_access_levels.yml
@@ -5,3 +5,4 @@ feature_category: source_code_management
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/134337
 milestone: '16.6'
 queued_migration_version: 20231016194943
+finalized_by: 20240111194603
diff --git a/db/docs/deleted_tables/ci_editor_ai_conversation_messages.yml b/db/docs/deleted_tables/ci_editor_ai_conversation_messages.yml
index e37b9bfc75c..dd7a4a6b45c 100644
--- a/db/docs/deleted_tables/ci_editor_ai_conversation_messages.yml
+++ b/db/docs/deleted_tables/ci_editor_ai_conversation_messages.yml
@@ -4,8 +4,7 @@ classes:
 - Ci::Editor::AiConversation::Message
 feature_categories:
 - pipeline_composition
-description: Represents an ai message for a user and project for the pipeline editor
-  bot.
+description: Represents an ai message for a user and project for the pipeline editor bot.
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/119840
 milestone: '16.0'
 gitlab_schema: gitlab_ci
diff --git a/db/post_migrate/20231215130625_schedule_index_to_events_author_group_action_target_type_created_at.rb b/db/post_migrate/20231215130625_schedule_index_to_events_author_group_action_target_type_created_at.rb
index 063aff58176..1e3658aa697 100644
--- a/db/post_migrate/20231215130625_schedule_index_to_events_author_group_action_target_type_created_at.rb
+++ b/db/post_migrate/20231215130625_schedule_index_to_events_author_group_action_target_type_created_at.rb
@@ -6,7 +6,6 @@ class ScheduleIndexToEventsAuthorGroupActionTargetTypeCreatedAt < Gitlab::Databa
   INDEX_NAME = 'index_events_author_id_group_id_action_target_type_created_at'
   COLUMNS = [:author_id, :group_id, :action, :target_type, :created_at]
 
-  # TODO: Index to be created synchronously in https://gitlab.com/gitlab-org/gitlab/-/issues/435524
   def up
     prepare_async_index :events, COLUMNS, name: INDEX_NAME
   end
diff --git a/db/post_migrate/20240111131500_add_async_index_merge_request_metrics_on_merged_by_id_target_project_id_m_r_id.rb b/db/post_migrate/20240111131500_add_async_index_merge_request_metrics_on_merged_by_id_target_project_id_m_r_id.rb
new file mode 100644
index 00000000000..cec9020f20b
--- /dev/null
+++ b/db/post_migrate/20240111131500_add_async_index_merge_request_metrics_on_merged_by_id_target_project_id_m_r_id.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class AddAsyncIndexMergeRequestMetricsOnMergedByIdTargetProjectIdMRId < Gitlab::Database::Migration[2.2]
+  milestone '16.9'
+
+  TABLE_NAME = :merge_request_metrics
+  INDEX_NAME = :idx_merge_request_metrics_on_merged_by_project_and_mr
+  INDEX_COLUMNS = %i[merged_by_id target_project_id merge_request_id]
+
+  def up
+    prepare_async_index TABLE_NAME, INDEX_COLUMNS, name: INDEX_NAME
+  end
+
+  def down
+    unprepare_async_index TABLE_NAME, INDEX_COLUMNS, name: INDEX_NAME
+  end
+end
diff --git a/db/post_migrate/20240111194603_finalize_delete_invalid_protected_tag_create_access_levels.rb b/db/post_migrate/20240111194603_finalize_delete_invalid_protected_tag_create_access_levels.rb
new file mode 100644
index 00000000000..9aeb09f8dde
--- /dev/null
+++ b/db/post_migrate/20240111194603_finalize_delete_invalid_protected_tag_create_access_levels.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class FinalizeDeleteInvalidProtectedTagCreateAccessLevels < Gitlab::Database::Migration[2.2]
+  milestone '16.8'
+  MIGRATION = 'DeleteInvalidProtectedTagCreateAccessLevels'
+
+  disable_ddl_transaction!
+  restrict_gitlab_migration gitlab_schema: :gitlab_main
+
+  def up
+    ensure_batched_background_migration_is_finished(
+      job_class_name: MIGRATION,
+      table_name: :protected_tag_create_access_levels,
+      column_name: :id,
+      job_arguments: [],
+      finalize: true
+    )
+  end
+
+  def down
+    # no-op
+  end
+end
diff --git a/db/post_migrate/20240111194658_drop_temp_index_on_protected_tag_create_access_levels.rb b/db/post_migrate/20240111194658_drop_temp_index_on_protected_tag_create_access_levels.rb
new file mode 100644
index 00000000000..6e20d4e4a11
--- /dev/null
+++ b/db/post_migrate/20240111194658_drop_temp_index_on_protected_tag_create_access_levels.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class DropTempIndexOnProtectedTagCreateAccessLevels < Gitlab::Database::Migration[2.2]
+  disable_ddl_transaction!
+  milestone '16.8'
+
+  INDEX_NAME = 'tmp_idx_protected_tag_create_access_levels_on_id_with_group'
+
+  def up
+    remove_concurrent_index_by_name(
+      :protected_tag_create_access_levels,
+      INDEX_NAME
+    )
+  end
+
+  def down
+    add_concurrent_index(
+      :protected_tag_create_access_levels,
+      %i[id],
+      where: 'group_id IS NOT NULL',
+      name: INDEX_NAME
+    )
+  end
+end
diff --git a/db/post_migrate/20240111194808_finalize_delete_invalid_protected_branch_push_access_levels.rb b/db/post_migrate/20240111194808_finalize_delete_invalid_protected_branch_push_access_levels.rb
new file mode 100644
index 00000000000..1af26ecf62a
--- /dev/null
+++ b/db/post_migrate/20240111194808_finalize_delete_invalid_protected_branch_push_access_levels.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class FinalizeDeleteInvalidProtectedBranchPushAccessLevels < Gitlab::Database::Migration[2.2]
+  milestone '16.8'
+  MIGRATION = 'DeleteInvalidProtectedBranchPushAccessLevels'
+
+  disable_ddl_transaction!
+  restrict_gitlab_migration gitlab_schema: :gitlab_main
+
+  def up
+    ensure_batched_background_migration_is_finished(
+      job_class_name: MIGRATION,
+      table_name: :protected_branch_push_access_levels,
+      column_name: :id,
+      job_arguments: [],
+      finalize: true
+    )
+  end
+
+  def down
+    # no-op
+  end
+end
diff --git a/db/post_migrate/20240111194925_drop_temp_index_on_protected_branch_push_access_levels.rb b/db/post_migrate/20240111194925_drop_temp_index_on_protected_branch_push_access_levels.rb
new file mode 100644
index 00000000000..4c6f96a97c1
--- /dev/null
+++ b/db/post_migrate/20240111194925_drop_temp_index_on_protected_branch_push_access_levels.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class DropTempIndexOnProtectedBranchPushAccessLevels < Gitlab::Database::Migration[2.2]
+  disable_ddl_transaction!
+  milestone '16.8'
+
+  INDEX_NAME = 'tmp_idx_protected_branch_push_access_levels_on_id_with_group'
+
+  def up
+    remove_concurrent_index_by_name(
+      :protected_branch_push_access_levels,
+      INDEX_NAME
+    )
+  end
+
+  def down
+    add_concurrent_index(
+      :protected_branch_push_access_levels,
+      %i[id],
+      where: 'group_id IS NOT NULL',
+      name: INDEX_NAME
+    )
+  end
+end
diff --git a/db/post_migrate/20240111195101_finalize_delete_invalid_protected_branch_merge_access_levels.rb b/db/post_migrate/20240111195101_finalize_delete_invalid_protected_branch_merge_access_levels.rb
new file mode 100644
index 00000000000..ca5f520c50f
--- /dev/null
+++ b/db/post_migrate/20240111195101_finalize_delete_invalid_protected_branch_merge_access_levels.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class FinalizeDeleteInvalidProtectedBranchMergeAccessLevels < Gitlab::Database::Migration[2.2]
+  milestone '16.8'
+  MIGRATION = 'DeleteInvalidProtectedBranchMergeAccessLevels'
+
+  disable_ddl_transaction!
+  restrict_gitlab_migration gitlab_schema: :gitlab_main
+
+  def up
+    ensure_batched_background_migration_is_finished(
+      job_class_name: MIGRATION,
+      table_name: :protected_branch_merge_access_levels,
+      column_name: :id,
+      job_arguments: [],
+      finalize: true
+    )
+  end
+
+  def down
+    # no-op
+  end
+end
diff --git a/db/post_migrate/20240111195145_drop_temp_index_on_protected_branch_merge_access_levels.rb b/db/post_migrate/20240111195145_drop_temp_index_on_protected_branch_merge_access_levels.rb
new file mode 100644
index 00000000000..f2eb2aa0ea1
--- /dev/null
+++ b/db/post_migrate/20240111195145_drop_temp_index_on_protected_branch_merge_access_levels.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class DropTempIndexOnProtectedBranchMergeAccessLevels < Gitlab::Database::Migration[2.2]
+  disable_ddl_transaction!
+  milestone '16.8'
+
+  INDEX_NAME = 'tmp_idx_protected_branch_merge_access_levels_on_id_with_group'
+
+  def up
+    remove_concurrent_index_by_name(
+      :protected_branch_merge_access_levels,
+      INDEX_NAME
+    )
+  end
+
+  def down
+    add_concurrent_index(
+      :protected_branch_merge_access_levels,
+      %i[id],
+      where: 'group_id IS NOT NULL',
+      name: INDEX_NAME
+    )
+  end
+end
diff --git a/db/post_migrate/20240112143548_add_index_to_events_author_group_action_target_type.rb b/db/post_migrate/20240112143548_add_index_to_events_author_group_action_target_type.rb
new file mode 100644
index 00000000000..88e5f95ef68
--- /dev/null
+++ b/db/post_migrate/20240112143548_add_index_to_events_author_group_action_target_type.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class AddIndexToEventsAuthorGroupActionTargetType < Gitlab::Database::Migration[2.2]
+  milestone '16.9'
+
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'index_events_author_id_group_id_action_target_type_created_at'
+  COLUMNS = [:author_id, :group_id, :action, :target_type, :created_at]
+
+  def up
+    add_concurrent_index :events, COLUMNS, name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :events, INDEX_NAME
+  end
+end
diff --git a/db/schema_migrations/20240111131500 b/db/schema_migrations/20240111131500
new file mode 100644
index 00000000000..77d7e821e64
--- /dev/null
+++ b/db/schema_migrations/20240111131500
@@ -0,0 +1 @@
+dc8b922002930d7227e75b50617c3f4d4949fae4c1444c92767316d97fab49be
\ No newline at end of file
diff --git a/db/schema_migrations/20240111194603 b/db/schema_migrations/20240111194603
new file mode 100644
index 00000000000..8c9da9a87cb
--- /dev/null
+++ b/db/schema_migrations/20240111194603
@@ -0,0 +1 @@
+cd476c18128010183f94f09c4543716f2e3bdd78c11209dcec500d5506e3f7b2
\ No newline at end of file
diff --git a/db/schema_migrations/20240111194658 b/db/schema_migrations/20240111194658
new file mode 100644
index 00000000000..90a6a94f4d8
--- /dev/null
+++ b/db/schema_migrations/20240111194658
@@ -0,0 +1 @@
+5f10603c43a631785057b2ad8c5f4e0de84c2499b3f862b5b851e28a3c9769ea
\ No newline at end of file
diff --git a/db/schema_migrations/20240111194808 b/db/schema_migrations/20240111194808
new file mode 100644
index 00000000000..69f70190f2c
--- /dev/null
+++ b/db/schema_migrations/20240111194808
@@ -0,0 +1 @@
+80a807f90560abd387b7579255e2301ba3b8bde4f25ca4ff1ecb83e4b3b42163
\ No newline at end of file
diff --git a/db/schema_migrations/20240111194925 b/db/schema_migrations/20240111194925
new file mode 100644
index 00000000000..c6272136c31
--- /dev/null
+++ b/db/schema_migrations/20240111194925
@@ -0,0 +1 @@
+7cf8a9d321e212f49de7419a3827e3c18504f7a98900738c1fb7e82661743c92
\ No newline at end of file
diff --git a/db/schema_migrations/20240111195101 b/db/schema_migrations/20240111195101
new file mode 100644
index 00000000000..b1c6f09b587
--- /dev/null
+++ b/db/schema_migrations/20240111195101
@@ -0,0 +1 @@
+560e3a35eee5ace51635ebc504d785b6008914316e8078a68f7cb735e14fc6c1
\ No newline at end of file
diff --git a/db/schema_migrations/20240111195145 b/db/schema_migrations/20240111195145
new file mode 100644
index 00000000000..965dd52f86e
--- /dev/null
+++ b/db/schema_migrations/20240111195145
@@ -0,0 +1 @@
+6e6b2669c731d4922cbe42adf003b5b934f4690a5466b166b9112f05681db62b
\ No newline at end of file
diff --git a/db/schema_migrations/20240112143548 b/db/schema_migrations/20240112143548
new file mode 100644
index 00000000000..0c3c4f8cc20
--- /dev/null
+++ b/db/schema_migrations/20240112143548
@@ -0,0 +1 @@
+efb53be25e54b05e213256b985a9b28b3af5754763436772b6552688c066a591
\ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index 10e32e7cdca..f17904b1c64 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -33624,6 +33624,8 @@ CREATE INDEX index_et_errors_on_project_id_and_status_first_seen_at_id_desc ON e
 
 CREATE INDEX index_et_errors_on_project_id_and_status_last_seen_at_id_desc ON error_tracking_errors USING btree (project_id, status, last_seen_at DESC, id DESC);
 
+CREATE INDEX index_events_author_id_group_id_action_target_type_created_at ON events USING btree (author_id, group_id, action, target_type, created_at);
+
 CREATE INDEX index_events_author_id_project_id_action_target_type_created_at ON events USING btree (author_id, project_id, action, target_type, created_at);
 
 CREATE INDEX index_events_for_followed_users ON events USING btree (author_id, target_type, action, id);
@@ -36130,12 +36132,6 @@ CREATE INDEX tmp_idx_orphaned_approval_merge_request_rules ON approval_merge_req
 
 CREATE INDEX tmp_idx_orphaned_approval_project_rules ON approval_project_rules USING btree (id) WHERE ((report_type = ANY (ARRAY[2, 4])) AND (security_orchestration_policy_configuration_id IS NULL));
 
-CREATE INDEX tmp_idx_protected_branch_merge_access_levels_on_id_with_group ON protected_branch_merge_access_levels USING btree (id) WHERE (group_id IS NOT NULL);
-
-CREATE INDEX tmp_idx_protected_branch_push_access_levels_on_id_with_group ON protected_branch_push_access_levels USING btree (id) WHERE (group_id IS NOT NULL);
-
-CREATE INDEX tmp_idx_protected_tag_create_access_levels_on_id_with_group ON protected_tag_create_access_levels USING btree (id) WHERE (group_id IS NOT NULL);
-
 CREATE INDEX tmp_index_ci_job_artifacts_on_expire_at_where_locked_unknown ON ci_job_artifacts USING btree (expire_at, job_id) WHERE ((locked = 2) AND (expire_at IS NOT NULL));
 
 CREATE INDEX tmp_index_cis_vulnerability_reads_on_id ON vulnerability_reads USING btree (id) WHERE (report_type = 7);