Add latest changes from gitlab-org/gitlab@master

7 files changed, 105 insertions, 0 deletions

diff --git a/db/post_migrate/20230317075852_add_tmp_unique_packages_index_when_debian.rb b/db/post_migrate/20230317075852_add_tmp_unique_packages_index_when_debian.rb
new file mode 100644
index 00000000000..9ea8d358366
--- /dev/null
+++ b/db/post_migrate/20230317075852_add_tmp_unique_packages_index_when_debian.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class AddTmpUniquePackagesIndexWhenDebian < Gitlab::Database::Migration[2.1]
+  TABLE_NAME = :packages_packages
+  PACKAGE_TYPE_DEBIAN = 9
+  PACKAGE_STATUS_PENDING_DESTRUCTION = 4
+  TMP_DEBIAN_UNIQUE_INDEX_NAME = 'tmp_unique_packages_project_id_and_name_and_version_when_debian'
+
+  disable_ddl_transaction!
+
+  def up
+    # This index will disallow further duplicates while we're deduplicating the data.
+    add_concurrent_index TABLE_NAME, [:project_id, :name, :version],
+      where: "package_type = #{PACKAGE_TYPE_DEBIAN} AND status != #{PACKAGE_STATUS_PENDING_DESTRUCTION} AND
+        created_at > TIMESTAMP WITH TIME ZONE '#{Time.now.utc}'",
+      unique: true,
+      name: TMP_DEBIAN_UNIQUE_INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name TABLE_NAME, TMP_DEBIAN_UNIQUE_INDEX_NAME
+  end
+end
diff --git a/db/post_migrate/20230317080000_ensure_unique_debian_packages.rb b/db/post_migrate/20230317080000_ensure_unique_debian_packages.rb
new file mode 100644
index 00000000000..250e0cb2369
--- /dev/null
+++ b/db/post_migrate/20230317080000_ensure_unique_debian_packages.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+class EnsureUniqueDebianPackages < Gitlab::Database::Migration[2.1]
+  BATCH_SIZE = 1_000
+
+  disable_ddl_transaction!
+
+  restrict_gitlab_migration gitlab_schema: :gitlab_main
+
+  class Package < MigrationRecord
+    include EachBatch
+
+    self.table_name = 'packages_packages'
+
+    enum package_type: { debian: 9 }
+
+    enum status: { pending_destruction: 4 }
+  end
+
+  def up
+    Package.distinct_each_batch(column: :project_id) do |package_projects|
+      project_ids = package_projects.pluck(:project_id)
+      duplicates = Package.debian
+                          .not_pending_destruction
+                          .where(project_id: project_ids)
+                          .select('project_id, name, version, MAX(id) as last_id')
+                          .group(:project_id, :name, :version)
+                          .having('count(id) > 1')
+      loop do
+        duplicates.limit(BATCH_SIZE).each do |duplicate|
+          Package.debian
+                .not_pending_destruction
+                .where(
+                  project_id: duplicate.project_id,
+                  name: duplicate.name,
+                  version: duplicate.version,
+                  id: ..duplicate.last_id - 1
+                ).update_all status: :pending_destruction
+        end
+        break unless duplicates.exists?
+      end
+    end
+  end
+
+  def down
+    # nothing to do
+  end
+end
diff --git a/db/post_migrate/20230317080137_add_unique_packages_index_when_debian.rb b/db/post_migrate/20230317080137_add_unique_packages_index_when_debian.rb
new file mode 100644
index 00000000000..b079e860c0f
--- /dev/null
+++ b/db/post_migrate/20230317080137_add_unique_packages_index_when_debian.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class AddUniquePackagesIndexWhenDebian < Gitlab::Database::Migration[2.1]
+  TABLE_NAME = :packages_packages
+  PACKAGE_TYPE_DEBIAN = 9
+  PACKAGE_STATUS_PENDING_DESTRUCTION = 4
+  TMP_DEBIAN_UNIQUE_INDEX_NAME = 'tmp_unique_packages_project_id_and_name_and_version_when_debian'
+  DEBIAN_UNIQUE_INDEX_NAME = 'unique_packages_project_id_and_name_and_version_when_debian'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index TABLE_NAME, [:project_id, :name, :version],
+      where: "package_type = #{PACKAGE_TYPE_DEBIAN} AND status != #{PACKAGE_STATUS_PENDING_DESTRUCTION}",
+      unique: true,
+      name: DEBIAN_UNIQUE_INDEX_NAME
+    remove_concurrent_index_by_name TABLE_NAME, TMP_DEBIAN_UNIQUE_INDEX_NAME
+  end
+
+  def down
+    # This index will disallow further duplicates while we're deduplicating the data.
+    add_concurrent_index TABLE_NAME, [:project_id, :name, :version],
+      where: "package_type = #{PACKAGE_TYPE_DEBIAN} AND status != #{PACKAGE_STATUS_PENDING_DESTRUCTION} AND
+        created_at > TIMESTAMP WITH TIME ZONE '#{Time.now.utc}'",
+      unique: true,
+      name: TMP_DEBIAN_UNIQUE_INDEX_NAME
+    remove_concurrent_index_by_name TABLE_NAME, DEBIAN_UNIQUE_INDEX_NAME
+  end
+end
diff --git a/db/schema_migrations/20230317075852 b/db/schema_migrations/20230317075852
new file mode 100644
index 00000000000..7506d9c5414
--- /dev/null
+++ b/db/schema_migrations/20230317075852
@@ -0,0 +1 @@
+5b9e269f3354b1f054a3e3d8f9a011ea2c1a19c72a4cecdfb8a3e6e80ab83ab3
\ No newline at end of file
diff --git a/db/schema_migrations/20230317080000 b/db/schema_migrations/20230317080000
new file mode 100644
index 00000000000..33b3252a977
--- /dev/null
+++ b/db/schema_migrations/20230317080000
@@ -0,0 +1 @@
+a33f941480ce270532dca961cf72ce31c5c2d732b1c1ab361c95bac6e278a443
\ No newline at end of file
diff --git a/db/schema_migrations/20230317080137 b/db/schema_migrations/20230317080137
new file mode 100644
index 00000000000..90914f8a533
--- /dev/null
+++ b/db/schema_migrations/20230317080137
@@ -0,0 +1 @@
+238730785306bb43677be1e86ae3cca210d651eb5397a7fbf90a12a7ec252614
\ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index 4af994590ad..ed121f987ea 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -32838,6 +32838,8 @@ CREATE UNIQUE INDEX unique_index_for_project_pages_unique_domain ON project_sett
 
 CREATE UNIQUE INDEX unique_merge_request_metrics_by_merge_request_id ON merge_request_metrics USING btree (merge_request_id);
 
+CREATE UNIQUE INDEX unique_packages_project_id_and_name_and_version_when_debian ON packages_packages USING btree (project_id, name, version) WHERE ((package_type = 9) AND (status <> 4));
+
 CREATE UNIQUE INDEX unique_postgres_async_fk_validations_name_and_table_name ON postgres_async_foreign_key_validations USING btree (name, table_name);
 
 CREATE UNIQUE INDEX unique_projects_on_name_namespace_id ON projects USING btree (name, namespace_id);