Add latest changes from gitlab-org/gitlab@master

5 files changed, 98 insertions, 6 deletions

diff --git a/lib/gitlab/database/partitioning/detached_partition_dropper.rb b/lib/gitlab/database/partitioning/detached_partition_dropper.rb
new file mode 100644
index 00000000000..dc63d93fd07
--- /dev/null
+++ b/lib/gitlab/database/partitioning/detached_partition_dropper.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+module Gitlab
+  module Database
+    module Partitioning
+      class DetachedPartitionDropper
+        def perform
+          return unless Feature.enabled?(:drop_detached_partitions, default_enabled: :yaml)
+
+          Gitlab::AppLogger.info(message: "Checking for previously detached partitions to drop")
+          Postgresql::DetachedPartition.ready_to_drop.find_each do |detached_partition|
+            conn.transaction do
+              # Another process may have already dropped the table and deleted this entry
+              next unless (detached_partition = Postgresql::DetachedPartition.lock.find_by(id: detached_partition.id))
+
+              unless check_partition_detached?(detached_partition)
+                Gitlab::AppLogger.error(message: "Attempt to drop attached database partition", partition_name: detached_partition.table_name)
+                detached_partition.destroy!
+                next
+              end
+
+              drop_one(detached_partition)
+            end
+          rescue StandardError => e
+            Gitlab::AppLogger.error(message: "Failed to drop previously detached partition",
+                                    partition_name: detached_partition.table_name,
+                                    exception_class: e.class,
+                                    exception_message: e.message)
+          end
+        end
+
+        private
+
+        def drop_one(detached_partition)
+          conn.transaction do
+            conn.execute(<<~SQL)
+              DROP TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{conn.quote_table_name(detached_partition.table_name)}
+            SQL
+
+            detached_partition.destroy!
+          end
+          Gitlab::AppLogger.info(message: "Dropped previously detached partition", partition_name: detached_partition.table_name)
+        end
+
+        def check_partition_detached?(detached_partition)
+          # PostgresPartition checks the pg_inherits view, so our partition will only show here if it's still attached
+          # and thus should not be dropped
+          !PostgresPartition.for_identifier("#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{detached_partition.table_name}").exists?
+        end
+
+        def conn
+          @conn ||= ApplicationRecord.connection
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb
index 4c68399cb68..7992c2fdaa7 100644
--- a/lib/gitlab/database/partitioning/monthly_strategy.rb
+++ b/lib/gitlab/database/partitioning/monthly_strategy.rb
@@ -86,7 +86,7 @@ module Gitlab
         end
 
         def pruning_old_partitions?
-          Feature.enabled?(:partition_pruning_dry_run) && retain_for.present?
+          retain_for.present?
         end
 
         def oldest_active_date
diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb
index c2a9422a42a..7e433ecdd39 100644
--- a/lib/gitlab/database/partitioning/partition_manager.rb
+++ b/lib/gitlab/database/partitioning/partition_manager.rb
@@ -4,6 +4,8 @@ module Gitlab
   module Database
     module Partitioning
       class PartitionManager
+        UnsafeToDetachPartitionError = Class.new(StandardError)
+
         def self.register(model)
           raise ArgumentError, "Only models with a #partitioning_strategy can be registered." unless model.respond_to?(:partitioning_strategy)
 
@@ -16,6 +18,7 @@ module Gitlab
 
         LEASE_TIMEOUT = 1.minute
         MANAGEMENT_LEASE_KEY = 'database_partition_management_%s'
+        RETAIN_DETACHED_PARTITIONS_FOR = 1.week
 
         attr_reader :models
 
@@ -35,13 +38,16 @@ module Gitlab
               partitions_to_create = missing_partitions(model)
               create(partitions_to_create) unless partitions_to_create.empty?
 
-              if Feature.enabled?(:partition_pruning_dry_run)
+              if Feature.enabled?(:partition_pruning, default_enabled: :yaml)
                 partitions_to_detach = extra_partitions(model)
                 detach(partitions_to_detach) unless partitions_to_detach.empty?
               end
             end
           rescue StandardError => e
-            Gitlab::AppLogger.error("Failed to create / detach partition(s) for #{model.table_name}: #{e.class}: #{e.message}")
+            Gitlab::AppLogger.error(message: "Failed to create / detach partition(s)",
+                                    table_name: model.table_name,
+                                    exception_class: e.class,
+                                    exception_message: e.message)
           end
         end
 
@@ -54,7 +60,6 @@ module Gitlab
         end
 
         def extra_partitions(model)
-          return [] unless Feature.enabled?(:partition_pruning_dry_run)
           return [] unless connection.table_exists?(model.table_name)
 
           model.partitioning_strategy.extra_partitions
@@ -74,7 +79,9 @@ module Gitlab
               partitions.each do |partition|
                 connection.execute partition.to_sql
 
-                Gitlab::AppLogger.info("Created partition #{partition.partition_name} for table #{partition.table}")
+                Gitlab::AppLogger.info(message: "Created partition",
+                                       partition_name: partition.partition_name,
+                                       table_name: partition.table)
               end
             end
           end
@@ -89,7 +96,24 @@ module Gitlab
         end
 
         def detach_one_partition(partition)
-          Gitlab::AppLogger.info("Planning to detach #{partition.partition_name} for table #{partition.table}")
+          assert_partition_detachable!(partition)
+
+          connection.execute partition.to_detach_sql
+
+          Postgresql::DetachedPartition.create!(table_name: partition.partition_name,
+                                                drop_after: RETAIN_DETACHED_PARTITIONS_FOR.from_now)
+
+          Gitlab::AppLogger.info(message: "Detached Partition",
+                                 partition_name: partition.partition_name,
+                                 table_name: partition.table)
+        end
+
+        def assert_partition_detachable!(partition)
+          parent_table_identifier = "#{connection.current_schema}.#{partition.table}"
+
+          if (example_fk = PostgresForeignKey.by_referenced_table_identifier(parent_table_identifier).first)
+            raise UnsafeToDetachPartitionError, "Cannot detach #{partition.partition_name}, it would block while checking foreign key #{example_fk.name} on #{example_fk.constrained_table_identifier}"
+          end
         end
 
         def with_lock_retries(&block)
diff --git a/lib/gitlab/database/partitioning/partition_monitoring.rb b/lib/gitlab/database/partitioning/partition_monitoring.rb
index ad122fd47fe..6963ecd2cc1 100644
--- a/lib/gitlab/database/partitioning/partition_monitoring.rb
+++ b/lib/gitlab/database/partitioning/partition_monitoring.rb
@@ -16,6 +16,7 @@ module Gitlab
 
             gauge_present.set({ table: model.table_name }, strategy.current_partitions.size)
             gauge_missing.set({ table: model.table_name }, strategy.missing_partitions.size)
+            gauge_extra.set({ table: model.table_name }, strategy.extra_partitions.size)
           end
         end
 
@@ -28,6 +29,10 @@ module Gitlab
         def gauge_missing
           @gauge_missing ||= Gitlab::Metrics.gauge(:db_partitions_missing, 'Number of database partitions currently expected, but not present')
         end
+
+        def gauge_extra
+          @gauge_extra ||= Gitlab::Metrics.gauge(:db_partitions_extra, 'Number of database partitions currently attached to tables, but outside of their retention window and scheduled to be dropped')
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/partitioning/time_partition.rb b/lib/gitlab/database/partitioning/time_partition.rb
index 7dca60c0854..1221f042530 100644
--- a/lib/gitlab/database/partitioning/time_partition.rb
+++ b/lib/gitlab/database/partitioning/time_partition.rb
@@ -47,6 +47,13 @@ module Gitlab
           SQL
         end
 
+        def to_detach_sql
+          <<~SQL
+            ALTER TABLE #{conn.quote_table_name(table)}
+            DETACH PARTITION #{fully_qualified_partition}
+          SQL
+        end
+
         def ==(other)
           table == other.table && partition_name == other.partition_name && from == other.from && to == other.to
         end