diff options
Diffstat (limited to 'lib/gitlab/database/partitioning')
| -rw-r--r-- | lib/gitlab/database/partitioning/monthly_strategy.rb | 96 | ||||
| -rw-r--r-- | lib/gitlab/database/partitioning/partition_creator.rb | 87 | ||||
| -rw-r--r-- | lib/gitlab/database/partitioning/time_partition.rb | 84 |
3 files changed, 267 insertions, 0 deletions
diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb new file mode 100644 index 00000000000..ecc05d9654a --- /dev/null +++ b/lib/gitlab/database/partitioning/monthly_strategy.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class MonthlyStrategy + attr_reader :model, :partitioning_key + + # We create this many partitions in the future + HEADROOM = 6.months + + delegate :table_name, to: :model + + def initialize(model, partitioning_key) + @model = model + @partitioning_key = partitioning_key + end + + def current_partitions + result = connection.select_all(<<~SQL) + select + pg_class.relname, + parent_class.relname as base_table, + pg_get_expr(pg_class.relpartbound, inhrelid) as condition + from pg_class + inner join pg_inherits i on pg_class.oid = inhrelid + inner join pg_class parent_class on parent_class.oid = inhparent + inner join pg_namespace ON pg_namespace.oid = pg_class.relnamespace + where pg_namespace.nspname = #{connection.quote(Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA)} + and parent_class.relname = #{connection.quote(table_name)} + and pg_class.relispartition + order by pg_class.relname + SQL + + result.map do |record| + TimePartition.from_sql(table_name, record['relname'], record['condition']) + end + end + + # Check the currently existing partitions and determine which ones are missing + def missing_partitions + desired_partitions - current_partitions + end + + private + + def desired_partitions + [].tap do |parts| + min_date, max_date = relevant_range + + parts << partition_for(upper_bound: min_date) + + while min_date < max_date + next_date = min_date.next_month + + parts << partition_for(lower_bound: min_date, upper_bound: next_date) + + min_date = next_date + end + end + end + + # This determines the relevant time range for which we expect to have data + # (and therefore need to create partitions for). + # + # Note: We typically expect the first partition to be half-unbounded, i.e. + # to start from MINVALUE to a specific date `x`. The range returned + # does not include the range of the first, half-unbounded partition. + def relevant_range + if first_partition = current_partitions.min + # Case 1: First partition starts with MINVALUE, i.e. from is nil -> start with first real partition + # Case 2: Rather unexpectedly, first partition does not start with MINVALUE, i.e. from is not nil + # In this case, use first partition beginning as a start + min_date = first_partition.from || first_partition.to + end + + # In case we don't have a partition yet + min_date ||= Date.today + min_date = min_date.beginning_of_month + + max_date = Date.today.end_of_month + HEADROOM + + [min_date, max_date] + end + + def partition_for(lower_bound: nil, upper_bound:) + TimePartition.new(table_name, lower_bound, upper_bound) + end + + def connection + ActiveRecord::Base.connection + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/partition_creator.rb b/lib/gitlab/database/partitioning/partition_creator.rb new file mode 100644 index 00000000000..348dd1ba660 --- /dev/null +++ b/lib/gitlab/database/partitioning/partition_creator.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class PartitionCreator + def self.register(model) + raise ArgumentError, "Only models with a #partitioning_strategy can be registered." unless model.respond_to?(:partitioning_strategy) + + models << model + end + + def self.models + @models ||= Set.new + end + + LEASE_TIMEOUT = 1.minute + LEASE_KEY = 'database_partition_creation_%s' + + attr_reader :models + + def initialize(models = self.class.models) + @models = models + end + + def create_partitions + return unless Feature.enabled?(:postgres_dynamic_partition_creation, default_enabled: true) + + models.each do |model| + # Double-checking before getting the lease: + # The prevailing situation is no missing partitions + next if missing_partitions(model).empty? + + only_with_exclusive_lease(model) do + partitions_to_create = missing_partitions(model) + + next if partitions_to_create.empty? + + create(model, partitions_to_create) + end + rescue => e + Gitlab::AppLogger.error("Failed to create partition(s) for #{model.table_name}: #{e.class}: #{e.message}") + end + end + + private + + def missing_partitions(model) + return [] unless connection.table_exists?(model.table_name) + + model.partitioning_strategy.missing_partitions + end + + def only_with_exclusive_lease(model) + lease = Gitlab::ExclusiveLease.new(LEASE_KEY % model.table_name, timeout: LEASE_TIMEOUT) + + yield if lease.try_obtain + ensure + lease&.cancel + end + + def create(model, partitions) + connection.transaction do + with_lock_retries do + partitions.each do |partition| + connection.execute partition.to_sql + + Gitlab::AppLogger.info("Created partition #{partition.partition_name} for table #{partition.table}") + end + end + end + end + + def with_lock_retries(&block) + Gitlab::Database::WithLockRetries.new({ + klass: self.class, + logger: Gitlab::AppLogger + }).run(&block) + end + + def connection + ActiveRecord::Base.connection + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/time_partition.rb b/lib/gitlab/database/partitioning/time_partition.rb new file mode 100644 index 00000000000..7dca60c0854 --- /dev/null +++ b/lib/gitlab/database/partitioning/time_partition.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class TimePartition + include Comparable + + def self.from_sql(table, partition_name, definition) + matches = definition.match(/FOR VALUES FROM \('?(?<from>.+)'?\) TO \('?(?<to>.+)'?\)/) + + raise ArgumentError, "Unknown partition definition: #{definition}" unless matches + + raise NotImplementedError, "Open-end time partitions with MAXVALUE are not supported yet" if matches[:to] == 'MAXVALUE' + + from = matches[:from] == 'MINVALUE' ? nil : matches[:from] + to = matches[:to] + + new(table, from, to, partition_name: partition_name) + end + + attr_reader :table, :from, :to + + def initialize(table, from, to, partition_name: nil) + @table = table.to_s + @from = date_or_nil(from) + @to = date_or_nil(to) + @partition_name = partition_name + end + + def partition_name + return @partition_name if @partition_name + + suffix = from&.strftime('%Y%m') || '000000' + + "#{table}_#{suffix}" + end + + def to_sql + from_sql = from ? conn.quote(from.strftime('%Y-%m-%d')) : 'MINVALUE' + to_sql = conn.quote(to.strftime('%Y-%m-%d')) + + <<~SQL + CREATE TABLE IF NOT EXISTS #{fully_qualified_partition} + PARTITION OF #{conn.quote_table_name(table)} + FOR VALUES FROM (#{from_sql}) TO (#{to_sql}) + SQL + end + + def ==(other) + table == other.table && partition_name == other.partition_name && from == other.from && to == other.to + end + alias_method :eql?, :== + + def hash + [table, partition_name, from, to].hash + end + + def <=>(other) + return if table != other.table + + partition_name <=> other.partition_name + end + + private + + def date_or_nil(obj) + return unless obj + return obj if obj.is_a?(Date) + + Date.parse(obj) + end + + def fully_qualified_partition + "%s.%s" % [conn.quote_table_name(Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA), conn.quote_table_name(partition_name)] + end + + def conn + @conn ||= ActiveRecord::Base.connection + end + end + end + end +end |