Add latest changes from gitlab-org/gitlab@master

10 files changed, 124 insertions, 51 deletions

diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb
index 73d27399680..09c75a2b3f1 100644
--- a/lib/gitlab/ci/variables/collection.rb
+++ b/lib/gitlab/ci/variables/collection.rb
@@ -90,7 +90,7 @@ module Gitlab
         end
 
         def sort_and_expand_all(project, keep_undefined: false)
-          return self if Feature.disabled?(:variable_inside_variable, project)
+          return self if Feature.disabled?(:variable_inside_variable, project, default_enabled: :yaml)
 
           sorted = Sort.new(self)
           return self.class.new(self, sorted.errors) unless sorted.valid?
diff --git a/lib/gitlab/database/partitioning.rb b/lib/gitlab/database/partitioning.rb
new file mode 100644
index 00000000000..bbde2063c41
--- /dev/null
+++ b/lib/gitlab/database/partitioning.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module Partitioning
+      def self.register_models(models)
+        registered_models.merge(models)
+      end
+
+      def self.registered_models
+        @registered_models ||= Set.new
+      end
+
+      def self.sync_partitions(models_to_sync = registered_models)
+        MultiDatabasePartitionManager.new(models_to_sync).sync_partitions
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/partitioning/multi_database_partition_manager.rb b/lib/gitlab/database/partitioning/multi_database_partition_manager.rb
new file mode 100644
index 00000000000..5a93e3fb1fb
--- /dev/null
+++ b/lib/gitlab/database/partitioning/multi_database_partition_manager.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module Partitioning
+      class MultiDatabasePartitionManager
+        def initialize(models)
+          @models = models
+        end
+
+        def sync_partitions
+          Gitlab::AppLogger.info(message: "Syncing dynamic postgres partitions")
+
+          models.each do |model|
+            Gitlab::Database::SharedModel.using_connection(model.connection) do
+              Gitlab::AppLogger.debug(message: "Switched database connection",
+                                      connection_name: connection_name,
+                                      table_name: model.table_name)
+
+              PartitionManager.new(model).sync_partitions
+            end
+          end
+
+          Gitlab::AppLogger.info(message: "Finished sync of dynamic postgres partitions")
+        end
+
+        private
+
+        attr_reader :models
+
+        def connection_name
+          Gitlab::Database::SharedModel.connection.pool.db_config.name
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb
index 2ba5b35d6b9..8742c0ff166 100644
--- a/lib/gitlab/database/partitioning/partition_manager.rb
+++ b/lib/gitlab/database/partitioning/partition_manager.rb
@@ -6,60 +6,49 @@ module Gitlab
       class PartitionManager
         UnsafeToDetachPartitionError = Class.new(StandardError)
 
-        def self.register(model)
-          raise ArgumentError, "Only models with a #partitioning_strategy can be registered." unless model.respond_to?(:partitioning_strategy)
-
-          models << model
-        end
-
-        def self.models
-          @models ||= Set.new
-        end
-
         LEASE_TIMEOUT = 1.minute
         MANAGEMENT_LEASE_KEY = 'database_partition_management_%s'
         RETAIN_DETACHED_PARTITIONS_FOR = 1.week
 
-        attr_reader :models
-
-        def initialize(models = self.class.models)
-          @models = models
+        def initialize(model)
+          @model = model
         end
 
         def sync_partitions
-          Gitlab::AppLogger.info("Checking state of dynamic postgres partitions")
+          Gitlab::AppLogger.info(message: "Checking state of dynamic postgres partitions", table_name: model.table_name)
 
-          models.each do |model|
-            # Double-checking before getting the lease:
-            # The prevailing situation is no missing partitions and no extra partitions
-            next if missing_partitions(model).empty? && extra_partitions(model).empty?
+          # Double-checking before getting the lease:
+          # The prevailing situation is no missing partitions and no extra partitions
+          return if missing_partitions.empty? && extra_partitions.empty?
 
-            only_with_exclusive_lease(model, lease_key: MANAGEMENT_LEASE_KEY) do
-              partitions_to_create = missing_partitions(model)
-              create(partitions_to_create) unless partitions_to_create.empty?
+          only_with_exclusive_lease(model, lease_key: MANAGEMENT_LEASE_KEY) do
+            partitions_to_create = missing_partitions
+            create(partitions_to_create) unless partitions_to_create.empty?
 
-              if Feature.enabled?(:partition_pruning, default_enabled: :yaml)
-                partitions_to_detach = extra_partitions(model)
-                detach(partitions_to_detach) unless partitions_to_detach.empty?
-              end
+            if Feature.enabled?(:partition_pruning, default_enabled: :yaml)
+              partitions_to_detach = extra_partitions
+              detach(partitions_to_detach) unless partitions_to_detach.empty?
             end
-          rescue StandardError => e
-            Gitlab::AppLogger.error(message: "Failed to create / detach partition(s)",
-                                    table_name: model.table_name,
-                                    exception_class: e.class,
-                                    exception_message: e.message)
           end
+        rescue StandardError => e
+          Gitlab::AppLogger.error(message: "Failed to create / detach partition(s)",
+                                  table_name: model.table_name,
+                                  exception_class: e.class,
+                                  exception_message: e.message)
         end
 
         private
 
-        def missing_partitions(model)
+        attr_reader :model
+        delegate :connection, to: :model
+
+        def missing_partitions
           return [] unless connection.table_exists?(model.table_name)
 
           model.partitioning_strategy.missing_partitions
         end
 
-        def extra_partitions(model)
+        def extra_partitions
           return [] unless connection.table_exists?(model.table_name)
 
           model.partitioning_strategy.extra_partitions
@@ -121,13 +110,10 @@ module Gitlab
         def with_lock_retries(&block)
           Gitlab::Database::WithLockRetries.new(
             klass: self.class,
-            logger: Gitlab::AppLogger
+            logger: Gitlab::AppLogger,
+            connection: connection
           ).run(&block)
         end
-
-        def connection
-          ActiveRecord::Base.connection
-        end
       end
     end
   end
diff --git a/lib/gitlab/database/partitioning/partition_monitoring.rb b/lib/gitlab/database/partitioning/partition_monitoring.rb
index 6963ecd2cc1..e5b561fc447 100644
--- a/lib/gitlab/database/partitioning/partition_monitoring.rb
+++ b/lib/gitlab/database/partitioning/partition_monitoring.rb
@@ -6,7 +6,7 @@ module Gitlab
       class PartitionMonitoring
         attr_reader :models
 
-        def initialize(models = PartitionManager.models)
+        def initialize(models = Gitlab::Database::Partitioning.registered_models)
           @models = models
         end
 
diff --git a/lib/gitlab/database/postgres_foreign_key.rb b/lib/gitlab/database/postgres_foreign_key.rb
index 94f74724295..72640f8785d 100644
--- a/lib/gitlab/database/postgres_foreign_key.rb
+++ b/lib/gitlab/database/postgres_foreign_key.rb
@@ -2,7 +2,7 @@
 
 module Gitlab
   module Database
-    class PostgresForeignKey < ApplicationRecord
+    class PostgresForeignKey < SharedModel
       self.primary_key = :oid
 
       scope :by_referenced_table_identifier, ->(identifier) do
diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb
index 7da60d8375d..eb080904f73 100644
--- a/lib/gitlab/database/postgres_partition.rb
+++ b/lib/gitlab/database/postgres_partition.rb
@@ -2,7 +2,7 @@
 
 module Gitlab
   module Database
-    class PostgresPartition < ActiveRecord::Base
+    class PostgresPartition < SharedModel
       self.primary_key = :identifier
 
       belongs_to :postgres_partitioned_table, foreign_key: 'parent_identifier', primary_key: 'identifier'
diff --git a/lib/gitlab/database/postgres_partitioned_table.rb b/lib/gitlab/database/postgres_partitioned_table.rb
index 5d2eaa22ee4..3bd342f940f 100644
--- a/lib/gitlab/database/postgres_partitioned_table.rb
+++ b/lib/gitlab/database/postgres_partitioned_table.rb
@@ -2,7 +2,7 @@
 
 module Gitlab
   module Database
-    class PostgresPartitionedTable < ActiveRecord::Base
+    class PostgresPartitionedTable < SharedModel
       DYNAMIC_PARTITION_STRATEGIES = %w[range list].freeze
 
       self.primary_key = :identifier
diff --git a/lib/gitlab/database/shared_model.rb b/lib/gitlab/database/shared_model.rb
new file mode 100644
index 00000000000..8f256758961
--- /dev/null
+++ b/lib/gitlab/database/shared_model.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    class SharedModel < ActiveRecord::Base
+      self.abstract_class = true
+
+      class << self
+        def using_connection(connection)
+          raise 'cannot nest connection overrides for shared models' unless overriding_connection.nil?
+
+          self.overriding_connection = connection
+
+          yield
+        ensure
+          self.overriding_connection = nil
+        end
+
+        def connection
+          if connection = self.overriding_connection
+            connection
+          else
+            super
+          end
+        end
+
+        private
+
+        def overriding_connection
+          Thread.current[:overriding_connection]
+        end
+
+        def overriding_connection=(connection)
+          Thread.current[:overriding_connection] = connection
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/devise_failure.rb b/lib/gitlab/devise_failure.rb
index eb475307f27..111ea697ec2 100644
--- a/lib/gitlab/devise_failure.rb
+++ b/lib/gitlab/devise_failure.rb
@@ -2,18 +2,10 @@
 
 module Gitlab
   class DeviseFailure < Devise::FailureApp
-    include ::SessionsHelper
-
     # If the request format is not known, send a redirect instead of a 401
     # response, since this is the outcome we're most likely to want
     def http_auth?
       request_format && super
     end
-
-    def respond
-      limit_session_time
-
-      super
-    end
   end
 end