Add latest changes from gitlab-org/gitlab@master

3 files changed, 163 insertions, 0 deletions

diff --git a/lib/gitlab/database/async_indexes/index_creator.rb b/lib/gitlab/database/async_indexes/index_creator.rb
new file mode 100644
index 00000000000..82f2a921f10
--- /dev/null
+++ b/lib/gitlab/database/async_indexes/index_creator.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module AsyncIndexes
+      class IndexCreator
+        include ExclusiveLeaseGuard
+
+        TIMEOUT_PER_ACTION = 1.day
+        STATEMENT_TIMEOUT = 9.hours
+
+        def initialize(async_index)
+          @async_index = async_index
+        end
+
+        def perform
+          try_obtain_lease do
+            if index_exists?
+              log_index_info('Skipping index creation as the index exists')
+            else
+              log_index_info('Creating async index')
+
+              set_statement_timeout do
+                connection.execute(async_index.definition)
+              end
+            end
+
+            async_index.destroy
+          end
+        end
+
+        private
+
+        attr_reader :async_index
+
+        def index_exists?
+          connection.indexes(async_index.table_name).any? { |index| index.name == async_index.name }
+        end
+
+        def connection
+          @connection ||= ApplicationRecord.connection
+        end
+
+        def lease_timeout
+          TIMEOUT_PER_ACTION
+        end
+
+        def set_statement_timeout
+          connection.execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT)
+          yield
+        ensure
+          connection.execute('RESET statement_timeout')
+        end
+
+        def log_index_info(message)
+          Gitlab::AppLogger.info(message: message, table_name: async_index.table_name, index_name: async_index.name)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/async_indexes/migration_helpers.rb b/lib/gitlab/database/async_indexes/migration_helpers.rb
new file mode 100644
index 00000000000..dff6376270a
--- /dev/null
+++ b/lib/gitlab/database/async_indexes/migration_helpers.rb
@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module AsyncIndexes
+      module MigrationHelpers
+        def unprepare_async_index(table_name, column_name, **options)
+          return unless async_index_creation_available?
+
+          index_name = options[:name] || index_name(table_name, column_name)
+
+          raise 'Specifying index name is mandatory - specify name: argument' unless index_name
+
+          unprepare_async_index_by_name(table_name, index_name)
+        end
+
+        def unprepare_async_index_by_name(table_name, index_name, **options)
+          return unless async_index_creation_available?
+
+          PostgresAsyncIndex.find_by(name: index_name).try do |async_index|
+            async_index.destroy
+          end
+        end
+
+        # Prepares an index for asynchronous creation.
+        #
+        # Stores the index information in the postgres_async_indexes table to be created later. The
+        # index will be always be created CONCURRENTLY, so that option does not need to be given.
+        # If an existing asynchronous definition exists with the same name, the existing entry will be
+        # updated with the new definition.
+        #
+        # If the requested index has already been created, it is not stored in the table for
+        # asynchronous creation.
+        def prepare_async_index(table_name, column_name, **options)
+          return unless async_index_creation_available?
+
+          index_name = options[:name] || index_name(table_name, column_name)
+
+          raise 'Specifying index name is mandatory - specify name: argument' unless index_name
+
+          options = options.merge({ algorithm: :concurrently })
+
+          if index_exists?(table_name, column_name, **options)
+            Gitlab::AppLogger.warn(
+              message: 'Index not prepared because it already exists',
+              table_name: table_name,
+              index_name: index_name)
+
+            return
+          end
+
+          index, algorithm, if_not_exists = add_index_options(table_name, column_name, **options)
+
+          create_index = ActiveRecord::ConnectionAdapters::CreateIndexDefinition.new(index, algorithm, if_not_exists)
+          schema_creation = ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaCreation.new(ApplicationRecord.connection)
+          definition = schema_creation.accept(create_index)
+
+          async_index = PostgresAsyncIndex.safe_find_or_create_by!(name: index_name) do |rec|
+            rec.table_name = table_name
+            rec.definition = definition
+          end
+
+          Gitlab::AppLogger.info(
+            message: 'Prepared index for async creation',
+            table_name: async_index.table_name,
+            index_name: async_index.name)
+
+          async_index
+        end
+
+        private
+
+        def async_index_creation_available?
+          ApplicationRecord.connection.table_exists?(:postgres_async_indexes) &&
+            Feature.enabled?(:database_async_index_creation, type: :ops)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/async_indexes/postgres_async_index.rb b/lib/gitlab/database/async_indexes/postgres_async_index.rb
new file mode 100644
index 00000000000..236459e6216
--- /dev/null
+++ b/lib/gitlab/database/async_indexes/postgres_async_index.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module AsyncIndexes
+      class PostgresAsyncIndex < ApplicationRecord
+        self.table_name = 'postgres_async_indexes'
+
+        MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH
+        MAX_DEFINITION_LENGTH = 2048
+
+        validates :name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH }
+        validates :table_name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH }
+        validates :definition, presence: true, length: { maximum: MAX_DEFINITION_LENGTH }
+
+        def to_s
+          definition
+        end
+      end
+    end
+  end
+end