Add latest changes from gitlab-org/gitlab@master

9 files changed, 299 insertions, 6 deletions

diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 2adcdb0598a..197bd420295 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -504,12 +504,12 @@ module API
 
     def render_validation_error!(model, status = 400)
       if model.errors.any?
-        render_api_error!(model_error_messages(model) || '400 Bad Request', status)
+        render_api_error!(model_errors(model).messages || '400 Bad Request', status)
       end
     end
 
-    def model_error_messages(model)
-      model.errors.messages
+    def model_errors(model)
+      model.errors
     end
 
     def render_api_error_with_reason!(status, message, reason)
diff --git a/lib/api/helpers/users_helpers.rb b/lib/api/helpers/users_helpers.rb
index e80b89488a2..f97071d9a97 100644
--- a/lib/api/helpers/users_helpers.rb
+++ b/lib/api/helpers/users_helpers.rb
@@ -12,10 +12,14 @@ module API
       params :optional_index_params_ee do
       end
 
-      def model_error_messages(model)
-        super.tap do |error_messages|
+      def model_errors(model)
+        super.tap do |errors|
           # Remapping errors from nested associations.
-          error_messages[:bio] = error_messages.delete(:"user_detail.bio") if error_messages.has_key?(:"user_detail.bio")
+          next unless errors.has_key?(:"user_detail.bio")
+
+          errors.delete(:"user_detail.bio").each do |message|
+            errors.add(:bio, message)
+          end
         end
       end
 
diff --git a/lib/feature.rb b/lib/feature.rb
index d012639d489..892d32c9b73 100644
--- a/lib/feature.rb
+++ b/lib/feature.rb
@@ -206,6 +206,7 @@ module Feature
     # to register Flipper groups.
     # See https://docs.gitlab.com/ee/development/feature_flags/index.html
     def register_feature_groups
+      Flipper.register(:gitlab_team_members) { |actor| FeatureGroups::GitlabTeamMembers.enabled?(actor.thing) }
     end
 
     def register_definitions
diff --git a/lib/feature_groups/gitlab_team_members.rb b/lib/feature_groups/gitlab_team_members.rb
new file mode 100644
index 00000000000..7f4c597fddd
--- /dev/null
+++ b/lib/feature_groups/gitlab_team_members.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module FeatureGroups
+  class GitlabTeamMembers
+    GITLAB_COM_GROUP_ID = 6543
+
+    class << self
+      def enabled?(thing)
+        return false unless Gitlab.com?
+
+        team_member?(thing)
+      end
+
+      private
+
+      def team_member?(thing)
+        thing.is_a?(::User) && gitlab_com_member_ids.include?(thing.id)
+      end
+
+      def gitlab_com
+        @gitlab_com ||= ::Group.find(GITLAB_COM_GROUP_ID)
+      end
+
+      def gitlab_com_member_ids
+        Rails.cache.fetch("gitlab_team_members", expires_in: 1.hour) do
+          gitlab_com.members.pluck_user_ids.to_set
+        end
+      end
+    end
+  end
+end
diff --git a/lib/generators/gitlab/partitioning/foreign_keys_generator.rb b/lib/generators/gitlab/partitioning/foreign_keys_generator.rb
new file mode 100644
index 00000000000..e013188c753
--- /dev/null
+++ b/lib/generators/gitlab/partitioning/foreign_keys_generator.rb
@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+require 'rails/generators/active_record/migration'
+
+module Gitlab
+  module Partitioning
+    class ForeignKeysGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      desc 'This generator creates the migrations needed for updating the foreign keys when partitioning the tables'
+
+      source_root File.expand_path('templates', __dir__)
+
+      class_option :target, type: :string, required: true, desc: 'Target table name'
+      class_option :source, type: :string, required: true, desc: 'Source table name'
+      class_option :partitioning_column, type: :string, default: :partition_id,
+        desc: 'The column that is used for partitioning'
+      class_option :database, type: :string, default: :ci,
+        desc: 'Database name connection'
+
+      def create_fk_index_migration
+        migration_template(
+          '../templates/foreign_key_index.rb.template',
+         fk_index_file_name)
+      end
+
+      def create_fk_definition_migration
+        migration_template(
+          '../templates/foreign_key_definition.rb.template',
+          fk_definition_file_name)
+      end
+
+      def create_fk_validation_migration
+        migration_template(
+          '../templates/foreign_key_validation.rb.template',
+          fk_validation_file_name)
+      end
+
+      def remove_old_fk_migration
+        migration_template(
+          '../templates/foreign_key_removal.rb.template',
+          fk_removal_file_name)
+      end
+
+      private
+
+      def fk_index_file_name
+        post_migration_file_path(
+          "add_fk_index_to_#{source_table_name}_on_#{partitioning_column}_and_#{foreign_key_column}.rb")
+      end
+
+      def fk_definition_file_name
+        post_migration_file_path(
+          "add_fk_to_#{source_table_name}_on_#{partitioning_column}_and_#{foreign_key_column}.rb")
+      end
+
+      def fk_validation_file_name
+        post_migration_file_path(
+          "validate_fk_on_#{source_table_name}_#{partitioning_column}_and_#{foreign_key_column}.rb")
+      end
+
+      def fk_removal_file_name
+        post_migration_file_path(
+          "remove_fk_to_#{target_table_name}_#{source_table_name}_on_#{foreign_key_column}.rb")
+      end
+
+      def post_migration_file_path(name)
+        File.join(db_migrate_path, name)
+      end
+
+      def db_migrate_path
+        super.sub('migrate', 'post_migrate')
+      end
+
+      def source_table_name
+        options[:source]
+      end
+
+      def target_table_name
+        options[:target]
+      end
+
+      def partitioning_column
+        options[:partitioning_column]
+      end
+
+      def foreign_keys_candidates
+        connection
+          .foreign_keys(source_table_name)
+          .select { |fk| fk.to_table == target_table_name }
+          .reject { |fk| fk.name.end_with?('_p') }
+      end
+
+      def fk_candidate
+        @fk_candidate ||= select_foreign_key
+      end
+
+      def foreign_key_name
+        fk_candidate.name
+      end
+
+      def partitioned_foreign_key_name
+        "#{foreign_key_name}_p"
+      end
+
+      def foreign_key_column
+        fk_candidate.column
+      end
+
+      def fk_on_delete_option
+        fk_candidate.on_delete
+      end
+
+      def fk_target_column
+        fk_candidate.primary_key
+      end
+
+      def connection
+        Gitlab::Database
+          .database_base_models
+          .fetch(options[:database])
+          .connection
+      end
+
+      def select_foreign_key
+        case foreign_keys_candidates.size
+        when 0
+          raise Thor::InvocationError, "No FK found between #{source_table_name} and #{target_table_name}"
+        when 1
+          foreign_keys_candidates.first
+        else
+          select_fk_from_user_input
+        end
+      end
+
+      def select_fk_from_user_input
+        options = (0...foreign_keys_candidates.size).to_a.map(&:to_s)
+
+        say "There are multiple FKs between #{source_table_name} and #{target_table_name}:"
+        foreign_keys_candidates.each.with_index do |fk, index|
+          say "\t#{index} : #{fk}"
+        end
+
+        input = ask("Please select one:", limited_to: options, default: '0')
+
+        foreign_keys_candidates.fetch(input.to_i)
+      end
+    end
+  end
+end
diff --git a/lib/generators/gitlab/partitioning/templates/foreign_key_definition.rb.template b/lib/generators/gitlab/partitioning/templates/foreign_key_definition.rb.template
new file mode 100644
index 00000000000..f2f9acea923
--- /dev/null
+++ b/lib/generators/gitlab/partitioning/templates/foreign_key_definition.rb.template
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class <%= migration_class_name %> < Gitlab::Database::Migration[<%= Gitlab::Database::Migration.current_version %>]
+  disable_ddl_transaction!
+
+  SOURCE_TABLE_NAME = :<%= source_table_name %>
+  TARGET_TABLE_NAME = :<%= target_table_name %>
+  COLUMN = :<%= foreign_key_column %>
+  TARGET_COLUMN = :<%= fk_target_column %>
+  FK_NAME = :<%= partitioned_foreign_key_name %>
+  PARTITION_COLUMN = :<%= partitioning_column %>
+
+  def up
+    add_concurrent_foreign_key(
+      SOURCE_TABLE_NAME,
+      TARGET_TABLE_NAME,
+      column: [PARTITION_COLUMN, COLUMN],
+      target_column: [PARTITION_COLUMN, TARGET_COLUMN],
+      validate: false,
+      reverse_lock_order: true,
+      on_update: :cascade,
+      on_delete: :<%= fk_on_delete_option %>,
+      name: FK_NAME
+    )
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key_if_exists(
+        SOURCE_TABLE_NAME,
+        TARGET_TABLE_NAME,
+        name: FK_NAME,
+        reverse_lock_order: true
+      )
+    end
+  end
+end
diff --git a/lib/generators/gitlab/partitioning/templates/foreign_key_index.rb.template b/lib/generators/gitlab/partitioning/templates/foreign_key_index.rb.template
new file mode 100644
index 00000000000..4896d931333
--- /dev/null
+++ b/lib/generators/gitlab/partitioning/templates/foreign_key_index.rb.template
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class <%= migration_class_name %> < Gitlab::Database::Migration[<%= Gitlab::Database::Migration.current_version %>]
+  disable_ddl_transaction!
+
+  INDEX_NAME = :index_<%= source_table_name -%>_on_<%= partitioning_column -%>_<%= foreign_key_column %>
+  TABLE_NAME = :<%= source_table_name %>
+  COLUMNS = [:<%= partitioning_column -%>, :<%= foreign_key_column -%>]
+
+  def up
+    add_concurrent_index(TABLE_NAME, COLUMNS, name: INDEX_NAME)
+  end
+
+  def down
+    remove_concurrent_index_by_name(TABLE_NAME, INDEX_NAME)
+  end
+end
diff --git a/lib/generators/gitlab/partitioning/templates/foreign_key_removal.rb.template b/lib/generators/gitlab/partitioning/templates/foreign_key_removal.rb.template
new file mode 100644
index 00000000000..b4e881074ad
--- /dev/null
+++ b/lib/generators/gitlab/partitioning/templates/foreign_key_removal.rb.template
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class <%= migration_class_name %> < Gitlab::Database::Migration[<%= Gitlab::Database::Migration.current_version %>]
+  disable_ddl_transaction!
+
+  SOURCE_TABLE_NAME = :<%= source_table_name %>
+  TARGET_TABLE_NAME = :<%= target_table_name %>
+  COLUMN = :<%= foreign_key_column %>
+  TARGET_COLUMN = :<%= fk_target_column %>
+  FK_NAME = :<%= foreign_key_name %>
+
+  def up
+    with_lock_retries do
+      remove_foreign_key_if_exists(
+        SOURCE_TABLE_NAME,
+        TARGET_TABLE_NAME,
+        name: FK_NAME,
+        reverse_lock_order: true
+      )
+    end
+  end
+
+  def down
+    add_concurrent_foreign_key(
+      SOURCE_TABLE_NAME,
+      TARGET_TABLE_NAME,
+      column: COLUMN,
+      target_column: TARGET_COLUMN,
+      validate: true,
+      reverse_lock_order: true,
+      on_delete: :<%= fk_on_delete_option %>,
+      name: FK_NAME
+    )
+  end
+end
diff --git a/lib/generators/gitlab/partitioning/templates/foreign_key_validation.rb.template b/lib/generators/gitlab/partitioning/templates/foreign_key_validation.rb.template
new file mode 100644
index 00000000000..bad7d17a51b
--- /dev/null
+++ b/lib/generators/gitlab/partitioning/templates/foreign_key_validation.rb.template
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class <%= migration_class_name %> < Gitlab::Database::Migration[<%= Gitlab::Database::Migration.current_version %>]
+  disable_ddl_transaction!
+
+  TABLE_NAME = :<%= source_table_name %>
+  FK_NAME = :<%= partitioned_foreign_key_name %>
+  COLUMNS = [:<%= partitioning_column -%>, :<%= foreign_key_column -%>]
+
+  def up
+    validate_foreign_key(TABLE_NAME, COLUMNS, name: FK_NAME)
+  end
+
+  def down
+    # no-op
+  end
+end