Add latest changes from gitlab-org/gitlab@master

18 files changed, 257 insertions, 37 deletions

diff --git a/lib/gitlab/branch_push_merge_commit_analyzer.rb b/lib/gitlab/branch_push_merge_commit_analyzer.rb
index a8f601f2451..ddf2086363c 100644
--- a/lib/gitlab/branch_push_merge_commit_analyzer.rb
+++ b/lib/gitlab/branch_push_merge_commit_analyzer.rb
@@ -114,7 +114,7 @@ module Gitlab
     # If child commit is a direct ancestor, its first parent is also a direct ancestor.
     # We assume direct ancestors matches the trail of the target branch over time,
     # This assumption is correct most of the time, especially for gitlab managed merges,
-    # but there are exception cases which can't be solved (https://stackoverflow.com/a/49754723/474597)
+    # but there are exception cases which can't be solved.
     def mark_all_direct_ancestors(commit)
       loop do
         commit = get_commit(commit.parent_ids.first)
diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml
index dd88953b9a4..841f17767eb 100644
--- a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml
@@ -21,7 +21,7 @@
 #
 # The deploy stage copies the exe and msi from build stage to a network drive
 # You need to have the network drive mapped as Local System user for gitlab-runner service to see it
-# The best way to persist the mapping is via a scheduled task (see: https://stackoverflow.com/a/7867064/1288473),
+# The best way to persist the mapping is via a scheduled task
 # running the following batch command: net use P: \\x.x.x.x\Projects /u:your_user your_pass /persistent:yes
 
 # place project specific paths in variables to make the rest of the script more generic
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
index fdc598c025a..2d31049a0c9 100644
--- a/lib/gitlab/ci/trace/stream.rb
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -3,7 +3,6 @@
 module Gitlab
   module Ci
     class Trace
-      # This was inspired from: http://stackoverflow.com/a/10219411/1520132
       class Stream
         BUFFER_SIZE = 4096
         LIMIT_SIZE = 500.kilobytes
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index f3f0d0305d3..385ac40cf13 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -198,14 +198,30 @@ module Gitlab
       ::ActiveRecord::Base.configurations.configs_for(env_name: Rails.env).map(&:name)
     end
 
-    def self.db_config_name(ar_connection)
-      if ar_connection.respond_to?(:pool) &&
-          ar_connection.pool.respond_to?(:db_config) &&
-          ar_connection.pool.db_config.respond_to?(:name)
-        return ar_connection.pool.db_config.name
-      end
+    def self.db_config_for_connection(connection)
+      return unless connection
+
+      # The LB connection proxy does not have a direct db_config
+      # that can be referenced
+      return if connection.is_a?(::Gitlab::Database::LoadBalancing::ConnectionProxy)
+
+      # During application init we might receive `NullPool`
+      return unless connection.respond_to?(:pool) &&
+        connection.pool.respond_to?(:db_config)
+
+      connection.pool.db_config
+    end
 
-      'unknown'
+    # At the moment, the connection can only be retrieved by
+    # Gitlab::Database::LoadBalancer#read or #read_write or from the
+    # ActiveRecord directly. Therefore, if the load balancer doesn't
+    # recognize the connection, this method returns the primary role
+    # directly. In future, we may need to check for other sources.
+    # Expected returned names:
+    # main, main_replica, ci, ci_replica, unknown
+    def self.db_config_name(connection)
+      db_config = db_config_for_connection(connection)
+      db_config&.name || 'unknown'
     end
 
     def self.read_only?
diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb
index d4f168ba188..bbfbf83222f 100644
--- a/lib/gitlab/database/load_balancing.rb
+++ b/lib/gitlab/database/load_balancing.rb
@@ -79,24 +79,12 @@ module Gitlab
       ].freeze
 
       # Returns the role (primary/replica) of the database the connection is
-      # connecting to. At the moment, the connection can only be retrieved by
-      # Gitlab::Database::LoadBalancer#read or #read_write or from the
-      # ActiveRecord directly. Therefore, if the load balancer doesn't
-      # recognize the connection, this method returns the primary role
-      # directly. In future, we may need to check for other sources.
+      # connecting to.
       def self.db_role_for_connection(connection)
-        return ROLE_UNKNOWN unless connection
+        db_config = Database.db_config_for_connection(connection)
+        return ROLE_UNKNOWN unless db_config
 
-        # The connection proxy does not have a role assigned
-        # as this is dependent on a execution context
-        return ROLE_UNKNOWN if connection.is_a?(ConnectionProxy)
-
-        # During application init we might receive `NullPool`
-        return ROLE_UNKNOWN unless connection.respond_to?(:pool) &&
-          connection.pool.respond_to?(:db_config) &&
-          connection.pool.db_config.respond_to?(:name)
-
-        if connection.pool.db_config.name.ends_with?(LoadBalancer::REPLICA_SUFFIX)
+        if db_config.name.ends_with?(LoadBalancer::REPLICA_SUFFIX)
           ROLE_REPLICA
         else
           ROLE_PRIMARY
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 2a996dc77c2..9968096b1f6 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -73,6 +73,7 @@ module Gitlab
         end
       end
 
+      # @deprecated Use `create_table` in V2 instead
       #
       # Creates a new table, optionally allowing the caller to add check constraints to the table.
       # Aside from that addition, this method should behave identically to Rails' `create_table` method.
diff --git a/lib/gitlab/database/migration_helpers/v2.rb b/lib/gitlab/database/migration_helpers/v2.rb
index 79d69676c44..855c9908e4a 100644
--- a/lib/gitlab/database/migration_helpers/v2.rb
+++ b/lib/gitlab/database/migration_helpers/v2.rb
@@ -6,6 +6,65 @@ module Gitlab
       module V2
         include Gitlab::Database::MigrationHelpers
 
+        # Superseded by `create_table` override below
+        def create_table_with_constraints(*_)
+          raise <<~EOM
+            #create_table_with_constraints is not supported anymore - use #create_table instead, for example:
+
+              create_table :db_guides do |t|
+                t.bigint :stars, default: 0, null: false
+                t.text :title, limit: 128
+                t.text :notes, limit: 1024
+
+                t.check_constraint 'stars > 1000', name: 'so_many_stars'
+              end
+
+            See https://docs.gitlab.com/ee/development/database/strings_and_the_text_data_type.html
+          EOM
+        end
+
+        # Creates a new table, optionally allowing the caller to add text limit constraints to the table.
+        # This method only extends Rails' `create_table` method
+        #
+        # Example:
+        #
+        #    create_table :db_guides do |t|
+        #      t.bigint :stars, default: 0, null: false
+        #      t.text :title, limit: 128
+        #      t.text :notes, limit: 1024
+        #
+        #      t.check_constraint 'stars > 1000', name: 'so_many_stars'
+        #    end
+        #
+        # See Rails' `create_table` for more info on the available arguments.
+        #
+        # When adding foreign keys to other tables, consider wrapping the call into a with_lock_retries block
+        # to avoid traffic stalls.
+        def create_table(table_name, *args, **kwargs, &block)
+          helper_context = self
+
+          super do |t|
+            t.define_singleton_method(:text) do |column_name, **kwargs|
+              limit = kwargs.delete(:limit)
+
+              super(column_name, **kwargs)
+
+              if limit
+                # rubocop:disable GitlabSecurity/PublicSend
+                name = helper_context.send(:text_limit_name, table_name, column_name)
+                # rubocop:enable GitlabSecurity/PublicSend
+
+                column_name = helper_context.quote_column_name(column_name)
+                definition = "char_length(#{column_name}) <= #{limit}"
+
+                t.check_constraint(definition, name: name)
+              end
+            end
+
+            t.instance_eval(&block) unless block.nil?
+          end
+        end
+
         # Executes the block with a retry mechanism that alters the +lock_timeout+ and +sleep_time+ between attempts.
         # The timings can be controlled via the +timing_configuration+ parameter.
         # If the lock was not acquired within the retry period, a last attempt is made without using +lock_timeout+.
diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb
index b110d39818d..4b490ae0d26 100644
--- a/lib/gitlab/email/handler/create_issue_handler.rb
+++ b/lib/gitlab/email/handler/create_issue_handler.rb
@@ -55,7 +55,7 @@ module Gitlab
         private
 
         def create_issue
-          Issues::CreateService.new(
+          ::Issues::CreateService.new(
             project: project,
             current_user: author,
             params: {
diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb
index 84b55079cea..74c8d0a1fd7 100644
--- a/lib/gitlab/email/handler/service_desk_handler.rb
+++ b/lib/gitlab/email/handler/service_desk_handler.rb
@@ -71,7 +71,7 @@ module Gitlab
         end
 
         def create_issue!
-          @issue = Issues::CreateService.new(
+          @issue = ::Issues::CreateService.new(
             project: project,
             current_user: User.support_bot,
             params: {
diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb
index d5ced2045f5..a1855132b0c 100644
--- a/lib/gitlab/git.rb
+++ b/lib/gitlab/git.rb
@@ -5,7 +5,6 @@ require_dependency 'gitlab/encoding_helper'
 module Gitlab
   module Git
     # The ID of empty tree.
-    # See http://stackoverflow.com/a/40884093/1856239 and
     # https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012
     EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'
     BLANK_SHA = ('0' * 40).freeze
diff --git a/lib/gitlab/issues/rebalancing/state.rb b/lib/gitlab/issues/rebalancing/state.rb
new file mode 100644
index 00000000000..dce165a3489
--- /dev/null
+++ b/lib/gitlab/issues/rebalancing/state.rb
@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Issues
+    module Rebalancing
+      class State
+        REDIS_EXPIRY_TIME = 10.days
+        MAX_NUMBER_OF_CONCURRENT_REBALANCES = 5
+        NAMESPACE = 1
+        PROJECT = 2
+
+        def initialize(root_namespace, projects)
+          @root_namespace = root_namespace
+          @projects = projects
+          @rebalanced_container_type = @root_namespace.is_a?(Group) ? NAMESPACE : PROJECT
+          @rebalanced_container_id = @rebalanced_container_type == NAMESPACE ? @root_namespace.id : projects.take.id # rubocop:disable CodeReuse/ActiveRecord
+        end
+
+        def track_new_running_rebalance
+          with_redis do |redis|
+            redis.multi do |multi|
+              # we trigger re-balance for namespaces(groups) or specific user project
+              value = "#{rebalanced_container_type}/#{rebalanced_container_id}"
+              multi.sadd(concurrent_running_rebalances_key, value)
+              multi.expire(concurrent_running_rebalances_key, REDIS_EXPIRY_TIME)
+            end
+          end
+        end
+
+        def concurrent_running_rebalances_count
+          with_redis { |redis| redis.scard(concurrent_running_rebalances_key).to_i }
+        end
+
+        def rebalance_in_progress?
+          all_rebalanced_containers = with_redis { |redis| redis.smembers(concurrent_running_rebalances_key) }
+
+          is_running = case rebalanced_container_type
+                       when NAMESPACE
+                         namespace_ids = all_rebalanced_containers.map {|string| string.split("#{NAMESPACE}/").second.to_i }.compact
+                         namespace_ids.include?(root_namespace.id)
+                       when PROJECT
+                         project_ids = all_rebalanced_containers.map {|string| string.split("#{PROJECT}/").second.to_i }.compact
+                         project_ids.include?(projects.take.id) # rubocop:disable CodeReuse/ActiveRecord
+                       else
+                         false
+                       end
+
+          refresh_keys_expiration if is_running
+
+          is_running
+        end
+
+        def can_start_rebalance?
+          rebalance_in_progress? || too_many_rebalances_running?
+        end
+
+        def cache_issue_ids(issue_ids)
+          with_redis do |redis|
+            values = issue_ids.map { |issue| [issue.relative_position, issue.id] }
+
+            redis.multi do |multi|
+              multi.zadd(issue_ids_key, values) unless values.blank?
+              multi.expire(issue_ids_key, REDIS_EXPIRY_TIME)
+            end
+          end
+        end
+
+        def get_cached_issue_ids(index, limit)
+          with_redis do |redis|
+            redis.zrange(issue_ids_key, index, index + limit - 1)
+          end
+        end
+
+        def cache_current_index(index)
+          with_redis { |redis| redis.set(current_index_key, index, ex: REDIS_EXPIRY_TIME) }
+        end
+
+        def get_current_index
+          with_redis { |redis| redis.get(current_index_key).to_i }
+        end
+
+        def cache_current_project_id(project_id)
+          with_redis { |redis| redis.set(current_project_key, project_id, ex: REDIS_EXPIRY_TIME) }
+        end
+
+        def get_current_project_id
+          with_redis { |redis| redis.get(current_project_key) }
+        end
+
+        def issue_count
+          @issue_count ||= with_redis { |redis| redis.zcard(issue_ids_key)}
+        end
+
+        def remove_current_project_id_cache
+          with_redis { |redis| redis.del(current_project_key)}
+        end
+
+        def refresh_keys_expiration
+          with_redis do |redis|
+            redis.multi do |multi|
+              multi.expire(issue_ids_key, REDIS_EXPIRY_TIME)
+              multi.expire(current_index_key, REDIS_EXPIRY_TIME)
+              multi.expire(current_project_key, REDIS_EXPIRY_TIME)
+              multi.expire(concurrent_running_rebalances_key, REDIS_EXPIRY_TIME)
+            end
+          end
+        end
+
+        def cleanup_cache
+          with_redis do |redis|
+            redis.multi do |multi|
+              multi.del(issue_ids_key)
+              multi.del(current_index_key)
+              multi.del(current_project_key)
+              multi.srem(concurrent_running_rebalances_key, "#{rebalanced_container_type}/#{rebalanced_container_id}")
+            end
+          end
+        end
+
+        private
+
+        attr_accessor :root_namespace, :projects, :rebalanced_container_type, :rebalanced_container_id
+
+        def too_many_rebalances_running?
+          concurrent_running_rebalances_count <= MAX_NUMBER_OF_CONCURRENT_REBALANCES
+        end
+
+        def redis_key_prefix
+          "gitlab:issues-position-rebalances"
+        end
+
+        def issue_ids_key
+          "#{redis_key_prefix}:#{root_namespace.id}"
+        end
+
+        def current_index_key
+          "#{issue_ids_key}:current_index"
+        end
+
+        def current_project_key
+          "#{issue_ids_key}:current_project_id"
+        end
+
+        def concurrent_running_rebalances_key
+          "#{redis_key_prefix}:running_rebalances"
+        end
+
+        def with_redis(&blk)
+          Gitlab::Redis::SharedState.with(&blk) # rubocop: disable CodeReuse/ActiveRecord
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/marginalia/comment.rb b/lib/gitlab/marginalia/comment.rb
index ee15d3b1812..f635f41ec39 100644
--- a/lib/gitlab/marginalia/comment.rb
+++ b/lib/gitlab/marginalia/comment.rb
@@ -41,6 +41,10 @@ module Gitlab
       def endpoint_id
         Labkit::Context.current&.get_attribute(:caller_id)
       end
+
+      def db_config_name
+        ::Gitlab::Database.db_config_name(marginalia_adapter)
+      end
     end
   end
 end
diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb
index 4a12e612cea..80726fc8efd 100644
--- a/lib/gitlab/pagination/keyset/order.rb
+++ b/lib/gitlab/pagination/keyset/order.rb
@@ -219,7 +219,7 @@ module Gitlab
             column_definition.column_expression.dup.as(column_definition.attribute_name).to_sql
           end
 
-          scope = scope.select(*scope.arel.projections, *additional_projections) if additional_projections
+          scope = scope.reselect(*scope.arel.projections, *additional_projections) unless additional_projections.blank?
           scope
         end
 
diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb
index ff17ecf8024..c5cf3262039 100644
--- a/lib/gitlab/quick_actions/issue_actions.rb
+++ b/lib/gitlab/quick_actions/issue_actions.rb
@@ -267,7 +267,7 @@ module Gitlab
         private
 
         def zoom_link_service
-          Issues::ZoomLinkService.new(project: quick_action_target.project, current_user: current_user, params: { issue: quick_action_target })
+          ::Issues::ZoomLinkService.new(project: quick_action_target.project, current_user: current_user, params: { issue: quick_action_target })
         end
       end
     end
diff --git a/lib/gitlab/slash_commands/issue_close.rb b/lib/gitlab/slash_commands/issue_close.rb
index 3dad7216983..5d33f2fe62d 100644
--- a/lib/gitlab/slash_commands/issue_close.rb
+++ b/lib/gitlab/slash_commands/issue_close.rb
@@ -29,7 +29,7 @@ module Gitlab
       private
 
       def close_issue(issue:)
-        Issues::CloseService.new(project: project, current_user: current_user).execute(issue)
+        ::Issues::CloseService.new(project: project, current_user: current_user).execute(issue)
       end
 
       def presenter(issue)
diff --git a/lib/gitlab/slash_commands/issue_move.rb b/lib/gitlab/slash_commands/issue_move.rb
index 0612663017c..9f10da247d7 100644
--- a/lib/gitlab/slash_commands/issue_move.rb
+++ b/lib/gitlab/slash_commands/issue_move.rb
@@ -29,11 +29,11 @@ module Gitlab
           return Gitlab::SlashCommands::Presenters::Access.new.not_found
         end
 
-        new_issue = Issues::MoveService.new(project: project, current_user: current_user)
+        new_issue = ::Issues::MoveService.new(project: project, current_user: current_user)
                       .execute(old_issue, target_project)
 
         presenter(new_issue).present(old_issue)
-      rescue Issues::MoveService::MoveError => e
+      rescue ::Issues::MoveService::MoveError => e
         presenter(old_issue).display_move_error(e.message)
       end
 
diff --git a/lib/gitlab/slash_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb
index fab016d2e1b..ef368767689 100644
--- a/lib/gitlab/slash_commands/issue_new.rb
+++ b/lib/gitlab/slash_commands/issue_new.rb
@@ -33,7 +33,7 @@ module Gitlab
       private
 
       def create_issue(title:, description:)
-        Issues::CreateService.new(project: project, current_user: current_user, params: { title: title, description: description }, spam_params: nil).execute
+        ::Issues::CreateService.new(project: project, current_user: current_user, params: { title: title, description: description }, spam_params: nil).execute
       end
 
       def presenter(issue)
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 910c8397f20..9a091e8819c 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -918,7 +918,7 @@ module Gitlab
           jira: count(::JiraImportState.where(time_period)), # rubocop: disable CodeReuse/ActiveRecord
           fogbugz: projects_imported_count('fogbugz', time_period),
           phabricator: projects_imported_count('phabricator', time_period),
-          csv: count(Issues::CsvImport.where(time_period)) # rubocop: disable CodeReuse/ActiveRecord
+          csv: count(::Issues::CsvImport.where(time_period)) # rubocop: disable CodeReuse/ActiveRecord
         }
       end
 
@@ -934,7 +934,7 @@ module Gitlab
         project_imports = distinct_count(::Project.where(time_period).where.not(import_type: nil), :creator_id)
         bulk_imports = distinct_count(::BulkImport.where(time_period), :user_id)
         jira_issue_imports = distinct_count(::JiraImportState.where(time_period), :user_id)
-        csv_issue_imports = distinct_count(Issues::CsvImport.where(time_period), :user_id)
+        csv_issue_imports = distinct_count(::Issues::CsvImport.where(time_period), :user_id)
         group_imports = distinct_count(::GroupImportState.where(time_period), :user_id)
 
         add(project_imports, bulk_imports, jira_issue_imports, csv_issue_imports, group_imports)