Add latest changes from gitlab-org/gitlab@master

9 files changed, 296 insertions, 8 deletions

diff --git a/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb b/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb
new file mode 100644
index 00000000000..e951b44b036
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    module UserMentions
+      class CreateResourceUserMention
+        # Resources that have mentions to be migrated:
+        # issue, merge_request, epic, commit, snippet, design
+
+        BULK_INSERT_SIZE = 5000
+        ISOLATION_MODULE = 'Gitlab::BackgroundMigration::UserMentions::Models'
+
+        def perform(resource_model, join, conditions, with_notes, start_id, end_id)
+          resource_model = "#{ISOLATION_MODULE}::#{resource_model}".constantize if resource_model.is_a?(String)
+          model = with_notes ? "#{ISOLATION_MODULE}::Note".constantize : resource_model
+          resource_user_mention_model = resource_model.user_mention_model
+
+          records = model.joins(join).where(conditions).where(id: start_id..end_id)
+
+          records.in_groups_of(BULK_INSERT_SIZE, false).each do |records|
+            mentions = []
+            records.each do |record|
+              mentions << record.build_mention_values
+            end
+
+            no_quote_columns = [:note_id]
+            no_quote_columns << resource_user_mention_model.resource_foreign_key
+
+            Gitlab::Database.bulk_insert(
+              resource_user_mention_model.table_name,
+              mentions,
+              return_ids: true,
+              disable_quote: no_quote_columns,
+              on_conflict: :do_nothing
+            )
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/models/epic.rb b/lib/gitlab/background_migration/user_mentions/models/epic.rb
new file mode 100644
index 00000000000..019d8f0ea8b
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/models/epic.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    module UserMentions
+      module Models
+        class Epic < ActiveRecord::Base
+          include IsolatedMentionable
+          include CacheMarkdownField
+
+          attr_mentionable :title, pipeline: :single_line
+          attr_mentionable :description
+          cache_markdown_field :title, pipeline: :single_line
+          cache_markdown_field :description, issuable_state_filter_enabled: true
+
+          self.table_name = 'epics'
+
+          belongs_to :author, class_name: "User"
+          belongs_to :project
+          belongs_to :group
+
+          def self.user_mention_model
+            Gitlab::BackgroundMigration::UserMentions::Models::EpicUserMention
+          end
+
+          def user_mention_model
+            self.class.user_mention_model
+          end
+
+          def project
+            nil
+          end
+
+          def mentionable_params
+            { group: group, label_url_method: :group_epics_url }
+          end
+
+          def user_mention_resource_id
+            id
+          end
+
+          def user_mention_note_id
+            'NULL'
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb
new file mode 100644
index 00000000000..4e3ce9bf3a7
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    module UserMentions
+      module Models
+        class EpicUserMention < ActiveRecord::Base
+          self.table_name = 'epic_user_mentions'
+
+          def self.resource_foreign_key
+            :epic_id
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/models/isolated_mentionable.rb b/lib/gitlab/background_migration/user_mentions/models/isolated_mentionable.rb
new file mode 100644
index 00000000000..40aab896212
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/models/isolated_mentionable.rb
@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module UserMentions
+      module Models
+        # == IsolatedMentionable concern
+        #
+        # Shortcutted for isolation version of Mentionable to be used in mentions migrations
+        #
+        module IsolatedMentionable
+          extend ::ActiveSupport::Concern
+
+          class_methods do
+            # Indicate which attributes of the Mentionable to search for GFM references.
+            def attr_mentionable(attr, options = {})
+              attr = attr.to_s
+              mentionable_attrs << [attr, options]
+            end
+          end
+
+          included do
+            # Accessor for attributes marked mentionable.
+            cattr_accessor :mentionable_attrs, instance_accessor: false do
+              []
+            end
+
+            if self < Participable
+              participant -> (user, ext) { all_references(user, extractor: ext) }
+            end
+          end
+
+          def all_references(current_user = nil, extractor: nil)
+            # Use custom extractor if it's passed in the function parameters.
+            if extractor
+              extractors[current_user] = extractor
+            else
+              extractor = extractors[current_user] ||= ::Gitlab::ReferenceExtractor.new(project, current_user)
+
+              extractor.reset_memoized_values
+            end
+
+            self.class.mentionable_attrs.each do |attr, options|
+              text    = __send__(attr) # rubocop:disable GitlabSecurity/PublicSend
+              options = options.merge(
+                cache_key: [self, attr],
+                author: author,
+                skip_project_check: skip_project_check?
+              ).merge(mentionable_params)
+
+              cached_html = self.try(:updated_cached_html_for, attr.to_sym)
+              options[:rendered] = cached_html if cached_html
+
+              extractor.analyze(text, options)
+            end
+
+            extractor
+          end
+
+          def extractors
+            @extractors ||= {}
+          end
+
+          def skip_project_check?
+            false
+          end
+
+          def build_mention_values
+            refs = all_references(author)
+
+            {
+              "#{self.user_mention_model.resource_foreign_key}": user_mention_resource_id,
+              note_id: user_mention_note_id,
+              mentioned_users_ids: array_to_sql(refs.mentioned_users.pluck(:id)),
+              mentioned_projects_ids: array_to_sql(refs.mentioned_projects.pluck(:id)),
+              mentioned_groups_ids: array_to_sql(refs.mentioned_groups.pluck(:id))
+            }
+          end
+
+          def array_to_sql(ids_array)
+            return unless ids_array.present?
+
+            '{' + ids_array.join(", ") + '}'
+          end
+
+          private
+
+          def mentionable_params
+            {}
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/models/note.rb b/lib/gitlab/background_migration/user_mentions/models/note.rb
new file mode 100644
index 00000000000..c2828202907
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/models/note.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    module UserMentions
+      module Models
+        class Note < ActiveRecord::Base
+          include IsolatedMentionable
+          include CacheMarkdownField
+
+          self.table_name = 'notes'
+          self.inheritance_column = :_type_disabled
+
+          attr_mentionable :note, pipeline: :note
+          cache_markdown_field :note, pipeline: :note, issuable_state_filter_enabled: true
+
+          belongs_to :author, class_name: "User"
+          belongs_to :noteable, polymorphic: true
+          belongs_to :project
+
+          def user_mention_model
+            "#{CreateResourceUserMention::ISOLATION_MODULE}::#{noteable.class}".constantize.user_mention_model
+          end
+
+          def for_personal_snippet?
+            noteable.class.name == 'PersonalSnippet'
+          end
+
+          def for_project_noteable?
+            !for_personal_snippet?
+          end
+
+          def skip_project_check?
+            !for_project_noteable?
+          end
+
+          def for_epic?
+            noteable.class.name == 'Epic'
+          end
+
+          def user_mention_resource_id
+            noteable_id || commit_id
+          end
+
+          def user_mention_note_id
+            id
+          end
+
+          private
+
+          def mentionable_params
+            return super unless for_epic?
+
+            super.merge(banzai_context_params)
+          end
+
+          def banzai_context_params
+            { group: noteable.group, label_url_method: :group_epics_url }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb
index f984d7d397a..571e056e096 100644
--- a/lib/gitlab/ci/config/entry/reports.rb
+++ b/lib/gitlab/ci/config/entry/reports.rb
@@ -11,7 +11,7 @@ module Gitlab
           include ::Gitlab::Config::Entry::Validatable
           include ::Gitlab::Config::Entry::Attributable
 
-          ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast performance license_management license_scanning metrics].freeze
+          ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast performance license_management license_scanning metrics lsif].freeze
 
           attributes ALLOWED_KEYS
 
@@ -30,6 +30,7 @@ module Gitlab
               validates :license_management, array_of_strings_or_string: true
               validates :license_scanning, array_of_strings_or_string: true
               validates :metrics, array_of_strings_or_string: true
+              validates :lsif, array_of_strings_or_string: true
             end
           end
 
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 5077143e15e..b53e01e6ff2 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -1119,6 +1119,20 @@ into similar problems in the future (e.g. when new tables are created).
         SQL
       end
 
+      # Note this should only be used with very small tables
+      def backfill_iids(table)
+        sql = <<-END
+          UPDATE #{table}
+          SET iid = #{table}_with_calculated_iid.iid_num
+          FROM (
+            SELECT id, ROW_NUMBER() OVER (PARTITION BY project_id ORDER BY id ASC) AS iid_num FROM #{table}
+          ) AS #{table}_with_calculated_iid
+          WHERE #{table}.id = #{table}_with_calculated_iid.id
+        END
+
+        execute(sql)
+      end
+
       private
 
       def tables_match?(target_table, foreign_key_table)
diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb
index d27da186de0..d126fdb2be2 100644
--- a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb
+++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb
@@ -47,11 +47,7 @@ module Gitlab
         private
 
         def cache
-          @cache ||= if Feature.enabled?(:hset_redis_diff_caching, project, default_enabled: true)
-                       Gitlab::Diff::HighlightCache.new(self)
-                     else
-                       Gitlab::Diff::DeprecatedHighlightCache.new(self)
-                     end
+          @cache ||= Gitlab::Diff::HighlightCache.new(self)
         end
       end
     end
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index 7e9ec097ef7..906350e57c5 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -50,8 +50,8 @@ module Gitlab
       @project  = project
       @protocol = protocol
       @authentication_abilities = authentication_abilities
-      @namespace_path = namespace_path
-      @project_path = project_path
+      @namespace_path = namespace_path || project&.namespace&.full_path
+      @project_path = project_path || project&.path
       @redirected_path = redirected_path
       @auth_result_type = auth_result_type
     end
@@ -60,6 +60,7 @@ module Gitlab
       @logger = Checks::TimedLogger.new(timeout: INTERNAL_TIMEOUT, header: LOG_HEADER)
       @changes = changes
 
+      check_namespace!
       check_protocol!
       check_valid_actor!
       check_active_user!
@@ -136,6 +137,12 @@ module Gitlab
       end
     end
 
+    def check_namespace!
+      return if namespace_path.present?
+
+      raise NotFoundError, ERROR_MESSAGES[:project_not_found]
+    end
+
     def check_active_user!
       return unless user