Add latest changes from gitlab-org/gitlab@13-5-stable-eev13.5.0-rc42

14 files changed, 456 insertions, 5 deletions

diff --git a/lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule.rb b/lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule.rb
new file mode 100644
index 00000000000..2148e96f6b4
--- /dev/null
+++ b/lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Compare all current rules to project rules
+    class AddModifiedToApprovalMergeRequestRule
+      # Stubbed class to access the Group table
+      class Group < ActiveRecord::Base
+        self.table_name = 'namespaces'
+        self.inheritance_column = :_type_disabled
+      end
+
+      # Stubbed class to access the ApprovalMergeRequestRule table
+      class ApprovalMergeRequestRule < ActiveRecord::Base
+        self.table_name = 'approval_merge_request_rules'
+
+        has_one :approval_merge_request_rule_source, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalMergeRequestRuleSource'
+        has_one :approval_project_rule, through: :approval_merge_request_rule_source
+        has_and_belongs_to_many :groups,
+          class_name: 'AddModifiedToApprovalMergeRequestRule::Group', join_table: "#{self.table_name}_groups"
+      end
+
+      # Stubbed class to access the ApprovalProjectRule table
+      class ApprovalProjectRule < ActiveRecord::Base
+        self.table_name = 'approval_project_rules'
+
+        has_many :approval_merge_request_rule_sources, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalMergeRequestRuleSource'
+        has_and_belongs_to_many :groups,
+          class_name: 'AddModifiedToApprovalMergeRequestRule::Group', join_table: "#{self.table_name}_groups"
+      end
+
+      # Stubbed class to access the ApprovalMergeRequestRuleSource table
+      class ApprovalMergeRequestRuleSource < ActiveRecord::Base
+        self.table_name = 'approval_merge_request_rule_sources'
+
+        belongs_to :approval_merge_request_rule, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalMergeRequestRule'
+        belongs_to :approval_project_rule, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalProjectRule'
+      end
+
+      def perform(start_id, stop_id)
+        approval_merge_requests_rules = ApprovalMergeRequestRule
+          .joins(:groups, :approval_merge_request_rule_source)
+          .where(id: start_id..stop_id)
+          .pluck(
+            'approval_merge_request_rule_sources.id as ars_id',
+            'approval_merge_request_rules_groups.id as amrg_id'
+          )
+
+        approval_project_rules = ApprovalProjectRule
+          .joins(:groups, approval_merge_request_rule_sources: :approval_merge_request_rule)
+          .where(approval_merge_request_rules: { id: start_id..stop_id })
+          .pluck(
+            'approval_merge_request_rule_sources.id as ars_id',
+            'approval_project_rules_groups.id as apg_id'
+          )
+
+        different_names_or_approval_sources = ApprovalMergeRequestRule.joins(:approval_project_rule, :approval_merge_request_rule_source)
+          .where(id: start_id..stop_id)
+          .where('approval_merge_request_rules.name != approval_project_rules.name OR ' \
+                'approval_merge_request_rules.approvals_required != approval_project_rules.approvals_required')
+          .pluck('approval_merge_request_rule_sources.id as ars_id')
+
+        intersected_set = approval_merge_requests_rules.to_set ^ approval_project_rules.to_set
+        source_ids = intersected_set.collect { |rule| rule[0] }.uniq
+
+        rule_sources = ApprovalMergeRequestRuleSource.where(id: source_ids + different_names_or_approval_sources)
+        changed_merge_request_rules = ApprovalMergeRequestRule.where(id: rule_sources.select(:approval_merge_request_rule_id))
+
+        changed_merge_request_rules.update_all(modified_from_project_rule: true)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/backfill_snippet_repositories.rb b/lib/gitlab/background_migration/backfill_snippet_repositories.rb
index 21538000fec..8befade8c3a 100644
--- a/lib/gitlab/background_migration/backfill_snippet_repositories.rb
+++ b/lib/gitlab/background_migration/backfill_snippet_repositories.rb
@@ -109,7 +109,7 @@ module Gitlab
       end
 
       def create_commit(snippet)
-        snippet.snippet_repository.multi_files_action(commit_author(snippet), snippet_action(snippet), commit_attrs)
+        snippet.snippet_repository.multi_files_action(commit_author(snippet), snippet_action(snippet), **commit_attrs)
       end
 
       # If the user is not allowed to access git or update the snippet
diff --git a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb
new file mode 100644
index 00000000000..b8c14aa2573
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+require "webauthn/u2f_migrator"
+
+module Gitlab
+  module BackgroundMigration
+    class MigrateU2fWebauthn
+      class U2fRegistration < ActiveRecord::Base
+        self.table_name = 'u2f_registrations'
+      end
+
+      class WebauthnRegistration < ActiveRecord::Base
+        self.table_name = 'webauthn_registrations'
+      end
+
+      def perform(start_id, end_id)
+        old_registrations = U2fRegistration.where(id: start_id..end_id)
+        old_registrations.each_slice(100) do |slice|
+          now = Time.now
+          values = slice.map do |u2f_registration|
+            converted_credential = WebAuthn::U2fMigrator.new(
+              app_id: Gitlab.config.gitlab.url,
+              certificate: u2f_registration.certificate,
+              key_handle: u2f_registration.key_handle,
+              public_key: u2f_registration.public_key,
+              counter: u2f_registration.counter
+            ).credential
+
+            {
+                credential_xid: Base64.strict_encode64(converted_credential.id),
+                public_key: Base64.strict_encode64(converted_credential.public_key),
+                counter: u2f_registration.counter || 0,
+                name: u2f_registration.name || '',
+                user_id: u2f_registration.user_id,
+                u2f_registration_id: u2f_registration.id,
+                created_at: now,
+                updated_at: now
+            }
+          end
+
+          WebauthnRegistration.insert_all(values, unique_by: :credential_xid, returning: false)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb b/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb
index ca64d13b118..bbe2164ae4e 100644
--- a/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb
+++ b/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb
@@ -13,8 +13,6 @@ module Gitlab
       end
 
       def perform(start_id, stop_id)
-        return if Feature.disabled?(:migrate_bio_to_user_details, default_enabled: true)
-
         relation = User
           .select("id AS user_id", "substring(COALESCE(bio, '') from 1 for 255) AS bio")
           .where("(COALESCE(bio, '') IS DISTINCT FROM '')")
diff --git a/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb b/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb
new file mode 100644
index 00000000000..cd305adc7cd
--- /dev/null
+++ b/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class RemoveDuplicatedCsFindingsWithoutVulnerabilityId
+      def perform(start_id, stop_id)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId')
diff --git a/lib/gitlab/background_migration/replace_blocked_by_links.rb b/lib/gitlab/background_migration/replace_blocked_by_links.rb
new file mode 100644
index 00000000000..26626aaef79
--- /dev/null
+++ b/lib/gitlab/background_migration/replace_blocked_by_links.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class ReplaceBlockedByLinks
+      class IssueLink < ActiveRecord::Base
+        self.table_name = 'issue_links'
+      end
+
+      def perform(start_id, stop_id)
+        blocked_by_links = IssueLink.where(id: start_id..stop_id).where(link_type: 2)
+
+        ActiveRecord::Base.transaction do
+          # if there is duplicit bi-directional relation (issue2 is blocked by issue1
+          # and issue1 already links issue2), then we can just delete 'blocked by'.
+          # This should be rare as we have a pre-create check which checks if issues are
+          # already linked
+          blocked_by_links
+            .joins('INNER JOIN issue_links as opposite_links ON issue_links.source_id = opposite_links.target_id AND issue_links.target_id = opposite_links.source_id')
+            .where('opposite_links.link_type': 1)
+            .delete_all
+
+          blocked_by_links.update_all('source_id=target_id,target_id=source_id,link_type=1')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/sync_blocking_issues_count.rb b/lib/gitlab/background_migration/sync_blocking_issues_count.rb
new file mode 100644
index 00000000000..6262320128c
--- /dev/null
+++ b/lib/gitlab/background_migration/sync_blocking_issues_count.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class SyncBlockingIssuesCount
+      def perform(start_id, end_id)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::SyncBlockingIssuesCount.prepend_if_ee('EE::Gitlab::BackgroundMigration::SyncBlockingIssuesCount')
diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb
new file mode 100644
index 00000000000..3def5eb3369
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module UserMentions
+      module Lib
+        module Banzai
+          # isolated Banzai::ReferenceParser
+          module ReferenceParser
+            # Returns the reference parser class for the given type
+            #
+            # Example:
+            #
+            #     Banzai::ReferenceParser['isolated_mentioned_group']
+            #
+            # This would return the `::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser::IsolatedMentionedGroupParser` class.
+            def self.[](name)
+              const_get("::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser::#{name.to_s.camelize}Parser", false)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb
new file mode 100644
index 00000000000..d3d032ba433
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module UserMentions
+      module Lib
+        module Banzai
+          module ReferenceParser
+            # isolated Banzai::ReferenceParser::MentionedGroupParser
+            class IsolatedMentionedGroupParser < ::Banzai::ReferenceParser::MentionedGroupParser
+              extend ::Gitlab::Utils::Override
+
+              self.reference_type = :user
+
+              override :references_relation
+              def references_relation
+                ::Gitlab::BackgroundMigration::UserMentions::Models::Group
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb
new file mode 100644
index 00000000000..1d3a3af81a1
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module UserMentions
+      module Lib
+        module Gitlab
+          # Extract possible GFM references from an arbitrary String for further processing.
+          class IsolatedReferenceExtractor < ::Gitlab::ReferenceExtractor
+            REFERABLES = %i(isolated_mentioned_group).freeze
+
+            REFERABLES.each do |type|
+              define_method("#{type}s") do
+                @references[type] ||= isolated_references(type)
+              end
+            end
+
+            def isolated_references(type)
+              context = ::Banzai::RenderContext.new(project, current_user)
+              processor = ::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser[type].new(context)
+
+              refs = processor.process(html_documents)
+              refs[:visible]
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb
index 69ba3f9132b..be9c0ad2b3a 100644
--- a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb
@@ -36,7 +36,8 @@ module Gitlab
               if extractor
                 extractors[current_user] = extractor
               else
-                extractor = extractors[current_user] ||= ::Gitlab::ReferenceExtractor.new(project, current_user)
+                extractor = extractors[current_user] ||=
+                  Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedReferenceExtractor.new(project, current_user)
 
                 extractor.reset_memoized_values
               end
@@ -71,7 +72,7 @@ module Gitlab
 
               mentioned_users_ids = array_to_sql(refs.mentioned_users.pluck(:id))
               mentioned_projects_ids = array_to_sql(refs.mentioned_projects.pluck(:id))
-              mentioned_groups_ids = array_to_sql(refs.mentioned_groups.pluck(:id))
+              mentioned_groups_ids = array_to_sql(refs.isolated_mentioned_groups.pluck(:id))
 
               return if mentioned_users_ids.blank? && mentioned_projects_ids.blank? && mentioned_groups_ids.blank?
 
diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb
new file mode 100644
index 00000000000..5cadfa45b5b
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module UserMentions
+      module Models
+        module Concerns
+          module Namespace
+            # extracted methods for recursive traversing of namespace hierarchy
+            module RecursiveTraversal
+              extend ActiveSupport::Concern
+
+              def root_ancestor
+                return self if persisted? && parent_id.nil?
+
+                strong_memoize(:root_ancestor) do
+                  Gitlab::ObjectHierarchy
+                    .new(self.class.where(id: id))
+                    .base_and_ancestors
+                    .reorder(nil)
+                    .find_by(parent_id: nil)
+                end
+              end
+
+              # Returns all ancestors, self, and descendants of the current namespace.
+              def self_and_hierarchy
+                Gitlab::ObjectHierarchy
+                  .new(self.class.where(id: id))
+                  .all_objects
+              end
+
+              # Returns all the ancestors of the current namespaces.
+              def ancestors
+                return self.class.none unless parent_id
+
+                Gitlab::ObjectHierarchy
+                  .new(self.class.where(id: parent_id))
+                  .base_and_ancestors
+              end
+
+              # returns all ancestors upto but excluding the given namespace
+              # when no namespace is given, all ancestors upto the top are returned
+              def ancestors_upto(top = nil, hierarchy_order: nil)
+                Gitlab::ObjectHierarchy.new(self.class.where(id: id))
+                  .ancestors(upto: top, hierarchy_order: hierarchy_order)
+              end
+
+              def self_and_ancestors(hierarchy_order: nil)
+                return self.class.where(id: id) unless parent_id
+
+                Gitlab::ObjectHierarchy
+                  .new(self.class.where(id: id))
+                  .base_and_ancestors(hierarchy_order: hierarchy_order)
+              end
+
+              # Returns all the descendants of the current namespace.
+              def descendants
+                Gitlab::ObjectHierarchy
+                  .new(self.class.where(parent_id: id))
+                  .base_and_descendants
+              end
+
+              def self_and_descendants
+                Gitlab::ObjectHierarchy
+                  .new(self.class.where(id: id))
+                  .base_and_descendants
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/models/group.rb b/lib/gitlab/background_migration/user_mentions/models/group.rb
new file mode 100644
index 00000000000..bc04172b9a2
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/models/group.rb
@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module UserMentions
+      module Models
+        # isolated Group model
+        class Group < ::Gitlab::BackgroundMigration::UserMentions::Models::Namespace
+          self.store_full_sti_class = false
+          has_one :saml_provider
+
+          def self.declarative_policy_class
+            "GroupPolicy"
+          end
+
+          def max_member_access_for_user(user)
+            return GroupMember::NO_ACCESS unless user
+
+            return GroupMember::OWNER if user.admin?
+
+            max_member_access = members_with_parents.where(user_id: user)
+                                  .reorder(access_level: :desc)
+                                  .first
+                                  &.access_level
+
+            max_member_access || GroupMember::NO_ACCESS
+          end
+
+          def members_with_parents
+            # Avoids an unnecessary SELECT when the group has no parents
+            source_ids =
+              if has_parent?
+                self_and_ancestors.reorder(nil).select(:id)
+              else
+                id
+              end
+
+            group_hierarchy_members = GroupMember.active_without_invites_and_requests
+                                        .where(source_id: source_ids)
+
+            GroupMember.from_union([group_hierarchy_members,
+                                    members_from_self_and_ancestor_group_shares])
+          end
+
+          # rubocop: disable Metrics/AbcSize
+          def members_from_self_and_ancestor_group_shares
+            group_group_link_table = GroupGroupLink.arel_table
+            group_member_table = GroupMember.arel_table
+
+            source_ids =
+              if has_parent?
+                self_and_ancestors.reorder(nil).select(:id)
+              else
+                id
+              end
+
+            group_group_links_query = GroupGroupLink.where(shared_group_id: source_ids)
+            cte = Gitlab::SQL::CTE.new(:group_group_links_cte, group_group_links_query)
+            cte_alias = cte.table.alias(GroupGroupLink.table_name)
+
+            # Instead of members.access_level, we need to maximize that access_level at
+            # the respective group_group_links.group_access.
+            member_columns = GroupMember.attribute_names.map do |column_name|
+              if column_name == 'access_level'
+                smallest_value_arel([cte_alias[:group_access], group_member_table[:access_level]],
+                                    'access_level')
+              else
+                group_member_table[column_name]
+              end
+            end
+
+            GroupMember
+              .with(cte.to_arel)
+              .select(*member_columns)
+              .from([group_member_table, cte.alias_to(group_group_link_table)])
+              .where(group_member_table[:requested_at].eq(nil))
+              .where(group_member_table[:source_id].eq(group_group_link_table[:shared_with_group_id]))
+              .where(group_member_table[:source_type].eq('Namespace'))
+          end
+          # rubocop: enable Metrics/AbcSize
+
+          def smallest_value_arel(args, column_alias)
+            Arel::Nodes::As.new(
+              Arel::Nodes::NamedFunction.new('LEAST', args),
+              Arel::Nodes::SqlLiteral.new(column_alias))
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb
new file mode 100644
index 00000000000..6d7b9a86e69
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/models/namespace.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module UserMentions
+      module Models
+        # isolated Namespace model
+        class Namespace < ApplicationRecord
+          include ::Gitlab::VisibilityLevel
+          include ::Gitlab::Utils::StrongMemoize
+          include Gitlab::BackgroundMigration::UserMentions::Models::Concerns::Namespace::RecursiveTraversal
+
+          belongs_to :parent, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Namespace"
+
+          def visibility_level_field
+            :visibility_level
+          end
+
+          def has_parent?
+            parent_id.present? || parent.present?
+          end
+
+          # Overridden in EE::Namespace
+          def feature_available?(_feature)
+            false
+          end
+        end
+      end
+    end
+  end
+end
+
+Namespace.prepend_if_ee('::EE::Namespace')