Add latest changes from gitlab-org/gitlab@master

9 files changed, 36 insertions, 552 deletions

diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb
index 41f94e79f91..fc0e4ab5a0d 100644
--- a/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb
+++ b/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb
@@ -50,8 +50,7 @@ module Gitlab
           def filter_author(query)
             return query if params[:author_username].blank?
 
-            user = User.by_username(params[:author_username]).first
-
+            user = find_user(params[:author_username])
             return query.none if user.blank?
 
             query.authored(user)
@@ -60,11 +59,7 @@ module Gitlab
           def filter_milestone_ids(query)
             return query if params[:milestone_title].blank?
 
-            milestone = MilestonesFinder
-              .new(group_ids: root_ancestor.self_and_descendant_ids, project_ids: root_ancestor.all_projects.select(:id), title: params[:milestone_title])
-              .execute
-              .first
-
+            milestone = find_milestone(params[:milestone_title])
             return query.none if milestone.blank?
 
             query.with_milestone_id(milestone.id)
@@ -115,6 +110,17 @@ module Gitlab
           private
 
           attr_reader :stage, :params, :root_ancestor, :stage_event_model
+
+          def find_milestone(title)
+            MilestonesFinder
+              .new(group_ids: root_ancestor.self_and_descendant_ids, project_ids: root_ancestor.all_projects.select(:id), title: title)
+              .execute
+              .first
+          end
+
+          def find_user(username)
+            User.by_username(username).first
+          end
         end
         # rubocop: enable CodeReuse/ActiveRecord
       end
diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb
index 005758b3db0..cea25ba2db4 100644
--- a/lib/gitlab/analytics/cycle_analytics/request_params.rb
+++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb
@@ -12,6 +12,24 @@ module Gitlab
         MAX_RANGE_DAYS = 180.days.freeze
         DEFAULT_DATE_RANGE = 29.days # 30 including Date.today
 
+        NEGATABLE_PARAMS = [
+          :assignee_username,
+          :author_username,
+          :epic_id,
+          :iteration_id,
+          :label_name,
+          :milestone_title,
+          :my_reaction_emoji,
+          :weight
+        ].freeze
+
+        LICENSED_PARAMS = [
+          :weight,
+          :epic_id,
+          :my_reaction_emoji,
+          :iteration_id
+        ].freeze
+
         STRONG_PARAMS_DEFINITION = [
           :created_before,
           :created_after,
@@ -22,13 +40,11 @@ module Gitlab
           :page,
           :stage_id,
           :end_event_filter,
-          :weight,
-          :epic_id,
-          :my_reaction_emoji,
-          :iteration_id,
+          *LICENSED_PARAMS,
           label_name: [].freeze,
           assignee_username: [].freeze,
-          project_ids: [].freeze
+          project_ids: [].freeze,
+          not: NEGATABLE_PARAMS
         ].freeze
 
         FINDER_PARAM_NAMES = [
@@ -54,6 +70,7 @@ module Gitlab
         attribute :epic_id
         attribute :my_reaction_emoji
         attribute :iteration_id
+        attribute :not, default: -> { {} }
 
         FINDER_PARAM_NAMES.each do |param_name|
           attribute param_name
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb
deleted file mode 100644
index 2314246da55..00000000000
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb
+++ /dev/null
@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-# This module can be included in migrations to make it easier to rename paths
-# of `Namespace` & `Project` models certain paths would become `reserved`.
-#
-# If the way things are stored on the filesystem related to namespaces and
-# projects ever changes. Don't update this module, or anything nested in `V1`,
-# since it needs to keep functioning for all migrations using it using the state
-# that the data is in at the time. Instead, create a `V2` module that implements
-# the new way of reserving paths.
-module Gitlab
-  module Database
-    module RenameReservedPathsMigration
-      module V1
-        def self.included(kls)
-          kls.include(MigrationHelpers)
-        end
-
-        def rename_wildcard_paths(one_or_more_paths)
-          rename_child_paths(one_or_more_paths)
-          paths = Array(one_or_more_paths)
-          RenameProjects.new(paths, self).rename_projects
-        end
-
-        def rename_child_paths(one_or_more_paths)
-          paths = Array(one_or_more_paths)
-          RenameNamespaces.new(paths, self).rename_namespaces(type: :child)
-        end
-
-        def rename_root_paths(paths)
-          paths = Array(paths)
-          RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level)
-        end
-
-        def revert_renames
-          RenameProjects.new([], self).revert_renames
-          RenameNamespaces.new([], self).revert_renames
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb
deleted file mode 100644
index f1dc3ed74fe..00000000000
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb
+++ /dev/null
@@ -1,99 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Database
-    module RenameReservedPathsMigration
-      module V1
-        module MigrationClasses
-          module Routable
-            def full_path
-              if route && route.path.present?
-                @full_path ||= route.path # rubocop:disable Gitlab/ModuleWithInstanceVariables
-              else
-                update_route if persisted?
-
-                build_full_path
-              end
-            end
-
-            def build_full_path
-              if parent && path
-                parent.full_path + '/' + path
-              else
-                path
-              end
-            end
-
-            def update_route
-              prepare_route
-              route.save
-            end
-
-            def prepare_route
-              route || build_route(source: self)
-              route.path = build_full_path
-              @full_path = nil # rubocop:disable Gitlab/ModuleWithInstanceVariables
-            end
-          end
-
-          class Namespace < ActiveRecord::Base
-            include MigrationClasses::Routable
-            self.table_name = 'namespaces'
-            self.inheritance_column = :_type_disabled
-            belongs_to :parent,
-                       class_name: "#{MigrationClasses.name}::Namespace"
-            has_one :route, as: :source
-            has_many :children,
-                     class_name: "#{MigrationClasses.name}::Namespace",
-                     foreign_key: :parent_id
-
-            # Overridden to have the correct `source_type` for the `route` relation
-            def self.name
-              'Namespace'
-            end
-
-            def kind
-              type == 'Group' ? 'group' : 'user'
-            end
-          end
-
-          class User < ActiveRecord::Base
-            self.table_name = 'users'
-          end
-
-          class Route < ActiveRecord::Base
-            self.table_name = 'routes'
-            belongs_to :source, polymorphic: true
-          end
-
-          class Project < ActiveRecord::Base
-            include MigrationClasses::Routable
-            has_one :route, as: :source
-            self.table_name = 'projects'
-
-            HASHED_STORAGE_FEATURES = {
-              repository: 1,
-              attachments: 2
-            }.freeze
-
-            def repository_storage_path
-              Gitlab.config.repositories.storages[repository_storage].legacy_disk_path
-            end
-
-            # Overridden to have the correct `source_type` for the `route` relation
-            def self.name
-              'Project'
-            end
-
-            def hashed_storage?(feature)
-              raise ArgumentError, "Invalid feature" unless HASHED_STORAGE_FEATURES.include?(feature)
-              return false unless respond_to?(:storage_version)
-
-              self.storage_version && self.storage_version >= HASHED_STORAGE_FEATURES[feature]
-            end
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
deleted file mode 100644
index 2c9d0d6c0d1..00000000000
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
+++ /dev/null
@@ -1,196 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Database
-    module RenameReservedPathsMigration
-      module V1
-        class RenameBase
-          attr_reader :paths, :migration
-
-          delegate :update_column_in_batches,
-                   :execute,
-                   :replace_sql,
-                   :quote_string,
-                   :say,
-                   to: :migration
-
-          def initialize(paths, migration)
-            @paths = paths
-            @migration = migration
-          end
-
-          def path_patterns
-            @path_patterns ||= paths.flat_map { |path| ["%/#{path}", path] }
-          end
-
-          def rename_path_for_routable(routable)
-            old_path = routable.path
-            old_full_path = routable.full_path
-            # Only remove the last occurrence of the path name to get the parent namespace path
-            namespace_path = remove_last_occurrence(old_full_path, old_path)
-            new_path = rename_path(namespace_path, old_path)
-            new_full_path = join_routable_path(namespace_path, new_path)
-
-            perform_rename(routable, old_full_path, new_full_path)
-
-            [old_full_path, new_full_path]
-          end
-
-          def perform_rename(routable, old_full_path, new_full_path)
-            # skips callbacks & validations
-            new_path = new_full_path.split('/').last
-            routable.class.where(id: routable)
-              .update_all(path: new_path)
-
-            rename_routes(old_full_path, new_full_path)
-          end
-
-          def rename_routes(old_full_path, new_full_path)
-            routes = Route.arel_table
-
-            quoted_old_full_path = quote_string(old_full_path)
-            quoted_old_wildcard_path = quote_string("#{old_full_path}/%")
-
-            filter =
-              "routes.id IN "\
-              "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\
-              "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )"
-
-            replace_statement = replace_sql(Route.arel_table[:path],
-                                            old_full_path,
-                                            new_full_path)
-
-            update = Arel::UpdateManager.new
-              .table(routes)
-              .set([[routes[:path], replace_statement]])
-              .where(Arel::Nodes::SqlLiteral.new(filter))
-
-            execute(update.to_sql)
-          end
-
-          def rename_path(namespace_path, path_was)
-            counter = 0
-            path = "#{path_was}#{counter}"
-
-            while route_exists?(join_routable_path(namespace_path, path))
-              counter += 1
-              path = "#{path_was}#{counter}"
-            end
-
-            path
-          end
-
-          def remove_last_occurrence(string, pattern)
-            string.reverse.sub(pattern.reverse, "").reverse
-          end
-
-          def join_routable_path(namespace_path, top_level)
-            if namespace_path.present?
-              File.join(namespace_path, top_level)
-            else
-              top_level
-            end
-          end
-
-          def route_exists?(full_path)
-            MigrationClasses::Route.where(Route.arel_table[:path].matches(full_path)).any?
-          end
-
-          def move_pages(old_path, new_path)
-            move_folders(pages_dir, old_path, new_path)
-          end
-
-          def move_uploads(old_path, new_path)
-            return unless file_storage?
-
-            move_folders(uploads_dir, old_path, new_path)
-          end
-
-          def move_folders(directory, old_relative_path, new_relative_path)
-            old_path = File.join(directory, old_relative_path)
-            unless File.directory?(old_path)
-              say "#{old_path} doesn't exist, skipping"
-              return
-            end
-
-            new_path = File.join(directory, new_relative_path)
-            FileUtils.mv(old_path, new_path)
-          end
-
-          def remove_cached_html_for_projects(project_ids)
-            project_ids.each do |project_id|
-              update_column_in_batches(:projects, :description_html, nil) do |table, query|
-                query.where(table[:id].eq(project_id))
-              end
-
-              update_column_in_batches(:issues, :description_html, nil) do |table, query|
-                query.where(table[:project_id].eq(project_id))
-              end
-
-              update_column_in_batches(:merge_requests, :description_html, nil) do |table, query|
-                query.where(table[:target_project_id].eq(project_id))
-              end
-
-              update_column_in_batches(:notes, :note_html, nil) do |table, query|
-                query.where(table[:project_id].eq(project_id))
-              end
-
-              update_column_in_batches(:milestones, :description_html, nil) do |table, query|
-                query.where(table[:project_id].eq(project_id))
-              end
-            end
-          end
-
-          def track_rename(type, old_path, new_path)
-            key = redis_key_for_type(type)
-            Gitlab::Redis::SharedState.with do |redis|
-              redis.lpush(key, [old_path, new_path].to_json)
-              redis.expire(key, 2.weeks.to_i)
-            end
-            say "tracked rename: #{key}: #{old_path} -> #{new_path}"
-          end
-
-          def reverts_for_type(type)
-            key = redis_key_for_type(type)
-
-            Gitlab::Redis::SharedState.with do |redis|
-              failed_reverts = []
-
-              while rename_info = redis.lpop(key)
-                path_before_rename, path_after_rename = Gitlab::Json.parse(rename_info)
-                say "renaming #{type} from #{path_after_rename} back to #{path_before_rename}"
-                begin
-                  yield(path_before_rename, path_after_rename)
-                rescue StandardError => e
-                  failed_reverts << rename_info
-                  say "Renaming #{type} from #{path_after_rename} back to "\
-                      "#{path_before_rename} failed. Review the error and try "\
-                      "again by running the `down` action. \n"\
-                      "#{e.message}: \n #{e.backtrace.join("\n")}"
-                end
-              end
-
-              failed_reverts.each { |rename_info| redis.lpush(key, rename_info) }
-            end
-          end
-
-          def redis_key_for_type(type)
-            "rename:#{migration.name}:#{type}"
-          end
-
-          def file_storage?
-            CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
-          end
-
-          def uploads_dir
-            File.join(CarrierWave.root, "uploads")
-          end
-
-          def pages_dir
-            Settings.pages.path
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
deleted file mode 100644
index 72ae2849911..00000000000
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
+++ /dev/null
@@ -1,106 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Database
-    module RenameReservedPathsMigration
-      module V1
-        class RenameNamespaces < RenameBase
-          include Gitlab::ShellAdapter
-
-          def rename_namespaces(type:)
-            namespaces_for_paths(type: type).each do |namespace|
-              rename_namespace(namespace)
-            end
-          end
-
-          def namespaces_for_paths(type:)
-            namespaces = case type
-                         when :child
-                           MigrationClasses::Namespace.where.not(parent_id: nil)
-                         when :top_level
-                           MigrationClasses::Namespace.where(parent_id: nil)
-                         end
-            with_paths = MigrationClasses::Route.arel_table[:path]
-                           .matches_any(path_patterns)
-            namespaces.joins(:route).where(with_paths)
-              .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046")
-          end
-
-          def rename_namespace(namespace)
-            old_full_path, new_full_path = rename_path_for_routable(namespace)
-
-            track_rename('namespace', old_full_path, new_full_path)
-
-            rename_namespace_dependencies(namespace, old_full_path, new_full_path)
-          end
-
-          def rename_namespace_dependencies(namespace, old_full_path, new_full_path)
-            move_repositories(namespace, old_full_path, new_full_path)
-            move_uploads(old_full_path, new_full_path)
-            move_pages(old_full_path, new_full_path)
-            rename_user(old_full_path, new_full_path) if namespace.kind == 'user'
-            remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id))
-          end
-
-          def revert_renames
-            reverts_for_type('namespace') do |path_before_rename, current_path|
-              matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path)
-              namespace = MigrationClasses::Namespace.joins(:route)
-                            .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046")
-                            .find_by(matches_path)&.becomes(MigrationClasses::Namespace) # rubocop: disable Cop/AvoidBecomes
-
-              if namespace
-                perform_rename(namespace, current_path, path_before_rename)
-
-                rename_namespace_dependencies(namespace, current_path, path_before_rename)
-              else
-                say "Couldn't rename namespace from #{current_path} back to #{path_before_rename}, "\
-                    "namespace was renamed, or no longer exists at the expected path"
-              end
-            end
-          end
-
-          def rename_user(old_username, new_username)
-            MigrationClasses::User.where(username: old_username)
-              .update_all(username: new_username)
-          end
-
-          def move_repositories(namespace, old_full_path, new_full_path)
-            repo_shards_for_namespace(namespace).each do |repository_storage|
-              # Ensure old directory exists before moving it
-              Gitlab::GitalyClient::NamespaceService.allow do
-                gitlab_shell.add_namespace(repository_storage, old_full_path)
-
-                unless gitlab_shell.mv_namespace(repository_storage, old_full_path, new_full_path)
-                  message = "Exception moving on shard #{repository_storage} from #{old_full_path} to #{new_full_path}"
-                  Gitlab::AppLogger.error message
-                end
-              end
-            end
-          end
-
-          def repo_shards_for_namespace(namespace)
-            projects_for_namespace(namespace).distinct.select(:repository_storage)
-              .map(&:repository_storage)
-          end
-
-          def projects_for_namespace(namespace)
-            namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id])
-            namespace_or_children = MigrationClasses::Project
-                                      .arel_table[:namespace_id]
-                                      .in(namespace_ids)
-            MigrationClasses::Project.where(namespace_or_children)
-          end
-
-          def child_ids_for_parent(namespace, ids: [])
-            namespace.children.each do |child|
-              ids << child.id
-              child_ids_for_parent(child, ids: ids) if child.children.any?
-            end
-            ids
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
deleted file mode 100644
index 155e35b64f4..00000000000
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
+++ /dev/null
@@ -1,78 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Database
-    module RenameReservedPathsMigration
-      module V1
-        class RenameProjects < RenameBase
-          include Gitlab::ShellAdapter
-
-          def rename_projects
-            projects_for_paths.each do |project|
-              rename_project(project)
-            end
-
-            remove_cached_html_for_projects(projects_for_paths.map(&:id))
-          end
-
-          def rename_project(project)
-            old_full_path, new_full_path = rename_path_for_routable(project)
-
-            track_rename('project', old_full_path, new_full_path)
-
-            move_project_folders(project, old_full_path, new_full_path)
-          end
-
-          def move_project_folders(project, old_full_path, new_full_path)
-            unless project.hashed_storage?(:repository)
-              move_repository(project, old_full_path, new_full_path)
-              move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki")
-            end
-
-            move_uploads(old_full_path, new_full_path) unless project.hashed_storage?(:attachments)
-            move_pages(old_full_path, new_full_path)
-          end
-
-          def revert_renames
-            reverts_for_type('project') do |path_before_rename, current_path|
-              matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path)
-              project = MigrationClasses::Project.joins(:route)
-                          .allow_cross_joins_across_databases(url:
-                            'https://gitlab.com/gitlab-org/gitlab/-/issues/421843')
-                          .find_by(matches_path)
-
-              if project
-                perform_rename(project, current_path, path_before_rename)
-
-                move_project_folders(project, current_path, path_before_rename)
-              else
-                say "Couldn't rename project from #{current_path} back to "\
-                    "#{path_before_rename}, project was renamed or no longer "\
-                    "exists at the expected path."
-
-              end
-            end
-          end
-
-          def move_repository(project, old_path, new_path)
-            unless gitlab_shell.mv_repository(project.repository_storage,
-                                              old_path,
-                                              new_path)
-              Gitlab::AppLogger.error "Error moving #{old_path} to #{new_path}"
-            end
-          end
-
-          def projects_for_paths
-            return @projects_for_paths if @projects_for_paths
-
-            with_paths = MigrationClasses::Route.arel_table[:path]
-                           .matches_any(path_patterns)
-
-            @projects_for_paths = MigrationClasses::Project.joins(:route).where(with_paths)
-              .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843')
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/graphql/pagination/active_record_array_connection.rb b/lib/gitlab/graphql/pagination/active_record_array_connection.rb
index 9e40f79b2fd..ce16693cf89 100644
--- a/lib/gitlab/graphql/pagination/active_record_array_connection.rb
+++ b/lib/gitlab/graphql/pagination/active_record_array_connection.rb
@@ -59,6 +59,7 @@ module Gitlab
         def dup
           self.class.new(
             items.dup,
+            context: context,
             first: first,
             after: after,
             max_page_size: max_page_size,
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index c3378856633..4109e952c8f 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -379,7 +379,6 @@ module Gitlab
           bulk_imports: {
             gitlab_v1: count(::BulkImport.where(**time_period, source_type: :gitlab))
           },
-          project_imports: project_imports(time_period),
           issue_imports: issue_imports(time_period),
           group_imports: group_imports(time_period)
         }
@@ -569,24 +568,6 @@ module Gitlab
         omniauth_provider_names.reject { |name| name.starts_with?('ldap') }
       end
 
-      def project_imports(time_period)
-        time_frame = metric_time_period(time_period)
-        counters = {
-          gitlab_project: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'gitlab_project' }),
-          github: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'github' }),
-          bitbucket: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'bitbucket' }),
-          bitbucket_server: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'bitbucket_server' }),
-          gitea: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'gitea' }),
-          git: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'git' }),
-          manifest: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'manifest' }),
-          gitlab_migration: add_metric('CountBulkImportsEntitiesMetric', time_frame: time_frame, options: { source_type: :project_entity })
-        }
-
-        counters[:total] = add_metric('CountImportedProjectsTotalMetric', time_frame: time_frame)
-
-        counters
-      end
-
       def issue_imports(time_period)
         time_frame = metric_time_period(time_period)
         {