Add latest changes from gitlab-org/gitlab@12-4-stable-ee

275 files changed, 5495 insertions, 1520 deletions

diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb
index a3fa7cd5cf9..02ea321df67 100644
--- a/lib/api/api_guard.rb
+++ b/lib/api/api_guard.rb
@@ -17,6 +17,8 @@ module API
         request.access_token
       end
 
+      use AdminModeMiddleware
+
       helpers HelperMethods
 
       install_error_responders(base)
@@ -52,6 +54,11 @@ module API
           forbidden!(api_access_denied_message(user))
         end
 
+        # Set admin mode for API requests (if admin)
+        if Feature.enabled?(:user_mode_in_session)
+          Gitlab::Auth::CurrentUserMode.new(user).enable_admin_mode!(skip_password_validation: true)
+        end
+
         user
       end
 
@@ -141,5 +148,22 @@ module API
         end
       end
     end
+
+    class AdminModeMiddleware < ::Grape::Middleware::Base
+      def initialize(app, **options)
+        super
+      end
+
+      def call(env)
+        if Feature.enabled?(:user_mode_in_session)
+          session = {}
+          Gitlab::Session.with_session(session) do
+            app.call(env)
+          end
+        else
+          app.call(env)
+        end
+      end
+    end
   end
 end
diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb
index d58a5e214ed..d108c811f4b 100644
--- a/lib/api/commit_statuses.rb
+++ b/lib/api/commit_statuses.rb
@@ -58,7 +58,6 @@ module API
       post ':id/statuses/:sha' do
         authorize! :create_commit_status, user_project
 
-        commit = @project.commit(params[:sha])
         not_found! 'Commit' unless commit
 
         # Since the CommitStatus is attached to Ci::Pipeline (in the future Pipeline)
@@ -68,14 +67,15 @@ module API
         # If we don't receive it, we will attach the CommitStatus to
         # the first found branch on that commit
 
+        pipeline = all_matching_pipelines.first
+
         ref = params[:ref]
+        ref ||= pipeline&.ref
         ref ||= @project.repository.branch_names_contains(commit.sha).first
         not_found! 'References for commit' unless ref
 
         name = params[:name] || params[:context] || 'default'
 
-        pipeline = @project.pipeline_for(ref, commit.sha, params[:pipeline_id])
-
         unless pipeline
           pipeline = @project.ci_pipelines.create!(
             source: :external,
@@ -126,6 +126,20 @@ module API
         end
       end
       # rubocop: enable CodeReuse/ActiveRecord
+      helpers do
+        def commit
+          strong_memoize(:commit) do
+            user_project.commit(params[:sha])
+          end
+        end
+
+        def all_matching_pipelines
+          pipelines = user_project.ci_pipelines.newest_first(sha: commit.sha)
+          pipelines = pipelines.for_ref(params[:ref]) if params[:ref]
+          pipelines = pipelines.for_id(params[:pipeline_id]) if params[:pipeline_id]
+          pipelines
+        end
+      end
     end
   end
 end
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index a2f3e87ebd2..ffff40141de 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -37,6 +37,7 @@ module API
         optional :path, type: String, desc: 'The file path'
         optional :all, type: Boolean, desc: 'Every commit will be returned'
         optional :with_stats, type: Boolean, desc: 'Stats about each commit will be added to the response'
+        optional :first_parent, type: Boolean, desc: 'Only include the first parent of merges'
         use :pagination
       end
       get ':id/repository/commits' do
@@ -47,6 +48,7 @@ module API
         offset = (params[:page] - 1) * params[:per_page]
         all = params[:all]
         with_stats = params[:with_stats]
+        first_parent = params[:first_parent]
 
         commits = user_project.repository.commits(ref,
                                                   path: path,
@@ -54,11 +56,12 @@ module API
                                                   offset: offset,
                                                   before: before,
                                                   after: after,
-                                                  all: all)
+                                                  all: all,
+                                                  first_parent: first_parent)
 
         commit_count =
-          if all || path || before || after
-            user_project.repository.count_commits(ref: ref, path: path, before: before, after: after, all: all)
+          if all || path || before || after || first_parent
+            user_project.repository.count_commits(ref: ref, path: path, before: before, after: after, all: all, first_parent: first_parent)
           else
             # Cacheable commit count.
             user_project.repository.commit_count_for_ref(ref)
diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb
index df6d2721977..e86bcc19b2b 100644
--- a/lib/api/deploy_keys.rb
+++ b/lib/api/deploy_keys.rb
@@ -115,14 +115,20 @@ module API
       put ":id/deploy_keys/:key_id" do
         deploy_keys_project = find_by_deploy_key(user_project, params[:key_id])
 
-        authorize!(:update_deploy_key, deploy_keys_project.deploy_key)
+        if !can?(current_user, :update_deploy_key, deploy_keys_project.deploy_key) &&
+            !can?(current_user, :update_deploy_keys_project, deploy_keys_project)
+          forbidden!(nil)
+        end
+
+        update_params = {}
+        update_params[:can_push] = params[:can_push] if params.key?(:can_push)
+        update_params[:deploy_key_attributes] = { id: params[:key_id] }
 
-        can_push = params[:can_push].nil? ? deploy_keys_project.can_push : params[:can_push]
-        title = params[:title] || deploy_keys_project.deploy_key.title
+        if can?(current_user, :update_deploy_key, deploy_keys_project.deploy_key)
+          update_params[:deploy_key_attributes][:title] = params[:title] if params.key?(:title)
+        end
 
-        result = deploy_keys_project.update(can_push: can_push,
-                                            deploy_key_attributes: { id: params[:key_id],
-                                                                     title: title })
+        result = deploy_keys_project.update(update_params)
 
         if result
           present deploy_keys_project, with: Entities::DeployKeysProject
diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb
index eb45df31ff9..da882547071 100644
--- a/lib/api/deployments.rb
+++ b/lib/api/deployments.rb
@@ -42,6 +42,88 @@ module API
 
         present deployment, with: Entities::Deployment
       end
+
+      desc 'Creates a new deployment' do
+        detail 'This feature was introduced in GitLab 12.4'
+        success Entities::Deployment
+      end
+      params do
+        requires :environment,
+          type: String,
+          desc: 'The name of the environment to deploy to'
+
+        requires :sha,
+          type: String,
+          desc: 'The SHA of the commit that was deployed'
+
+        requires :ref,
+          type: String,
+          desc: 'The name of the branch or tag that was deployed'
+
+        requires :tag,
+          type: Boolean,
+          desc: 'A boolean indicating if the deployment ran for a tag'
+
+        requires :status,
+          type: String,
+          desc: 'The status of the deployment',
+          values: %w[running success failed canceled]
+      end
+      post ':id/deployments' do
+        authorize!(:create_deployment, user_project)
+        authorize!(:create_environment, user_project)
+
+        environment = user_project
+          .environments
+          .find_or_create_by_name(params[:environment])
+
+        unless environment.persisted?
+          render_validation_error!(deployment)
+        end
+
+        authorize!(:create_deployment, environment)
+
+        service = ::Deployments::CreateService
+          .new(environment, current_user, declared_params)
+
+        deployment = service.execute
+
+        if deployment.persisted?
+          present(deployment, with: Entities::Deployment, current_user: current_user)
+        else
+          render_validation_error!(deployment)
+        end
+      end
+
+      desc 'Updates an existing deployment' do
+        detail 'This feature was introduced in GitLab 12.4'
+        success Entities::Deployment
+      end
+      params do
+        requires :status,
+          type: String,
+          desc: 'The new status of the deployment',
+          values: %w[running success failed canceled]
+      end
+      put ':id/deployments/:deployment_id' do
+        authorize!(:read_deployment, user_project)
+
+        deployment = user_project.deployments.find(params[:deployment_id])
+
+        authorize!(:update_deployment, deployment)
+
+        if deployment.deployable
+          forbidden!('Deployments created using GitLab CI can not be updated using the API')
+        end
+
+        service = ::Deployments::UpdateService.new(deployment, declared_params)
+
+        if service.execute
+          present(deployment, with: Entities::Deployment, current_user: current_user)
+        else
+          render_validation_error!(deployment)
+        end
+      end
     end
   end
 end
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 89951498489..91811efacd7 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -378,6 +378,13 @@ module API
 
     class Group < BasicGroupDetails
       expose :path, :description, :visibility
+      expose :share_with_group_lock
+      expose :require_two_factor_authentication
+      expose :two_factor_grace_period
+      expose :project_creation_level_str, as: :project_creation_level
+      expose :auto_devops_enabled
+      expose :subgroup_creation_level_str, as: :subgroup_creation_level
+      expose :emails_disabled
       expose :lfs_enabled?, as: :lfs_enabled
       expose :avatar_url do |group, options|
         group.avatar_url(only_path: false)
@@ -682,6 +689,7 @@ module API
 
     class PipelineBasic < Grape::Entity
       expose :id, :sha, :ref, :status
+      expose :created_at, :updated_at
 
       expose :web_url do |pipeline, _options|
         Gitlab::Routing.url_helpers.project_pipeline_url(pipeline.project, pipeline)
@@ -771,7 +779,7 @@ module API
     end
 
     class MergeRequest < MergeRequestBasic
-      expose :subscribed do |merge_request, options|
+      expose :subscribed, if: -> (_, options) { options.fetch(:include_subscribed, true) } do |merge_request, options|
         merge_request.subscribed?(options[:current_user], options[:project])
       end
 
@@ -925,8 +933,8 @@ module API
     end
 
     class PushEventPayload < Grape::Entity
-      expose :commit_count, :action, :ref_type, :commit_from, :commit_to
-      expose :ref, :commit_title
+      expose :commit_count, :action, :ref_type, :commit_from, :commit_to, :ref,
+             :commit_title, :ref_count
     end
 
     class Event < Grape::Entity
@@ -965,13 +973,7 @@ module API
       end
 
       expose :target_url do |todo, options|
-        target_type   = todo.target_type.underscore
-        target_url    = "#{todo.parent.class.to_s.underscore}_#{target_type}_url"
-        target_anchor = "note_#{todo.note_id}" if todo.note_id?
-
-        Gitlab::Routing
-          .url_helpers
-          .public_send(target_url, todo.parent, todo.target, anchor: target_anchor) # rubocop:disable GitlabSecurity/PublicSend
+        todo_target_url(todo)
       end
 
       expose :body
@@ -983,6 +985,19 @@ module API
         # see also https://gitlab.com/gitlab-org/gitlab-foss/issues/59719
         ::API::Entities.const_get(target_type, false)
       end
+
+      def todo_target_url(todo)
+        target_type = todo.target_type.underscore
+        target_url = "#{todo.resource_parent.class.to_s.underscore}_#{target_type}_url"
+
+        Gitlab::Routing
+          .url_helpers
+          .public_send(target_url, todo.resource_parent, todo.target, anchor: todo_target_anchor(todo)) # rubocop:disable GitlabSecurity/PublicSend
+      end
+
+      def todo_target_anchor(todo)
+        "note_#{todo.note_id}" if todo.note_id?
+      end
     end
 
     class NamespaceBasic < Grape::Entity
@@ -1045,7 +1060,7 @@ module API
       expose :job_events
       # Expose serialized properties
       expose :properties do |service, options|
-        # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab-ce/issues/63084
+        # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404
         if service.data_fields_present?
           service.data_fields.as_json.slice(*service.api_field_names)
         else
@@ -1276,7 +1291,7 @@ module API
 
     class Release < Grape::Entity
       expose :name
-      expose :tag, as: :tag_name, if: lambda { |_, _| can_download_code? }
+      expose :tag, as: :tag_name, if: ->(_, _) { can_download_code? }
       expose :description
       expose :description_html do |entity|
         MarkupHelper.markdown_field(entity, :description)
@@ -1284,26 +1299,61 @@ module API
       expose :created_at
       expose :released_at
       expose :author, using: Entities::UserBasic, if: -> (release, _) { release.author.present? }
-      expose :commit, using: Entities::Commit, if: lambda { |_, _| can_download_code? }
+      expose :commit, using: Entities::Commit, if: ->(_, _) { can_download_code? }
       expose :upcoming_release?, as: :upcoming_release
       expose :milestones, using: Entities::Milestone, if: -> (release, _) { release.milestones.present? }
-
+      expose :commit_path, if: ->(_, _) { can_download_code? }
+      expose :tag_path, if: ->(_, _) { can_download_code? }
       expose :assets do
         expose :assets_count, as: :count do |release, _|
           assets_to_exclude = can_download_code? ? [] : [:sources]
           release.assets_count(except: assets_to_exclude)
         end
-        expose :sources, using: Entities::Releases::Source, if: lambda { |_, _| can_download_code? }
+        expose :sources, using: Entities::Releases::Source, if: ->(_, _) { can_download_code? }
         expose :links, using: Entities::Releases::Link do |release, options|
           release.links.sorted
         end
       end
+      expose :_links do
+        expose :merge_requests_url, if: -> (_) { release_mr_issue_urls_available? }
+        expose :issues_url, if: -> (_) { release_mr_issue_urls_available? }
+      end
 
       private
 
       def can_download_code?
         Ability.allowed?(options[:current_user], :download_code, object.project)
       end
+
+      def commit_path
+        return unless object.commit
+
+        Gitlab::Routing.url_helpers.project_commit_path(project, object.commit.id)
+      end
+
+      def tag_path
+        Gitlab::Routing.url_helpers.project_tag_path(project, object.tag)
+      end
+
+      def merge_requests_url
+        Gitlab::Routing.url_helpers.project_merge_requests_url(project, params_for_issues_and_mrs)
+      end
+
+      def issues_url
+        Gitlab::Routing.url_helpers.project_issues_url(project, params_for_issues_and_mrs)
+      end
+
+      def params_for_issues_and_mrs
+        { scope: 'all', state: 'opened', release_tag: object.tag }
+      end
+
+      def release_mr_issue_urls_available?
+        ::Feature.enabled?(:release_mr_issue_urls, project)
+      end
+
+      def project
+        @project ||= object.project
+      end
     end
 
     class Tag < Grape::Entity
@@ -1448,15 +1498,17 @@ module API
     end
 
     class Deployment < Grape::Entity
-      expose :id, :iid, :ref, :sha, :created_at
+      expose :id, :iid, :ref, :sha, :created_at, :updated_at
       expose :user,        using: Entities::UserBasic
       expose :environment, using: Entities::EnvironmentBasic
       expose :deployable,  using: Entities::Job
+      expose :status
     end
 
     class Environment < EnvironmentBasic
       expose :project, using: Entities::BasicProjectDetails
       expose :last_deployment, using: Entities::Deployment, if: { last_deployment: true }
+      expose :state
     end
 
     class LicenseBasic < Grape::Entity
diff --git a/lib/api/group_labels.rb b/lib/api/group_labels.rb
index 79a44941c81..7585293031f 100644
--- a/lib/api/group_labels.rb
+++ b/lib/api/group_labels.rb
@@ -18,10 +18,24 @@ module API
       params do
         optional :with_counts, type: Boolean, default: false,
                  desc: 'Include issue and merge request counts'
+        optional :include_ancestor_groups, type: Boolean, default: true,
+                 desc: 'Include ancestor groups'
         use :pagination
       end
       get ':id/labels' do
-        get_labels(user_group, Entities::GroupLabel)
+        get_labels(user_group, Entities::GroupLabel, include_ancestor_groups: params[:include_ancestor_groups])
+      end
+
+      desc 'Get a single label' do
+        detail 'This feature was added in GitLab 12.4.'
+        success Entities::GroupLabel
+      end
+      params do
+        optional :include_ancestor_groups, type: Boolean, default: true,
+                 desc: 'Include ancestor groups'
+      end
+      get ':id/labels/:name' do
+        get_label(user_group, Entities::GroupLabel, include_ancestor_groups: params[:include_ancestor_groups])
       end
 
       desc 'Create a new label' do
@@ -36,22 +50,21 @@ module API
       end
 
       desc 'Update an existing label. At least one optional parameter is required.' do
-        detail 'This feature was added in GitLab 11.8'
+        detail 'This feature was added in GitLab 11.8 and deprecated in GitLab 12.4.'
         success Entities::GroupLabel
       end
       params do
-        requires :name, type: String, desc: 'The name of the label to be updated'
-        optional :new_name, type: String, desc: 'The new name of the label'
-        optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names"
-        optional :description, type: String, desc: 'The new description of label'
-        at_least_one_of :new_name, :color, :description
+        optional :label_id, type: Integer, desc: 'The id of the label to be updated'
+        optional :name, type: String, desc: 'The name of the label to be updated'
+        use :group_label_update_params
+        exactly_one_of :label_id, :name
       end
       put ':id/labels' do
         update_label(user_group, Entities::GroupLabel)
       end
 
       desc 'Delete an existing label' do
-        detail 'This feature was added in GitLab 11.8'
+        detail 'This feature was added in GitLab 11.8 and deprecated in GitLab 12.4.'
         success Entities::GroupLabel
       end
       params do
@@ -60,6 +73,29 @@ module API
       delete ':id/labels' do
         delete_label(user_group)
       end
+
+      desc 'Update an existing label. At least one optional parameter is required.' do
+        detail 'This feature was added in GitLab 12.4.'
+        success Entities::GroupLabel
+      end
+      params do
+        requires :name, type: String, desc: 'The name or id of the label to be updated'
+        use :group_label_update_params
+      end
+      put ':id/labels/:name' do
+        update_label(user_group, Entities::GroupLabel)
+      end
+
+      desc 'Delete an existing label' do
+        detail 'This feature was added in GitLab 12.4.'
+        success Entities::GroupLabel
+      end
+      params do
+        requires :name, type: String, desc: 'The name or id of the label to be deleted'
+      end
+      delete ':id/labels/:name' do
+        delete_label(user_group)
+      end
     end
   end
 end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index fad8bb13150..19c29847ce3 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -350,7 +350,7 @@ module API
       render_api_error!(message || '409 Conflict', 409)
     end
 
-    def file_to_large!
+    def file_too_large!
       render_api_error!('413 Request Entity Too Large', 413)
     end
 
diff --git a/lib/api/helpers/graphql_helpers.rb b/lib/api/helpers/graphql_helpers.rb
index bd60470fbd6..3ddef0c16b3 100644
--- a/lib/api/helpers/graphql_helpers.rb
+++ b/lib/api/helpers/graphql_helpers.rb
@@ -6,7 +6,7 @@ module API
     # against the graphql API. Helper code for the graphql server implementation
     # should be in app/graphql/ or lib/gitlab/graphql/
     module GraphqlHelpers
-      def conditionally_graphql!(fallback:, query:, context: {}, transform: nil)
+      def run_graphql!(query:, context: {}, transform: nil)
         result = GitlabSchema.execute(query, context: context)
 
         if transform
diff --git a/lib/api/helpers/groups_helpers.rb b/lib/api/helpers/groups_helpers.rb
index 585ae1eb5c4..2cc18acb7ec 100644
--- a/lib/api/helpers/groups_helpers.rb
+++ b/lib/api/helpers/groups_helpers.rb
@@ -10,12 +10,16 @@ module API
         optional :description, type: String, desc: 'The description of the group'
         optional :visibility, type: String,
                  values: Gitlab::VisibilityLevel.string_values,
-                 default: Gitlab::VisibilityLevel.string_level(
-                   Gitlab::CurrentSettings.current_application_settings.default_group_visibility),
                  desc: 'The visibility of the group'
+        optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group'
+        optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users in this group to setup Two-factor authentication'
+        optional :two_factor_grace_period, type: Integer, desc: 'Time before Two-factor authentication is enforced'
+        optional :project_creation_level, type: String, values: ::Gitlab::Access.project_creation_string_values, desc: 'Determine if developers can create projects in the group', as: :project_creation_level_str
+        optional :auto_devops_enabled, type: Boolean, desc: 'Default to Auto DevOps pipeline for all projects within this group'
+        optional :subgroup_creation_level, type: String, values: ::Gitlab::Access.subgroup_creation_string_values, desc: 'Allowed to create subgroups', as: :subgroup_creation_level_str
+        optional :emails_disabled, type: Boolean, desc: 'Disable email notifications'
         optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group'
         optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
-        optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group'
       end
 
       params :optional_params_ee do
diff --git a/lib/api/helpers/label_helpers.rb b/lib/api/helpers/label_helpers.rb
index ec5b688dd1c..2fb2d9b79cf 100644
--- a/lib/api/helpers/label_helpers.rb
+++ b/lib/api/helpers/label_helpers.rb
@@ -11,6 +11,23 @@ module API
         optional :description, type: String, desc: 'The description of label to be created'
       end
 
+      params :label_update_params do
+        optional :new_name, type: String, desc: 'The new name of the label'
+        optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names"
+        optional :description, type: String, desc: 'The new description of label'
+      end
+
+      params :project_label_update_params do
+        use :label_update_params
+        optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true
+        at_least_one_of :new_name, :color, :description, :priority
+      end
+
+      params :group_label_update_params do
+        use :label_update_params
+        at_least_one_of :new_name, :color, :description
+      end
+
       def find_label(parent, id_or_title, include_ancestor_groups: true)
         labels = available_labels_for(parent, include_ancestor_groups: include_ancestor_groups)
         label = labels.find_by_id(id_or_title) || labels.find_by_title(id_or_title)
@@ -18,14 +35,20 @@ module API
         label || not_found!('Label')
       end
 
-      def get_labels(parent, entity)
-        present paginate(available_labels_for(parent)),
+      def get_labels(parent, entity, include_ancestor_groups: true)
+        present paginate(available_labels_for(parent, include_ancestor_groups: include_ancestor_groups)),
                 with: entity,
                 current_user: current_user,
                 parent: parent,
                 with_counts: params[:with_counts]
       end
 
+      def get_label(parent, entity, include_ancestor_groups: true)
+        label = find_label(parent, params_id_or_title, include_ancestor_groups: include_ancestor_groups)
+
+        present label, with: entity, current_user: current_user, parent: parent
+      end
+
       def create_label(parent, entity)
         authorize! :admin_label, parent
 
@@ -57,6 +80,7 @@ module API
 
         # params is used to update the label so we need to remove this field here
         params.delete(:label_id)
+        params.delete(:name)
 
         label = ::Labels::UpdateService.new(declared_params(include_missing: false)).execute(label)
         render_validation_error!(label) unless label.valid?
@@ -80,6 +104,24 @@ module API
         destroy_conditionally!(label)
       end
 
+      def promote_label(parent)
+        authorize! :admin_label, parent
+
+        label = find_label(parent, params[:name], include_ancestor_groups: false)
+
+        begin
+          group_label = ::Labels::PromoteService.new(parent, current_user).execute(label)
+
+          if group_label
+            present group_label, with: Entities::GroupLabel, current_user: current_user, parent: parent.group
+          else
+            render_api_error!('Failed to promote project label to group label', 400)
+          end
+        rescue => error
+          render_api_error!(error.to_s, 400)
+        end
+      end
+
       def params_id_or_title
         @params_id_or_title ||= params[:label_id] || params[:name]
       end
diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb
index 11631378137..fa8b9ad79bd 100644
--- a/lib/api/helpers/runner.rb
+++ b/lib/api/helpers/runner.rb
@@ -59,8 +59,9 @@ module API
         token && job.valid_token?(token)
       end
 
-      def max_artifacts_size
-        Gitlab::CurrentSettings.max_artifacts_size.megabytes.to_i
+      def max_artifacts_size(job)
+        max_size = job.project.closest_setting(:max_artifacts_size)
+        max_size.megabytes.to_i
       end
 
       def job_forbidden!(job, reason)
diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb
index d5f0ddb0805..d9a22484c1f 100644
--- a/lib/api/internal/base.rb
+++ b/lib/api/internal/base.rb
@@ -26,20 +26,11 @@ module API
         def ee_post_receive_response_hook(response)
           # Hook for EE to add messages
         end
-      end
 
-      namespace 'internal' do
-        # Check if git command is allowed for project
-        #
-        # Params:
-        #   key_id - ssh key id for Git over SSH
-        #   user_id - user id for Git over HTTP or over SSH in keyless SSH CERT mode
-        #   username - user name for Git over SSH in keyless SSH cert mode
-        #   protocol - Git access protocol being used, e.g. HTTP or SSH
-        #   project - project full_path (not path on disk)
-        #   action - git action (git-upload-pack or git-receive-pack)
-        #   changes - changes as "oldrev newrev ref", see Gitlab::ChangesList
-        post "/allowed" do
+        def check_allowed(params)
+          # This is a separate method so that EE can alter its behaviour more
+          # easily.
+
           # Stores some Git-specific env thread-safely
           env = parse_env
           Gitlab::Git::HookEnv.set(gl_repository, env) if project
@@ -53,11 +44,11 @@ module API
                            @project ||= access_checker.project
                            result
                          rescue Gitlab::GitAccess::UnauthorizedError => e
-                           break response_with_status(code: 401, success: false, message: e.message)
+                           return response_with_status(code: 401, success: false, message: e.message)
                          rescue Gitlab::GitAccess::TimeoutError => e
-                           break response_with_status(code: 503, success: false, message: e.message)
+                           return response_with_status(code: 503, success: false, message: e.message)
                          rescue Gitlab::GitAccess::NotFoundError => e
-                           break response_with_status(code: 404, success: false, message: e.message)
+                           return response_with_status(code: 404, success: false, message: e.message)
                          end
 
           log_user_activity(actor.user)
@@ -78,6 +69,10 @@ module API
             receive_max_input_size = Gitlab::CurrentSettings.receive_max_input_size.to_i
             if receive_max_input_size > 0
               payload[:git_config_options] << "receive.maxInputSize=#{receive_max_input_size.megabytes}"
+
+              if Feature.enabled?(:gitaly_upload_pack_filter, project)
+                payload[:git_config_options] << "uploadpack.allowFilter=true" << "uploadpack.allowAnySHA1InWant=true"
+              end
             end
 
             response_with_status(**payload)
@@ -87,6 +82,26 @@ module API
             response_with_status(code: 500, success: false, message: UNKNOWN_CHECK_RESULT_ERROR)
           end
         end
+      end
+
+      namespace 'internal' do
+        # Check if git command is allowed for project
+        #
+        # Params:
+        #   key_id - ssh key id for Git over SSH
+        #   user_id - user id for Git over HTTP or over SSH in keyless SSH CERT mode
+        #   username - user name for Git over SSH in keyless SSH cert mode
+        #   protocol - Git access protocol being used, e.g. HTTP or SSH
+        #   project - project full_path (not path on disk)
+        #   action - git action (git-upload-pack or git-receive-pack)
+        #   changes - changes as "oldrev newrev ref", see Gitlab::ChangesList
+        #   check_ip - optional, only in EE version, may limit access to
+        #     group resources based on its IP restrictions
+        post "/allowed" do
+          # It was moved to a separate method so that EE can alter its behaviour more
+          # easily.
+          check_allowed(params)
+        end
 
         # rubocop: disable CodeReuse/ActiveRecord
         post "/lfs_authenticate" do
@@ -108,10 +123,6 @@ module API
         end
         # rubocop: enable CodeReuse/ActiveRecord
 
-        get "/merge_request_urls" do
-          merge_request_urls
-        end
-
         #
         # Get a ssh key using the fingerprint
         #
@@ -129,20 +140,15 @@ module API
         #
         # Discover user by ssh key, user id or username
         #
-        # rubocop: disable CodeReuse/ActiveRecord
-        get "/discover" do
+        get '/discover' do
           if params[:key_id]
-            key = Key.find(params[:key_id])
-            user = key.user
-          elsif params[:user_id]
-            user = User.find_by(id: params[:user_id])
+            user = UserFinder.new(params[:key_id]).find_by_ssh_key_id
           elsif params[:username]
             user = UserFinder.new(params[:username]).find_by_username
           end
 
           present user, with: Entities::UserSafe
         end
-        # rubocop: enable CodeReuse/ActiveRecord
 
         get "/check" do
           {
@@ -153,22 +159,6 @@ module API
           }
         end
 
-        get "/broadcast_messages" do
-          if messages = BroadcastMessage.current
-            present messages, with: Entities::BroadcastMessage
-          else
-            []
-          end
-        end
-
-        get "/broadcast_message" do
-          if message = BroadcastMessage.current&.last
-            present message, with: Entities::BroadcastMessage
-          else
-            {}
-          end
-        end
-
         # rubocop: disable CodeReuse/ActiveRecord
         post '/two_factor_recovery_codes' do
           status 200
diff --git a/lib/api/internal/pages.rb b/lib/api/internal/pages.rb
index eaa434cff51..003af7f6dd4 100644
--- a/lib/api/internal/pages.rb
+++ b/lib/api/internal/pages.rb
@@ -17,11 +17,18 @@ module API
 
       namespace 'internal' do
         namespace 'pages' do
+          desc 'Get GitLab Pages domain configuration by hostname' do
+            detail 'This feature was introduced in GitLab 12.3.'
+          end
+          params do
+            requires :host, type: String, desc: 'The host to query for'
+          end
           get "/" do
-            host = PagesDomain.find_by_domain(params[:host])
+            host = Namespace.find_by_pages_host(params[:host]) || PagesDomain.find_by_domain(params[:host])
             not_found! unless host
 
             virtual_domain = host.pages_virtual_domain
+            no_content! unless virtual_domain
 
             present virtual_domain, with: Entities::Internal::Pages::VirtualDomain
           end
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index de6af980896..4208385a48d 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -343,7 +343,8 @@ module API
         present paginate(::Kaminari.paginate_array(merge_requests)),
           with: Entities::MergeRequest,
           current_user: current_user,
-          project: user_project
+          project: user_project,
+          include_subscribed: false
       end
 
       desc 'List merge requests closing issue' do
diff --git a/lib/api/labels.rb b/lib/api/labels.rb
index de89e94b0c0..2b283d82e4a 100644
--- a/lib/api/labels.rb
+++ b/lib/api/labels.rb
@@ -17,10 +17,24 @@ module API
       params do
         optional :with_counts, type: Boolean, default: false,
                  desc: 'Include issue and merge request counts'
+        optional :include_ancestor_groups, type: Boolean, default: true,
+                 desc: 'Include ancestor groups'
         use :pagination
       end
       get ':id/labels' do
-        get_labels(user_project, Entities::ProjectLabel)
+        get_labels(user_project, Entities::ProjectLabel, include_ancestor_groups: params[:include_ancestor_groups])
+      end
+
+      desc 'Get a single label' do
+        detail 'This feature was added in GitLab 12.4.'
+        success Entities::ProjectLabel
+      end
+      params do
+        optional :include_ancestor_groups, type: Boolean, default: true,
+                 desc: 'Include ancestor groups'
+      end
+      get ':id/labels/:name' do
+        get_label(user_project, Entities::ProjectLabel, include_ancestor_groups: params[:include_ancestor_groups])
       end
 
       desc 'Create a new label' do
@@ -35,23 +49,21 @@ module API
       end
 
       desc 'Update an existing label. At least one optional parameter is required.' do
+        detail 'This feature was deprecated in GitLab 12.4.'
         success Entities::ProjectLabel
       end
       params do
         optional :label_id, type: Integer, desc: 'The id of the label to be updated'
         optional :name, type: String, desc: 'The name of the label to be updated'
-        optional :new_name, type: String, desc: 'The new name of the label'
-        optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names"
-        optional :description, type: String, desc: 'The new description of label'
-        optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true
+        use :project_label_update_params
         exactly_one_of :label_id, :name
-        at_least_one_of :new_name, :color, :description, :priority
       end
       put ':id/labels' do
         update_label(user_project, Entities::ProjectLabel)
       end
 
       desc 'Delete an existing label' do
+        detail 'This feature was deprecated in GitLab 12.4.'
         success Entities::ProjectLabel
       end
       params do
@@ -64,28 +76,48 @@ module API
       end
 
       desc 'Promote a label to a group label' do
-        detail 'This feature was added in GitLab 12.3'
+        detail 'This feature was added in GitLab 12.3 and deprecated in GitLab 12.4.'
         success Entities::GroupLabel
       end
       params do
         requires :name, type: String, desc: 'The name of the label to be promoted'
       end
       put ':id/labels/promote' do
-        authorize! :admin_label, user_project
+        promote_label(user_project)
+      end
 
-        label = find_label(user_project, params[:name], include_ancestor_groups: false)
+      desc 'Update an existing label. At least one optional parameter is required.' do
+        detail 'This feature was added in GitLab 12.4.'
+        success Entities::ProjectLabel
+      end
+      params do
+        requires :name, type: String, desc: 'The name or id of the label to be updated'
+        use :project_label_update_params
+      end
+      put ':id/labels/:name' do
+        update_label(user_project, Entities::ProjectLabel)
+      end
 
-        begin
-          group_label = ::Labels::PromoteService.new(user_project, current_user).execute(label)
+      desc 'Delete an existing label' do
+        detail 'This feature was added in GitLab 12.4.'
+        success Entities::ProjectLabel
+      end
+      params do
+        requires :name, type: String, desc: 'The name or id of the label to be deleted'
+      end
+      delete ':id/labels/:name' do
+        delete_label(user_project)
+      end
 
-          if group_label
-            present group_label, with: Entities::GroupLabel, current_user: current_user, parent: user_project.group
-          else
-            render_api_error!('Failed to promote project label to group label', 400)
-          end
-        rescue => error
-          render_api_error!(error.to_s, 400)
-        end
+      desc 'Promote a label to a group label' do
+        detail 'This feature was added in GitLab 12.4.'
+        success Entities::GroupLabel
+      end
+      params do
+        requires :name, type: String, desc: 'The name or id of the label to be promoted'
+      end
+      put ':id/labels/:name/promote' do
+        promote_label(user_project)
       end
     end
   end
diff --git a/lib/api/members.rb b/lib/api/members.rb
index 461ffe71a62..1d4616fed52 100644
--- a/lib/api/members.rb
+++ b/lib/api/members.rb
@@ -18,6 +18,7 @@ module API
         end
         params do
           optional :query, type: String, desc: 'A query string to search for members'
+          optional :user_ids, type: Array[Integer], desc: 'Array of user ids to look up for membership'
           use :pagination
         end
         # rubocop: disable CodeReuse/ActiveRecord
@@ -26,6 +27,7 @@ module API
 
           members = source.members.where.not(user_id: nil).includes(:user)
           members = members.joins(:user).merge(User.search(params[:query])) if params[:query].present?
+          members = members.where(user_id: params[:user_ids]) if params[:user_ids].present?
           members = paginate(members)
 
           present members, with: Entities::Member
@@ -37,6 +39,7 @@ module API
         end
         params do
           optional :query, type: String, desc: 'A query string to search for members'
+          optional :user_ids, type: Array[Integer], desc: 'Array of user ids to look up for membership'
           use :pagination
         end
         # rubocop: disable CodeReuse/ActiveRecord
@@ -45,6 +48,7 @@ module API
 
           members = find_all_members(source_type, source)
           members = members.includes(:user).references(:user).merge(User.search(params[:query])) if params[:query].present?
+          members = members.where(user_id: params[:user_ids]) if params[:user_ids].present?
           members = paginate(members)
 
           present members, with: Entities::Member
@@ -68,6 +72,23 @@ module API
         end
         # rubocop: enable CodeReuse/ActiveRecord
 
+        desc 'Gets a member of a group or project, including those who gained membership through ancestor group' do
+          success Entities::Member
+        end
+        params do
+          requires :user_id, type: Integer, desc: 'The user ID of the member'
+        end
+        # rubocop: disable CodeReuse/ActiveRecord
+        get ":id/members/all/:user_id" do
+          source = find_source(source_type, params[:id])
+
+          members = find_all_members(source_type, source)
+          member = members.find_by!(user_id: params[:user_id])
+
+          present member, with: Entities::Member
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
         desc 'Adds a member to a group or project.' do
           success Entities::Member
         end
diff --git a/lib/api/notes.rb b/lib/api/notes.rb
index 16fca9acccb..89e4da5a42e 100644
--- a/lib/api/notes.rb
+++ b/lib/api/notes.rb
@@ -80,7 +80,7 @@ module API
           note = create_note(noteable, opts)
 
           if note.valid?
-            present note, with: Entities.const_get(note.class.name)
+            present note, with: Entities.const_get(note.class.name, false)
           else
             bad_request!("Note #{note.errors.messages}")
           end
diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb
index c10ef96922c..2a05974509a 100644
--- a/lib/api/project_container_repositories.rb
+++ b/lib/api/project_container_repositories.rb
@@ -106,9 +106,15 @@ module API
         authorize_destroy_container_image!
         validate_tag!
 
-        tag.delete
-
-        status :ok
+        result = ::Projects::ContainerRepository::DeleteTagsService
+          .new(repository.project, current_user, tags: [declared_params[:tag_name]])
+          .execute(repository)
+
+        if result[:status] == :success
+          status :ok
+        else
+          status :bad_request
+        end
       end
     end
 
diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb
index 7f1ae5ffbe6..b3f17447ea0 100644
--- a/lib/api/project_import.rb
+++ b/lib/api/project_import.rb
@@ -29,6 +29,7 @@ module API
         requires :path, type: String, desc: 'The new project path and name'
         # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960
         requires :file, type: File, desc: 'The project export file to be imported' # rubocop:disable Scalability/FileUploads
+        optional :name, type: String, desc: 'The name of the project to be imported. Defaults to the path of the project if not provided.'
         optional :namespace, type: String, desc: "The ID or name of the namespace that the project will be imported into. Defaults to the current user's namespace."
         optional :overwrite, type: Boolean, default: false, desc: 'If there is a project in the same namespace and with the same name overwrite it'
         optional :override_params,
@@ -55,6 +56,7 @@ module API
         project_params = {
             path: import_params[:path],
             namespace_id: namespace.id,
+            name: import_params[:name],
             file: import_params[:file]['tempfile'],
             overwrite: import_params[:overwrite]
         }
diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb
index ca75ee906ce..c7665c20234 100644
--- a/lib/api/protected_branches.rb
+++ b/lib/api/protected_branches.rb
@@ -42,7 +42,7 @@ module API
       end
       # rubocop: enable CodeReuse/ActiveRecord
 
-      desc 'Protect a single branch or wildcard' do
+      desc 'Protect a single branch' do
         success Entities::ProtectedBranch
       end
       params do
@@ -93,3 +93,5 @@ module API
     end
   end
 end
+
+API::ProtectedBranches.prepend_if_ee('EE::API::ProtectedBranches')
diff --git a/lib/api/runner.rb b/lib/api/runner.rb
index fdf4904e9f5..f383c541f8a 100644
--- a/lib/api/runner.rb
+++ b/lib/api/runner.rb
@@ -221,14 +221,16 @@ module API
         job = authenticate_job!
         forbidden!('Job is not running') unless job.running?
 
+        max_size = max_artifacts_size(job)
+
         if params[:filesize]
           file_size = params[:filesize].to_i
-          file_to_large! unless file_size < max_artifacts_size
+          file_too_large! unless file_size < max_size
         end
 
         status 200
         content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
-        JobArtifactUploader.workhorse_authorize(has_length: false, maximum_size: max_artifacts_size)
+        JobArtifactUploader.workhorse_authorize(has_length: false, maximum_size: max_size)
       end
 
       desc 'Upload artifacts for job' do
@@ -268,7 +270,7 @@ module API
         metadata = UploadedFile.from_params(params, :metadata, JobArtifactUploader.workhorse_local_upload_path)
 
         bad_request!('Missing artifacts file!') unless artifacts
-        file_to_large! unless artifacts.size < max_artifacts_size
+        file_too_large! unless artifacts.size < max_artifacts_size(job)
 
         expire_in = params['expire_in'] ||
           Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in
diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index e4ef507228b..c90ba0c9b5d 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -101,6 +101,8 @@ module API
       optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.'
       optional :project_export_enabled, type: Boolean, desc: 'Enable project export'
       optional :prometheus_metrics_enabled, type: Boolean, desc: 'Enable Prometheus metrics'
+      optional :push_event_hooks_limit, type: Integer, desc: "Number of changes (branches or tags) in a single push to determine whether webhooks and services will be fired or not. Webhooks and services won't be submitted if it surpasses that value."
+      optional :push_event_activities_limit, type: Integer, desc: 'Number of changes (branches or tags) in a single push to determine whether individual push events or bulk push event will be created. Bulk push event will be created if it surpasses that value.'
       optional :recaptcha_enabled, type: Boolean, desc: 'Helps prevent bots from creating accounts'
       given recaptcha_enabled: ->(val) { val } do
         requires :recaptcha_site_key, type: String, desc: 'Generate site key at http://www.google.com/recaptcha'
diff --git a/lib/api/todos.rb b/lib/api/todos.rb
index 404675bfaec..e3f3aca27df 100644
--- a/lib/api/todos.rb
+++ b/lib/api/todos.rb
@@ -49,7 +49,7 @@ module API
     resource :todos do
       helpers do
         def issuable_and_awardable?(type)
-          obj_type = Object.const_get(type)
+          obj_type = Object.const_get(type, false)
 
           (obj_type < Issuable) && (obj_type < Awardable)
         rescue NameError
diff --git a/lib/api/users.rb b/lib/api/users.rb
index ff8b82e1898..ff0b1e87b03 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -459,6 +459,42 @@ module API
       end
       # rubocop: enable CodeReuse/ActiveRecord
 
+      desc 'Activate a deactivated user. Available only for admins.'
+      params do
+        requires :id, type: Integer, desc: 'The ID of the user'
+      end
+      # rubocop: disable CodeReuse/ActiveRecord
+      post ':id/activate' do
+        authenticated_as_admin!
+
+        user = User.find_by(id: params[:id])
+        not_found!('User') unless user
+        forbidden!('A blocked user must be unblocked to be activated') if user.blocked?
+
+        user.activate
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+      desc 'Deactivate an active user. Available only for admins.'
+      params do
+        requires :id, type: Integer, desc: 'The ID of the user'
+      end
+      # rubocop: disable CodeReuse/ActiveRecord
+      post ':id/deactivate' do
+        authenticated_as_admin!
+        user = User.find_by(id: params[:id])
+        not_found!('User') unless user
+
+        break if user.deactivated?
+
+        unless user.can_be_deactivated?
+          forbidden!('A blocked user cannot be deactivated by the API') if user.blocked?
+          forbidden!("The user you are trying to deactivate has been active in the past #{::User::MINIMUM_INACTIVE_DAYS} days and cannot be deactivated")
+        end
+
+        user.deactivate
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
       desc 'Block a user. Available only for admins.'
       params do
         requires :id, type: Integer, desc: 'The ID of the user'
@@ -489,6 +525,8 @@ module API
 
         if user.ldap_blocked?
           forbidden!('LDAP blocked users cannot be unblocked by the API')
+        elsif user.deactivated?
+          forbidden!('Deactivated users cannot be unblocked by the API')
         else
           user.activate
         end
diff --git a/lib/api/version.rb b/lib/api/version.rb
index eca1b529094..f79bb3428f2 100644
--- a/lib/api/version.rb
+++ b/lib/api/version.rb
@@ -19,11 +19,10 @@ module API
       detail 'This feature was introduced in GitLab 8.13.'
     end
     get '/version' do
-      conditionally_graphql!(
+      run_graphql!(
         query: METADATA_QUERY,
         context: { current_user: current_user },
-        transform: ->(result) { result.dig('data', 'metadata') },
-        fallback: -> { { version: Gitlab::VERSION, revision: Gitlab.revision } }
+        transform: ->(result) { result.dig('data', 'metadata') }
       )
     end
   end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index c0390959269..ce0c4c5d974 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -127,7 +127,7 @@ module Backup
         end
 
         tar_file = if ENV['BACKUP'].present?
-                     "#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}"
+                     File.basename(ENV['BACKUP']) + FILE_NAME_SUFFIX
                    else
                      backup_file_list.first
                    end
@@ -235,8 +235,8 @@ module Backup
     end
 
     def tar_file
-      @tar_file ||= if ENV['BACKUP']
-                      ENV['BACKUP'] + "#{FILE_NAME_SUFFIX}"
+      @tar_file ||= if ENV['BACKUP'].present?
+                      File.basename(ENV['BACKUP']) + FILE_NAME_SUFFIX
                     else
                       "#{backup_information[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{backup_information[:gitlab_version]}#{FILE_NAME_SUFFIX}"
                     end
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index 22ed1d8e7b4..974e32ce17c 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -41,12 +41,6 @@ module Backup
       end
     end
 
-    def prepare_directories
-      Gitlab.config.repositories.storages.each do |name, _repository_storage|
-        Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories
-      end
-    end
-
     def backup_project(project)
       path_to_project_bundle = path_to_bundle(project)
       Gitlab::GitalyClient::RepositoryService.new(project.repository)
@@ -75,14 +69,13 @@ module Backup
     end
 
     def restore
-      prepare_directories
-
       Project.find_each(batch_size: 1000) do |project|
         progress.print " * #{project.full_path} ... "
         path_to_project_bundle = path_to_bundle(project)
-        project.ensure_storage_path_exists
 
+        project.repository.remove rescue nil
         restore_repo_success = nil
+
         if File.exist?(path_to_project_bundle)
           begin
             project.repository.create_from_bundle(path_to_project_bundle)
diff --git a/lib/banzai/filter.rb b/lib/banzai/filter.rb
index 7d9766c906c..2438cb3c166 100644
--- a/lib/banzai/filter.rb
+++ b/lib/banzai/filter.rb
@@ -3,7 +3,7 @@
 module Banzai
   module Filter
     def self.[](name)
-      const_get("#{name.to_s.camelize}Filter")
+      const_get("#{name.to_s.camelize}Filter", false)
     end
   end
 end
diff --git a/lib/banzai/filter/ascii_doc_sanitization_filter.rb b/lib/banzai/filter/ascii_doc_sanitization_filter.rb
index 9105e86ad04..e41f7d8488a 100644
--- a/lib/banzai/filter/ascii_doc_sanitization_filter.rb
+++ b/lib/banzai/filter/ascii_doc_sanitization_filter.rb
@@ -22,6 +22,10 @@ module Banzai
       CHECKLIST_CLASSES = %w(fa fa-check-square-o fa-square-o).freeze
       LIST_CLASSES = %w(checklist none no-bullet unnumbered unstyled).freeze
 
+      TABLE_FRAME_CLASSES = %w(frame-all frame-topbot frame-sides frame-ends frame-none).freeze
+      TABLE_GRID_CLASSES = %w(grid-all grid-rows grid-cols grid-none).freeze
+      TABLE_STRIPES_CLASSES = %w(stripes-all stripes-odd stripes-even stripes-hover stripes-none).freeze
+
       ELEMENT_CLASSES_WHITELIST = {
         span: %w(big small underline overline line-through).freeze,
         div: ['admonitionblock'].freeze,
@@ -29,7 +33,8 @@ module Banzai
         i: ADMONITION_CLASSES + CALLOUT_CLASSES + CHECKLIST_CLASSES,
         ul: LIST_CLASSES,
         ol: LIST_CLASSES,
-        a: ['anchor'].freeze
+        a: ['anchor'].freeze,
+        table: TABLE_FRAME_CLASSES + TABLE_GRID_CLASSES + TABLE_STRIPES_CLASSES
       }.freeze
 
       def customize_whitelist(whitelist)
@@ -45,6 +50,7 @@ module Banzai
         whitelist[:attributes]['ul'] = %w(class)
         whitelist[:attributes]['ol'] = %w(class)
         whitelist[:attributes]['a'].push('class')
+        whitelist[:attributes]['table'] = %w(class)
         whitelist[:transformers].push(self.class.remove_element_classes)
 
         # Allow `id` in heading elements for section anchors
diff --git a/lib/banzai/filter/audio_link_filter.rb b/lib/banzai/filter/audio_link_filter.rb
new file mode 100644
index 00000000000..50472c3cf81
--- /dev/null
+++ b/lib/banzai/filter/audio_link_filter.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/audio.js
+module Banzai
+  module Filter
+    class AudioLinkFilter < PlayableLinkFilter
+      private
+
+      def media_type
+        "audio"
+      end
+
+      def safe_media_ext
+        Gitlab::FileTypeDetection::SAFE_AUDIO_EXT
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/playable_link_filter.rb b/lib/banzai/filter/playable_link_filter.rb
new file mode 100644
index 00000000000..0a043aa809c
--- /dev/null
+++ b/lib/banzai/filter/playable_link_filter.rb
@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Banzai
+  module Filter
+    # Find every image that isn't already wrapped in an `a` tag, and that has
+    # a `src` attribute ending with an audio or video extension, add a new audio or video node and
+    # a "Download" link in the case the media cannot be played.
+    class PlayableLinkFilter < HTML::Pipeline::Filter
+      def call
+        doc.xpath('descendant-or-self::img[not(ancestor::a)]').each do |el|
+          el.replace(media_node(doc, el)) if has_media_extension?(el)
+        end
+
+        doc
+      end
+
+      private
+
+      def media_type
+        raise NotImplementedError
+      end
+
+      def safe_media_ext
+        raise NotImplementedError
+      end
+
+      def extra_element_attrs
+        {}
+      end
+
+      def has_media_extension?(element)
+        src = element.attr('data-canonical-src').presence || element.attr('src')
+
+        return unless src.present?
+
+        src_ext = File.extname(src).sub('.', '').downcase
+        safe_media_ext.include?(src_ext)
+      end
+
+      def media_element(doc, element)
+        media_element_attrs = {
+            src: element['src'],
+            controls: true,
+            'data-setup': '{}',
+            'data-title': element['title'] || element['alt']
+        }.merge!(extra_element_attrs)
+
+        if element['data-canonical-src']
+          media_element_attrs['data-canonical-src'] = element['data-canonical-src']
+        end
+
+        doc.document.create_element(media_type, media_element_attrs)
+      end
+
+      def download_paragraph(doc, element)
+        link_content = element['title'] || element['alt']
+
+        link_element_attrs = {
+          href: element['src'],
+          target: '_blank',
+          rel: 'noopener noreferrer',
+          title: "Download '#{link_content}'"
+        }
+
+        # make sure the original non-proxied src carries over
+        if element['data-canonical-src']
+          link_element_attrs['data-canonical-src'] = element['data-canonical-src']
+        end
+
+        link = doc.document.create_element('a', link_content, link_element_attrs)
+
+        doc.document.create_element('p').tap do |paragraph|
+          paragraph.children = link
+        end
+      end
+
+      def media_node(doc, element)
+        container_element_attrs = { class: "#{media_type}-container" }
+
+        doc.document.create_element( "div", container_element_attrs).tap do |container|
+          container.add_child(media_element(doc, element))
+          container.add_child(download_paragraph(doc, element))
+        end
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb
index e8001889ca3..c7589e69262 100644
--- a/lib/banzai/filter/relative_link_filter.rb
+++ b/lib/banzai/filter/relative_link_filter.rb
@@ -20,16 +20,13 @@ module Banzai
       def call
         return doc if context[:system_note]
 
-        @uri_types = {}
         clear_memoization(:linkable_files)
+        clear_memoization(:linkable_attributes)
 
-        doc.search('a:not(.gfm)').each do |el|
-          process_link_attr el.attribute('href')
-        end
+        load_uri_types
 
-        doc.css('img, video').each do |el|
-          process_link_attr el.attribute('src')
-          process_link_attr el.attribute('data-src')
+        linkable_attributes.each do |attr|
+          process_link_attr(attr)
         end
 
         doc
@@ -37,16 +34,80 @@ module Banzai
 
       protected
 
+      def load_uri_types
+        return unless linkable_files?
+        return unless linkable_attributes.present?
+        return {} unless repository
+
+        @uri_types = request_path.present? ? get_uri_types([request_path]) : {}
+
+        paths = linkable_attributes.flat_map do |attr|
+          [get_uri(attr).to_s, relative_file_path(get_uri(attr))]
+        end
+
+        paths.reject!(&:blank?)
+        paths.uniq!
+
+        @uri_types.merge!(get_uri_types(paths))
+      end
+
       def linkable_files?
         strong_memoize(:linkable_files) do
           context[:project_wiki].nil? && repository.try(:exists?) && !repository.empty?
         end
       end
 
-      def process_link_attr(html_attr)
-        return if html_attr.blank?
-        return if html_attr.value.start_with?('//')
+      def linkable_attributes
+        strong_memoize(:linkable_attributes) do
+          attrs = []
+
+          attrs += doc.search('a:not(.gfm)').map do |el|
+            el.attribute('href')
+          end
+
+          attrs += doc.search('img, video, audio').flat_map do |el|
+            [el.attribute('src'), el.attribute('data-src')]
+          end
+
+          attrs.reject do |attr|
+            attr.blank? || attr.value.start_with?('//')
+          end
+        end
+      end
+
+      def get_uri_types(paths)
+        return {} if paths.empty?
+
+        uri_types = Hash[paths.collect { |name| [name, nil] }]
+
+        get_blob_types(paths).each do |name, type|
+          if type == :blob
+            blob = ::Blob.decorate(Gitlab::Git::Blob.new(name: name), project)
+            uri_types[name] = blob.image? || blob.video? || blob.audio? ? :raw : :blob
+          else
+            uri_types[name] = type
+          end
+        end
+
+        uri_types
+      end
 
+      def get_blob_types(paths)
+        revision_paths = paths.collect do |path|
+          [current_commit.sha, path.chomp("/")]
+        end
+
+        Gitlab::GitalyClient::BlobService.new(repository).get_blob_types(revision_paths, 1)
+      end
+
+      def get_uri(html_attr)
+        uri = URI(html_attr.value)
+
+        uri if uri.relative? && uri.path.present?
+      rescue URI::Error, Addressable::URI::InvalidURIError
+      end
+
+      def process_link_attr(html_attr)
         if html_attr.value.start_with?('/uploads/')
           process_link_to_upload_attr(html_attr)
         elsif linkable_files? && repo_visible_to_user?
@@ -81,6 +142,7 @@ module Banzai
 
       def process_link_to_repository_attr(html_attr)
         uri = URI(html_attr.value)
+
         if uri.relative? && uri.path.present?
           html_attr.value = rebuild_relative_uri(uri).to_s
         end
@@ -89,7 +151,7 @@ module Banzai
       end
 
       def rebuild_relative_uri(uri)
-        file_path = relative_file_path(uri)
+        file_path = nested_file_path_if_exists(uri)
 
         uri.path = [
           relative_url_root,
@@ -102,13 +164,29 @@ module Banzai
         uri
       end
 
-      def relative_file_path(uri)
-        path = Addressable::URI.unescape(uri.path).delete("\0")
-        request_path = Addressable::URI.unescape(context[:requested_path])
-        nested_path = build_relative_path(path, request_path)
+      def nested_file_path_if_exists(uri)
+        path = cleaned_file_path(uri)
+        nested_path = relative_file_path(uri)
+
         file_exists?(nested_path) ? nested_path : path
       end
 
+      def cleaned_file_path(uri)
+        Addressable::URI.unescape(uri.path).delete("\0").chomp("/")
+      end
+
+      def relative_file_path(uri)
+        return if uri.nil?
+
+        build_relative_path(cleaned_file_path(uri), request_path)
+      end
+
+      def request_path
+        return unless context[:requested_path]
+
+        Addressable::URI.unescape(context[:requested_path]).chomp("/")
+      end
+
       # Convert a relative path into its correct location based on the currently
       # requested path
       #
@@ -136,6 +214,7 @@ module Banzai
         return path[1..-1] if path.start_with?('/')
 
         parts = request_path.split('/')
+
         parts.pop if uri_type(request_path) != :tree
 
         path.sub!(%r{\A\./}, '')
@@ -149,14 +228,11 @@ module Banzai
       end
 
       def file_exists?(path)
-        path.present? && !!uri_type(path)
+        path.present? && uri_type(path).present?
       end
 
       def uri_type(path)
-        # https://gitlab.com/gitlab-org/gitlab-foss/issues/58657
-        Gitlab::GitalyClient.allow_n_plus_1_calls do
-          @uri_types[path] ||= current_commit.uri_type(path)
-        end
+        @uri_types[path] == :unknown ? "" : @uri_types[path]
       end
 
       def current_commit
diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb
index ade4d260be1..a2c8e92e560 100644
--- a/lib/banzai/filter/table_of_contents_filter.rb
+++ b/lib/banzai/filter/table_of_contents_filter.rb
@@ -56,7 +56,8 @@ module Banzai
       private
 
       def anchor_tag(href)
-        %Q{<a id="user-content-#{href}" class="anchor" href="##{href}" aria-hidden="true"></a>}
+        escaped_href = CGI.escape(href) # account for non-ASCII characters
+        %Q{<a id="user-content-#{href}" class="anchor" href="##{escaped_href}" aria-hidden="true"></a>}
       end
 
       def push_toc(children, root: false)
@@ -80,7 +81,7 @@ module Banzai
 
         def initialize(node: nil, href: nil, previous_header: nil)
           @node = node
-          @href = href
+          @href = CGI.escape(href) if href
           @children = []
 
           @parent = find_parent(previous_header)
diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb
index a278fcfdb47..ed82fbc1f94 100644
--- a/lib/banzai/filter/video_link_filter.rb
+++ b/lib/banzai/filter/video_link_filter.rb
@@ -3,73 +3,19 @@
 # Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/video.js
 module Banzai
   module Filter
-    # Find every image that isn't already wrapped in an `a` tag, and that has
-    # a `src` attribute ending with a video extension, add a new video node and
-    # a "Download" link in the case the video cannot be played.
-    class VideoLinkFilter < HTML::Pipeline::Filter
-      def call
-        doc.xpath(query).each do |el|
-          el.replace(video_node(doc, el))
-        end
-
-        doc
-      end
-
+    class VideoLinkFilter < PlayableLinkFilter
       private
 
-      def query
-        @query ||= begin
-          src_query = UploaderHelper::VIDEO_EXT.map do |ext|
-            "'.#{ext}' = substring(@src, string-length(@src) - #{ext.size})"
-          end
-
-          if context[:asset_proxy_enabled].present?
-            src_query.concat(
-              UploaderHelper::VIDEO_EXT.map do |ext|
-                "'.#{ext}' = substring(@data-canonical-src, string-length(@data-canonical-src) - #{ext.size})"
-              end
-            )
-          end
-
-          "descendant-or-self::img[not(ancestor::a) and (#{src_query.join(' or ')})]"
-        end
+      def media_type
+        "video"
       end
 
-      def video_node(doc, element)
-        container = doc.document.create_element(
-          'div',
-          class: 'video-container'
-        )
-
-        video = doc.document.create_element(
-          'video',
-          src: element['src'],
-          width: '400',
-          controls: true,
-          'data-setup' => '{}',
-          'data-title' => element['title'] || element['alt'])
-
-        link = doc.document.create_element(
-          'a',
-          element['title'] || element['alt'],
-          href: element['src'],
-          target: '_blank',
-          rel: 'noopener noreferrer',
-          title: "Download '#{element['title'] || element['alt']}'")
-
-        # make sure the original non-proxied src carries over
-        if element['data-canonical-src']
-          video['data-canonical-src'] = element['data-canonical-src']
-          link['data-canonical-src']  = element['data-canonical-src']
-        end
-
-        download_paragraph = doc.document.create_element('p')
-        download_paragraph.children = link
-
-        container.add_child(video)
-        container.add_child(download_paragraph)
+      def safe_media_ext
+        Gitlab::FileTypeDetection::SAFE_VIDEO_EXT
+      end
 
-        container
+      def extra_element_attrs
+        { width: "100%" }
       end
     end
   end
diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb
index 18947679b69..205f777bc90 100644
--- a/lib/banzai/filter/wiki_link_filter.rb
+++ b/lib/banzai/filter/wiki_link_filter.rb
@@ -15,7 +15,7 @@ module Banzai
 
         doc.search('a:not(.gfm)').each { |el| process_link(el.attribute('href'), el) }
 
-        doc.search('video').each { |el| process_link(el.attribute('src'), el) }
+        doc.search('video, audio').each { |el| process_link(el.attribute('src'), el) }
 
         doc.search('img').each do |el|
           attr = el.attribute('data-src') || el.attribute('src')
diff --git a/lib/banzai/pipeline.rb b/lib/banzai/pipeline.rb
index e8a81bebaa9..497d3f27542 100644
--- a/lib/banzai/pipeline.rb
+++ b/lib/banzai/pipeline.rb
@@ -4,7 +4,7 @@ module Banzai
   module Pipeline
     def self.[](name)
       name ||= :full
-      const_get("#{name.to_s.camelize}Pipeline")
+      const_get("#{name.to_s.camelize}Pipeline", false)
     end
   end
 end
diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb
index bb0d1eaa1e1..08e27257fdf 100644
--- a/lib/banzai/pipeline/gfm_pipeline.rb
+++ b/lib/banzai/pipeline/gfm_pipeline.rb
@@ -26,6 +26,7 @@ module Banzai
           Filter::ColorFilter,
           Filter::MermaidFilter,
           Filter::VideoLinkFilter,
+          Filter::AudioLinkFilter,
           Filter::ImageLazyLoadFilter,
           Filter::ImageLinkFilter,
           Filter::InlineMetricsFilter,
diff --git a/lib/banzai/reference_parser.rb b/lib/banzai/reference_parser.rb
index efe15096f08..c08d3364a87 100644
--- a/lib/banzai/reference_parser.rb
+++ b/lib/banzai/reference_parser.rb
@@ -10,7 +10,7 @@ module Banzai
     #
     # This would return the `Banzai::ReferenceParser::IssueParser` class.
     def self.[](name)
-      const_get("#{name.to_s.camelize}Parser")
+      const_get("#{name.to_s.camelize}Parser", false)
     end
   end
 end
diff --git a/lib/banzai/reference_parser/mentioned_user_parser.rb b/lib/banzai/reference_parser/mentioned_user_parser.rb
new file mode 100644
index 00000000000..4b1bcb3ca09
--- /dev/null
+++ b/lib/banzai/reference_parser/mentioned_user_parser.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Banzai
+  module ReferenceParser
+    class MentionedUserParser < BaseParser
+      self.reference_type = :user
+
+      def references_relation
+        User
+      end
+
+      # any user can be mentioned by username
+      def can_read_reference?(user, ref_attr, node)
+        true
+      end
+    end
+  end
+end
diff --git a/lib/banzai/reference_parser/mentioned_users_by_group_parser.rb b/lib/banzai/reference_parser/mentioned_users_by_group_parser.rb
new file mode 100644
index 00000000000..d4ff6a12cd0
--- /dev/null
+++ b/lib/banzai/reference_parser/mentioned_users_by_group_parser.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Banzai
+  module ReferenceParser
+    class MentionedUsersByGroupParser < BaseParser
+      GROUP_ATTR = 'data-group'
+
+      self.reference_type = :user
+
+      def self.data_attribute
+        @data_attribute ||= GROUP_ATTR
+      end
+
+      def references_relation
+        Group
+      end
+
+      def nodes_visible_to_user(user, nodes)
+        groups = lazy { grouped_objects_for_nodes(nodes, Group, GROUP_ATTR) }
+
+        nodes.select do |node|
+          node.has_attribute?(GROUP_ATTR) && can_read_group_reference?(node, user, groups)
+        end
+      end
+
+      def can_read_group_reference?(node, user, groups)
+        node_group = groups[node]
+
+        node_group && can?(user, :read_group, node_group)
+      end
+    end
+  end
+end
diff --git a/lib/banzai/reference_parser/mentioned_users_by_project_parser.rb b/lib/banzai/reference_parser/mentioned_users_by_project_parser.rb
new file mode 100644
index 00000000000..79258d81cc3
--- /dev/null
+++ b/lib/banzai/reference_parser/mentioned_users_by_project_parser.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Banzai
+  module ReferenceParser
+    class MentionedUsersByProjectParser < ProjectParser
+      PROJECT_ATTR = 'data-project'
+
+      self.reference_type = :user
+
+      def self.data_attribute
+        @data_attribute ||= PROJECT_ATTR
+      end
+
+      def references_relation
+        Project
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb
index 1343f424c51..92894575ec2 100644
--- a/lib/bitbucket/client.rb
+++ b/lib/bitbucket/client.rb
@@ -38,8 +38,10 @@ module Bitbucket
       Representation::Repo.new(parsed_response)
     end
 
-    def repos
+    def repos(filter: nil)
       path = "/repositories?role=member"
+      path += "&q=name~\"#{filter}\"" if filter
+
       get_collection(path, :repo)
     end
 
diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb
index 7cc1342ad65..38c689628dd 100644
--- a/lib/bitbucket/page.rb
+++ b/lib/bitbucket/page.rb
@@ -30,7 +30,7 @@ module Bitbucket
     end
 
     def representation_class(type)
-      Bitbucket::Representation.const_get(type.to_s.camelize)
+      Bitbucket::Representation.const_get(type.to_s.camelize, false)
     end
   end
 end
diff --git a/lib/bitbucket_server/page.rb b/lib/bitbucket_server/page.rb
index 5d9a3168876..304f7cd9d72 100644
--- a/lib/bitbucket_server/page.rb
+++ b/lib/bitbucket_server/page.rb
@@ -30,7 +30,7 @@ module BitbucketServer
     end
 
     def representation_class(type)
-      BitbucketServer::Representation.const_get(type.to_s.camelize)
+      BitbucketServer::Representation.const_get(type.to_s.camelize, false)
     end
   end
 end
diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb
index 15f40993ea3..92861c567a8 100644
--- a/lib/container_registry/client.rb
+++ b/lib/container_registry/client.rb
@@ -2,6 +2,7 @@
 
 require 'faraday'
 require 'faraday_middleware'
+require 'digest'
 
 module ContainerRegistry
   class Client
@@ -9,6 +10,8 @@ module ContainerRegistry
 
     DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE = 'application/vnd.docker.distribution.manifest.v2+json'
     OCI_MANIFEST_V1_TYPE = 'application/vnd.oci.image.manifest.v1+json'
+    CONTAINER_IMAGE_V1_TYPE = 'application/vnd.docker.container.image.v1+json'
+
     ACCEPTED_TYPES = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE].freeze
 
     # Taken from: FaradayMiddleware::FollowRedirects
@@ -33,7 +36,48 @@ module ContainerRegistry
     end
 
     def delete_repository_tag(name, reference)
-      faraday.delete("/v2/#{name}/manifests/#{reference}").success?
+      result = faraday.delete("/v2/#{name}/manifests/#{reference}")
+
+      result.success? || result.status == 404
+    end
+
+    def upload_raw_blob(path, blob)
+      digest = "sha256:#{Digest::SHA256.hexdigest(blob)}"
+
+      if upload_blob(path, blob, digest).success?
+        [blob, digest]
+      end
+    end
+
+    def upload_blob(name, content, digest)
+      upload = faraday.post("/v2/#{name}/blobs/uploads/")
+      return unless upload.success?
+
+      location = URI(upload.headers['location'])
+
+      faraday.put("#{location.path}?#{location.query}") do |req|
+        req.params['digest'] = digest
+        req.headers['Content-Type'] = 'application/octet-stream'
+        req.body = content
+      end
+    end
+
+    def generate_empty_manifest(path)
+      image = {
+        config: {}
+      }
+      image, image_digest = upload_raw_blob(path, JSON.pretty_generate(image))
+      return unless image
+
+      {
+        schemaVersion: 2,
+        mediaType: DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE,
+        config: {
+          mediaType: CONTAINER_IMAGE_V1_TYPE,
+          size: image.size,
+          digest: image_digest
+        }
+      }
     end
 
     def blob(name, digest, type = nil)
@@ -42,7 +86,18 @@ module ContainerRegistry
     end
 
     def delete_blob(name, digest)
-      faraday.delete("/v2/#{name}/blobs/#{digest}").success?
+      result = faraday.delete("/v2/#{name}/blobs/#{digest}")
+
+      result.success? || result.status == 404
+    end
+
+    def put_tag(name, reference, manifest)
+      response = faraday.put("/v2/#{name}/manifests/#{reference}") do |req|
+        req.headers['Content-Type'] = DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE
+        req.body = JSON.pretty_generate(manifest)
+      end
+
+      response.headers['docker-content-digest'] if response.success?
     end
 
     private
diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb
index ebea84fa1ca..2cc4c8d8b1c 100644
--- a/lib/container_registry/tag.rb
+++ b/lib/container_registry/tag.rb
@@ -98,6 +98,10 @@ module ContainerRegistry
       end
     end
 
+    def put(digests)
+      repository.client.put_tag(repository.path, name, digests)
+    end
+
     # rubocop: disable CodeReuse/ActiveRecord
     def total_size
       return unless layers
@@ -106,7 +110,10 @@ module ContainerRegistry
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
-    def delete
+    # Deletes the image associated with this tag
+    # Note this will delete the image and all tags associated with it.
+    # Consider using DeleteTagsService instead.
+    def unsafe_delete
       return unless digest
 
       client.delete_repository_tag(repository.path, digest)
diff --git a/lib/event_filter.rb b/lib/event_filter.rb
index 85bf9c14f26..e062e3ddb1c 100644
--- a/lib/event_filter.rb
+++ b/lib/event_filter.rb
@@ -9,12 +9,11 @@ class EventFilter
   ISSUE = 'issue'
   COMMENTS = 'comments'
   TEAM = 'team'
-  FILTERS = [ALL, PUSH, MERGED, ISSUE, COMMENTS, TEAM].freeze
 
   def initialize(filter)
     # Split using comma to maintain backward compatibility Ex/ "filter1,filter2"
     filter = filter.to_s.split(',')[0].to_s
-    @filter = FILTERS.include?(filter) ? filter : ALL
+    @filter = filters.include?(filter) ? filter : ALL
   end
 
   def active?(key)
@@ -39,4 +38,12 @@ class EventFilter
     end
   end
   # rubocop: enable CodeReuse/ActiveRecord
+
+  private
+
+  def filters
+    [ALL, PUSH, MERGED, ISSUE, COMMENTS, TEAM]
+  end
 end
+
+EventFilter.prepend_if_ee('EE::EventFilter')
diff --git a/lib/gitlab.rb b/lib/gitlab.rb
index b337f5cbf2c..ad8e693ccbc 100644
--- a/lib/gitlab.rb
+++ b/lib/gitlab.rb
@@ -65,14 +65,18 @@ module Gitlab
 
   def self.ee?
     @is_ee ||=
-      if ENV['IS_GITLAB_EE'] && !ENV['IS_GITLAB_EE'].empty?
-        Gitlab::Utils.to_boolean(ENV['IS_GITLAB_EE'])
-      else
-        # We may use this method when the Rails environment is not loaded. This
-        # means that checking the presence of the License class could result in
-        # this method returning `false`, even for an EE installation.
-        root.join('ee/app/models/license.rb').exist?
-      end
+      # We use this method when the Rails environment is not loaded. This
+      # means that checking the presence of the License class could result in
+      # this method returning `false`, even for an EE installation.
+      #
+      # The `FOSS_ONLY` is always `string` or `nil`
+      # Thus the nil or empty string will result
+      # in using default value: false
+      #
+      # The behavior needs to be synchronised with
+      # config/helpers/is_ee_env.js
+      root.join('ee/app/models/license.rb').exist? &&
+        !%w[true 1].include?(ENV['FOSS_ONLY'].to_s)
   end
 
   def self.ee
diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb
index ed5816482a9..6492ccc286a 100644
--- a/lib/gitlab/access.rb
+++ b/lib/gitlab/access.rb
@@ -103,10 +103,22 @@ module Gitlab
         }
       end
 
+      def project_creation_string_options
+        {
+          'noone'       => NO_ONE_PROJECT_ACCESS,
+          'maintainer'  => MAINTAINER_PROJECT_ACCESS,
+          'developer'   => DEVELOPER_MAINTAINER_PROJECT_ACCESS
+        }
+      end
+
       def project_creation_values
         project_creation_options.values
       end
 
+      def project_creation_string_values
+        project_creation_string_options.keys
+      end
+
       def project_creation_level_name(name)
         project_creation_options.key(name)
       end
@@ -117,6 +129,21 @@ module Gitlab
           s_('SubgroupCreationlevel|Maintainers') => MAINTAINER_SUBGROUP_ACCESS
         }
       end
+
+      def subgroup_creation_string_options
+        {
+          'owner'      => OWNER_SUBGROUP_ACCESS,
+          'maintainer' => MAINTAINER_SUBGROUP_ACCESS
+        }
+      end
+
+      def subgroup_creation_values
+        subgroup_creation_options.values
+      end
+
+      def subgroup_creation_string_values
+        subgroup_creation_string_options.keys
+      end
     end
 
     def human_access
diff --git a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb
new file mode 100644
index 00000000000..33cbe1a62ef
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb
@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      class BaseQueryBuilder
+        include Gitlab::CycleAnalytics::MetricsTables
+
+        delegate :subject_class, to: :stage
+
+        # rubocop: disable CodeReuse/ActiveRecord
+
+        def initialize(stage:, params: {})
+          @stage = stage
+          @params = params
+        end
+
+        def build
+          query = subject_class
+          query = filter_by_parent_model(query)
+          query = filter_by_time_range(query)
+          query = stage.start_event.apply_query_customization(query)
+          query = stage.end_event.apply_query_customization(query)
+          query.where(duration_condition)
+        end
+
+        private
+
+        attr_reader :stage, :params
+
+        def duration_condition
+          stage.end_event.timestamp_projection.gteq(stage.start_event.timestamp_projection)
+        end
+
+        def filter_by_parent_model(query)
+          if parent_class.eql?(Project)
+            if subject_class.eql?(Issue)
+              query.where(project_id: stage.parent_id)
+            elsif subject_class.eql?(MergeRequest)
+              query.where(target_project_id: stage.parent_id)
+            else
+              raise ArgumentError, "unknown subject_class: #{subject_class}"
+            end
+          else
+            raise ArgumentError, "unknown parent_class: #{parent_class}"
+          end
+        end
+
+        def filter_by_time_range(query)
+          from = params.fetch(:from, 30.days.ago)
+          to = params[:to]
+
+          query = query.where(subject_table[:created_at].gteq(from))
+          query = query.where(subject_table[:created_at].lteq(to)) if to
+          query
+        end
+
+        def subject_table
+          subject_class.arel_table
+        end
+
+        def parent_class
+          stage.parent.class
+        end
+
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb
new file mode 100644
index 00000000000..0c0f737f2c9
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      # Arguments:
+      #   stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::GroupStage
+      #   params:
+      #     current_user: an instance of User
+      #     from: DateTime
+      #     to: DateTime
+      class DataCollector
+        include Gitlab::Utils::StrongMemoize
+
+        def initialize(stage:, params: {})
+          @stage = stage
+          @params = params
+        end
+
+        def records_fetcher
+          strong_memoize(:records_fetcher) do
+            RecordsFetcher.new(stage: stage, query: query, params: params)
+          end
+        end
+
+        def median
+          strong_memoize(:median) do
+            Median.new(stage: stage, query: query)
+          end
+        end
+
+        private
+
+        attr_reader :stage, :params
+
+        def query
+          BaseQueryBuilder.new(stage: stage, params: params).build
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/default_stages.rb b/lib/gitlab/analytics/cycle_analytics/default_stages.rb
index 286c393005f..8e70236ce75 100644
--- a/lib/gitlab/analytics/cycle_analytics/default_stages.rb
+++ b/lib/gitlab/analytics/cycle_analytics/default_stages.rb
@@ -23,6 +23,10 @@ module Gitlab
           ]
         end
 
+        def self.names
+          all.map { |stage| stage[:name] }
+        end
+
         def self.params_for_issue_stage
           {
             name: 'issue',
@@ -88,8 +92,8 @@ module Gitlab
             name: 'production',
             custom: false,
             relative_position: 7,
-            start_event_identifier: :merge_request_merged,
-            end_event_identifier: :merge_request_first_deployed_to_production
+            start_event_identifier: :issue_created,
+            end_event_identifier: :production_stage_end
           }
         end
       end
diff --git a/lib/gitlab/analytics/cycle_analytics/median.rb b/lib/gitlab/analytics/cycle_analytics/median.rb
new file mode 100644
index 00000000000..41883a80338
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/median.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      class Median
+        include StageQueryHelpers
+
+        def initialize(stage:, query:)
+          @stage = stage
+          @query = query
+        end
+
+        def seconds
+          @query = @query.select(median_duration_in_seconds.as('median'))
+          result = execute_query(@query).first || {}
+
+          result['median'] ? result['median'].to_i : nil
+        end
+
+        private
+
+        attr_reader :stage
+
+        def percentile_cont
+          percentile_cont_ordering = Arel::Nodes::UnaryOperation.new(Arel::Nodes::SqlLiteral.new('ORDER BY'), duration)
+          Arel::Nodes::NamedFunction.new(
+            'percentile_cont(0.5) WITHIN GROUP',
+            [percentile_cont_ordering]
+          )
+        end
+
+        def median_duration_in_seconds
+          Arel::Nodes::Extract.new(percentile_cont, :epoch)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
new file mode 100644
index 00000000000..90d03142b2a
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      class RecordsFetcher
+        include Gitlab::Utils::StrongMemoize
+        include StageQueryHelpers
+        include Gitlab::CycleAnalytics::MetricsTables
+
+        MAX_RECORDS = 20
+
+        MAPPINGS = {
+          Issue => {
+            finder_class: IssuesFinder,
+            serializer_class: AnalyticsIssueSerializer,
+            includes_for_query: { project: [:namespace], author: [] },
+            columns_for_select: %I[title iid id created_at author_id project_id]
+          },
+          MergeRequest => {
+            finder_class: MergeRequestsFinder,
+            serializer_class: AnalyticsMergeRequestSerializer,
+            includes_for_query: { target_project: [:namespace], author: [] },
+            columns_for_select: %I[title iid id created_at author_id state target_project_id]
+          }
+        }.freeze
+
+        delegate :subject_class, to: :stage
+
+        def initialize(stage:, query:, params: {})
+          @stage = stage
+          @query = query
+          @params = params
+        end
+
+        def serialized_records
+          strong_memoize(:serialized_records) do
+            # special case (legacy): 'Test' and 'Staging' stages should show Ci::Build records
+            if default_test_stage? || default_staging_stage?
+              AnalyticsBuildSerializer.new.represent(ci_build_records.map { |e| e['build'] })
+            else
+              records.map do |record|
+                project = record.project
+                attributes = record.attributes.merge({
+                  project_path: project.path,
+                  namespace_path: project.namespace.path,
+                  author: record.author
+                })
+                serializer.represent(attributes)
+              end
+            end
+          end
+        end
+
+        private
+
+        attr_reader :stage, :query, :params
+
+        def finder_query
+          MAPPINGS
+            .fetch(subject_class)
+            .fetch(:finder_class)
+            .new(params.fetch(:current_user), finder_params.fetch(stage.parent.class))
+            .execute
+        end
+
+        def columns
+          MAPPINGS.fetch(subject_class).fetch(:columns_for_select).map do |column_name|
+            subject_class.arel_table[column_name]
+          end
+        end
+
+        # EE will override this to include Group rules
+        def finder_params
+          {
+            Project => { project_id: stage.parent_id }
+          }
+        end
+
+        def default_test_stage?
+          stage.matches_with_stage_params?(Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_test_stage)
+        end
+
+        def default_staging_stage?
+          stage.matches_with_stage_params?(Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_staging_stage)
+        end
+
+        def serializer
+          MAPPINGS.fetch(subject_class).fetch(:serializer_class).new
+        end
+
+        # Loading Ci::Build records instead of MergeRequest records
+        # rubocop: disable CodeReuse/ActiveRecord
+        def ci_build_records
+          ci_build_join = mr_metrics_table
+            .join(build_table)
+            .on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id]))
+            .join_sources
+
+          q = ordered_and_limited_query
+            .joins(ci_build_join)
+            .select(build_table[:id], round_duration_to_seconds.as('total_time'))
+
+          results = execute_query(q).to_a
+
+          Gitlab::CycleAnalytics::Updater.update!(results, from: 'id', to: 'build', klass: ::Ci::Build.includes({ project: [:namespace], user: [], pipeline: [] }))
+        end
+
+        def ordered_and_limited_query
+          query
+            .reorder(stage.end_event.timestamp_projection.desc)
+            .limit(MAX_RECORDS)
+        end
+
+        def records
+          results = finder_query
+            .merge(ordered_and_limited_query)
+            .select(*columns, round_duration_to_seconds.as('total_time'))
+
+          # using preloader instead of includes to avoid AR generating a large column list
+          ActiveRecord::Associations::Preloader.new.preload(
+            results,
+            MAPPINGS.fetch(subject_class).fetch(:includes_for_query)
+          )
+
+          results
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb
index d21f344f483..58572446de6 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb
@@ -18,7 +18,8 @@ module Gitlab
           StageEvents::MergeRequestMerged => 104,
           StageEvents::CodeStageStart => 1_000,
           StageEvents::IssueStageEnd => 1_001,
-          StageEvents::PlanStageStart => 1_002
+          StageEvents::PlanStageStart => 1_002,
+          StageEvents::ProductionStageEnd => 1_003
         }.freeze
 
         EVENTS = ENUM_MAPPING.keys.freeze
@@ -32,7 +33,8 @@ module Gitlab
             StageEvents::MergeRequestCreated
           ],
           StageEvents::IssueCreated => [
-            StageEvents::IssueStageEnd
+            StageEvents::IssueStageEnd,
+            StageEvents::ProductionStageEnd
           ],
           StageEvents::MergeRequestCreated => [
             StageEvents::MergeRequestMerged
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb
index ff9c8a79225..6af1b90bccc 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb
@@ -16,6 +16,21 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def timestamp_projection
+            issue_metrics_table[:first_mentioned_in_commit_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            issue_metrics_join = mr_closing_issues_table
+              .join(issue_metrics_table)
+              .on(mr_closing_issues_table[:issue_id].eq(issue_metrics_table[:issue_id]))
+              .join_sources
+
+            query.joins(:merge_requests_closing_issues).joins(issue_metrics_join)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb
index a601c9797f8..8c9a80740a9 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb
@@ -16,6 +16,10 @@ module Gitlab
           def object_type
             Issue
           end
+
+          def timestamp_projection
+            issue_table[:created_at]
+          end
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb
index 7424043ef7b..fe7f2d85f8b 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb
@@ -16,6 +16,16 @@ module Gitlab
           def object_type
             Issue
           end
+
+          def timestamp_projection
+            issue_metrics_table[:first_mentioned_in_commit_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(:metrics)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb
index ceb229c552f..77e4092b9ab 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb
@@ -16,6 +16,19 @@ module Gitlab
           def object_type
             Issue
           end
+
+          def timestamp_projection
+            Arel::Nodes::NamedFunction.new('COALESCE', [
+              issue_metrics_table[:first_associated_with_milestone_at],
+              issue_metrics_table[:first_added_to_board_at]
+            ])
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(:metrics).where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil)))
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb
index 8be00831b4f..7059c425b8f 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb
@@ -16,6 +16,10 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def timestamp_projection
+            mr_table[:created_at]
+          end
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb
index 6d7a2c023ff..3d7482eaaf0 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb
@@ -16,6 +16,16 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def timestamp_projection
+            mr_metrics_table[:first_deployed_to_production_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(:metrics).where(timestamp_projection.gteq(mr_table[:created_at]))
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb
index 12d82fe2c62..36bb4d6fc8d 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb
@@ -16,6 +16,16 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def timestamp_projection
+            mr_metrics_table[:latest_build_finished_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(:metrics)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb
index 9e749b0fdfa..468d9899cc7 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb
@@ -16,6 +16,16 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def timestamp_projection
+            mr_metrics_table[:latest_build_started_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(:metrics)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb
index bbfb5d12992..82ecaf1cd6b 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb
@@ -16,6 +16,16 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def timestamp_projection
+            mr_metrics_table[:merged_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(:metrics)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
index 803317d8b55..7ece7d62faa 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
@@ -16,6 +16,22 @@ module Gitlab
           def object_type
             Issue
           end
+
+          def timestamp_projection
+            Arel::Nodes::NamedFunction.new('COALESCE', [
+              issue_metrics_table[:first_associated_with_milestone_at],
+              issue_metrics_table[:first_added_to_board_at]
+            ])
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query
+              .joins(:metrics)
+              .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil)))
+              .where(issue_metrics_table[:first_mentioned_in_commit_at].not_eq(nil))
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb
new file mode 100644
index 00000000000..607371a32e8
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageEvents
+        class ProductionStageEnd < SimpleStageEvent
+          def self.name
+            PlanStageStart.name
+          end
+
+          def self.identifier
+            :production_stage_end
+          end
+
+          def object_type
+            Issue
+          end
+
+          def timestamp_projection
+            mr_metrics_table[:first_deployed_to_production_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(merge_requests_closing_issues: { merge_request: [:metrics] }).where(mr_metrics_table[:first_deployed_to_production_at].gteq(mr_table[:created_at]))
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb
index a55eee048c2..aa392140eb5 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb
@@ -6,6 +6,8 @@ module Gitlab
       module StageEvents
         # Base class for expressing an event that can be used for a stage.
         class StageEvent
+          include Gitlab::CycleAnalytics::MetricsTables
+
           def initialize(params)
             @params = params
           end
@@ -21,6 +23,21 @@ module Gitlab
           def object_type
             raise NotImplementedError
           end
+
+          # Each StageEvent must expose a timestamp or a timestamp like expression in order to build a range query.
+          # Example: get me all the Issue records between start event end end event
+          def timestamp_projection
+            raise NotImplementedError
+          end
+
+          # Optionally a StageEvent may apply additional filtering or join other tables on the base query.
+          def apply_query_customization(query)
+            query
+          end
+
+          private
+
+          attr_reader :params
         end
       end
     end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
new file mode 100644
index 00000000000..34c726b2254
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageQueryHelpers
+        def execute_query(query)
+          ActiveRecord::Base.connection.execute(query.to_sql)
+        end
+
+        def zero_interval
+          Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")])
+        end
+
+        def round_duration_to_seconds
+          Arel::Nodes::Extract.new(duration, :epoch)
+        end
+
+        def duration
+          Arel::Nodes::Subtraction.new(
+            stage.end_event.timestamp_projection,
+            stage.start_event.timestamp_projection
+          )
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/artifacts/migration_helper.rb b/lib/gitlab/artifacts/migration_helper.rb
new file mode 100644
index 00000000000..4f047ab3ea8
--- /dev/null
+++ b/lib/gitlab/artifacts/migration_helper.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Artifacts
+    class MigrationHelper
+      def migrate_to_remote_storage(&block)
+        artifacts = ::Ci::JobArtifact.with_files_stored_locally
+        migrate(artifacts, ObjectStorage::Store::REMOTE, &block)
+      end
+
+      def migrate_to_local_storage(&block)
+        artifacts = ::Ci::JobArtifact.with_files_stored_remotely
+        migrate(artifacts, ObjectStorage::Store::LOCAL, &block)
+      end
+
+      private
+
+      def batch_size
+        ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i
+      end
+
+      def migrate(artifacts, store, &block)
+        artifacts.find_each(batch_size: batch_size) do |artifact| # rubocop:disable CodeReuse/ActiveRecord
+          artifact.file.migrate!(store)
+
+          yield artifact if block
+        rescue => e
+          raise StandardError.new("Failed to transfer artifact of type #{artifact.file_type} and ID #{artifact.id} with error: #{e.message}")
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 53c1398d6ab..4217859f9fb 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -69,7 +69,7 @@ module Gitlab
         Gitlab::Auth::UniqueIpsLimiter.limit_user! do
           user = User.by_login(login)
 
-          break if user && !user.active?
+          break if user && !user.can?(:log_in)
 
           authenticators = []
 
@@ -231,7 +231,7 @@ module Gitlab
 
         authentication_abilities =
           if token_handler.user?
-            full_authentication_abilities
+            read_write_project_authentication_abilities
           elsif token_handler.deploy_key_pushable?(project)
             read_write_authentication_abilities
           else
@@ -272,10 +272,21 @@ module Gitlab
         ]
       end
 
-      def read_only_authentication_abilities
+      def read_only_project_authentication_abilities
         [
           :read_project,
-          :download_code,
+          :download_code
+        ]
+      end
+
+      def read_write_project_authentication_abilities
+        read_only_project_authentication_abilities + [
+          :push_code
+        ]
+      end
+
+      def read_only_authentication_abilities
+        read_only_project_authentication_abilities + [
           :read_container_image
         ]
       end
diff --git a/lib/gitlab/auth/current_user_mode.rb b/lib/gitlab/auth/current_user_mode.rb
new file mode 100644
index 00000000000..df5039f50c1
--- /dev/null
+++ b/lib/gitlab/auth/current_user_mode.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Auth
+    # Keeps track of the current session user mode
+    #
+    # In order to perform administrative tasks over some interfaces,
+    # an administrator must have explicitly enabled admin-mode
+    # e.g. on web access require re-authentication
+    class CurrentUserMode
+      SESSION_STORE_KEY = :current_user_mode
+      ADMIN_MODE_START_TIME_KEY = 'admin_mode'
+      MAX_ADMIN_MODE_TIME = 6.hours
+
+      def initialize(user)
+        @user = user
+      end
+
+      def admin_mode?
+        return false unless user
+
+        Gitlab::SafeRequestStore.fetch(request_store_key) do
+          user&.admin? && any_session_with_admin_mode?
+        end
+      end
+
+      def enable_admin_mode!(password: nil, skip_password_validation: false)
+        return unless user&.admin?
+        return unless skip_password_validation || user&.valid_password?(password)
+
+        current_session_data[ADMIN_MODE_START_TIME_KEY] = Time.now
+      end
+
+      def disable_admin_mode!
+        current_session_data[ADMIN_MODE_START_TIME_KEY] = nil
+        Gitlab::SafeRequestStore.delete(request_store_key)
+      end
+
+      private
+
+      attr_reader :user
+
+      def request_store_key
+        @request_store_key ||= { res: :current_user_mode, user: user.id }
+      end
+
+      def current_session_data
+        @current_session ||= Gitlab::NamespacedSessionStore.new(SESSION_STORE_KEY)
+      end
+
+      def any_session_with_admin_mode?
+        return true if current_session_data.initiated? && current_session_data[ADMIN_MODE_START_TIME_KEY].to_i > MAX_ADMIN_MODE_TIME.ago.to_i
+
+        all_sessions.any? do |session|
+          session[ADMIN_MODE_START_TIME_KEY].to_i > MAX_ADMIN_MODE_TIME.ago.to_i
+        end
+      end
+
+      def all_sessions
+        @all_sessions ||= ActiveSession.list_sessions(user).lazy.map do |session|
+          Gitlab::NamespacedSessionStore.new(SESSION_STORE_KEY, session.with_indifferent_access )
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/auth/ip_rate_limiter.rb b/lib/gitlab/auth/ip_rate_limiter.rb
index 0b7055b3256..74d359bcd28 100644
--- a/lib/gitlab/auth/ip_rate_limiter.rb
+++ b/lib/gitlab/auth/ip_rate_limiter.rb
@@ -24,6 +24,7 @@ module Gitlab
         # Allow2Ban.filter will return false if this IP has not failed too often yet
         @banned = Rack::Attack::Allow2Ban.filter(ip, config) do
           # If we return false here, the failure for this IP is ignored by Allow2Ban
+          # If we return true here, the count for the IP is incremented.
           ip_can_be_banned?
         end
       end
diff --git a/lib/gitlab/auth/user_access_denied_reason.rb b/lib/gitlab/auth/user_access_denied_reason.rb
index fd09fe76c02..e73f6ca808c 100644
--- a/lib/gitlab/auth/user_access_denied_reason.rb
+++ b/lib/gitlab/auth/user_access_denied_reason.rb
@@ -14,6 +14,9 @@ module Gitlab
         when :terms_not_accepted
           "You (#{@user.to_reference}) must accept the Terms of Service in order to perform this action. "\
           "Please access GitLab from a web browser to accept these terms."
+        when :deactivated
+          "Your account has been deactivated by your administrator. "\
+          "Please log back in from a web browser to reactivate your account at #{Gitlab.config.gitlab.url}"
         else
           "Your account has been blocked."
         end
@@ -26,6 +29,8 @@ module Gitlab
           :internal
         elsif @user.required_terms_not_accepted?
           :terms_not_accepted
+        elsif @user.deactivated?
+          :deactivated
         else
           :blocked
         end
diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb
index 2e3a4f3b869..61e0a075018 100644
--- a/lib/gitlab/background_migration.rb
+++ b/lib/gitlab/background_migration.rb
@@ -78,7 +78,7 @@ module Gitlab
     end
 
     def self.migration_class_for(class_name)
-      const_get(class_name)
+      const_get(class_name, false)
     end
 
     def self.enqueued_job?(queues, migration_class)
diff --git a/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb b/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb
index 29fa0f18448..3c142327e94 100644
--- a/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb
+++ b/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb
@@ -171,7 +171,11 @@ module Gitlab
         end
 
         def schedule_retry(project, retry_count)
-          BackgroundMigrationWorker.perform_in(RETRY_DELAY, self.class::RetryOne.name, [project.id, retry_count])
+          # Constants provided to BackgroundMigrationWorker must be within the
+          # scope of Gitlab::BackgroundMigration
+          retry_class_name = self.class::RetryOne.name.sub('Gitlab::BackgroundMigration::', '')
+
+          BackgroundMigrationWorker.perform_in(RETRY_DELAY, retry_class_name, [project.id, retry_count])
         end
       end
 
diff --git a/lib/gitlab/background_migration/legacy_upload_mover.rb b/lib/gitlab/background_migration/legacy_upload_mover.rb
index 051c1176edb..c9e47f210be 100644
--- a/lib/gitlab/background_migration/legacy_upload_mover.rb
+++ b/lib/gitlab/background_migration/legacy_upload_mover.rb
@@ -92,7 +92,7 @@ module Gitlab
 
       def legacy_file_uploader
         strong_memoize(:legacy_file_uploader) do
-          uploader = upload.build_uploader
+          uploader = upload.retrieve_uploader
           uploader.retrieve_from_store!(File.basename(upload.path))
           uploader
         end
diff --git a/lib/gitlab/background_migration/migrate_pages_metadata.rb b/lib/gitlab/background_migration/migrate_pages_metadata.rb
new file mode 100644
index 00000000000..68fd0c17d29
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_pages_metadata.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Class that will insert record into project_pages_metadata
+    # for each existing project
+    class MigratePagesMetadata
+      def perform(start_id, stop_id)
+        perform_on_relation(Project.where(id: start_id..stop_id))
+      end
+
+      def perform_on_relation(relation)
+        successful_pages_deploy = <<~SQL
+          SELECT TRUE
+          FROM ci_builds
+          WHERE ci_builds.type = 'GenericCommitStatus'
+            AND ci_builds.status = 'success'
+            AND ci_builds.stage = 'deploy'
+            AND ci_builds.name = 'pages:deploy'
+            AND ci_builds.project_id = projects.id
+          LIMIT 1
+        SQL
+
+        select_from = relation
+          .select("projects.id", "COALESCE((#{successful_pages_deploy}), FALSE)")
+          .to_sql
+
+        ActiveRecord::Base.connection_pool.with_connection do |connection|
+          connection.execute <<~SQL
+            INSERT INTO project_pages_metadata (project_id, deployed)
+          #{select_from}
+            ON CONFLICT (project_id) DO NOTHING
+          SQL
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/badge/pipeline/template.rb b/lib/gitlab/badge/pipeline/template.rb
index 2c5f9654496..0d3d44135e7 100644
--- a/lib/gitlab/badge/pipeline/template.rb
+++ b/lib/gitlab/badge/pipeline/template.rb
@@ -15,7 +15,7 @@ module Gitlab
           failed: '#e05d44',
           running: '#dfb317',
           pending: '#dfb317',
-          preparing: '#dfb317',
+          preparing: '#a7a7a7',
           canceled: '#9f9f9f',
           skipped: '#9f9f9f',
           unknown: '#9f9f9f'
diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb
index 24bc73e0de5..e01ffb631ba 100644
--- a/lib/gitlab/bitbucket_import/importer.rb
+++ b/lib/gitlab/bitbucket_import/importer.rb
@@ -104,7 +104,7 @@ module Gitlab
             iid: issue.iid,
             title: issue.title,
             description: description,
-            state: issue.state,
+            state_id: Issue.available_states[issue.state],
             author_id: gitlab_user_id(project, issue.author),
             milestone: milestone,
             created_at: issue.created_at,
diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb
index f1a653a9d95..5382bdab7eb 100644
--- a/lib/gitlab/blame.rb
+++ b/lib/gitlab/blame.rb
@@ -17,6 +17,7 @@ module Gitlab
       i = 0
       blame.each do |commit, line|
         commit = Commit.new(commit, project)
+        commit.lazy_author # preload author
 
         sha = commit.sha
         if prev_sha != sha
diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb
index 4c658dc0b8d..6e48ca90054 100644
--- a/lib/gitlab/cache/request_cache.rb
+++ b/lib/gitlab/cache/request_cache.rb
@@ -23,7 +23,7 @@ module Gitlab
       end
 
       def request_cache(method_name, &method_key_block)
-        const_get(:RequestCacheExtension).module_eval do
+        const_get(:RequestCacheExtension, false).module_eval do
           cache_key_method_name = "#{method_name}_cache_key"
 
           define_method(method_name) do |*args|
diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb
index b7886114e9c..eb5d78ebcd4 100644
--- a/lib/gitlab/ci/ansi2html.rb
+++ b/lib/gitlab/ci/ansi2html.rb
@@ -178,6 +178,8 @@ module Gitlab
 
           close_open_tags
 
+          # TODO: replace OpenStruct with a better type
+          # https://gitlab.com/gitlab-org/gitlab/issues/34305
           OpenStruct.new(
             html: @out.force_encoding(Encoding.default_external),
             state: state,
diff --git a/lib/gitlab/ci/ansi2json.rb b/lib/gitlab/ci/ansi2json.rb
new file mode 100644
index 00000000000..79114d35916
--- /dev/null
+++ b/lib/gitlab/ci/ansi2json.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+# Convert terminal stream to JSON
+module Gitlab
+  module Ci
+    module Ansi2json
+      def self.convert(ansi, state = nil)
+        Converter.new.convert(ansi, state)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb
new file mode 100644
index 00000000000..8d25b66af9c
--- /dev/null
+++ b/lib/gitlab/ci/ansi2json/converter.rb
@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Ansi2json
+      class Converter
+        def convert(stream, new_state)
+          @lines = []
+          @state = State.new(new_state, stream.size)
+
+          append = false
+          truncated = false
+
+          cur_offset = stream.tell
+          if cur_offset > @state.offset
+            @state.offset = cur_offset
+            truncated = true
+          else
+            stream.seek(@state.offset)
+            append = @state.offset > 0
+          end
+
+          start_offset = @state.offset
+
+          @state.set_current_line!(style: Style.new(@state.inherited_style))
+
+          stream.each_line do |line|
+            s = StringScanner.new(line)
+            convert_line(s)
+          end
+
+          # This must be assigned before flushing the current line
+          # or the @current_line.offset will advance to the very end
+          # of the trace. Instead we want @last_line_offset to always
+          # point to the beginning of last line.
+          @state.set_last_line_offset
+
+          flush_current_line
+
+          # TODO: replace OpenStruct with a better type
+          # https://gitlab.com/gitlab-org/gitlab/issues/34305
+          OpenStruct.new(
+            lines: @lines,
+            state: @state.encode,
+            append: append,
+            truncated: truncated,
+            offset: start_offset,
+            size: stream.tell - start_offset,
+            total: stream.size
+          )
+        end
+
+        private
+
+        def convert_line(scanner)
+          until scanner.eos?
+
+            if scanner.scan(Gitlab::Regex.build_trace_section_regex)
+              handle_section(scanner)
+            elsif scanner.scan(/\e([@-_])(.*?)([@-~])/)
+              handle_sequence(scanner)
+            elsif scanner.scan(/\e(([@-_])(.*?)?)?$/)
+              break
+            elsif scanner.scan(/</)
+              @state.current_line << '&lt;'
+            elsif scanner.scan(/\r?\n/)
+              # we advance the offset of the next current line
+              # so it does not start from \n
+              flush_current_line(advance_offset: scanner.matched_size)
+            else
+              @state.current_line << scanner.scan(/./m)
+            end
+
+            @state.offset += scanner.matched_size
+          end
+        end
+
+        def handle_sequence(scanner)
+          indicator = scanner[1]
+          commands = scanner[2].split ';'
+          terminator = scanner[3]
+
+          # We are only interested in color and text style changes - triggered by
+          # sequences starting with '\e[' and ending with 'm'. Any other control
+          # sequence gets stripped (including stuff like "delete last line")
+          return unless indicator == '[' && terminator == 'm'
+
+          @state.update_style(commands)
+        end
+
+        def handle_section(scanner)
+          action = scanner[1]
+          timestamp = scanner[2]
+          section = scanner[3]
+
+          section_name = sanitize_section_name(section)
+
+          if action == "start"
+            handle_section_start(section_name, timestamp)
+          elsif action == "end"
+            handle_section_end(section_name, timestamp)
+          end
+        end
+
+        def handle_section_start(section, timestamp)
+          flush_current_line unless @state.current_line.empty?
+          @state.open_section(section, timestamp)
+        end
+
+        def handle_section_end(section, timestamp)
+          return unless @state.section_open?(section)
+
+          flush_current_line unless @state.current_line.empty?
+          @state.close_section(section, timestamp)
+
+          # ensure that section end is detached from the last
+          # line in the section
+          flush_current_line
+        end
+
+        def flush_current_line(advance_offset: 0)
+          @lines << @state.current_line.to_h
+
+          @state.set_current_line!(advance_offset: advance_offset)
+        end
+
+        def sanitize_section_name(section)
+          section.to_s.downcase.gsub(/[^a-z0-9]/, '-')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb
new file mode 100644
index 00000000000..173fb1df88e
--- /dev/null
+++ b/lib/gitlab/ci/ansi2json/line.rb
@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Ansi2json
+      # Line class is responsible for keeping the internal state of
+      # a log line and to finally serialize it as Hash.
+      class Line
+        # Line::Segment is a portion of a line that has its own style
+        # and text. Multiple segments make the line content.
+        class Segment
+          attr_accessor :text, :style
+
+          def initialize(style:)
+            @text = +''
+            @style = style
+          end
+
+          def empty?
+            text.empty?
+          end
+
+          def to_h
+            # Without force encoding to UTF-8 we could get an error
+            # when serializing the Hash to JSON.
+            # Encoding::UndefinedConversionError:
+            #   "\xE2" from ASCII-8BIT to UTF-8
+            { text: text.force_encoding('UTF-8') }.tap do |result|
+              result[:style] = style.to_s if style.set?
+            end
+          end
+        end
+
+        attr_reader :offset, :sections, :segments, :current_segment,
+                    :section_header, :section_duration
+
+        def initialize(offset:, style:, sections: [])
+          @offset = offset
+          @segments = []
+          @sections = sections
+          @section_header = false
+          @duration = nil
+          @current_segment = Segment.new(style: style)
+        end
+
+        def <<(data)
+          @current_segment.text << data
+        end
+
+        def style
+          @current_segment.style
+        end
+
+        def empty?
+          @segments.empty? && @current_segment.empty?
+        end
+
+        def update_style(ansi_commands)
+          @current_segment.style.update(ansi_commands)
+        end
+
+        def add_section(section)
+          @sections << section
+        end
+
+        def set_as_section_header
+          @section_header = true
+        end
+
+        def set_section_duration(duration)
+          @section_duration = Time.at(duration.to_i).strftime('%M:%S')
+        end
+
+        def flush_current_segment!
+          return if @current_segment.empty?
+
+          @segments << @current_segment.to_h
+          @current_segment = Segment.new(style: @current_segment.style)
+        end
+
+        def to_h
+          flush_current_segment!
+
+          { offset: offset, content: @segments }.tap do |result|
+            result[:section] = sections.last if sections.any?
+            result[:section_header] = true if @section_header
+            result[:section_duration] = @section_duration if @section_duration
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/ansi2json/parser.rb b/lib/gitlab/ci/ansi2json/parser.rb
new file mode 100644
index 00000000000..d428680fb2a
--- /dev/null
+++ b/lib/gitlab/ci/ansi2json/parser.rb
@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+# This Parser translates ANSI escape codes into human readable format.
+# It considers color and format changes.
+# Inspired by http://en.wikipedia.org/wiki/ANSI_escape_code
+module Gitlab
+  module Ci
+    module Ansi2json
+      class Parser
+        # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107)
+        COLOR = {
+          0 => 'black', # not that this is gray in the intense color table
+          1 => 'red',
+          2 => 'green',
+          3 => 'yellow',
+          4 => 'blue',
+          5 => 'magenta',
+          6 => 'cyan',
+          7 => 'white' # not that this is gray in the dark (aka default) color table
+        }.freeze
+
+        STYLE_SWITCHES = {
+          bold:       0x01,
+          italic:     0x02,
+          underline:  0x04,
+          conceal:    0x08,
+          cross:      0x10
+        }.freeze
+
+        def self.bold?(mask)
+          mask & STYLE_SWITCHES[:bold] != 0
+        end
+
+        def self.matching_formats(mask)
+          formats = []
+          STYLE_SWITCHES.each do |text_format, flag|
+            formats << "term-#{text_format}" if mask & flag != 0
+          end
+
+          formats
+        end
+
+        def initialize(command, ansi_stack = nil)
+          @command = command
+          @ansi_stack = ansi_stack
+        end
+
+        def changes
+          if self.respond_to?("on_#{@command}")
+            send("on_#{@command}", @ansi_stack) # rubocop:disable GitlabSecurity/PublicSend
+          end
+        end
+
+        # rubocop:disable Style/SingleLineMethods
+        def on_0(_) { reset: true } end
+
+        def on_1(_) { enable: STYLE_SWITCHES[:bold] } end
+
+        def on_3(_) { enable: STYLE_SWITCHES[:italic] } end
+
+        def on_4(_) { enable: STYLE_SWITCHES[:underline] } end
+
+        def on_8(_) { enable: STYLE_SWITCHES[:conceal] } end
+
+        def on_9(_) { enable: STYLE_SWITCHES[:cross] } end
+
+        def on_21(_) { disable: STYLE_SWITCHES[:bold] } end
+
+        def on_22(_) { disable: STYLE_SWITCHES[:bold] } end
+
+        def on_23(_) { disable: STYLE_SWITCHES[:italic] } end
+
+        def on_24(_) { disable: STYLE_SWITCHES[:underline] } end
+
+        def on_28(_) { disable: STYLE_SWITCHES[:conceal] } end
+
+        def on_29(_) { disable: STYLE_SWITCHES[:cross] } end
+
+        def on_30(_) { fg: fg_color(0) } end
+
+        def on_31(_) { fg: fg_color(1) } end
+
+        def on_32(_) { fg: fg_color(2) } end
+
+        def on_33(_) { fg: fg_color(3) } end
+
+        def on_34(_) { fg: fg_color(4) } end
+
+        def on_35(_) { fg: fg_color(5) } end
+
+        def on_36(_) { fg: fg_color(6) } end
+
+        def on_37(_) { fg: fg_color(7) } end
+
+        def on_38(stack) { fg: fg_color_256(stack) } end
+
+        def on_39(_) { fg: fg_color(9) } end
+
+        def on_40(_) { bg: bg_color(0) } end
+
+        def on_41(_) { bg: bg_color(1) } end
+
+        def on_42(_) { bg: bg_color(2) } end
+
+        def on_43(_) { bg: bg_color(3) } end
+
+        def on_44(_) { bg: bg_color(4) } end
+
+        def on_45(_) { bg: bg_color(5) } end
+
+        def on_46(_) { bg: bg_color(6) } end
+
+        def on_47(_) { bg: bg_color(7) } end
+
+        def on_48(stack) { bg: bg_color_256(stack) } end
+
+        # TODO: all the x9 never get called?
+        def on_49(_) { fg: fg_color(9) } end
+
+        def on_90(_) { fg: fg_color(0, 'l') } end
+
+        def on_91(_) { fg: fg_color(1, 'l') } end
+
+        def on_92(_) { fg: fg_color(2, 'l') } end
+
+        def on_93(_) { fg: fg_color(3, 'l') } end
+
+        def on_94(_) { fg: fg_color(4, 'l') } end
+
+        def on_95(_) { fg: fg_color(5, 'l') } end
+
+        def on_96(_) { fg: fg_color(6, 'l') } end
+
+        def on_97(_) { fg: fg_color(7, 'l') } end
+
+        def on_99(_) { fg: fg_color(9, 'l') } end
+
+        def on_100(_) { fg: bg_color(0, 'l') } end
+
+        def on_101(_) { fg: bg_color(1, 'l') } end
+
+        def on_102(_) { fg: bg_color(2, 'l') } end
+
+        def on_103(_) { fg: bg_color(3, 'l') } end
+
+        def on_104(_) { fg: bg_color(4, 'l') } end
+
+        def on_105(_) { fg: bg_color(5, 'l') } end
+
+        def on_106(_) { fg: bg_color(6, 'l') } end
+
+        def on_107(_) { fg: bg_color(7, 'l') } end
+
+        def on_109(_) { fg: bg_color(9, 'l') } end
+        # rubocop:enable Style/SingleLineMethods
+
+        def fg_color(color_index, prefix = nil)
+          term_color_class(color_index, ['fg', prefix])
+        end
+
+        def fg_color_256(command_stack)
+          xterm_color_class(command_stack, 'fg')
+        end
+
+        def bg_color(color_index, prefix = nil)
+          term_color_class(color_index, ['bg', prefix])
+        end
+
+        def bg_color_256(command_stack)
+          xterm_color_class(command_stack, 'bg')
+        end
+
+        def term_color_class(color_index, prefix)
+          color_name = COLOR[color_index]
+          return if color_name.nil?
+
+          color_class(['term', prefix, color_name])
+        end
+
+        def xterm_color_class(command_stack, prefix)
+          # the 38 and 48 commands have to be followed by "5" and the color index
+          return unless command_stack.length >= 2
+          return unless command_stack[0] == "5"
+
+          command_stack.shift # ignore the "5" command
+          color_index = command_stack.shift.to_i
+
+          return unless color_index >= 0
+          return unless color_index <= 255
+
+          color_class(["xterm", prefix, color_index])
+        end
+
+        def color_class(segments)
+          [segments].flatten.compact.join('-')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb
new file mode 100644
index 00000000000..db7a9035b8b
--- /dev/null
+++ b/lib/gitlab/ci/ansi2json/state.rb
@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+# In this class we keep track of the state changes that the
+# Converter makes as it scans through the log stream.
+module Gitlab
+  module Ci
+    module Ansi2json
+      class State
+        attr_accessor :offset, :current_line, :inherited_style, :open_sections, :last_line_offset
+
+        def initialize(new_state, stream_size)
+          @offset = 0
+          @inherited_style = {}
+          @open_sections = {}
+          @stream_size = stream_size
+
+          restore_state!(new_state)
+        end
+
+        def encode
+          state = {
+            offset: @last_line_offset,
+            style: @current_line.style.to_h,
+            open_sections: @open_sections
+          }
+          Base64.urlsafe_encode64(state.to_json)
+        end
+
+        def open_section(section, timestamp)
+          @open_sections[section] = timestamp
+
+          @current_line.add_section(section)
+          @current_line.set_as_section_header
+        end
+
+        def close_section(section, timestamp)
+          return unless section_open?(section)
+
+          duration = timestamp.to_i - @open_sections[section].to_i
+          @current_line.set_section_duration(duration)
+
+          @open_sections.delete(section)
+        end
+
+        def section_open?(section)
+          @open_sections.key?(section)
+        end
+
+        def set_current_line!(style: nil, advance_offset: 0)
+          new_line = Line.new(
+            offset: @offset + advance_offset,
+            style: style || @current_line.style,
+            sections: @open_sections.keys
+          )
+          @current_line = new_line
+        end
+
+        def set_last_line_offset
+          @last_line_offset = @current_line.offset
+        end
+
+        def update_style(commands)
+          @current_line.flush_current_segment!
+          @current_line.update_style(commands)
+        end
+
+        private
+
+        def restore_state!(encoded_state)
+          state = decode_state(encoded_state)
+
+          return unless state
+          return if state['offset'].to_i > @stream_size
+
+          @offset = state['offset'].to_i if state['offset']
+          @open_sections = state['open_sections'] if state['open_sections']
+
+          if state['style']
+            @inherited_style = {
+              fg: state.dig('style', 'fg'),
+              bg: state.dig('style', 'bg'),
+              mask: state.dig('style', 'mask')
+            }
+          end
+        end
+
+        def decode_state(state)
+          return unless state.present?
+
+          decoded_state = Base64.urlsafe_decode64(state)
+          return unless decoded_state.present?
+
+          JSON.parse(decoded_state)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/ansi2json/style.rb b/lib/gitlab/ci/ansi2json/style.rb
new file mode 100644
index 00000000000..2739ffdfa5d
--- /dev/null
+++ b/lib/gitlab/ci/ansi2json/style.rb
@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Ansi2json
+      class Style
+        attr_reader :fg, :bg, :mask
+
+        def initialize(fg: nil, bg: nil, mask: 0)
+          @fg = fg
+          @bg = bg
+          @mask = mask
+
+          update_formats
+        end
+
+        def update(ansi_commands)
+          command = ansi_commands.shift
+          return unless command
+
+          if changes = Gitlab::Ci::Ansi2json::Parser.new(command, ansi_commands).changes
+            apply_changes(changes)
+          end
+
+          update(ansi_commands)
+        end
+
+        def set?
+          @fg || @bg || @formats.any?
+        end
+
+        def reset!
+          @fg = nil
+          @bg = nil
+          @mask = 0
+          @formats = []
+        end
+
+        def ==(other)
+          self.to_h == other.to_h
+        end
+
+        def to_s
+          [@fg, @bg, @formats].flatten.compact.join(' ')
+        end
+
+        def to_h
+          { fg: @fg, bg: @bg, mask: @mask }
+        end
+
+        private
+
+        def apply_changes(changes)
+          case
+          when changes[:reset]
+            reset!
+          when changes[:fg]
+            @fg = changes[:fg]
+          when changes[:bg]
+            @bg = changes[:bg]
+          when changes[:enable]
+            @mask |= changes[:enable]
+          when changes[:disable]
+            @mask &= ~changes[:disable]
+          else
+            return
+          end
+
+          update_formats
+        end
+
+        def update_formats
+          # Most terminals show bold colored text in the light color variant
+          # Let's mimic that here
+          if @fg.present? && Gitlab::Ci::Ansi2json::Parser.bold?(@mask)
+            @fg = @fg.sub(/fg-([a-z]{2,}+)/, 'fg-l-\1')
+          end
+
+          @formats = Gitlab::Ci::Ansi2json::Parser.matching_formats(@mask)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/policy.rb b/lib/gitlab/ci/build/policy.rb
index 43c46ad74af..ebeebe7fb5b 100644
--- a/lib/gitlab/ci/build/policy.rb
+++ b/lib/gitlab/ci/build/policy.rb
@@ -6,7 +6,7 @@ module Gitlab
       module Policy
         def self.fabricate(specs)
           specifications = specs.to_h.map do |spec, value|
-            self.const_get(spec.to_s.camelize).new(value)
+            self.const_get(spec.to_s.camelize, false).new(value)
           end
 
           specifications.compact
diff --git a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb
index f448d55f00a..9950e1dec55 100644
--- a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb
+++ b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb
@@ -36,7 +36,7 @@ module Gitlab
               Clusters::KubernetesNamespaceFinder.new(
                 deployment_cluster,
                 project: environment.project,
-                environment_slug: environment.slug,
+                environment_name: environment.name,
                 allow_blank_token: true
               ).execute
             end
diff --git a/lib/gitlab/ci/build/rules/rule/clause/exists.rb b/lib/gitlab/ci/build/rules/rule/clause/exists.rb
new file mode 100644
index 00000000000..62f8371283f
--- /dev/null
+++ b/lib/gitlab/ci/build/rules/rule/clause/exists.rb
@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Build
+      class Rules::Rule::Clause::Exists < Rules::Rule::Clause
+        # The maximum number of patterned glob comparisons that will be
+        # performed before the rule assumes that it has a match
+        MAX_PATTERN_COMPARISONS = 10_000
+
+        def initialize(globs)
+          globs = Array(globs)
+
+          @top_level_only = globs.all?(&method(:top_level_glob?))
+          @exact_globs, @pattern_globs = globs.partition(&method(:exact_glob?))
+        end
+
+        def satisfied_by?(pipeline, seed)
+          paths = worktree_paths(pipeline)
+
+          exact_matches?(paths) || pattern_matches?(paths)
+        end
+
+        private
+
+        def worktree_paths(pipeline)
+          if @top_level_only
+            pipeline.top_level_worktree_paths
+          else
+            pipeline.all_worktree_paths
+          end
+        end
+
+        def exact_matches?(paths)
+          @exact_globs.any? { |glob| paths.bsearch { |path| glob <=> path } }
+        end
+
+        def pattern_matches?(paths)
+          comparisons = 0
+          @pattern_globs.any? do |glob|
+            paths.any? do |path|
+              comparisons += 1
+              comparisons > MAX_PATTERN_COMPARISONS || pattern_match?(glob, path)
+            end
+          end
+        end
+
+        def pattern_match?(glob, path)
+          File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB)
+        end
+
+        # matches glob patterns that only match files in the top level directory
+        def top_level_glob?(glob)
+          !glob.include?('/') && !glob.include?('**')
+        end
+
+        # matches glob patterns that have no metacharacters for File#fnmatch?
+        def exact_glob?(glob)
+          !glob.include?('*') && !glob.include?('?') && !glob.include?('[') && !glob.include?('{')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 668e4a5e246..9c1e6277e95 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -7,6 +7,8 @@ module Gitlab
     #
     class Config
       ConfigError = Class.new(StandardError)
+      TIMEOUT_SECONDS = 30.seconds
+      TIMEOUT_MESSAGE = 'Resolving config took longer than expected'
 
       RESCUE_ERRORS = [
         Gitlab::Config::Loader::FormatError,
@@ -17,17 +19,17 @@ module Gitlab
       attr_reader :root
 
       def initialize(config, project: nil, sha: nil, user: nil)
-        @config = Config::Extendable
-          .new(build_config(config, project: project, sha: sha, user: user))
-          .to_hash
+        @context = build_context(project: project, sha: sha, user: user)
+
+        if Feature.enabled?(:ci_limit_yaml_expansion, project, default_enabled: true)
+          @context.set_deadline(TIMEOUT_SECONDS)
+        end
+
+        @config = expand_config(config)
 
         @root = Entry::Root.new(@config)
         @root.compose!
 
-      rescue Gitlab::Config::Loader::Yaml::DataTooLargeError => e
-        Gitlab::Sentry.track_exception(e, extra: { user: user.inspect, project: project.inspect })
-        raise Config::ConfigError, e.message
-
       rescue *rescue_errors => e
         raise Config::ConfigError, e.message
       end
@@ -61,18 +63,39 @@ module Gitlab
 
       private
 
-      def build_config(config, project:, sha:, user:)
+      def expand_config(config)
+        build_config(config)
+
+      rescue Gitlab::Config::Loader::Yaml::DataTooLargeError => e
+        track_exception(e)
+        raise Config::ConfigError, e.message
+
+      rescue Gitlab::Ci::Config::External::Context::TimeoutError => e
+        track_exception(e)
+        raise Config::ConfigError, TIMEOUT_MESSAGE
+      end
+
+      def build_config(config)
         initial_config = Gitlab::Config::Loader::Yaml.new(config).load!
+        initial_config = Config::External::Processor.new(initial_config, @context).perform
+        initial_config = Config::Extendable.new(initial_config).to_hash
 
-        process_external_files(initial_config, project: project, sha: sha, user: user)
+        if Feature.enabled?(:ci_pre_post_pipeline_stages, @context.project, default_enabled: true)
+          initial_config = Config::EdgeStagesInjector.new(initial_config).to_hash
+        end
+
+        initial_config
       end
 
-      def process_external_files(config, project:, sha:, user:)
-        Config::External::Processor.new(config,
+      def build_context(project:, sha:, user:)
+        Config::External::Context.new(
           project: project,
           sha: sha || project&.repository&.root_ref_sha,
-          user: user,
-          expandset: Set.new).perform
+          user: user)
+      end
+
+      def track_exception(error)
+        Gitlab::Sentry.track_exception(error, extra: @context.sentry_payload)
       end
 
       # Overriden in EE
diff --git a/lib/gitlab/ci/config/edge_stages_injector.rb b/lib/gitlab/ci/config/edge_stages_injector.rb
new file mode 100644
index 00000000000..64ff9f951e4
--- /dev/null
+++ b/lib/gitlab/ci/config/edge_stages_injector.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      class EdgeStagesInjector
+        PRE_PIPELINE = '.pre'
+        POST_PIPELINE = '.post'
+        EDGES = [PRE_PIPELINE, POST_PIPELINE].freeze
+
+        def self.wrap_stages(stages)
+          stages = stages.to_a - EDGES
+          stages.unshift PRE_PIPELINE
+          stages.push POST_PIPELINE
+
+          stages
+        end
+
+        def initialize(config)
+          @config = config.to_h.deep_dup
+        end
+
+        def to_hash
+          if config.key?(:stages)
+            process(:stages)
+          elsif config.key?(:types)
+            process(:types)
+          else
+            config
+          end
+        end
+
+        private
+
+        attr_reader :config
+
+        delegate :wrap_stages, to: :class
+
+        def process(keyword)
+          stages = extract_stages(keyword)
+          return config if stages.empty?
+
+          stages = wrap_stages(stages)
+          config[keyword] = stages
+          config
+        end
+
+        def extract_stages(keyword)
+          stages = config[keyword]
+          return [] unless stages.is_a?(Array)
+
+          stages
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/rules/rule.rb b/lib/gitlab/ci/config/entry/rules/rule.rb
index 1f2a34ec90e..5d6d1c026e3 100644
--- a/lib/gitlab/ci/config/entry/rules/rule.rb
+++ b/lib/gitlab/ci/config/entry/rules/rule.rb
@@ -8,11 +8,11 @@ module Gitlab
           include ::Gitlab::Config::Entry::Validatable
           include ::Gitlab::Config::Entry::Attributable
 
-          CLAUSES      = %i[if changes].freeze
-          ALLOWED_KEYS = %i[if changes when start_in].freeze
+          CLAUSES      = %i[if changes exists].freeze
+          ALLOWED_KEYS = %i[if changes exists when start_in].freeze
           ALLOWED_WHEN = %w[on_success on_failure always never manual delayed].freeze
 
-          attributes :if, :changes, :when, :start_in
+          attributes :if, :changes, :exists, :when, :start_in
 
           validations do
             validates :config, presence: true
@@ -24,7 +24,7 @@ module Gitlab
 
             with_options allow_nil: true do
               validates :if, expression: true
-              validates :changes, array_of_strings: true
+              validates :changes, :exists, array_of_strings: true, length: { maximum: 50 }
               validates :when, allowed_values: { in: ALLOWED_WHEN }
             end
           end
diff --git a/lib/gitlab/ci/config/entry/stages.rb b/lib/gitlab/ci/config/entry/stages.rb
index 2d715cbc6bb..7e431f0f8bb 100644
--- a/lib/gitlab/ci/config/entry/stages.rb
+++ b/lib/gitlab/ci/config/entry/stages.rb
@@ -15,7 +15,7 @@ module Gitlab
           end
 
           def self.default
-            %w[build test deploy]
+            Config::EdgeStagesInjector.wrap_stages %w[build test deploy]
           end
         end
       end
diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb
new file mode 100644
index 00000000000..bb4439cd069
--- /dev/null
+++ b/lib/gitlab/ci/config/external/context.rb
@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module External
+        class Context
+          TimeoutError = Class.new(StandardError)
+
+          attr_reader :project, :sha, :user
+          attr_reader :expandset, :execution_deadline
+
+          def initialize(project: nil, sha: nil, user: nil)
+            @project = project
+            @sha = sha
+            @user = user
+            @expandset = Set.new
+            @execution_deadline = 0
+
+            yield self if block_given?
+          end
+
+          def mutate(attrs = {})
+            self.class.new(**attrs) do |ctx|
+              ctx.expandset = expandset
+              ctx.execution_deadline = execution_deadline
+            end
+          end
+
+          def set_deadline(timeout_seconds)
+            @execution_deadline = current_monotonic_time + timeout_seconds.to_f
+          end
+
+          def check_execution_time!
+            raise TimeoutError if execution_expired?
+          end
+
+          def sentry_payload
+            {
+              user: user.inspect,
+              project: project.inspect
+            }
+          end
+
+          protected
+
+          attr_writer :expandset, :execution_deadline
+
+          private
+
+          def current_monotonic_time
+            Gitlab::Metrics::System.monotonic_time
+          end
+
+          def execution_expired?
+            return false if execution_deadline.zero?
+
+            current_monotonic_time > execution_deadline
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb
index c56d33544ba..4684a9eb981 100644
--- a/lib/gitlab/ci/config/external/file/base.rb
+++ b/lib/gitlab/ci/config/external/file/base.rb
@@ -12,8 +12,6 @@ module Gitlab
 
             YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i.freeze
 
-            Context = Struct.new(:project, :sha, :user, :expandset)
-
             def initialize(params, context)
               @params = params
               @context = context
@@ -69,11 +67,16 @@ module Gitlab
             end
 
             def validate!
+              validate_execution_time!
               validate_location!
               validate_content! if errors.none?
               validate_hash! if errors.none?
             end
 
+            def validate_execution_time!
+              context.check_execution_time!
+            end
+
             def validate_location!
               if invalid_location_type?
                 errors.push("Included file `#{location}` needs to be a string")
@@ -95,11 +98,11 @@ module Gitlab
             end
 
             def expand_includes(hash)
-              External::Processor.new(hash, **expand_context).perform
+              External::Processor.new(hash, context.mutate(expand_context_attrs)).perform
             end
 
-            def expand_context
-              { project: nil, sha: nil, user: nil, expandset: context.expandset }
+            def expand_context_attrs
+              {}
             end
           end
         end
diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb
index cac321ec4a6..8cb1575a3e1 100644
--- a/lib/gitlab/ci/config/external/file/local.rb
+++ b/lib/gitlab/ci/config/external/file/local.rb
@@ -6,6 +6,7 @@ module Gitlab
       module External
         module File
           class Local < Base
+            extend ::Gitlab::Utils::Override
             include Gitlab::Utils::StrongMemoize
 
             def initialize(params, context)
@@ -34,11 +35,13 @@ module Gitlab
               context.project.repository.blob_data_at(context.sha, location)
             end
 
-            def expand_context
-              super.merge(
+            override :expand_context_attrs
+            def expand_context_attrs
+              {
                 project: context.project,
                 sha: context.sha,
-                user: context.user)
+                user: context.user
+              }
             end
           end
         end
diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb
index b828f77835c..c7b49b495fa 100644
--- a/lib/gitlab/ci/config/external/file/project.rb
+++ b/lib/gitlab/ci/config/external/file/project.rb
@@ -6,11 +6,12 @@ module Gitlab
       module External
         module File
           class Project < Base
+            extend ::Gitlab::Utils::Override
             include Gitlab::Utils::StrongMemoize
 
             attr_reader :project_name, :ref_name
 
-            def initialize(params, context = {})
+            def initialize(params, context)
               @location = params[:file]
               @project_name = params[:project]
               @ref_name = params[:ref] || 'HEAD'
@@ -65,11 +66,13 @@ module Gitlab
               end
             end
 
-            def expand_context
-              super.merge(
+            override :expand_context_attrs
+            def expand_context_attrs
+              {
                 project: project,
                 sha: sha,
-                user: context.user)
+                user: context.user
+              }
             end
           end
         end
diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb
index aff5c5b9651..0143d7784fa 100644
--- a/lib/gitlab/ci/config/external/mapper.rb
+++ b/lib/gitlab/ci/config/external/mapper.rb
@@ -7,7 +7,7 @@ module Gitlab
         class Mapper
           include Gitlab::Utils::StrongMemoize
 
-          MAX_INCLUDES = 50
+          MAX_INCLUDES = 100
 
           FILE_CLASSES = [
             External::File::Remote,
@@ -21,14 +21,9 @@ module Gitlab
           DuplicateIncludesError = Class.new(Error)
           TooManyIncludesError = Class.new(Error)
 
-          def initialize(values, project:, sha:, user:, expandset:)
-            raise Error, 'Expanded needs to be `Set`' unless expandset.is_a?(Set)
-
+          def initialize(values, context)
             @locations = Array.wrap(values.fetch(:include, []))
-            @project = project
-            @sha = sha
-            @user = user
-            @expandset = expandset
+            @context = context
           end
 
           def process
@@ -43,7 +38,9 @@ module Gitlab
 
           private
 
-          attr_reader :locations, :project, :sha, :user, :expandset
+          attr_reader :locations, :context
+
+          delegate :expandset, to: :context
 
           # convert location if String to canonical form
           def normalize_location(location)
@@ -68,11 +65,11 @@ module Gitlab
             end
 
             # We scope location to context, as this allows us to properly support
-            # relative incldues, and similarly looking relative in another project
+            # relative includes, and similarly looking relative in another project
             # does not trigger duplicate error
             scoped_location = location.merge(
-              context_project: project,
-              context_sha: sha)
+              context_project: context.project,
+              context_sha: context.sha)
 
             unless expandset.add?(scoped_location)
               raise DuplicateIncludesError, "Include `#{location.to_json}` was already included!"
@@ -88,12 +85,6 @@ module Gitlab
 
             matching.first
           end
-
-          def context
-            strong_memoize(:context) do
-              External::File::Base::Context.new(project, sha, user, expandset)
-            end
-          end
         end
       end
     end
diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb
index 4a049ecae49..de69a1b1e8f 100644
--- a/lib/gitlab/ci/config/external/processor.rb
+++ b/lib/gitlab/ci/config/external/processor.rb
@@ -7,9 +7,9 @@ module Gitlab
         class Processor
           IncludeError = Class.new(StandardError)
 
-          def initialize(values, project:, sha:, user:, expandset:)
+          def initialize(values, context)
             @values = values
-            @external_files = External::Mapper.new(values, project: project, sha: sha, user: user, expandset: expandset).process
+            @external_files = External::Mapper.new(values, context).process
             @content = {}
           rescue External::Mapper::Error,
                  OpenSSL::SSL::SSLError => e
diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb
index dca60eabc1c..8f8cae0b5f2 100644
--- a/lib/gitlab/ci/parsers/test/junit.rb
+++ b/lib/gitlab/ci/parsers/test/junit.rb
@@ -49,6 +49,12 @@ module Gitlab
             if data['failure']
               status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED
               system_output = data['failure']
+            elsif data['error']
+              # For now, as an MVC, we are grouping error test cases together
+              # with failed ones. But we will improve this further on
+              # https://gitlab.com/gitlab-org/gitlab/issues/32046.
+              status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED
+              system_output = data['error']
             else
               status = ::Gitlab::Ci::Reports::TestCase::STATUS_SUCCESS
               system_output = nil
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index 1f6b3853069..fc9c540088b 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -73,7 +73,9 @@ module Gitlab
               if bridge?
                 ::Ci::Bridge.new(attributes)
               else
-                ::Ci::Build.new(attributes)
+                ::Ci::Build.new(attributes).tap do |job|
+                  job.deployment = Seed::Deployment.new(job).to_resource
+                end
               end
             end
           end
diff --git a/lib/gitlab/ci/pipeline/seed/deployment.rb b/lib/gitlab/ci/pipeline/seed/deployment.rb
new file mode 100644
index 00000000000..8c90f03cb1d
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/seed/deployment.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Seed
+        class Deployment < Seed::Base
+          attr_reader :job, :environment
+
+          def initialize(job)
+            @job = job
+            @environment = Seed::Environment.new(@job)
+          end
+
+          def to_resource
+            return job.deployment if job.deployment
+            return unless job.starts_environment?
+
+            deployment = ::Deployment.new(attributes)
+            deployment.environment = environment.to_resource
+
+            # If there is a validation error on environment creation, such as
+            # the name contains invalid character, the job will fall back to a
+            # non-environment job.
+            return unless deployment.valid? && deployment.environment.persisted?
+
+            deployment.cluster_id =
+              deployment.environment.deployment_platform&.cluster_id
+
+            # Allocate IID for deployments.
+            # This operation must be outside of transactions of pipeline creations.
+            deployment.ensure_project_iid!
+
+            deployment
+          end
+
+          private
+
+          def attributes
+            {
+              project: job.project,
+              user: job.user,
+              ref: job.ref,
+              tag: job.tag,
+              sha: job.sha,
+              on_stop: job.on_stop
+            }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/seed/environment.rb b/lib/gitlab/ci/pipeline/seed/environment.rb
new file mode 100644
index 00000000000..2d3a1e702f9
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/seed/environment.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Seed
+        class Environment < Seed::Base
+          attr_reader :job
+
+          def initialize(job)
+            @job = job
+          end
+
+          def to_resource
+            find_environment || ::Environment.create(attributes)
+          end
+
+          private
+
+          def find_environment
+            job.project.environments.find_by_name(expanded_environment_name)
+          end
+
+          def expanded_environment_name
+            job.expanded_environment_name
+          end
+
+          def attributes
+            {
+              project: job.project,
+              name: expanded_environment_name
+            }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb
new file mode 100644
index 00000000000..3c00b67911f
--- /dev/null
+++ b/lib/gitlab/ci/status/composite.rb
@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Status
+      class Composite
+        include Gitlab::Utils::StrongMemoize
+
+        # This class accepts an array of arrays/hashes/or objects
+        def initialize(all_statuses, with_allow_failure: true)
+          unless all_statuses.respond_to?(:pluck)
+            raise ArgumentError, "all_statuses needs to respond to `.pluck`"
+          end
+
+          @status_set = Set.new
+          @status_key = 0
+          @allow_failure_key = 1 if with_allow_failure
+
+          consume_all_statuses(all_statuses)
+        end
+
+        # The status calculation is order dependent,
+        # 1. In some cases we assume that that status is exact
+        #    if the we only have given statues,
+        # 2. In other cases we assume that status is of that type
+        #    based on what statuses are no longer valid based on the
+        #    data set that we have
+        def status
+          return if none?
+
+          strong_memoize(:status) do
+            if only_of?(:skipped, :ignored)
+              'skipped'
+            elsif only_of?(:success, :skipped, :success_with_warnings, :ignored)
+              'success'
+            elsif only_of?(:created, :success_with_warnings, :ignored)
+              'created'
+            elsif only_of?(:preparing, :success_with_warnings, :ignored)
+              'preparing'
+            elsif only_of?(:canceled, :success, :skipped, :success_with_warnings, :ignored)
+              'canceled'
+            elsif only_of?(:pending, :created, :skipped, :success_with_warnings, :ignored)
+              'pending'
+            elsif any_of?(:running, :pending)
+              'running'
+            elsif any_of?(:manual)
+              'manual'
+            elsif any_of?(:scheduled)
+              'scheduled'
+            elsif any_of?(:preparing)
+              'preparing'
+            elsif any_of?(:created)
+              'running'
+            else
+              'failed'
+            end
+          end
+        end
+
+        def warnings?
+          @status_set.include?(:success_with_warnings)
+        end
+
+        private
+
+        def none?
+          @status_set.empty?
+        end
+
+        def any_of?(*names)
+          names.any? { |name| @status_set.include?(name) }
+        end
+
+        def only_of?(*names)
+          matching = names.count { |name| @status_set.include?(name) }
+          matching > 0 &&
+            matching == @status_set.size
+        end
+
+        def consume_all_statuses(all_statuses)
+          columns = []
+          columns[@status_key] = :status
+          columns[@allow_failure_key] = :allow_failure if @allow_failure_key
+
+          all_statuses
+            .pluck(*columns) # rubocop: disable CodeReuse/ActiveRecord
+            .each(&method(:consume_status))
+        end
+
+        def consume_status(description)
+          # convert `"status"` into `["status"]`
+          description = Array(description)
+
+          status =
+            if success_with_warnings?(description)
+              :success_with_warnings
+            elsif ignored_status?(description)
+              :ignored
+            else
+              description[@status_key].to_sym
+            end
+
+          @status_set.add(status)
+        end
+
+        def success_with_warnings?(status)
+          @allow_failure_key &&
+            status[@allow_failure_key] &&
+            HasStatus::PASSED_WITH_WARNINGS_STATUSES.include?(status[@status_key])
+        end
+
+        def ignored_status?(status)
+          @allow_failure_key &&
+            status[@allow_failure_key] &&
+            HasStatus::EXCLUDE_IGNORED_STATUSES.include?(status[@status_key])
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb
index 2a0bf060c9b..c29dc51f076 100644
--- a/lib/gitlab/ci/status/factory.rb
+++ b/lib/gitlab/ci/status/factory.rb
@@ -20,7 +20,7 @@ module Gitlab
 
         def core_status
           Gitlab::Ci::Status
-            .const_get(@status.capitalize)
+            .const_get(@status.capitalize, false)
             .new(@subject, @user)
             .extend(self.class.common_helpers)
         end
diff --git a/lib/gitlab/ci/status/preparing.rb b/lib/gitlab/ci/status/preparing.rb
index 62985d0a9f9..1ebdbc482b7 100644
--- a/lib/gitlab/ci/status/preparing.rb
+++ b/lib/gitlab/ci/status/preparing.rb
@@ -12,20 +12,12 @@ module Gitlab
           s_('CiStatusLabel|preparing')
         end
 
-        ##
-        # TODO: shared with 'created'
-        # until we get one for 'preparing'
-        #
         def icon
-          'status_created'
+          'status_preparing'
         end
 
-        ##
-        # TODO: shared with 'created'
-        # until we get one for 'preparing'
-        #
         def favicon
-          'favicon_status_created'
+          'favicon_status_preparing'
         end
       end
     end
diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
index 1ad9dd2913e..5a7642d24ee 100644
--- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
@@ -77,15 +77,10 @@ include:
   - template: Jobs/Test.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml
   - template: Jobs/Code-Quality.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
   - template: Jobs/Deploy.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
+  - template: Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
   - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml
   - template: Security/DAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
   - template: Security/Container-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
   - template: Security/Dependency-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
   - template: Security/License-Management.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml
   - template: Security/SAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
-
-# Override DAST job to exclude master branch
-dast:
-  except:
-    refs:
-      - master
diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml
index f6d240b7b6d..15cdbf63cb1 100644
--- a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml
@@ -1,4 +1,4 @@
-build-master:
+docker-build-master:
   # Official docker image.
   image: docker:latest
   stage: build
@@ -12,7 +12,7 @@ build-master:
   only:
     - master
 
-build:
+docker-build:
   # Official docker image.
   image: docker:latest
   stage: build
diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
new file mode 100644
index 00000000000..ae2ff9992f9
--- /dev/null
+++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
@@ -0,0 +1,55 @@
+.auto-deploy:
+  image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.1.0"
+
+dast_environment_deploy:
+  extends: .auto-deploy
+  stage: review
+  script:
+    - auto-deploy check_kube_domain
+    - auto-deploy download_chart
+    - auto-deploy ensure_namespace
+    - auto-deploy initialize_tiller
+    - auto-deploy create_secret
+    - auto-deploy deploy
+    - auto-deploy persist_environment_url
+  environment:
+    name: dast-default
+    url: http://dast-$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
+    on_stop: stop_dast_environment
+  artifacts:
+    paths: [environment_url.txt]
+  only:
+    refs:
+      - branches
+    variables:
+      - $GITLAB_FEATURES =~ /\bdast\b/
+    kubernetes: active
+  except:
+    variables:
+      - $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME
+      - $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH
+      - $DAST_WEBSITE # we don't need to create a review app if a URL is already given
+
+stop_dast_environment:
+  extends: .auto-deploy
+  stage: cleanup
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - auto-deploy initialize_tiller
+    - auto-deploy delete
+  environment:
+    name: dast-default
+    action: stop
+  needs: ["dast"]
+  only:
+    refs:
+      - branches
+    variables:
+      - $GITLAB_FEATURES =~ /\bdast\b/
+    kubernetes: active
+  except:
+    variables:
+      - $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME
+      - $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH
+      - $DAST_WEBSITE
diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
index 7f9a7df2f31..f058468ed8e 100644
--- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
@@ -1,9 +1,12 @@
 # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/container_scanning/
 
+variables:
+  CS_MAJOR_VERSION: 1
+
 container_scanning:
   stage: test
   image:
-    name: registry.gitlab.com/gitlab-org/security-products/analyzers/klar:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable
+    name: registry.gitlab.com/gitlab-org/security-products/analyzers/klar:$CS_MAJOR_VERSION
     entrypoint: []
   variables:
     # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here
diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
index 4b55ffd3771..23c65a0cb67 100644
--- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
@@ -46,3 +46,4 @@ dast:
   except:
     variables:
       - $DAST_DISABLED
+      - $DAST_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
index 88f4b72044c..a0c2ab3aa26 100644
--- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
@@ -4,7 +4,13 @@
 # List of the variables: https://gitlab.com/gitlab-org/security-products/sast#settings
 # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
 
-.sast:
+variables:
+  SAST_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
+  SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex"
+  SAST_MAJOR_VERSION: 2
+  SAST_DISABLE_DIND: "false"
+
+sast:
   stage: test
   allow_failure: true
   artifacts:
@@ -15,13 +21,6 @@
       - branches
     variables:
       - $GITLAB_FEATURES =~ /\bsast\b/
-
-variables:
-  SAST_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
-  SAST_DISABLE_DIND: "false"
-
-sast:
-  extends: .sast
   image: docker:stable
   variables:
     DOCKER_DRIVER: overlay2
@@ -84,7 +83,8 @@ sast:
       - $SAST_DISABLE_DIND == 'true'
 
 .analyzer:
-  extends: .sast
+  extends: sast
+  services: []
   except:
     variables:
       - $SAST_DISABLE_DIND == 'false'
@@ -94,100 +94,128 @@ sast:
 bandit-sast:
   extends: .analyzer
   image:
-    name: "$SAST_ANALYZER_IMAGE_PREFIX/bandit"
+    name: "$SAST_ANALYZER_IMAGE_PREFIX/bandit:$SAST_MAJOR_VERSION"
   only:
     variables:
-      - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /python/'
+      - $GITLAB_FEATURES =~ /\bsast\b/ &&
+          $SAST_DEFAULT_ANALYZERS =~ /bandit/&&
+          $CI_PROJECT_REPOSITORY_LANGUAGES =~ /python/
 
 brakeman-sast:
   extends: .analyzer
   image:
-    name: "$SAST_ANALYZER_IMAGE_PREFIX/brakeman"
+    name: "$SAST_ANALYZER_IMAGE_PREFIX/brakeman:$SAST_MAJOR_VERSION"
   only:
     variables:
-      - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby/'
+      - $GITLAB_FEATURES =~ /\bsast\b/ &&
+          $SAST_DEFAULT_ANALYZERS =~ /brakeman/ &&
+          $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby/
 
 eslint-sast:
   extends: .analyzer
   image:
-    name: "$SAST_ANALYZER_IMAGE_PREFIX/eslint"
+    name: "$SAST_ANALYZER_IMAGE_PREFIX/eslint:$SAST_MAJOR_VERSION"
   only:
     variables:
-      - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /javascript/'
+      - $GITLAB_FEATURES =~ /\bsast\b/ &&
+          $SAST_DEFAULT_ANALYZERS =~ /eslint/ &&
+          $CI_PROJECT_REPOSITORY_LANGUAGES =~ /javascript/
 
 flawfinder-sast:
   extends: .analyzer
   image:
-    name: "$SAST_ANALYZER_IMAGE_PREFIX/flawfinder"
+    name: "$SAST_ANALYZER_IMAGE_PREFIX/flawfinder:$SAST_MAJOR_VERSION"
   only:
     variables:
-      - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(c\+\+|c\b)/'
+      - $GITLAB_FEATURES =~ /\bsast\b/ &&
+          $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ &&
+          $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(c\+\+|c)\b/
 
 gosec-sast:
   extends: .analyzer
   image:
-    name: "$SAST_ANALYZER_IMAGE_PREFIX/gosec"
+    name: "$SAST_ANALYZER_IMAGE_PREFIX/gosec:$SAST_MAJOR_VERSION"
   only:
     variables:
-      - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /go/'
+      - $GITLAB_FEATURES =~ /\bsast\b/ &&
+          $SAST_DEFAULT_ANALYZERS =~ /gosec/ &&
+          $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bgo\b/
 
 nodejs-scan-sast:
   extends: .analyzer
   image:
-    name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan"
+    name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_MAJOR_VERSION"
   only:
     variables:
-      - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /javascript/'
+      - $GITLAB_FEATURES =~ /\bsast\b/ &&
+          $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ &&
+          $CI_PROJECT_REPOSITORY_LANGUAGES =~ /javascript/
 
 phpcs-security-audit-sast:
   extends: .analyzer
   image:
-    name: "$SAST_ANALYZER_IMAGE_PREFIX/phpcs-security-audit"
+    name: "$SAST_ANALYZER_IMAGE_PREFIX/phpcs-security-audit:$SAST_MAJOR_VERSION"
   only:
     variables:
-      - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /php/'
+      - $GITLAB_FEATURES =~ /\bsast\b/ &&
+          $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/ &&
+          $CI_PROJECT_REPOSITORY_LANGUAGES =~ /php/
 
 pmd-apex-sast:
   extends: .analyzer
   image:
-    name: "$SAST_ANALYZER_IMAGE_PREFIX/pmd-apex"
+    name: "$SAST_ANALYZER_IMAGE_PREFIX/pmd-apex:$SAST_MAJOR_VERSION"
   only:
     variables:
-      - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /apex/'
+      - $GITLAB_FEATURES =~ /\bsast\b/ &&
+          $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/ &&
+          $CI_PROJECT_REPOSITORY_LANGUAGES =~ /apex/
 
 secrets-sast:
   extends: .analyzer
   image:
-    name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets"
+    name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets:$SAST_MAJOR_VERSION"
+  only:
+    variables:
+      - $GITLAB_FEATURES =~ /\bsast\b/ &&
+          $SAST_DEFAULT_ANALYZERS =~ /secrets/
 
 security-code-scan-sast:
   extends: .analyzer
   image:
-    name: "$SAST_ANALYZER_IMAGE_PREFIX/security-code-scan"
+    name: "$SAST_ANALYZER_IMAGE_PREFIX/security-code-scan:$SAST_MAJOR_VERSION"
   only:
     variables:
-      - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /c\#/ || $CI_PROJECT_REPOSITORY_LANGUAGES =~ /visual basic/'
+      - $GITLAB_FEATURES =~ /\bsast\b/ &&
+          $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/ &&
+          $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(c\#|visual basic\b)/
 
 sobelow-sast:
   extends: .analyzer
   image:
-    name: "$SAST_ANALYZER_IMAGE_PREFIX/sobelow"
+    name: "$SAST_ANALYZER_IMAGE_PREFIX/sobelow:$SAST_MAJOR_VERSION"
   only:
     variables:
-      - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /elixir/'
+      - $GITLAB_FEATURES =~ /\bsast\b/ &&
+          $SAST_DEFAULT_ANALYZERS =~ /sobelow/ &&
+          $CI_PROJECT_REPOSITORY_LANGUAGES =~ /elixir/
 
 spotbugs-sast:
   extends: .analyzer
   image:
-    name: "$SAST_ANALYZER_IMAGE_PREFIX/spotbugs"
+    name: "$SAST_ANALYZER_IMAGE_PREFIX/spotbugs:$SAST_MAJOR_VERSION"
   only:
     variables:
-      - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /java\b/'
+      - $GITLAB_FEATURES =~ /\bsast\b/ &&
+          $SAST_DEFAULT_ANALYZERS =~ /spotbugs/ &&
+          $CI_PROJECT_REPOSITORY_LANGUAGES =~ /java\b/
 
 tslint-sast:
   extends: .analyzer
   image:
-    name: "$SAST_ANALYZER_IMAGE_PREFIX/tslint"
+    name: "$SAST_ANALYZER_IMAGE_PREFIX/tslint:$SAST_MAJOR_VERSION"
   only:
     variables:
-      - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /typescript/'
+      - $GITLAB_FEATURES =~ /\bsast\b/ &&
+          $SAST_DEFAULT_ANALYZERS =~ /tslint/ &&
+          $CI_PROJECT_REPOSITORY_LANGUAGES =~ /typescript/
diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml
new file mode 100644
index 00000000000..eced181e966
--- /dev/null
+++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml
@@ -0,0 +1,29 @@
+# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html
+
+stages:
+  - build
+  - test
+  - deploy
+  - performance
+
+performance:
+  stage: performance
+  image: docker:git
+  variables:
+    URL: https://example.com
+    SITESPEED_VERSION: 6.3.1
+    SITESPEED_OPTIONS: ''
+  services:
+    - docker:stable-dind
+  script:
+    - mkdir gitlab-exporter
+    - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js
+    - mkdir sitespeed-results
+    - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS
+    - mv sitespeed-results/data/performance.json performance.json
+  artifacts:
+    paths:
+      - performance.json
+      - sitespeed-results/
+    reports:
+      performance: performance.json
diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
index 5b8c2d2f7c7..941f7178dac 100644
--- a/lib/gitlab/ci/trace.rb
+++ b/lib/gitlab/ci/trace.rb
@@ -4,6 +4,7 @@ module Gitlab
   module Ci
     class Trace
       include ::Gitlab::ExclusiveLeaseHelpers
+      include Checksummable
 
       LOCK_TTL = 10.minutes
       LOCK_RETRIES = 2
@@ -193,7 +194,7 @@ module Gitlab
             project: job.project,
             file_type: :trace,
             file: stream,
-            file_sha256: Digest::SHA256.file(path).hexdigest)
+            file_sha256: self.class.hexdigest(path))
         end
       end
 
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
index e61fb50a303..20f5620dd64 100644
--- a/lib/gitlab/ci/trace/stream.rb
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -63,10 +63,6 @@ module Gitlab
           end.force_encoding(Encoding.default_external)
         end
 
-        def html_with_state(state = nil)
-          ::Gitlab::Ci::Ansi2html.convert(stream, state)
-        end
-
         def html(last_lines: nil)
           text = raw(last_lines: last_lines)
           buffer = StringIO.new(text)
diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb
index 8f796748199..294ffad02ce 100644
--- a/lib/gitlab/cluster/lifecycle_events.rb
+++ b/lib/gitlab/cluster/lifecycle_events.rb
@@ -8,14 +8,50 @@ module Gitlab
     # watchdog threads. This lets us abstract away the Unix process
     # lifecycles of Unicorn, Sidekiq, Puma, Puma Cluster, etc.
     #
-    # We have three lifecycle events.
+    # We have the following lifecycle events.
     #
-    # - before_fork (only in forking processes)
-    #     In forking processes (Unicorn and Puma in multiprocess mode) this
-    #     will be called exactly once, on startup, before the workers are
-    #     forked. This will be called in the parent process.
-    # - worker_start
-    # - before_master_restart (only in forking processes)
+    # - on_master_start:
+    #
+    #     Unicorn/Puma Cluster: This will be called exactly once,
+    #       on startup, before the workers are forked. This is
+    #       called in the PARENT/MASTER process.
+    #
+    #     Sidekiq/Puma Single: This is called immediately.
+    #
+    # - on_before_fork:
+    #
+    #     Unicorn/Puma Cluster: This will be called exactly once,
+    #       on startup, before the workers are forked. This is
+    #       called in the PARENT/MASTER process.
+    #
+    #     Sidekiq/Puma Single: This is not called.
+    #
+    # - on_worker_start:
+    #
+    #     Unicorn/Puma Cluster: This is called in the worker process
+    #       exactly once before processing requests.
+    #
+    #     Sidekiq/Puma Single: This is called immediately.
+    #
+    # - on_before_phased_restart:
+    #
+    #     Unicorn/Puma Cluster: This will be called before a graceful
+    #       shutdown of workers starts happening.
+    #       This is called on `master` process.
+    #
+    #     Sidekiq/Puma Single: This is not called.
+    #
+    # - on_before_master_restart:
+    #
+    #     Unicorn: This will be called before a new master is spun up.
+    #       This is called on forked master before `execve` to become
+    #       a new masterfor Unicorn. This means that this does not really
+    #       affect old master process.
+    #
+    #     Puma Cluster: This will be called before a new master is spun up.
+    #       This is called on `master` process.
+    #
+    #     Sidekiq/Puma Single: This is not called.
     #
     # Blocks will be executed in the order in which they are registered.
     #
@@ -34,15 +70,17 @@ module Gitlab
         end
 
         def on_before_fork(&block)
-          return unless in_clustered_environment?
-
           # Defer block execution
           (@before_fork_hooks ||= []) << block
         end
 
-        def on_master_restart(&block)
-          return unless in_clustered_environment?
+        # Read the config/initializers/cluster_events_before_phased_restart.rb
+        def on_before_phased_restart(&block)
+          # Defer block execution
+          (@master_phased_restart ||= []) << block
+        end
 
+        def on_before_master_restart(&block)
           # Defer block execution
           (@master_restart_hooks ||= []) << block
         end
@@ -70,12 +108,21 @@ module Gitlab
           end
         end
 
-        def do_master_restart
-          @master_restart_hooks && @master_restart_hooks.each do |block|
+        def do_before_phased_restart
+          @master_phased_restart&.each do |block|
             block.call
           end
         end
 
+        def do_before_master_restart
+          @master_restart_hooks&.each do |block|
+            block.call
+          end
+        end
+
+        # DEPRECATED
+        alias_method :do_master_restart, :do_before_master_restart
+
         # Puma doesn't use singletons (which is good) but
         # this means we need to pass through whether the
         # puma server is running in single mode or cluster mode
diff --git a/lib/gitlab/cluster/mixins/puma_cluster.rb b/lib/gitlab/cluster/mixins/puma_cluster.rb
new file mode 100644
index 00000000000..e9157d9f1e4
--- /dev/null
+++ b/lib/gitlab/cluster/mixins/puma_cluster.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Cluster
+    module Mixins
+      module PumaCluster
+        def self.prepended(base)
+          raise 'missing method Puma::Cluster#stop_workers' unless base.method_defined?(:stop_workers)
+        end
+
+        def stop_workers
+          Gitlab::Cluster::LifecycleEvents.do_before_phased_restart
+
+          super
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cluster/mixins/unicorn_http_server.rb b/lib/gitlab/cluster/mixins/unicorn_http_server.rb
new file mode 100644
index 00000000000..765fd0c2baa
--- /dev/null
+++ b/lib/gitlab/cluster/mixins/unicorn_http_server.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Cluster
+    module Mixins
+      module UnicornHttpServer
+        def self.prepended(base)
+          raise 'missing method Unicorn::HttpServer#reexec' unless base.method_defined?(:reexec)
+        end
+
+        def reexec
+          Gitlab::Cluster::LifecycleEvents.do_before_phased_restart
+
+          super
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb
index 4affc52b7b0..a8440b63baa 100644
--- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb
+++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb
@@ -3,7 +3,7 @@
 module Gitlab
   module Cluster
     class PumaWorkerKillerInitializer
-      def self.start(puma_options, puma_per_worker_max_memory_mb: 650)
+      def self.start(puma_options, puma_per_worker_max_memory_mb: 850, puma_master_max_memory_mb: 550)
         require 'puma_worker_killer'
 
         PumaWorkerKiller.config do |config|
@@ -12,10 +12,9 @@ module Gitlab
           # not each worker as is the case with GITLAB_UNICORN_MEMORY_MAX
           worker_count = puma_options[:workers] || 1
           # The Puma Worker Killer checks the total RAM used by both the master
-          # and worker processes. Bump the limits to N+1 instead of N workers
-          # to account for this:
+          # and worker processes.
           # https://github.com/schneems/puma_worker_killer/blob/v0.1.0/lib/puma_worker_killer/puma_memory.rb#L57
-          config.ram = (worker_count + 1) * puma_per_worker_max_memory_mb
+          config.ram = puma_master_max_memory_mb + (worker_count * puma_per_worker_max_memory_mb)
 
           config.frequency = 20 # seconds
 
@@ -23,10 +22,9 @@ module Gitlab
           # of available RAM.
           config.percent_usage = 0.98
 
-          # Ideally we'll never hit the maximum amount of memory. If so the worker
-          # is restarted already, thus periodically restarting workers shouldn't be
-          # needed.
-          config.rolling_restart_frequency = false
+          # Ideally we'll never hit the maximum amount of memory. Restart the workers
+          # regularly rather than rely on OOM behavior for periodic restarting.
+          config.rolling_restart_frequency = 43200 # 12 hours in seconds.
 
           observer = Gitlab::Cluster::PumaWorkerKillerObserver.new
           config.pre_term = observer.callback
diff --git a/lib/gitlab/config/entry/simplifiable.rb b/lib/gitlab/config/entry/simplifiable.rb
index a56a89adb35..d58aba07d15 100644
--- a/lib/gitlab/config/entry/simplifiable.rb
+++ b/lib/gitlab/config/entry/simplifiable.rb
@@ -37,7 +37,7 @@ module Gitlab
 
         def self.entry_class(strategy)
           if strategy.present?
-            self.const_get(strategy.name)
+            self.const_get(strategy.name, false)
           else
             self::UnknownStrategy
           end
diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb
index 459bb5177b5..6aedbf64f26 100644
--- a/lib/gitlab/cycle_analytics/base_query.rb
+++ b/lib/gitlab/cycle_analytics/base_query.rb
@@ -23,6 +23,7 @@ module Gitlab
           .project(routes_table[:path].as("namespace_path"))
 
         query = limit_query(query, project_ids)
+        query = limit_query_by_date_range(query)
 
         # Load merge_requests
 
@@ -34,7 +35,12 @@ module Gitlab
       def limit_query(query, project_ids)
         query.where(issue_table[:project_id].in(project_ids))
           .where(routes_table[:source_type].eq('Namespace'))
-          .where(issue_table[:created_at].gteq(options[:from]))
+      end
+
+      def limit_query_by_date_range(query)
+        query = query.where(issue_table[:created_at].gteq(options[:from]))
+        query = query.where(issue_table[:created_at].lteq(options[:to])) if options[:to]
+        query
       end
 
       def load_merge_requests(query)
diff --git a/lib/gitlab/cycle_analytics/event_fetcher.rb b/lib/gitlab/cycle_analytics/event_fetcher.rb
index 98a30a8fc97..04f4b4f053f 100644
--- a/lib/gitlab/cycle_analytics/event_fetcher.rb
+++ b/lib/gitlab/cycle_analytics/event_fetcher.rb
@@ -4,7 +4,7 @@ module Gitlab
   module CycleAnalytics
     module EventFetcher
       def self.[](stage_name)
-        CycleAnalytics.const_get("#{stage_name.to_s.camelize}EventFetcher")
+        CycleAnalytics.const_get("#{stage_name.to_s.camelize}EventFetcher", false)
       end
     end
   end
diff --git a/lib/gitlab/cycle_analytics/issue_helper.rb b/lib/gitlab/cycle_analytics/issue_helper.rb
index 295eca5edca..f6f85b84ed8 100644
--- a/lib/gitlab/cycle_analytics/issue_helper.rb
+++ b/lib/gitlab/cycle_analytics/issue_helper.rb
@@ -12,14 +12,12 @@ module Gitlab
           .project(routes_table[:path].as("namespace_path"))
 
         query = limit_query(query, project_ids)
-
-        query
+        limit_query_by_date_range(query)
       end
 
       def limit_query(query, project_ids)
         query.where(issue_table[:project_id].in(project_ids))
           .where(routes_table[:source_type].eq('Namespace'))
-          .where(issue_table[:created_at].gteq(options[:from]))
           .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil)))
       end
     end
diff --git a/lib/gitlab/cycle_analytics/plan_helper.rb b/lib/gitlab/cycle_analytics/plan_helper.rb
index a63ae58ad21..af4bf6ed3eb 100644
--- a/lib/gitlab/cycle_analytics/plan_helper.rb
+++ b/lib/gitlab/cycle_analytics/plan_helper.rb
@@ -14,12 +14,11 @@ module Gitlab
           .where(routes_table[:source_type].eq('Namespace'))
         query = limit_query(query)
 
-        query
+        limit_query_by_date_range(query)
       end
 
       def limit_query(query)
-        query.where(issue_table[:created_at].gteq(options[:from]))
-          .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil)))
+        query.where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil)))
           .where(issue_metrics_table[:first_mentioned_in_commit_at].not_eq(nil))
       end
     end
diff --git a/lib/gitlab/cycle_analytics/stage.rb b/lib/gitlab/cycle_analytics/stage.rb
index 1bd40a7aa18..5cfd9ea4730 100644
--- a/lib/gitlab/cycle_analytics/stage.rb
+++ b/lib/gitlab/cycle_analytics/stage.rb
@@ -4,7 +4,7 @@ module Gitlab
   module CycleAnalytics
     module Stage
       def self.[](stage_name)
-        CycleAnalytics.const_get("#{stage_name.to_s.camelize}Stage")
+        CycleAnalytics.const_get("#{stage_name.to_s.camelize}Stage", false)
       end
     end
   end
diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb
index 5198dd5b4eb..ea440c441b7 100644
--- a/lib/gitlab/cycle_analytics/stage_summary.rb
+++ b/lib/gitlab/cycle_analytics/stage_summary.rb
@@ -3,16 +3,17 @@
 module Gitlab
   module CycleAnalytics
     class StageSummary
-      def initialize(project, from:, current_user:)
+      def initialize(project, from:, to: nil, current_user:)
         @project = project
         @from = from
+        @to = to
         @current_user = current_user
       end
 
       def data
-        [serialize(Summary::Issue.new(project: @project, from: @from, current_user: @current_user)),
-         serialize(Summary::Commit.new(project: @project, from: @from)),
-         serialize(Summary::Deploy.new(project: @project, from: @from))]
+        [serialize(Summary::Issue.new(project: @project, from: @from, to: @to, current_user: @current_user)),
+         serialize(Summary::Commit.new(project: @project, from: @from, to: @to)),
+         serialize(Summary::Deploy.new(project: @project, from: @from, to: @to))]
       end
 
       private
diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb
index 709221c648e..a825d48fb77 100644
--- a/lib/gitlab/cycle_analytics/summary/base.rb
+++ b/lib/gitlab/cycle_analytics/summary/base.rb
@@ -4,9 +4,10 @@ module Gitlab
   module CycleAnalytics
     module Summary
       class Base
-        def initialize(project:, from:)
+        def initialize(project:, from:, to: nil)
           @project = project
           @from = from
+          @to = to
         end
 
         def title
diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb
index f0019b26fa2..76049c6b742 100644
--- a/lib/gitlab/cycle_analytics/summary/commit.rb
+++ b/lib/gitlab/cycle_analytics/summary/commit.rb
@@ -21,7 +21,7 @@ module Gitlab
         def count_commits
           return unless ref
 
-          gitaly_commit_client.commit_count(ref, after: @from)
+          gitaly_commit_client.commit_count(ref, after: @from, before: @to)
         end
 
         def gitaly_commit_client
diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb
index 3b56dc2a7bc..5ff8d881143 100644
--- a/lib/gitlab/cycle_analytics/summary/deploy.rb
+++ b/lib/gitlab/cycle_analytics/summary/deploy.rb
@@ -4,12 +4,18 @@ module Gitlab
   module CycleAnalytics
     module Summary
       class Deploy < Base
+        include Gitlab::Utils::StrongMemoize
+
         def title
           n_('Deploy', 'Deploys', value)
         end
 
         def value
-          @value ||= @project.deployments.where("created_at > ?", @from).count
+          strong_memoize(:value) do
+            query = @project.deployments.success.where("created_at >= ?", @from)
+            query = query.where("created_at <= ?", @to) if @to
+            query.count
+          end
         end
       end
     end
diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb
index 51695c86192..52892eb5a1a 100644
--- a/lib/gitlab/cycle_analytics/summary/issue.rb
+++ b/lib/gitlab/cycle_analytics/summary/issue.rb
@@ -4,9 +4,10 @@ module Gitlab
   module CycleAnalytics
     module Summary
       class Issue < Base
-        def initialize(project:, from:, current_user:)
+        def initialize(project:, from:, to: nil, current_user:)
           @project = project
           @from = from
+          @to = to
           @current_user = current_user
         end
 
@@ -15,7 +16,7 @@ module Gitlab
         end
 
         def value
-          @value ||= IssuesFinder.new(@current_user, project_id: @project.id).execute.created_after(@from).count
+          @value ||= IssuesFinder.new(@current_user, project_id: @project.id, created_after: @from, created_before: @to).execute.count
         end
       end
     end
diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb
index 43c159fee27..8a253893892 100644
--- a/lib/gitlab/daemon.rb
+++ b/lib/gitlab/daemon.rb
@@ -34,7 +34,9 @@ module Gitlab
       @mutex.synchronize do
         break thread if thread?
 
-        @thread = Thread.new { start_working }
+        if start_working
+          @thread = Thread.new { run_thread }
+        end
       end
     end
 
@@ -57,10 +59,18 @@ module Gitlab
 
     private
 
+    # Executed in lock context before starting thread
+    # Needs to return success
     def start_working
+      true
+    end
+
+    # Executed in separate thread
+    def run_thread
       raise NotImplementedError
     end
 
+    # Executed in lock context
     def stop_working
       # no-ops
     end
diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb
index e2911b4e6c8..f22fc41a6d8 100644
--- a/lib/gitlab/danger/helper.rb
+++ b/lib/gitlab/danger/helper.rb
@@ -35,7 +35,8 @@ module Gitlab
       end
 
       def ee?
-        ENV['CI_PROJECT_NAME'] == 'gitlab-ee' || File.exist?('../../CHANGELOG-EE.md')
+        # Support former project name for `dev` and support local Danger run
+        %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) || Dir.exist?('../../ee')
       end
 
       def gitlab_helper
@@ -52,7 +53,7 @@ module Gitlab
       end
 
       def project_name
-        ee? ? 'gitlab-ee' : 'gitlab-ce'
+        ee? ? 'gitlab' : 'gitlab-foss'
       end
 
       def markdown_list(items)
@@ -89,7 +90,7 @@ module Gitlab
       end
 
       CATEGORY_LABELS = {
-        docs: "~Documentation", # Docs are reviewed along DevOps stages, so don't need roulette for now.
+        docs: "~documentation", # Docs are reviewed along DevOps stages, so don't need roulette for now.
         none: "",
         qa: "~QA",
         test: "~test for `spec/features/*`",
diff --git a/lib/gitlab/danger/request_helper.rb b/lib/gitlab/danger/request_helper.rb
new file mode 100644
index 00000000000..06da4ed9ad3
--- /dev/null
+++ b/lib/gitlab/danger/request_helper.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'json'
+
+module Gitlab
+  module Danger
+    module RequestHelper
+      HTTPError = Class.new(RuntimeError)
+
+      # @param [String] url
+      def self.http_get_json(url)
+        rsp = Net::HTTP.get_response(URI.parse(url))
+
+        unless rsp.is_a?(Net::HTTPOK)
+          raise HTTPError, "Failed to read #{url}: #{rsp.code} #{rsp.message}"
+        end
+
+        JSON.parse(rsp.body)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb
index 25de0a87c9d..dbf42912882 100644
--- a/lib/gitlab/danger/roulette.rb
+++ b/lib/gitlab/danger/roulette.rb
@@ -1,16 +1,11 @@
 # frozen_string_literal: true
 
-require 'net/http'
-require 'json'
-require 'cgi'
-
 require_relative 'teammate'
 
 module Gitlab
   module Danger
     module Roulette
       ROULETTE_DATA_URL = 'https://about.gitlab.com/roulette.json'
-      HTTPError = Class.new(RuntimeError)
 
       # Looks up the current list of GitLab team members and parses it into a
       # useful form
@@ -19,7 +14,7 @@ module Gitlab
       def team
         @team ||=
           begin
-            data = http_get_json(ROULETTE_DATA_URL)
+            data = Gitlab::Danger::RequestHelper.http_get_json(ROULETTE_DATA_URL)
             data.map { |hash| ::Gitlab::Danger::Teammate.new(hash) }
           rescue JSON::ParserError
             raise "Failed to parse JSON response from #{ROULETTE_DATA_URL}"
@@ -44,6 +39,7 @@ module Gitlab
 
       # Known issue: If someone is rejected due to OOO, and then becomes not OOO, the
       # selection will change on next spin
+      # @param [Array<Teammate>] people
       def spin_for_person(people, random:)
         people.shuffle(random: random)
           .find(&method(:valid_person?))
@@ -51,32 +47,17 @@ module Gitlab
 
       private
 
+      # @param [Teammate] person
+      # @return [Boolean]
       def valid_person?(person)
-        !mr_author?(person) && !out_of_office?(person)
+        !mr_author?(person) && person.available?
       end
 
+      # @param [Teammate] person
+      # @return [Boolean]
       def mr_author?(person)
         person.username == gitlab.mr_author
       end
-
-      def out_of_office?(person)
-        username = CGI.escape(person.username)
-        api_endpoint = "https://gitlab.com/api/v4/users/#{username}/status"
-        response = http_get_json(api_endpoint)
-        response["message"]&.match?(/OOO/i)
-      rescue HTTPError, JSON::ParserError
-        false # this is no worse than not checking for OOO
-      end
-
-      def http_get_json(url)
-        rsp = Net::HTTP.get_response(URI.parse(url))
-
-        unless rsp.is_a?(Net::HTTPSuccess)
-          raise HTTPError, "Failed to read #{url}: #{rsp.code} #{rsp.message}"
-        end
-
-        JSON.parse(rsp.body)
-      end
     end
   end
 end
diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb
index 4ad66f61c2b..5c2324836d7 100644
--- a/lib/gitlab/danger/teammate.rb
+++ b/lib/gitlab/danger/teammate.rb
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'cgi'
+
 module Gitlab
   module Danger
     class Teammate
@@ -34,8 +36,30 @@ module Gitlab
         has_capability?(project, category, :maintainer, labels)
       end
 
+      def status
+        api_endpoint = "https://gitlab.com/api/v4/users/#{CGI.escape(username)}/status"
+        @status ||= Gitlab::Danger::RequestHelper.http_get_json(api_endpoint)
+      rescue Gitlab::Danger::RequestHelper::HTTPError, JSON::ParserError
+        nil # better no status than a crashing Danger
+      end
+
+      # @return [Boolean]
+      def available?
+        !out_of_office? && has_capacity?
+      end
+
       private
 
+      # @return [Boolean]
+      def out_of_office?
+        status&.dig("message")&.match?(/OOO/i) || false
+      end
+
+      # @return [Boolean]
+      def has_capacity?
+        status&.dig("emoji") != 'red_circle'
+      end
+
       def has_capability?(project, category, kind, labels)
         case category
         when :test
diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb
index 3460e07fdc5..a83b03f540c 100644
--- a/lib/gitlab/data_builder/push.rb
+++ b/lib/gitlab/data_builder/push.rb
@@ -107,6 +107,14 @@ module Gitlab
         }
       end
 
+      def build_bulk(action:, ref_type:, changes:)
+        {
+          action: action,
+          ref_count: changes.count,
+          ref_type: ref_type
+        }
+      end
+
       # This method provides a sample data generated with
       # existing project and commits to test webhooks
       def build_sample(project, user)
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index bea9eb8cb31..50e23681de0 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -87,10 +87,6 @@ module Gitlab
       version.to_f < 10
     end
 
-    def self.join_lateral_supported?
-      version.to_f >= 9.3
-    end
-
     def self.replication_slots_supported?
       version.to_f >= 9.4
     end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 5a42952796c..ae29546cdac 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -1018,7 +1018,7 @@ into similar problems in the future (e.g. when new tables are created).
         end
 
         model_class.each_batch(of: batch_size) do |relation, index|
-          start_id, end_id = relation.pluck('MIN(id), MAX(id)').first
+          start_id, end_id = relation.pluck(Arel.sql('MIN(id), MAX(id)')).first
 
           # `BackgroundMigrationWorker.bulk_perform_in` schedules all jobs for
           # the same time, which is not helpful in most cases where we wish to
diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb
index 5422a8631a0..dfef158cc1d 100644
--- a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb
+++ b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb
@@ -33,7 +33,7 @@ module Gitlab
 
             if result[:status] == :success
               result
-            elsif STEPS_ALLOWED_TO_FAIL.include?(result[:failed_step])
+            elsif STEPS_ALLOWED_TO_FAIL.include?(result[:last_step])
               success
             else
               raise StandardError, result[:message]
@@ -42,121 +42,124 @@ module Gitlab
 
           private
 
-          def validate_application_settings
+          def validate_application_settings(_result)
             return success if application_settings
 
             log_error('No application_settings found')
             error(_('No application_settings found'))
           end
 
-          def validate_project_created
-            return success unless project_created?
+          def validate_project_created(result)
+            return success(result) unless project_created?
 
             log_error('Project already created')
             error(_('Project already created'))
           end
 
-          def validate_admins
+          def validate_admins(result)
             unless instance_admins.any?
               log_error('No active admin user found')
               return error(_('No active admin user found'))
             end
 
-            success
+            success(result)
           end
 
-          def create_group
+          def create_group(result)
             if project_created?
               log_info(_('Instance administrators group already exists'))
-              @group = application_settings.instance_administration_project.owner
-              return success(group: @group)
+              result[:group] = application_settings.instance_administration_project.owner
+              return success(result)
             end
 
-            @group = ::Groups::CreateService.new(group_owner, create_group_params).execute
+            result[:group] = ::Groups::CreateService.new(group_owner, create_group_params).execute
 
-            if @group.persisted?
-              success(group: @group)
+            if result[:group].persisted?
+              success(result)
             else
               error(_('Could not create group'))
             end
           end
 
-          def create_project
+          def create_project(result)
             if project_created?
               log_info('Instance administration project already exists')
-              @project = application_settings.instance_administration_project
-              return success(project: project)
+              result[:project] = application_settings.instance_administration_project
+              return success(result)
             end
 
-            @project = ::Projects::CreateService.new(group_owner, create_project_params).execute
+            result[:project] = ::Projects::CreateService.new(group_owner, create_project_params(result[:group])).execute
 
-            if project.persisted?
-              success(project: project)
+            if result[:project].persisted?
+              success(result)
             else
-              log_error("Could not create instance administration project. Errors: %{errors}" % { errors: project.errors.full_messages })
+              log_error("Could not create instance administration project. Errors: %{errors}" % { errors: result[:project].errors.full_messages })
               error(_('Could not create project'))
             end
           end
 
-          def save_project_id
+          def save_project_id(result)
             return success if project_created?
 
-            result = application_settings.update(instance_administration_project_id: @project.id)
+            response = application_settings.update(
+              instance_administration_project_id: result[:project].id
+            )
 
-            if result
-              success
+            if response
+              success(result)
             else
               log_error("Could not save instance administration project ID, errors: %{errors}" % { errors: application_settings.errors.full_messages })
               error(_('Could not save project ID'))
             end
           end
 
-          def add_group_members
-            members = @group.add_users(members_to_add, Gitlab::Access::MAINTAINER)
+          def add_group_members(result)
+            group = result[:group]
+            members = group.add_users(members_to_add(group), Gitlab::Access::MAINTAINER)
             errors = members.flat_map { |member| member.errors.full_messages }
 
             if errors.any?
               log_error('Could not add admins as members to self-monitoring project. Errors: %{errors}' % { errors: errors })
               error(_('Could not add admins as members'))
             else
-              success
+              success(result)
             end
           end
 
-          def add_to_whitelist
-            return success unless prometheus_enabled?
-            return success unless prometheus_listen_address.present?
+          def add_to_whitelist(result)
+            return success(result) unless prometheus_enabled?
+            return success(result) unless prometheus_listen_address.present?
 
             uri = parse_url(internal_prometheus_listen_address_uri)
             return error(_('Prometheus listen_address in config/gitlab.yml is not a valid URI')) unless uri
 
             application_settings.add_to_outbound_local_requests_whitelist([uri.normalized_host])
-            result = application_settings.save
+            response = application_settings.save
 
-            if result
+            if response
               # Expire the Gitlab::CurrentSettings cache after updating the whitelist.
               # This happens automatically in an after_commit hook, but in migrations,
               # the after_commit hook only runs at the end of the migration.
               Gitlab::CurrentSettings.expire_current_application_settings
-              success
+              success(result)
             else
               log_error("Could not add prometheus URL to whitelist, errors: %{errors}" % { errors: application_settings.errors.full_messages })
               error(_('Could not add prometheus URL to whitelist'))
             end
           end
 
-          def add_prometheus_manual_configuration
-            return success unless prometheus_enabled?
-            return success unless prometheus_listen_address.present?
+          def add_prometheus_manual_configuration(result)
+            return success(result) unless prometheus_enabled?
+            return success(result) unless prometheus_listen_address.present?
 
-            service = project.find_or_initialize_service('prometheus')
+            service = result[:project].find_or_initialize_service('prometheus')
 
             unless service.update(prometheus_service_attributes)
               log_error('Could not save prometheus manual configuration for self-monitoring project. Errors: %{errors}' % { errors: service.errors.full_messages })
               return error(_('Could not save prometheus manual configuration'))
             end
 
-            success
+            success(result)
           end
 
           def application_settings
@@ -196,11 +199,11 @@ module Gitlab
             instance_admins.first
           end
 
-          def members_to_add
+          def members_to_add(group)
             # Exclude admins who are already members of group because
-            # `@group.add_users(users)` returns an error if the users parameter contains
+            # `group.add_users(users)` returns an error if the users parameter contains
             # users who are already members of the group.
-            instance_admins - @group.members.collect(&:user)
+            instance_admins - group.members.collect(&:user)
           end
 
           def create_group_params
@@ -217,13 +220,13 @@ module Gitlab
             )
           end
 
-          def create_project_params
+          def create_project_params(group)
             {
               initialize_with_readme: true,
               visibility_level: VISIBILITY_LEVEL,
               name: PROJECT_NAME,
               description: "This project is automatically generated and will be used to help monitor this GitLab instance. [More information](#{docs_path})",
-              namespace_id: @group.id
+              namespace_id: group.id
             }
           end
 
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index c46087e65de..30fe7440148 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -428,8 +428,8 @@ module Gitlab
 
       def viewer_class_from(classes)
         return unless diffable?
-        return if different_type? || external_storage_error?
         return unless new_file? || deleted_file? || content_changed?
+        return if different_type? || external_storage_error?
 
         verify_binary = !stored_externally?
 
diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb
index e29bf75f341..c4288ca6408 100644
--- a/lib/gitlab/diff/file_collection/merge_request_diff.rb
+++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb
@@ -3,19 +3,7 @@
 module Gitlab
   module Diff
     module FileCollection
-      class MergeRequestDiff < Base
-        extend ::Gitlab::Utils::Override
-
-        def initialize(merge_request_diff, diff_options:)
-          @merge_request_diff = merge_request_diff
-
-          super(merge_request_diff,
-            project: merge_request_diff.project,
-            diff_options: diff_options,
-            diff_refs: merge_request_diff.diff_refs,
-            fallback_diff_refs: merge_request_diff.fallback_diff_refs)
-        end
-
+      class MergeRequestDiff < MergeRequestDiffBase
         def diff_files
           diff_files = super
 
diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb
new file mode 100644
index 00000000000..a747a6ed475
--- /dev/null
+++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Diff
+    module FileCollection
+      class MergeRequestDiffBase < Base
+        extend ::Gitlab::Utils::Override
+
+        def initialize(merge_request_diff, diff_options:)
+          @merge_request_diff = merge_request_diff
+
+          super(merge_request_diff,
+            project: merge_request_diff.project,
+            diff_options: diff_options,
+            diff_refs: merge_request_diff.diff_refs,
+            fallback_diff_refs: merge_request_diff.fallback_diff_refs)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb
new file mode 100644
index 00000000000..663326e01d5
--- /dev/null
+++ b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Diff
+    module FileCollection
+      # Builds a paginated diff file collection and collects pagination
+      # metadata.
+      #
+      # It doesn't handle caching yet as we're not prepared to write/read
+      # separate file keys (https://gitlab.com/gitlab-org/gitlab/issues/30550).
+      #
+      class MergeRequestDiffBatch < MergeRequestDiffBase
+        DEFAULT_BATCH_PAGE = 1
+        DEFAULT_BATCH_SIZE = 20
+
+        attr_reader :pagination_data
+
+        def initialize(merge_request_diff, batch_page, batch_size, diff_options:)
+          super(merge_request_diff, diff_options: diff_options)
+
+          batch_page ||= DEFAULT_BATCH_PAGE
+          batch_size ||= DEFAULT_BATCH_SIZE
+
+          @paginated_collection = relation.page(batch_page).per(batch_size)
+          @pagination_data = {
+            current_page: @paginated_collection.current_page,
+            next_page: @paginated_collection.next_page,
+            total_pages: @paginated_collection.total_pages
+          }
+        end
+
+        def diff_file_paths
+          diff_files.map(&:file_path)
+        end
+
+        override :diffs
+        def diffs
+          strong_memoize(:diffs) do
+            @merge_request_diff.opening_external_diff do
+              # Avoiding any extra queries.
+              collection = @paginated_collection.to_a
+
+              # The offset collection and calculation is required so that we
+              # know how much has been loaded in previous batches, collapsing
+              # the current paginated set accordingly (collection limit calculation).
+              # See: https://docs.gitlab.com/ee/development/diffs.html#diff-collection-limits
+              #
+              offset_index = collection.first&.index
+              options = diff_options.dup
+
+              collection =
+                if offset_index && offset_index > 0
+                  offset_collection = relation.limit(offset_index) # rubocop:disable CodeReuse/ActiveRecord
+                  options[:offset_index] = offset_index
+                  offset_collection + collection
+                else
+                  collection
+                end
+
+              Gitlab::Git::DiffCollection.new(collection.map(&:to_hash), options)
+            end
+          end
+        end
+
+        private
+
+        def relation
+          @merge_request_diff.merge_request_diff_files
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/lines_unfolder.rb b/lib/gitlab/diff/lines_unfolder.rb
index 0bd18fe9622..6def3a074a3 100644
--- a/lib/gitlab/diff/lines_unfolder.rb
+++ b/lib/gitlab/diff/lines_unfolder.rb
@@ -54,7 +54,7 @@ module Gitlab
       def unfold_required?
         strong_memoize(:unfold_required) do
           next false unless @diff_file.text?
-          next false unless @position.on_text? && @position.unchanged?
+          next false unless @position.unfoldable?
           next false if @diff_file.new_file? || @diff_file.deleted_file?
           next false unless @position.old_line
           # Invalid position (MR import scenario)
diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb
index dfa80eb4a64..8b99fd5cd42 100644
--- a/lib/gitlab/diff/position.rb
+++ b/lib/gitlab/diff/position.rb
@@ -79,6 +79,10 @@ module Gitlab
         formatter.line_age
       end
 
+      def unfoldable?
+        on_text? && unchanged?
+      end
+
       def unchanged?
         type.nil?
       end
@@ -118,8 +122,14 @@ module Gitlab
             path: file_path
           }
 
+          # Takes action when creating diff notes (multiple calls are
+          # submitted to this method).
           Gitlab::SafeRequestStore.fetch(key) { find_diff_file(repository) }
         end
+
+        # We need to unfold diff lines according to the position in order
+        # to correctly calculate the line code and trace position changes.
+        @diff_file&.tap { |file| file.unfold_diff_lines(self) }
       end
 
       def diff_options
@@ -152,13 +162,7 @@ module Gitlab
         return unless diff_refs.complete?
         return unless comparison = diff_refs.compare_in(repository.project)
 
-        file = comparison.diffs(diff_options).diff_files.first
-
-        # We need to unfold diff lines according to the position in order
-        # to correctly calculate the line code and trace position changes.
-        file&.unfold_diff_lines(self)
-
-        file
+        comparison.diffs(diff_options).diff_files.first
       end
 
       def get_formatter_class(type)
diff --git a/lib/gitlab/diff/position_collection.rb b/lib/gitlab/diff/position_collection.rb
new file mode 100644
index 00000000000..2112d347678
--- /dev/null
+++ b/lib/gitlab/diff/position_collection.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Diff
+    class PositionCollection
+      include Enumerable
+
+      # collection - An array of Gitlab::Diff::Position
+      def initialize(collection, diff_head_sha = nil)
+        @collection = collection
+        @diff_head_sha = diff_head_sha
+      end
+
+      def each(&block)
+        filtered_positions.each(&block)
+      end
+
+      def concat(positions)
+        tap { @collection.concat(positions) }
+      end
+
+      # Doing a lightweight filter in-memory given we're not prepared for querying
+      # positions (https://gitlab.com/gitlab-org/gitlab/issues/33271).
+      def unfoldable
+        select do |position|
+          position.unfoldable? && valid_head_sha?(position)
+        end
+      end
+
+      private
+
+      def filtered_positions
+        @collection.select { |item| item.is_a?(Position) }
+      end
+
+      def valid_head_sha?(position)
+        return true unless @diff_head_sha
+
+        position.head_sha == @diff_head_sha
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/discussions_diff/file_collection.rb b/lib/gitlab/discussions_diff/file_collection.rb
index 6692dd76438..7a9d4c5c0c2 100644
--- a/lib/gitlab/discussions_diff/file_collection.rb
+++ b/lib/gitlab/discussions_diff/file_collection.rb
@@ -27,12 +27,14 @@ module Gitlab
       # - The cache content is not updated (there's no need to do so)
       def load_highlight
         ids = highlightable_collection_ids
+        return if ids.empty?
+
         cached_content = read_cache(ids)
 
         uncached_ids = ids.select.each_with_index { |_, i| cached_content[i].nil? }
         mapping = highlighted_lines_by_ids(uncached_ids)
 
-        HighlightCache.write_multiple(mapping)
+        HighlightCache.write_multiple(mapping) if mapping.any?
 
         diffs = diff_files_indexed_by_id.values_at(*ids)
 
diff --git a/lib/gitlab/downtime_check.rb b/lib/gitlab/downtime_check.rb
index 31bb6810391..457a3c12206 100644
--- a/lib/gitlab/downtime_check.rb
+++ b/lib/gitlab/downtime_check.rb
@@ -58,13 +58,13 @@ module Gitlab
 
     # Returns true if the given migration can be performed without downtime.
     def online?(migration)
-      migration.const_get(DOWNTIME_CONST) == false
+      migration.const_get(DOWNTIME_CONST, false) == false
     end
 
     # Returns the downtime reason, or nil if none was defined.
     def downtime_reason(migration)
       if migration.const_defined?(DOWNTIME_REASON_CONST)
-        migration.const_get(DOWNTIME_REASON_CONST)
+        migration.const_get(DOWNTIME_REASON_CONST, false)
       else
         nil
       end
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index 7da8b385266..847260b2e0f 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -32,7 +32,7 @@ module Gitlab
 
         mail = build_mail
 
-        ignore_auto_submitted!(mail)
+        ignore_auto_reply!(mail)
 
         mail_key = extract_mail_key(mail)
         handler = Handler.for(mail, mail_key)
@@ -96,14 +96,25 @@ module Gitlab
         end
       end
 
-      def ignore_auto_submitted!(mail)
+      def ignore_auto_reply!(mail)
+        if auto_submitted?(mail) || auto_replied?(mail)
+          raise AutoGeneratedEmailError
+        end
+      end
+
+      def auto_submitted?(mail)
         # Mail::Header#[] is case-insensitive
         auto_submitted = mail.header['Auto-Submitted']&.value
 
         # Mail::Field#value would strip leading and trailing whitespace
-        raise AutoGeneratedEmailError if
-          # See also https://tools.ietf.org/html/rfc3834
-          auto_submitted && auto_submitted != 'no'
+        # See also https://tools.ietf.org/html/rfc3834
+        auto_submitted && auto_submitted != 'no'
+      end
+
+      def auto_replied?(mail)
+        autoreply = mail.header['X-Autoreply']&.value
+
+        autoreply && autoreply == 'yes'
       end
     end
   end
diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb
new file mode 100644
index 00000000000..895755376ee
--- /dev/null
+++ b/lib/gitlab/experimentation.rb
@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+# == Experimentation
+#
+# Utility module used for A/B testing experimental features. Define your experiments in the `EXPERIMENTS` constant.
+# The feature_toggle and environment keys are optional. If the feature_toggle is not set, a feature with the name of
+# the experiment will be checked, with a default value of true. The enabled_ratio is required and should be
+# the ratio for the number of users for which this experiment is enabled. For example: a ratio of 0.1 will
+# enable the experiment for 10% of the users (determined by the `experimentation_subject_index`).
+#
+module Gitlab
+  module Experimentation
+    EXPERIMENTS = {
+      signup_flow: {
+        feature_toggle: :experimental_separate_sign_up_flow,
+        environment: ::Gitlab.dev_env_or_com?,
+        enabled_ratio: 0.1
+      }
+    }.freeze
+
+    # Controller concern that checks if an experimentation_subject_id cookie is present and sets it if absent.
+    # Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name)` method
+    # to controllers and views.
+    #
+    module ControllerConcern
+      extend ActiveSupport::Concern
+
+      included do
+        before_action :set_experimentation_subject_id_cookie
+        helper_method :experiment_enabled?
+      end
+
+      def set_experimentation_subject_id_cookie
+        return if cookies[:experimentation_subject_id].present?
+
+        cookies.permanent.signed[:experimentation_subject_id] = {
+          value: SecureRandom.uuid,
+          domain: :all,
+          secure: ::Gitlab.config.gitlab.https
+        }
+      end
+
+      def experiment_enabled?(experiment_key)
+        Experimentation.enabled?(experiment_key, experimentation_subject_index)
+      end
+
+      private
+
+      def experimentation_subject_index
+        experimentation_subject_id = cookies.signed[:experimentation_subject_id]
+        return if experimentation_subject_id.blank?
+
+        experimentation_subject_id.delete('-').hex % 100
+      end
+    end
+
+    class << self
+      def experiment(key)
+        Experiment.new(EXPERIMENTS[key].merge(key: key))
+      end
+
+      def enabled?(experiment_key, experimentation_subject_index)
+        return false unless EXPERIMENTS.key?(experiment_key)
+
+        experiment = experiment(experiment_key)
+
+        experiment.feature_toggle_enabled? &&
+          experiment.enabled_for_environment? &&
+          experiment.enabled_for_experimentation_subject?(experimentation_subject_index)
+      end
+    end
+
+    Experiment = Struct.new(:key, :feature_toggle, :environment, :enabled_ratio, keyword_init: true) do
+      def feature_toggle_enabled?
+        return Feature.enabled?(key, default_enabled: true) if feature_toggle.nil?
+
+        Feature.enabled?(feature_toggle)
+      end
+
+      def enabled_for_environment?
+        return true if environment.nil?
+
+        environment
+      end
+
+      def enabled_for_experimentation_subject?(experimentation_subject_index)
+        return false if enabled_ratio.nil? || experimentation_subject_index.blank?
+
+        experimentation_subject_index <= enabled_ratio * 100
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/file_markdown_link_builder.rb b/lib/gitlab/file_markdown_link_builder.rb
index 180140e7da2..09d799b859d 100644
--- a/lib/gitlab/file_markdown_link_builder.rb
+++ b/lib/gitlab/file_markdown_link_builder.rb
@@ -10,14 +10,14 @@ module Gitlab
       return unless name = markdown_name
 
       markdown = "[#{name.gsub(']', '\\]')}](#{secure_url})"
-      markdown = "!#{markdown}" if image_or_video? || dangerous?
+      markdown = "!#{markdown}" if embeddable? || dangerous_embeddable?
       markdown
     end
 
     def markdown_name
       return unless filename.present?
 
-      image_or_video? ? File.basename(filename, File.extname(filename)) : filename
+      embeddable? ? File.basename(filename, File.extname(filename)) : filename
     end
   end
 end
diff --git a/lib/gitlab/file_type_detection.rb b/lib/gitlab/file_type_detection.rb
index 25ee07cf940..ca78d49f99b 100644
--- a/lib/gitlab/file_type_detection.rb
+++ b/lib/gitlab/file_type_detection.rb
@@ -1,34 +1,69 @@
 # frozen_string_literal: true
 
-# File helpers methods.
-# It needs the method filename to be defined.
+# The method `filename` must be defined in classes that use this module.
+#
+# This module is intended to be used as a helper and not a security gate
+# to validate that a file is safe, as it identifies files only by the
+# file extension and not its actual contents.
+#
+# An example useage of this module is in `FileMarkdownLinkBuilder` that
+# renders markdown depending on a file name.
+#
+# We use Workhorse to detect the real extension when we serve files with
+# the `SendsBlob` helper methods, and ask Workhorse to set the content
+# type when it serves the file:
+# https://gitlab.com/gitlab-org/gitlab/blob/33e5955/app/helpers/workhorse_helper.rb#L48.
+#
+# Because Workhorse has access to the content when it is downloaded, if
+# the type/extension doesn't match the real type, we adjust the
+# `Content-Type` and `Content-Disposition` to the one we get from the detection.
 module Gitlab
   module FileTypeDetection
-    IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico].freeze
+    SAFE_IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico].freeze
     # We recommend using the .mp4 format over .mov. Videos in .mov format can
     # still be used but you really need to make sure they are served with the
     # proper MIME type video/mp4 and not video/quicktime or your videos won't play
     # on IE >= 9.
     # http://archive.sublimevideo.info/20150912/docs.sublimevideo.net/troubleshooting.html
-    VIDEO_EXT = %w[mp4 m4v mov webm ogv].freeze
+    SAFE_VIDEO_EXT = %w[mp4 m4v mov webm ogv].freeze
+    SAFE_AUDIO_EXT = %w[mp3 oga ogg spx wav].freeze
+
     # These extension types can contain dangerous code and should only be embedded inline with
     # proper filtering. They should always be tagged as "Content-Disposition: attachment", not "inline".
-    DANGEROUS_EXT = %w[svg].freeze
+    DANGEROUS_IMAGE_EXT = %w[svg].freeze
+    DANGEROUS_VIDEO_EXT = [].freeze # None, yet
+    DANGEROUS_AUDIO_EXT = [].freeze # None, yet
 
     def image?
-      extension_match?(IMAGE_EXT)
+      extension_match?(SAFE_IMAGE_EXT)
     end
 
     def video?
-      extension_match?(VIDEO_EXT)
+      extension_match?(SAFE_VIDEO_EXT)
+    end
+
+    def audio?
+      extension_match?(SAFE_AUDIO_EXT)
+    end
+
+    def embeddable?
+      image? || video? || audio?
+    end
+
+    def dangerous_image?
+      extension_match?(DANGEROUS_IMAGE_EXT)
+    end
+
+    def dangerous_video?
+      extension_match?(DANGEROUS_VIDEO_EXT)
     end
 
-    def image_or_video?
-      image? || video?
+    def dangerous_audio?
+      extension_match?(DANGEROUS_AUDIO_EXT)
     end
 
-    def dangerous?
-      extension_match?(DANGEROUS_EXT)
+    def dangerous_embeddable?
+      dangerous_image? || dangerous_video? || dangerous_audio?
     end
 
     private
diff --git a/lib/gitlab/git/changes.rb b/lib/gitlab/git/changes.rb
new file mode 100644
index 00000000000..4e888eec44f
--- /dev/null
+++ b/lib/gitlab/git/changes.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Git
+    class Changes
+      include Enumerable
+
+      attr_reader :repository_data
+
+      def initialize
+        @refs = Set.new
+        @items = []
+        @branches_index = []
+        @tags_index = []
+        @repository_data = []
+      end
+
+      def includes_branches?
+        branches_index.any?
+      end
+
+      def includes_tags?
+        tags_index.any?
+      end
+
+      def add_branch_change(change)
+        @branches_index << add_change(change)
+        self
+      end
+
+      def add_tag_change(change)
+        @tags_index << add_change(change)
+        self
+      end
+
+      def each
+        items.each do |item|
+          yield item
+        end
+      end
+
+      def refs
+        @refs.to_a
+      end
+
+      def branch_changes
+        items.values_at(*branches_index)
+      end
+
+      def tag_changes
+        items.values_at(*tags_index)
+      end
+
+      private
+
+      attr_reader :items, :branches_index, :tags_index
+
+      def add_change(change)
+        # refs and repository_data are being cached when a change is added to
+        # the collection to remove the need to iterate through changes multiple
+        # times.
+        @refs << change[:ref]
+        @repository_data << build_change_repository_data(change)
+        @items << change
+
+        @items.size - 1
+      end
+
+      def build_change_repository_data(change)
+        DataBuilder::Repository.single_change(change[:oldrev], change[:newrev], change[:ref])
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb
index cb9154cb1e8..b79e30bff78 100644
--- a/lib/gitlab/git/diff_collection.rb
+++ b/lib/gitlab/git/diff_collection.rb
@@ -31,6 +31,7 @@ module Gitlab
         @limits = self.class.limits(options)
         @enforce_limits = !!options.fetch(:limits, true)
         @expanded = !!options.fetch(:expanded, true)
+        @offset_index = options.fetch(:offset_index, 0)
 
         @line_count = 0
         @byte_count = 0
@@ -128,7 +129,7 @@ module Gitlab
       def each_serialized_patch
         i = @array.length
 
-        @iterator.each do |raw|
+        @iterator.each_with_index do |raw, iterator_index|
           @empty = false
 
           if @enforce_limits && i >= max_files
@@ -154,8 +155,12 @@ module Gitlab
             break
           end
 
-          yield @array[i] = diff
-          i += 1
+          # We should not yield / memoize diffs before the offset index. Though,
+          # we still consider the limit buffers for diffs before it.
+          if iterator_index >= @offset_index
+            yield @array[i] = diff
+            i += 1
+          end
         end
       end
     end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 4ea618f063b..b2c22898079 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -131,6 +131,18 @@ module Gitlab
         end
       end
 
+      def rename(new_relative_path)
+        wrapped_gitaly_errors do
+          gitaly_repository_client.rename(new_relative_path)
+        end
+      end
+
+      def remove
+        wrapped_gitaly_errors do
+          gitaly_repository_client.remove
+        end
+      end
+
       def expire_has_local_branches_cache
         clear_memoization(:has_local_branches)
       end
diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb
index 2a8bcd015a8..5264bae47a1 100644
--- a/lib/gitlab/git_post_receive.rb
+++ b/lib/gitlab/git_post_receive.rb
@@ -8,7 +8,7 @@ module Gitlab
     def initialize(project, identifier, changes, push_options = {})
       @project = project
       @identifier = identifier
-      @changes = deserialize_changes(changes)
+      @changes = parse_changes(changes)
       @push_options = push_options
     end
 
@@ -16,27 +16,12 @@ module Gitlab
       super(identifier)
     end
 
-    def changes_refs
-      return changes unless block_given?
-
-      changes.each do |change|
-        change.strip!
-        oldrev, newrev, ref = change.split(' ')
-
-        yield oldrev, newrev, ref
-      end
-    end
-
     def includes_branches?
-      enum_for(:changes_refs).any? do |_oldrev, _newrev, ref|
-        Gitlab::Git.branch_ref?(ref)
-      end
+      changes.includes_branches?
     end
 
     def includes_tags?
-      enum_for(:changes_refs).any? do |_oldrev, _newrev, ref|
-        Gitlab::Git.tag_ref?(ref)
-      end
+      changes.includes_tags?
     end
 
     def includes_default_branch?
@@ -44,16 +29,28 @@ module Gitlab
       # first branch pushed will be the default.
       return true unless project.default_branch.present?
 
-      enum_for(:changes_refs).any? do |_oldrev, _newrev, ref|
-        Gitlab::Git.branch_ref?(ref) &&
-          Gitlab::Git.branch_name(ref) == project.default_branch
+      changes.branch_changes.any? do |change|
+        Gitlab::Git.branch_name(change[:ref]) == project.default_branch
       end
     end
 
     private
 
-    def deserialize_changes(changes)
-      utf8_encode_changes(changes).each_line
+    def parse_changes(changes)
+      deserialized_changes = utf8_encode_changes(changes).each_line
+
+      Git::Changes.new.tap do |collection|
+        deserialized_changes.each_with_index do |raw_change, index|
+          oldrev, newrev, ref = raw_change.strip.split(' ')
+          change = { index: index, oldrev: oldrev, newrev: newrev, ref: ref }
+
+          if Git.branch_ref?(ref)
+            collection.add_branch_change(change)
+          elsif Git.tag_ref?(ref)
+            collection.add_tag_change(change)
+          end
+        end
+      end
     end
 
     def utf8_encode_changes(changes)
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 2ac99b1ff02..b0f29d22ad4 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -86,7 +86,7 @@ module Gitlab
       if name == :health_check
         Grpc::Health::V1::Health::Stub
       else
-        Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub)
+        Gitaly.const_get(name.to_s.camelcase.to_sym, false).const_get(:Stub, false)
       end
     end
 
@@ -142,13 +142,13 @@ module Gitlab
     #   kwargs.merge(deadline: Time.now + 10)
     # end
     #
-    def self.call(storage, service, rpc, request, remote_storage: nil, timeout: nil)
+    def self.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout)
       start = Gitlab::Metrics::System.monotonic_time
       request_hash = request.is_a?(Google::Protobuf::MessageExts) ? request.to_h : {}
 
       enforce_gitaly_request_limits(:call)
 
-      kwargs = request_kwargs(storage, timeout, remote_storage: remote_storage)
+      kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage)
       kwargs = yield(kwargs) if block_given?
 
       stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend
@@ -200,7 +200,7 @@ module Gitlab
     end
     private_class_method :authorization_token
 
-    def self.request_kwargs(storage, timeout, remote_storage: nil)
+    def self.request_kwargs(storage, timeout:, remote_storage: nil)
       metadata = {
         'authorization' => "Bearer #{authorization_token(storage)}",
         'client_name' => CLIENT_NAME
@@ -216,14 +216,7 @@ module Gitlab
 
       result = { metadata: metadata }
 
-      # nil timeout indicates that we should use the default
-      timeout = default_timeout if timeout.nil?
-
-      return result unless timeout > 0
-
-      deadline = real_time + timeout
-      result[:deadline] = deadline
-
+      result[:deadline] = real_time + timeout if timeout > 0
       result
     end
 
@@ -357,8 +350,6 @@ module Gitlab
 
     # The default timeout on all Gitaly calls
     def self.default_timeout
-      return no_timeout if Sidekiq.server?
-
       timeout(:gitaly_timeout_default)
     end
 
@@ -370,8 +361,12 @@ module Gitlab
       timeout(:gitaly_timeout_medium)
     end
 
-    def self.no_timeout
-      0
+    def self.long_timeout
+      if Sidekiq.server?
+        6.hours
+      else
+        default_timeout
+      end
     end
 
     def self.storage_metadata_file_path(storage)
diff --git a/lib/gitlab/gitaly_client/attributes_bag.rb b/lib/gitlab/gitaly_client/attributes_bag.rb
index 3f1a0ef4888..f935281ac2e 100644
--- a/lib/gitlab/gitaly_client/attributes_bag.rb
+++ b/lib/gitlab/gitaly_client/attributes_bag.rb
@@ -8,7 +8,7 @@ module Gitlab
       extend ActiveSupport::Concern
 
       included do
-        attr_accessor(*const_get(:ATTRS))
+        attr_accessor(*const_get(:ATTRS, false))
       end
 
       def initialize(params)
@@ -26,7 +26,7 @@ module Gitlab
       end
 
       def attributes
-        self.class.const_get(:ATTRS)
+        self.class.const_get(:ATTRS, false)
       end
     end
   end
diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb
index 8ccefb00d20..5cde06bb6aa 100644
--- a/lib/gitlab/gitaly_client/blob_service.rb
+++ b/lib/gitlab/gitaly_client/blob_service.rb
@@ -76,6 +76,30 @@ module Gitlab
         GitalyClient::BlobsStitcher.new(response)
       end
 
+      def get_blob_types(revision_paths, limit = -1)
+        return {} if revision_paths.empty?
+
+        request_revision_paths = revision_paths.map do |rev, path|
+          Gitaly::GetBlobsRequest::RevisionPath.new(revision: rev, path: encode_binary(path))
+        end
+
+        request = Gitaly::GetBlobsRequest.new(
+          repository: @gitaly_repo,
+          revision_paths: request_revision_paths,
+          limit: limit
+        )
+
+        response = GitalyClient.call(
+          @gitaly_repo.storage_name,
+          :blob_service,
+          :get_blobs,
+          request,
+          timeout: GitalyClient.fast_timeout
+        )
+
+        map_blob_types(response)
+      end
+
       def get_new_lfs_pointers(revision, limit, not_in, dynamic_timeout = nil)
         request = Gitaly::GetNewLFSPointersRequest.new(
           repository: @gitaly_repo,
@@ -132,6 +156,16 @@ module Gitlab
           end
         end
       end
+
+      def map_blob_types(response)
+        types = {}
+
+        response.each do |msg|
+          types[msg.path.dup.force_encoding('utf-8')] = msg.type.downcase
+        end
+
+        types
+      end
     end
   end
 end
diff --git a/lib/gitlab/gitaly_client/cleanup_service.rb b/lib/gitlab/gitaly_client/cleanup_service.rb
index a56bc35f6d7..e2293d3121a 100644
--- a/lib/gitlab/gitaly_client/cleanup_service.rb
+++ b/lib/gitlab/gitaly_client/cleanup_service.rb
@@ -18,7 +18,7 @@ module Gitlab
           :cleanup_service,
           :apply_bfg_object_map_stream,
           build_object_map_enum(io),
-          timeout: GitalyClient.no_timeout
+          timeout: GitalyClient.long_timeout
         )
 
         responses.each(&blk)
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index a80ce462ab0..b0559729ff3 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -140,7 +140,8 @@ module Gitlab
         request = Gitaly::CountCommitsRequest.new(
           repository: @gitaly_repo,
           revision: encode_binary(ref),
-          all: !!options[:all]
+          all: !!options[:all],
+          first_parent: !!options[:first_parent]
         )
         request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present?
         request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present?
@@ -254,7 +255,7 @@ module Gitlab
 
       def languages(ref = nil)
         request = Gitaly::CommitLanguagesRequest.new(repository: @gitaly_repo, revision: ref || '')
-        response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request)
+        response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request, timeout: GitalyClient.long_timeout)
 
         response.languages.map { |l| { value: l.share.round(2), label: l.name, color: l.color, highlight: l.color } }
       end
@@ -297,18 +298,6 @@ module Gitlab
         Gitlab::SafeRequestStore[key] = commit
       end
 
-      # rubocop: disable CodeReuse/ActiveRecord
-      def patch(revision)
-        request = Gitaly::CommitPatchRequest.new(
-          repository: @gitaly_repo,
-          revision: encode_binary(revision)
-        )
-        response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request, timeout: GitalyClient.medium_timeout)
-
-        response.sum(&:data)
-      end
-      # rubocop: enable CodeReuse/ActiveRecord
-
       def commit_stats(revision)
         request = Gitaly::CommitStatsRequest.new(
           repository: @gitaly_repo,
@@ -325,6 +314,7 @@ module Gitlab
           follow:       options[:follow],
           skip_merges:  options[:skip_merges],
           all:          !!options[:all],
+          first_parent: !!options[:first_parent],
           disable_walk: true # This option is deprecated. The 'walk' implementation is being removed.
         )
         request.after    = GitalyClient.timestamp(options[:after]) if options[:after]
@@ -360,7 +350,7 @@ module Gitlab
 
       def extract_signature(commit_id)
         request = Gitaly::ExtractCommitSignatureRequest.new(repository: @gitaly_repo, commit_id: commit_id)
-        response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request)
+        response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request, timeout: GitalyClient.fast_timeout)
 
         signature = +''.b
         signed_text = +''.b
diff --git a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb
index 0e00f6e8c44..38ec910111c 100644
--- a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb
+++ b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb
@@ -5,8 +5,11 @@ module Gitlab
     class ConflictFilesStitcher
       include Enumerable
 
-      def initialize(rpc_response)
+      attr_reader :gitaly_repo
+
+      def initialize(rpc_response, gitaly_repo)
         @rpc_response = rpc_response
+        @gitaly_repo = gitaly_repo
       end
 
       def each
@@ -31,7 +34,7 @@ module Gitlab
 
       def file_from_gitaly_header(header)
         Gitlab::Git::Conflict::File.new(
-          Gitlab::GitalyClient::Util.git_repository(header.repository),
+          Gitlab::GitalyClient::Util.git_repository(gitaly_repo),
           header.commit_oid,
           conflict_from_gitaly_file_header(header),
           ''
diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb
index d16e45c964d..f7eb4b45197 100644
--- a/lib/gitlab/gitaly_client/conflicts_service.rb
+++ b/lib/gitlab/gitaly_client/conflicts_service.rb
@@ -20,9 +20,9 @@ module Gitlab
           our_commit_oid: @our_commit_oid,
           their_commit_oid: @their_commit_oid
         )
-        response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request)
+        response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request, timeout: GitalyClient.long_timeout)
 
-        GitalyClient::ConflictFilesStitcher.new(response)
+        GitalyClient::ConflictFilesStitcher.new(response, @gitaly_repo)
       end
 
       def conflicts?
diff --git a/lib/gitlab/gitaly_client/namespace_service.rb b/lib/gitlab/gitaly_client/namespace_service.rb
index f0be3cbebd2..0be214f3035 100644
--- a/lib/gitlab/gitaly_client/namespace_service.rb
+++ b/lib/gitlab/gitaly_client/namespace_service.rb
@@ -22,7 +22,7 @@ module Gitlab
       def remove(name)
         request = Gitaly::RemoveNamespaceRequest.new(storage_name: @storage, name: name)
 
-        gitaly_client_call(:remove_namespace, request, timeout: nil)
+        gitaly_client_call(:remove_namespace, request, timeout: GitalyClient.long_timeout)
       end
 
       def rename(from, to)
diff --git a/lib/gitlab/gitaly_client/object_pool_service.rb b/lib/gitlab/gitaly_client/object_pool_service.rb
index d7fac26bc13..786ef0ebebe 100644
--- a/lib/gitlab/gitaly_client/object_pool_service.rb
+++ b/lib/gitlab/gitaly_client/object_pool_service.rb
@@ -15,13 +15,15 @@ module Gitlab
           object_pool: object_pool,
           origin: repository.gitaly_repository)
 
-        GitalyClient.call(storage, :object_pool_service, :create_object_pool, request)
+        GitalyClient.call(storage, :object_pool_service, :create_object_pool,
+                          request, timeout: GitalyClient.medium_timeout)
       end
 
       def delete
         request = Gitaly::DeleteObjectPoolRequest.new(object_pool: object_pool)
 
-        GitalyClient.call(storage, :object_pool_service, :delete_object_pool, request)
+        GitalyClient.call(storage, :object_pool_service, :delete_object_pool,
+                          request, timeout: GitalyClient.long_timeout)
       end
 
       def link_repository(repository)
@@ -40,7 +42,8 @@ module Gitlab
           origin: repository.gitaly_repository
         )
 
-        GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, request)
+        GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool,
+                          request, timeout: GitalyClient.long_timeout)
       end
     end
   end
diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb
index 33ca428a942..6e486c763da 100644
--- a/lib/gitlab/gitaly_client/operation_service.rb
+++ b/lib/gitlab/gitaly_client/operation_service.rb
@@ -19,7 +19,7 @@ module Gitlab
           user: Gitlab::Git::User.from_gitlab(user).to_gitaly
         )
 
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.medium_timeout)
+        response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.long_timeout)
 
         if pre_receive_error = response.pre_receive_error.presence
           raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -35,7 +35,7 @@ module Gitlab
           message: encode_binary(message.to_s)
         )
 
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.medium_timeout)
+        response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.long_timeout)
         if pre_receive_error = response.pre_receive_error.presence
           raise Gitlab::Git::PreReceiveError, pre_receive_error
         elsif response.exists
@@ -55,7 +55,7 @@ module Gitlab
           start_point: encode_binary(start_point)
         )
         response = GitalyClient.call(@repository.storage, :operation_service,
-          :user_create_branch, request)
+                                     :user_create_branch, request, timeout: GitalyClient.long_timeout)
 
         if response.pre_receive_error.present?
           raise Gitlab::Git::PreReceiveError.new(response.pre_receive_error)
@@ -79,7 +79,8 @@ module Gitlab
           oldrev: encode_binary(oldrev)
         )
 
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_update_branch, request)
+        response = GitalyClient.call(@repository.storage, :operation_service,
+                                     :user_update_branch, request, timeout: GitalyClient.long_timeout)
 
         if pre_receive_error = response.pre_receive_error.presence
           raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -93,7 +94,8 @@ module Gitlab
           user: Gitlab::Git::User.from_gitlab(user).to_gitaly
         )
 
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_branch, request)
+        response = GitalyClient.call(@repository.storage, :operation_service,
+                                     :user_delete_branch, request, timeout: GitalyClient.long_timeout)
 
         if pre_receive_error = response.pre_receive_error.presence
           raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -111,7 +113,8 @@ module Gitlab
           first_parent_ref: encode_binary(first_parent_ref)
         )
 
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_merge_to_ref, request)
+        response = GitalyClient.call(@repository.storage, :operation_service,
+                                     :user_merge_to_ref, request, timeout: GitalyClient.long_timeout)
 
         if pre_receive_error = response.pre_receive_error.presence
           raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -126,7 +129,8 @@ module Gitlab
           @repository.storage,
           :operation_service,
           :user_merge_branch,
-          request_enum.each
+          request_enum.each,
+          timeout: GitalyClient.long_timeout
         )
 
         request_enum.push(
@@ -170,7 +174,8 @@ module Gitlab
           @repository.storage,
           :operation_service,
           :user_ff_branch,
-          request
+          request,
+          timeout: GitalyClient.long_timeout
         )
 
         Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
@@ -215,6 +220,7 @@ module Gitlab
           :operation_service,
           :user_rebase,
           request,
+          timeout: GitalyClient.long_timeout,
           remote_storage: remote_repository.storage
         )
 
@@ -236,6 +242,7 @@ module Gitlab
           :operation_service,
           :user_rebase_confirmable,
           request_enum.each,
+          timeout: GitalyClient.long_timeout,
           remote_storage: remote_repository.storage
         )
 
@@ -286,7 +293,8 @@ module Gitlab
           @repository.storage,
           :operation_service,
           :user_squash,
-          request
+          request,
+          timeout: GitalyClient.long_timeout
         )
 
         if response.git_error.presence
@@ -310,7 +318,8 @@ module Gitlab
           @repository.storage,
           :operation_service,
           :user_update_submodule,
-          request
+          request,
+          timeout: GitalyClient.long_timeout
         )
 
         if response.pre_receive_error.present?
@@ -352,7 +361,8 @@ module Gitlab
         end
 
         response = GitalyClient.call(@repository.storage, :operation_service,
-          :user_commit_files, req_enum, remote_storage: start_repository.storage)
+                                     :user_commit_files, req_enum, timeout: GitalyClient.long_timeout,
+                                     remote_storage: start_repository.storage)
 
         if (pre_receive_error = response.pre_receive_error.presence)
           raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -384,7 +394,8 @@ module Gitlab
           end
         end
 
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_apply_patch, chunks)
+        response = GitalyClient.call(@repository.storage, :operation_service,
+                                     :user_apply_patch, chunks, timeout: GitalyClient.long_timeout)
 
         Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
       end
@@ -424,7 +435,7 @@ module Gitlab
           :"user_#{rpc}",
           request,
           remote_storage: start_repository.storage,
-          timeout: GitalyClient.medium_timeout
+          timeout: GitalyClient.long_timeout
         )
 
         handle_cherry_pick_or_revert_response(response)
diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb
index b7d509dfa48..d1f848fae26 100644
--- a/lib/gitlab/gitaly_client/ref_service.rb
+++ b/lib/gitlab/gitaly_client/ref_service.rb
@@ -21,7 +21,7 @@ module Gitlab
 
       def remote_branches(remote_name)
         request = Gitaly::FindAllRemoteBranchesRequest.new(repository: @gitaly_repo, remote_name: remote_name)
-        response = GitalyClient.call(@repository.storage, :ref_service, :find_all_remote_branches, request)
+        response = GitalyClient.call(@repository.storage, :ref_service, :find_all_remote_branches, request, timeout: GitalyClient.medium_timeout)
 
         consume_find_all_remote_branches_response(remote_name, response)
       end
@@ -158,7 +158,7 @@ module Gitlab
           start_point: encode_binary(start_point)
         )
 
-        response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request)
+        response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request, timeout: GitalyClient.medium_timeout)
 
         case response.status
         when :OK
@@ -182,7 +182,7 @@ module Gitlab
           name: encode_binary(branch_name)
         )
 
-        GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request)
+        GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request, timeout: GitalyClient.medium_timeout)
       end
 
       def delete_refs(refs: [], except_with_prefixes: [])
@@ -192,7 +192,7 @@ module Gitlab
           except_with_prefix: except_with_prefixes.map { |r| encode_binary(r) }
         )
 
-        response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.default_timeout)
+        response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.medium_timeout)
 
         raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present?
       end
@@ -242,7 +242,7 @@ module Gitlab
       def pack_refs
         request = Gitaly::PackRefsRequest.new(repository: @gitaly_repo)
 
-        GitalyClient.call(@storage, :ref_service, :pack_refs, request)
+        GitalyClient.call(@storage, :ref_service, :pack_refs, request, timeout: GitalyClient.long_timeout)
       end
 
       private
diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb
index f3589fea39f..d01a29e1a05 100644
--- a/lib/gitlab/gitaly_client/remote_service.rb
+++ b/lib/gitlab/gitaly_client/remote_service.rb
@@ -38,9 +38,7 @@ module Gitlab
       def remove_remote(name)
         request = Gitaly::RemoveRemoteRequest.new(repository: @gitaly_repo, name: name)
 
-        response = GitalyClient.call(@storage, :remote_service, :remove_remote, request)
-
-        response.result
+        GitalyClient.call(@storage, :remote_service, :remove_remote, request, timeout: GitalyClient.long_timeout).result
       end
 
       def fetch_internal_remote(repository)
@@ -51,6 +49,7 @@ module Gitlab
 
         response = GitalyClient.call(@storage, :remote_service,
                                      :fetch_internal_remote, request,
+                                     timeout: GitalyClient.medium_timeout,
                                      remote_storage: repository.storage)
 
         response.result
@@ -63,7 +62,7 @@ module Gitlab
         )
 
         response = GitalyClient.call(@storage, :remote_service,
-                                     :find_remote_root_ref, request)
+                                     :find_remote_root_ref, request, timeout: GitalyClient.medium_timeout)
 
         encode_utf8(response.ref)
       end
@@ -95,7 +94,7 @@ module Gitlab
           end
         end
 
-        GitalyClient.call(@storage, :remote_service, :update_remote_mirror, req_enum)
+        GitalyClient.call(@storage, :remote_service, :update_remote_mirror, req_enum, timeout: GitalyClient.long_timeout)
       end
     end
   end
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index ca3e5b51ecc..d0e5e0db830 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -28,17 +28,17 @@ module Gitlab
 
       def garbage_collect(create_bitmap)
         request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap)
-        GitalyClient.call(@storage, :repository_service, :garbage_collect, request)
+        GitalyClient.call(@storage, :repository_service, :garbage_collect, request, timeout: GitalyClient.long_timeout)
       end
 
       def repack_full(create_bitmap)
         request = Gitaly::RepackFullRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap)
-        GitalyClient.call(@storage, :repository_service, :repack_full, request)
+        GitalyClient.call(@storage, :repository_service, :repack_full, request, timeout: GitalyClient.long_timeout)
       end
 
       def repack_incremental
         request = Gitaly::RepackIncrementalRequest.new(repository: @gitaly_repo)
-        GitalyClient.call(@storage, :repository_service, :repack_incremental, request)
+        GitalyClient.call(@storage, :repository_service, :repack_incremental, request, timeout: GitalyClient.long_timeout)
       end
 
       def repository_size
@@ -86,12 +86,12 @@ module Gitlab
           end
         end
 
-        GitalyClient.call(@storage, :repository_service, :fetch_remote, request)
+        GitalyClient.call(@storage, :repository_service, :fetch_remote, request, timeout: GitalyClient.long_timeout)
       end
 
       def create_repository
         request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo)
-        GitalyClient.call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.medium_timeout)
+        GitalyClient.call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.fast_timeout)
       end
 
       def has_local_branches?
@@ -123,7 +123,7 @@ module Gitlab
           :create_fork,
           request,
           remote_storage: source_repository.storage,
-          timeout: GitalyClient.default_timeout
+          timeout: GitalyClient.long_timeout
         )
       end
 
@@ -138,7 +138,7 @@ module Gitlab
           :repository_service,
           :create_repository_from_url,
           request,
-          timeout: GitalyClient.default_timeout
+          timeout: GitalyClient.long_timeout
         )
       end
 
@@ -189,6 +189,7 @@ module Gitlab
           :repository_service,
           :fetch_source_branch,
           request,
+          timeout: GitalyClient.long_timeout,
           remote_storage: source_repository.storage
         )
 
@@ -197,7 +198,7 @@ module Gitlab
 
       def fsck
         request = Gitaly::FsckRequest.new(repository: @gitaly_repo)
-        response = GitalyClient.call(@storage, :repository_service, :fsck, request, timeout: GitalyClient.no_timeout)
+        response = GitalyClient.call(@storage, :repository_service, :fsck, request, timeout: GitalyClient.long_timeout)
 
         if response.error.empty?
           return "", 0
@@ -211,7 +212,7 @@ module Gitlab
           save_path,
           :create_bundle,
           Gitaly::CreateBundleRequest,
-          GitalyClient.no_timeout
+          GitalyClient.long_timeout
         )
       end
 
@@ -229,7 +230,7 @@ module Gitlab
           bundle_path,
           :create_repository_from_bundle,
           Gitaly::CreateRepositoryFromBundleRequest,
-          GitalyClient.no_timeout
+          GitalyClient.long_timeout
         )
       end
 
@@ -254,7 +255,7 @@ module Gitlab
           :repository_service,
           :create_repository_from_snapshot,
           request,
-          timeout: GitalyClient.no_timeout
+          timeout: GitalyClient.long_timeout
         )
       end
 
@@ -333,7 +334,7 @@ module Gitlab
 
       def search_files_by_content(ref, query)
         request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query)
-        response = GitalyClient.call(@storage, :repository_service, :search_files_by_content, request)
+        response = GitalyClient.call(@storage, :repository_service, :search_files_by_content, request, timeout: GitalyClient.default_timeout)
 
         search_results_from_response(response)
       end
@@ -343,7 +344,19 @@ module Gitlab
           repository: @gitaly_repo
         )
 
-        GitalyClient.call(@storage, :object_pool_service, :disconnect_git_alternates, request)
+        GitalyClient.call(@storage, :object_pool_service, :disconnect_git_alternates, request, timeout: GitalyClient.long_timeout)
+      end
+
+      def rename(relative_path)
+        request = Gitaly::RenameRepositoryRequest.new(repository: @gitaly_repo, relative_path: relative_path)
+
+        GitalyClient.call(@storage, :repository_service, :rename_repository, request, timeout: GitalyClient.fast_timeout)
+      end
+
+      def remove
+        request = Gitaly::RemoveRepositoryRequest.new(repository: @gitaly_repo)
+
+        GitalyClient.call(@storage, :repository_service, :remove_repository, request, timeout: GitalyClient.long_timeout)
       end
 
       private
diff --git a/lib/gitlab/gitaly_client/storage_service.rb b/lib/gitlab/gitaly_client/storage_service.rb
deleted file mode 100644
index 4edcb0b8ba9..00000000000
--- a/lib/gitlab/gitaly_client/storage_service.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module GitalyClient
-    class StorageService
-      def initialize(storage)
-        @storage = storage
-      end
-
-      # Returns all directories in the git storage directory, lexically ordered
-      def list_directories(depth: 1)
-        request = Gitaly::ListDirectoriesRequest.new(storage_name: @storage, depth: depth)
-
-        GitalyClient.call(@storage, :storage_service, :list_directories, request)
-          .flat_map(&:paths)
-      end
-
-      # Delete all repositories in the storage. This is a slow and VERY DESTRUCTIVE operation.
-      def delete_all_repositories
-        request = Gitaly::DeleteAllRepositoriesRequest.new(storage_name: @storage)
-        GitalyClient.call(@storage, :storage_service, :delete_all_repositories, request)
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb
index 7d1206e551b..43848772947 100644
--- a/lib/gitlab/gitaly_client/storage_settings.rb
+++ b/lib/gitlab/gitaly_client/storage_settings.rb
@@ -53,7 +53,7 @@ module Gitlab
         @legacy_disk_path = File.expand_path(storage['path'], Rails.root) if storage['path']
 
         storage['path'] = Deprecated
-        @hash = storage
+        @hash = storage.with_indifferent_access
       end
 
       def gitaly_address
diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb
index ce9faad825c..15e0d7349dd 100644
--- a/lib/gitlab/gitaly_client/wiki_service.rb
+++ b/lib/gitlab/gitaly_client/wiki_service.rb
@@ -34,7 +34,7 @@ module Gitlab
           end
         end
 
-        response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_write_page, enum)
+        response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_write_page, enum, timeout: GitalyClient.medium_timeout)
         if error = response.duplicate_error.presence
           raise Gitlab::Git::Wiki::DuplicatePageError, error
         end
@@ -61,7 +61,7 @@ module Gitlab
           end
         end
 
-        GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum)
+        GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum, timeout: GitalyClient.medium_timeout)
       end
 
       def delete_page(page_path, commit_details)
@@ -187,7 +187,7 @@ module Gitlab
           directory: encode_binary(dir)
         )
 
-        response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_formatted_data, request)
+        response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_formatted_data, request, timeout: GitalyClient.medium_timeout)
         response.reduce([]) { |memo, msg| memo << msg.data }.join
       end
 
diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb
index 9d925581441..a3734ccf069 100644
--- a/lib/gitlab/github_import/importer/releases_importer.rb
+++ b/lib/gitlab/github_import/importer/releases_importer.rb
@@ -32,11 +32,13 @@ module Gitlab
 
         def build(release)
           {
+            name: release.name,
             tag: release.tag_name,
             description: description_for(release),
             created_at: release.created_at,
-            updated_at: release.updated_at,
-            released_at: release.published_at,
+            updated_at: release.created_at,
+            # Draft releases will have a null published_at
+            released_at: release.published_at || Time.current,
             project_id: project.id
           }
         end
@@ -46,11 +48,7 @@ module Gitlab
         end
 
         def description_for(release)
-          if release.body.present?
-            release.body
-          else
-            "Release for tag #{release.tag_name}"
-          end
+          release.body.presence || "Release for tag #{release.tag_name}"
         end
       end
     end
diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb
index 19915980d7f..01bc27f963b 100644
--- a/lib/gitlab/gl_repository/repo_type.rb
+++ b/lib/gitlab/gl_repository/repo_type.rb
@@ -40,3 +40,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::GlRepository::RepoType.prepend_if_ee('EE::Gitlab::GlRepository::RepoType')
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index 92917028851..f1e31a615a4 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -38,6 +38,13 @@ module Gitlab
         gon.current_user_fullname = current_user.name
         gon.current_user_avatar_url = current_user.avatar_url
       end
+
+      # Initialize gon.features with any flags that should be
+      # made globally available to the frontend
+      push_frontend_feature_flag(:suppress_ajax_navigation_errors, default_enabled: true)
+
+      # Flag controls a GFM feature used across many routes.
+      push_frontend_feature_flag(:gfm_grafana_integration)
     end
 
     # Exposes the state of a feature flag to the frontend code.
diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb
index 1e7203cb82a..4da2004b74f 100644
--- a/lib/gitlab/google_code_import/importer.rb
+++ b/lib/gitlab/google_code_import/importer.rb
@@ -117,7 +117,7 @@ module Gitlab
             description:  body,
             author_id:    project.creator_id,
             assignee_ids: [assignee_id],
-            state:        raw_issue['state'] == 'closed' ? 'closed' : 'opened'
+            state_id:     raw_issue['state'] == 'closed' ? Issue.available_states[:closed] : Issue.available_states[:opened]
           )
 
           issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true)
diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb
index 4b797a0e397..dc71d0b427a 100644
--- a/lib/gitlab/gpg/commit.rb
+++ b/lib/gitlab/gpg/commit.rb
@@ -10,6 +10,8 @@ module Gitlab
 
         repo = commit.project.repository.raw_repository
         @signature_data = Gitlab::Git::Commit.extract_signature_lazily(repo, commit.sha || commit.id)
+
+        lazy_signature
       end
 
       def signature_text
@@ -28,18 +30,16 @@ module Gitlab
         !!(signature_text && signed_text)
       end
 
-      # rubocop: disable CodeReuse/ActiveRecord
       def signature
         return unless has_signature?
 
         return @signature if @signature
 
-        cached_signature = GpgSignature.find_by(commit_sha: @commit.sha)
+        cached_signature = lazy_signature&.itself
         return @signature = cached_signature if cached_signature.present?
 
         @signature = create_cached_signature!
       end
-      # rubocop: enable CodeReuse/ActiveRecord
 
       def update_signature!(cached_signature)
         using_keychain do |gpg_key|
@@ -50,6 +50,14 @@ module Gitlab
 
       private
 
+      def lazy_signature
+        BatchLoader.for(@commit.sha).batch do |shas, loader|
+          GpgSignature.by_commit_sha(shas).each do |signature|
+            loader.call(signature.commit_sha, signature)
+          end
+        end
+      end
+
       def using_keychain
         Gitlab::Gpg.using_tmp_keychain do
           # first we need to get the fingerprint from the signature to query the gpg
diff --git a/lib/gitlab/graphql/docs/renderer.rb b/lib/gitlab/graphql/docs/renderer.rb
index f47a372aa19..41aef64f683 100644
--- a/lib/gitlab/graphql/docs/renderer.rb
+++ b/lib/gitlab/graphql/docs/renderer.rb
@@ -23,15 +23,12 @@ module Gitlab
           @parsed_schema = GraphQLDocs::Parser.new(schema, {}).parse
         end
 
-        def render
-          contents = @layout.render(self)
-
-          write_file(contents)
+        def contents
+          # Render and remove an extra trailing new line
+          @contents ||= @layout.render(self).sub!(/\n(?=\Z)/, '')
         end
 
-        private
-
-        def write_file(contents)
+        def write
           filename = File.join(@output_dir, 'index.md')
 
           FileUtils.mkdir_p(@output_dir)
diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml
index cc22d43ab4f..33acff38ef4 100644
--- a/lib/gitlab/graphql/docs/templates/default.md.haml
+++ b/lib/gitlab/graphql/docs/templates/default.md.haml
@@ -20,6 +20,3 @@
     - type[:fields].each do |field|
       = "| `#{field[:name]}` | #{render_field_type(field[:type][:info])} | #{field[:description]} |"
     \
-
-
-
diff --git a/lib/gitlab/health_checks/base_abstract_check.rb b/lib/gitlab/health_checks/base_abstract_check.rb
index 1d31f59999c..199cd2f9b2d 100644
--- a/lib/gitlab/health_checks/base_abstract_check.rb
+++ b/lib/gitlab/health_checks/base_abstract_check.rb
@@ -15,10 +15,6 @@ module Gitlab
         raise NotImplementedError
       end
 
-      def liveness
-        HealthChecks::Result.new(true)
-      end
-
       def metrics
         []
       end
diff --git a/lib/gitlab/health_checks/gitaly_check.rb b/lib/gitlab/health_checks/gitaly_check.rb
index e560f87bf98..e780bf8a986 100644
--- a/lib/gitlab/health_checks/gitaly_check.rb
+++ b/lib/gitlab/health_checks/gitaly_check.rb
@@ -5,7 +5,7 @@ module Gitlab
     class GitalyCheck
       extend BaseAbstractCheck
 
-      METRIC_PREFIX = 'gitaly_health_check'
+      METRIC_PREFIX = 'gitaly_health_check'.freeze
 
       class << self
         def readiness
@@ -29,7 +29,13 @@ module Gitlab
         def check(storage_name)
           serv = Gitlab::GitalyClient::HealthCheckService.new(storage_name)
           result = serv.check
-          HealthChecks::Result.new(result[:success], result[:message], shard: storage_name)
+
+          HealthChecks::Result.new(
+            name,
+            result[:success],
+            result[:message],
+            shard: storage_name
+          )
         end
 
         private
diff --git a/lib/gitlab/health_checks/metric.rb b/lib/gitlab/health_checks/metric.rb
index 184083de2bc..b697cb0d027 100644
--- a/lib/gitlab/health_checks/metric.rb
+++ b/lib/gitlab/health_checks/metric.rb
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
-module Gitlab::HealthChecks
-  Metric = Struct.new(:name, :value, :labels)
+module Gitlab
+  module HealthChecks
+    Metric = Struct.new(:name, :value, :labels)
+  end
 end
diff --git a/lib/gitlab/health_checks/probes/collection.rb b/lib/gitlab/health_checks/probes/collection.rb
new file mode 100644
index 00000000000..db3ef4834c2
--- /dev/null
+++ b/lib/gitlab/health_checks/probes/collection.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module HealthChecks
+    module Probes
+      class Collection
+        attr_reader :checks
+
+        # This accepts an array of objects implementing `:readiness`
+        # that returns `::Gitlab::HealthChecks::Result`
+        def initialize(*checks)
+          @checks = checks
+        end
+
+        def execute
+          readiness = probe_readiness
+          success = all_succeeded?(readiness)
+
+          Probes::Status.new(
+            success ? 200 : 503,
+            status(success).merge(payload(readiness))
+          )
+        end
+
+        private
+
+        def all_succeeded?(readiness)
+          readiness.all? do |name, probes|
+            probes.any?(&:success)
+          end
+        end
+
+        def status(success)
+          { status: success ? 'ok' : 'failed' }
+        end
+
+        def payload(readiness)
+          readiness.transform_values do |probes|
+            probes.map(&:payload)
+          end
+        end
+
+        def probe_readiness
+          checks
+            .flat_map(&:readiness)
+            .compact
+            .group_by(&:name)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/probes/status.rb b/lib/gitlab/health_checks/probes/status.rb
new file mode 100644
index 00000000000..192e9366001
--- /dev/null
+++ b/lib/gitlab/health_checks/probes/status.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module HealthChecks
+    module Probes
+      Status = Struct.new(:http_status, :json) do
+        # We accept 2xx
+        def success?
+          http_status / 100 == 2
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/prometheus_text_format.rb b/lib/gitlab/health_checks/prometheus_text_format.rb
deleted file mode 100644
index 2a8f9d31cd5..00000000000
--- a/lib/gitlab/health_checks/prometheus_text_format.rb
+++ /dev/null
@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module HealthChecks
-    class PrometheusTextFormat
-      def marshal(metrics)
-        "#{metrics_with_type_declarations(metrics).join("\n")}\n"
-      end
-
-      private
-
-      def metrics_with_type_declarations(metrics)
-        type_declaration_added = {}
-
-        metrics.flat_map do |metric|
-          metric_lines = []
-
-          unless type_declaration_added.key?(metric.name)
-            type_declaration_added[metric.name] = true
-            metric_lines << metric_type_declaration(metric)
-          end
-
-          metric_lines << metric_text(metric)
-        end
-      end
-
-      def metric_type_declaration(metric)
-        "# TYPE #{metric.name} gauge"
-      end
-
-      def metric_text(metric)
-        labels = metric.labels&.map { |key, value| "#{key}=\"#{value}\"" }&.join(',') || ''
-
-        if labels.empty?
-          "#{metric.name} #{metric.value}"
-        else
-          "#{metric.name}{#{labels}} #{metric.value}"
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/health_checks/puma_check.rb b/lib/gitlab/health_checks/puma_check.rb
new file mode 100644
index 00000000000..7aafe29fbae
--- /dev/null
+++ b/lib/gitlab/health_checks/puma_check.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module HealthChecks
+    # This check can only be run on Puma `master` process
+    class PumaCheck
+      extend SimpleAbstractCheck
+
+      class << self
+        private
+
+        def metric_prefix
+          'puma_check'
+        end
+
+        def successful?(result)
+          result > 0
+        end
+
+        def check
+          return unless defined?(::Puma)
+
+          stats = Puma.stats
+          stats = JSON.parse(stats)
+
+          # If `workers` is missing this means that
+          # Puma server is running in single mode
+          stats.fetch('workers', 1)
+        rescue NoMethodError
+          # server is not ready
+          0
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/result.rb b/lib/gitlab/health_checks/result.rb
index 4586b1d94a7..38a36100ec7 100644
--- a/lib/gitlab/health_checks/result.rb
+++ b/lib/gitlab/health_checks/result.rb
@@ -1,5 +1,15 @@
 # frozen_string_literal: true
 
-module Gitlab::HealthChecks
-  Result = Struct.new(:success, :message, :labels)
+module Gitlab
+  module HealthChecks
+    Result = Struct.new(:name, :success, :message, :labels) do
+      def payload
+        {
+          status: success ? 'ok' : 'failed',
+          message: message,
+          labels: labels
+        }.compact
+      end
+    end
+  end
 end
diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb
index 5a1e8c2a1dd..4e0b9296819 100644
--- a/lib/gitlab/health_checks/simple_abstract_check.rb
+++ b/lib/gitlab/health_checks/simple_abstract_check.rb
@@ -7,17 +7,23 @@ module Gitlab
 
       def readiness
         check_result = check
+        return if check_result.nil?
+
         if successful?(check_result)
-          HealthChecks::Result.new(true)
+          HealthChecks::Result.new(name, true)
         elsif check_result.is_a?(Timeout::Error)
-          HealthChecks::Result.new(false, "#{human_name} check timed out")
+          HealthChecks::Result.new(name, false, "#{human_name} check timed out")
         else
-          HealthChecks::Result.new(false, "unexpected #{human_name} check result: #{check_result}")
+          HealthChecks::Result.new(name, false, "unexpected #{human_name} check result: #{check_result}")
         end
+      rescue => e
+        HealthChecks::Result.new(name, false, "unexpected #{human_name} check result: #{e}")
       end
 
       def metrics
         result, elapsed = with_timing(&method(:check))
+        return if result.nil?
+
         Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless successful?(result) # rubocop:disable Gitlab/RailsLogger
         [
           metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0),
diff --git a/lib/gitlab/health_checks/unicorn_check.rb b/lib/gitlab/health_checks/unicorn_check.rb
new file mode 100644
index 00000000000..a30ae015257
--- /dev/null
+++ b/lib/gitlab/health_checks/unicorn_check.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module HealthChecks
+    # This check can only be run on Unicorn `master` process
+    class UnicornCheck
+      extend SimpleAbstractCheck
+
+      class << self
+        include Gitlab::Utils::StrongMemoize
+
+        private
+
+        def metric_prefix
+          'unicorn_check'
+        end
+
+        def successful?(result)
+          result > 0
+        end
+
+        def check
+          return unless http_servers
+
+          http_servers.sum(&:worker_processes) # rubocop: disable CodeReuse/ActiveRecord
+        end
+
+        # Traversal of ObjectSpace is expensive, on fully loaded application
+        # it takes around 80ms. The instances of HttpServers are not a subject
+        # to change so we can cache the list of servers.
+        def http_servers
+          strong_memoize(:http_servers) do
+            next unless defined?(::Unicorn::HttpServer)
+
+            ObjectSpace.each_object(::Unicorn::HttpServer).to_a
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb
index 1f64e440141..9d9db6cf94f 100644
--- a/lib/gitlab/hook_data/issue_builder.rb
+++ b/lib/gitlab/hook_data/issue_builder.rb
@@ -27,7 +27,7 @@ module Gitlab
           duplicated_to_id
           project_id
           relative_position
-          state
+          state_id
           time_estimate
           title
           updated_at
@@ -46,7 +46,8 @@ module Gitlab
             human_time_estimate: issue.human_time_estimate,
             assignee_ids: issue.assignee_ids,
             assignee_id: issue.assignee_ids.first, # This key is deprecated
-            labels: issue.labels_hook_attrs
+            labels: issue.labels_hook_attrs,
+            state: issue.state
         }
 
         issue.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes)
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index d08848a65a8..b2ac60fe825 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -38,6 +38,10 @@ module Gitlab
       "lfs-objects"
     end
 
+    def wiki_repo_bundle_filename
+      "project.wiki.bundle"
+    end
+
     def config_file
       Rails.root.join('lib/gitlab/import_export/import_export.yml')
     end
@@ -61,3 +65,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::ImportExport.prepend_if_ee('EE::Gitlab::ImportExport')
diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
index c5fb39b7b52..b30258123d4 100644
--- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
@@ -10,11 +10,9 @@ module Gitlab
 
         StrategyError = Class.new(StandardError)
 
-        AFTER_EXPORT_LOCK_FILE_NAME = '.after_export_action'
-
         private
 
-        attr_reader :project, :current_user
+        attr_reader :project, :current_user, :lock_file
 
         public
 
@@ -29,8 +27,9 @@ module Gitlab
         def execute(current_user, project)
           @project = project
 
-          return unless @project.export_status == :finished
-
+          ensure_export_ready!
+          ensure_lock_files_path!
+          @lock_file = File.join(lock_files_path, SecureRandom.hex)
           @current_user = current_user
 
           if invalid?
@@ -48,19 +47,32 @@ module Gitlab
           false
         ensure
           delete_after_export_lock
+          delete_export_file
+          delete_archive_path
         end
 
         def to_json(options = {})
           @options.to_h.merge!(klass: self.class.name).to_json
         end
 
-        def self.lock_file_path(project)
-          return unless project.export_path || export_file_exists?
+        def ensure_export_ready!
+          raise StrategyError unless project.export_file_exists?
+        end
+
+        def ensure_lock_files_path!
+          FileUtils.mkdir_p(lock_files_path) unless Dir.exist?(lock_files_path)
+        end
+
+        def lock_files_path
+          project.import_export_shared.lock_files_path
+        end
 
-          lock_path = project.import_export_shared.archive_path
+        def archive_path
+          project.import_export_shared.archive_path
+        end
 
-          mkdir_p(lock_path)
-          File.join(lock_path, AFTER_EXPORT_LOCK_FILE_NAME)
+        def locks_present?
+          project.import_export_shared.locks_present?
         end
 
         protected
@@ -69,25 +81,33 @@ module Gitlab
           raise NotImplementedError
         end
 
+        def delete_export?
+          true
+        end
+
         private
 
+        def delete_export_file
+          return if locks_present? || !delete_export?
+
+          project.remove_exports
+        end
+
+        def delete_archive_path
+          FileUtils.rm_rf(archive_path) if File.directory?(archive_path)
+        end
+
         def create_or_update_after_export_lock
-          FileUtils.touch(self.class.lock_file_path(project))
+          FileUtils.touch(lock_file)
         end
 
         def delete_after_export_lock
-          lock_file = self.class.lock_file_path(project)
-
           FileUtils.rm(lock_file) if lock_file.present? && File.exist?(lock_file)
         end
 
         def log_validation_errors
           errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) }
         end
-
-        def export_file_exists?
-          project.export_file_exists?
-        end
       end
     end
   end
diff --git a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb
index 1b391314a74..39a6090ad87 100644
--- a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb
@@ -4,6 +4,12 @@ module Gitlab
   module ImportExport
     module AfterExportStrategies
       class DownloadNotificationStrategy < BaseAfterExportStrategy
+        protected
+
+        def delete_export?
+          false
+        end
+
         private
 
         def strategy_execute
diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
index aaa70f0b36d..fd98bc2caad 100644
--- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
@@ -24,8 +24,6 @@ module Gitlab
 
         def strategy_execute
           handle_response_error(send_file)
-
-          project.remove_exports
         end
 
         def handle_response_error(response)
diff --git a/lib/gitlab/import_export/fast_hash_serializer.rb b/lib/gitlab/import_export/fast_hash_serializer.rb
index a6ab4f3a3d9..5a067b5c9f3 100644
--- a/lib/gitlab/import_export/fast_hash_serializer.rb
+++ b/lib/gitlab/import_export/fast_hash_serializer.rb
@@ -26,6 +26,51 @@ module Gitlab
     class FastHashSerializer
       attr_reader :subject, :tree
 
+      # Usage of this class results in delayed
+      # serialization of relation. The serialization
+      # will be triggered when the `JSON.generate`
+      # is exected.
+      #
+      # This class uses memory-optimised, lazily
+      # initialised, fast to recycle relation
+      # serialization.
+      #
+      # The `JSON.generate` does use `#to_json`,
+      # that returns raw JSON content that is written
+      # directly to file.
+      class JSONBatchRelation
+        include Gitlab::Utils::StrongMemoize
+
+        def initialize(relation, options, preloads)
+          @relation = relation
+          @options = options
+          @preloads = preloads
+        end
+
+        def raw_json
+          strong_memoize(:raw_json) do
+            result = +''
+
+            batch = @relation
+            batch = batch.preload(@preloads) if @preloads
+            batch.each do |item|
+              result.concat(",") unless result.empty?
+              result.concat(item.to_json(@options))
+            end
+
+            result
+          end
+        end
+
+        def to_json(options = {})
+          raw_json
+        end
+
+        def as_json(*)
+          raise NotImplementedError
+        end
+      end
+
       BATCH_SIZE = 100
 
       def initialize(subject, tree, batch_size: BATCH_SIZE)
@@ -34,8 +79,11 @@ module Gitlab
         @tree = tree
       end
 
-      # Serializes the subject into a Hash for the given option tree
-      # (e.g. Project#as_json)
+      # With the usage of `JSONBatchRelation`, it returns partially
+      # serialized hash which is not easily accessible.
+      # It means you can only manipulate and replace top-level objects.
+      # All future mutations of the hash (such as `fix_project_tree`)
+      # should be aware of that.
       def execute
         simple_serialize.merge(serialize_includes)
       end
@@ -85,12 +133,15 @@ module Gitlab
           return record.as_json(options)
         end
 
-        # has-many relation
         data = []
 
         record.in_batches(of: @batch_size) do |batch| # rubocop:disable Cop/InBatches
-          batch = batch.preload(preloads[key]) if preloads&.key?(key)
-          data += batch.as_json(options)
+          if Feature.enabled?(:export_fast_serialize_with_raw_json, default_enabled: true)
+            data.append(JSONBatchRelation.new(batch, options, preloads[key]).tap(&:raw_json))
+          else
+            batch = batch.preload(preloads[key]) if preloads&.key?(key)
+            data += batch.as_json(options)
+          end
         end
 
         data
diff --git a/lib/gitlab/import_export/group_project_object_builder.rb b/lib/gitlab/import_export/group_project_object_builder.rb
index 1c62591ed5a..de1629d0e28 100644
--- a/lib/gitlab/import_export/group_project_object_builder.rb
+++ b/lib/gitlab/import_export/group_project_object_builder.rb
@@ -26,30 +26,60 @@ module Gitlab
       end
 
       def find
-        find_object || @klass.create(project_attributes)
+        find_object || klass.create(project_attributes)
       end
 
       private
 
+      attr_reader :klass, :attributes, :group, :project
+
       def find_object
-        @klass.where(where_clause).first
+        klass.where(where_clause).first
       end
 
       def where_clause
-        @attributes.slice('title').map do |key, value|
-          scope_clause = table[:project_id].eq(@project.id)
-          scope_clause = scope_clause.or(table[:group_id].eq(@group.id)) if @group
+        where_clauses.reduce(:and)
+      end
+
+      def where_clauses
+        [
+          where_clause_base,
+          where_clause_for_title,
+          where_clause_for_klass
+        ].compact
+      end
+
+      # Returns Arel clause `"{table_name}"."project_id" = {project.id}`
+      # or, if group is present:
+      # `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}`
+      def where_clause_base
+        clause = table[:project_id].eq(project.id)
+        clause = clause.or(table[:group_id].eq(group.id)) if group
+
+        clause
+      end
 
-          table[key].eq(value).and(scope_clause)
-        end.reduce(:or)
+      # Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'`
+      # if attributes has 'title key, otherwise `nil`.
+      def where_clause_for_title
+        attrs_to_arel(attributes.slice('title'))
+      end
+
+      # Returns Arel clause:
+      # `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"`
+      # from the given Hash of attributes.
+      def attrs_to_arel(attrs)
+        attrs.map do |key, value|
+          table[key].eq(value)
+        end.reduce(:and)
       end
 
       def table
-        @table ||= @klass.arel_table
+        @table ||= klass.arel_table
       end
 
       def project_attributes
-        @attributes.except('group').tap do |atts|
+        attributes.except('group').tap do |atts|
           if label?
             atts['type'] = 'ProjectLabel' # Always create project labels
           elsif milestone?
@@ -60,15 +90,17 @@ module Gitlab
               claim_iid
             end
           end
+
+          atts['importing'] = true if klass.ancestors.include?(Importable)
         end
       end
 
       def label?
-        @klass == Label
+        klass == Label
       end
 
       def milestone?
-        @klass == Milestone
+        klass == Milestone
       end
 
       # If an existing group milestone used the IID
@@ -79,7 +111,7 @@ module Gitlab
       def claim_iid
         # The milestone has to be a group milestone, as it's the only case where
         # we set the IID as the maximum. The rest of them are fixed.
-        milestone = @project.milestones.find_by(iid: @attributes['iid'])
+        milestone = project.milestones.find_by(iid: attributes['iid'])
 
         return unless milestone
 
@@ -87,6 +119,15 @@ module Gitlab
         milestone.ensure_project_iid!
         milestone.save!
       end
+
+      protected
+
+      # Returns Arel clause for a particular model or `nil`.
+      def where_clause_for_klass
+        # no-op
+      end
     end
   end
 end
+
+Gitlab::ImportExport::GroupProjectObjectBuilder.prepend_if_ee('EE::Gitlab::ImportExport::GroupProjectObjectBuilder')
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 511b702553e..141e73e6a47 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -66,6 +66,7 @@ tree:
       - stages:
         - :statuses
       - :external_pull_request
+      - :merge_request
     - :external_pull_requests
     - :auto_devops
     - :triggers
@@ -138,11 +139,14 @@ excluded_attributes:
     - :mirror_trigger_builds
     - :only_mirror_protected_branches
     - :pull_mirror_available_overridden
+    - :pull_mirror_branch_prefix
     - :mirror_overwrites_diverged_branches
     - :packages_enabled
     - :mirror_last_update_at
     - :mirror_last_successful_update_at
     - :emails_disabled
+    - :max_pages_size
+    - :max_artifacts_size
   namespaces:
     - :runners_token
     - :runners_token_encrypted
@@ -166,6 +170,12 @@ excluded_attributes:
     - :external_diff_size
   issues:
     - :milestone_id
+  merge_request:
+    - :milestone_id
+    - :ref_fetched
+    - :merge_jid
+    - :rebase_jid
+    - :latest_merge_request_diff_id
   merge_requests:
     - :milestone_id
     - :ref_fetched
@@ -246,7 +256,16 @@ preloads:
 ee:
   tree:
     project:
-      protected_branches:
+      - issues:
+        - designs:
+          - notes:
+            - :author
+            - events:
+              - :push_event_payload
+        - design_versions:
+          - actions:
+            - :design # Duplicate export of issues.designs in order to link the record to both Issue and DesignVersion
+      - protected_branches:
         - :unprotect_access_levels
-      protected_environments:
+      - protected_environments:
         - :deploy_access_levels
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index 767f1b5de0e..62cf6c86906 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -19,9 +19,9 @@ module Gitlab
 
       def execute
         if import_file && check_version! && restorers.all?(&:restore) && overwrite_project
-          project_tree.restored_project
+          project
         else
-          raise Projects::ImportService::Error.new(@shared.errors.join(', '))
+          raise Projects::ImportService::Error.new(shared.errors.to_sentence)
         end
       rescue => e
         raise Projects::ImportService::Error.new(e.message)
@@ -31,70 +31,72 @@ module Gitlab
 
       private
 
+      attr_accessor :archive_file, :current_user, :project, :shared
+
       def restorers
         [repo_restorer, wiki_restorer, project_tree, avatar_restorer,
          uploads_restorer, lfs_restorer, statistics_restorer]
       end
 
       def import_file
-        Gitlab::ImportExport::FileImporter.import(project: @project,
-                                                  archive_file: @archive_file,
-                                                  shared: @shared)
+        Gitlab::ImportExport::FileImporter.import(project: project,
+                                                  archive_file: archive_file,
+                                                  shared: shared)
       end
 
       def check_version!
-        Gitlab::ImportExport::VersionChecker.check!(shared: @shared)
+        Gitlab::ImportExport::VersionChecker.check!(shared: shared)
       end
 
       def project_tree
-        @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user,
-                                                                        shared: @shared,
-                                                                        project: @project)
+        @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: current_user,
+                                                                        shared: shared,
+                                                                        project: project)
       end
 
       def avatar_restorer
-        Gitlab::ImportExport::AvatarRestorer.new(project: project_tree.restored_project, shared: @shared)
+        Gitlab::ImportExport::AvatarRestorer.new(project: project, shared: shared)
       end
 
       def repo_restorer
         Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path,
-                                               shared: @shared,
-                                               project: project_tree.restored_project)
+                                               shared: shared,
+                                               project: project)
       end
 
       def wiki_restorer
         Gitlab::ImportExport::WikiRestorer.new(path_to_bundle: wiki_repo_path,
-                                               shared: @shared,
-                                               project: ProjectWiki.new(project_tree.restored_project),
-                                               wiki_enabled: @project.wiki_enabled?)
+                                               shared: shared,
+                                               project: ProjectWiki.new(project),
+                                               wiki_enabled: project.wiki_enabled?)
       end
 
       def uploads_restorer
-        Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared)
+        Gitlab::ImportExport::UploadsRestorer.new(project: project, shared: shared)
       end
 
       def lfs_restorer
-        Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: @shared)
+        Gitlab::ImportExport::LfsRestorer.new(project: project, shared: shared)
       end
 
       def statistics_restorer
-        Gitlab::ImportExport::StatisticsRestorer.new(project: project_tree.restored_project, shared: @shared)
+        Gitlab::ImportExport::StatisticsRestorer.new(project: project, shared: shared)
       end
 
       def path_with_namespace
-        File.join(@project.namespace.full_path, @project.path)
+        File.join(project.namespace.full_path, project.path)
       end
 
       def repo_path
-        File.join(@shared.export_path, 'project.bundle')
+        File.join(shared.export_path, Gitlab::ImportExport.project_bundle_filename)
       end
 
       def wiki_repo_path
-        File.join(@shared.export_path, 'project.wiki.bundle')
+        File.join(shared.export_path, Gitlab::ImportExport.wiki_repo_bundle_filename)
       end
 
       def remove_import_file
-        upload = @project.import_export_upload
+        upload = project.import_export_upload
 
         return unless upload&.import_file&.file
 
@@ -103,12 +105,10 @@ module Gitlab
       end
 
       def overwrite_project
-        project = project_tree.restored_project
-
-        return unless can?(@current_user, :admin_namespace, project.namespace)
+        return unless can?(current_user, :admin_namespace, project.namespace)
 
         if overwrite_project?
-          ::Projects::OverwriteProjectService.new(project, @current_user)
+          ::Projects::OverwriteProjectService.new(project, current_user)
                                              .execute(project_to_overwrite)
         end
 
@@ -116,7 +116,7 @@ module Gitlab
       end
 
       def original_path
-        @project.import_data&.data&.fetch('original_path', nil)
+        project.import_data&.data&.fetch('original_path', nil)
       end
 
       def overwrite_project?
@@ -125,9 +125,11 @@ module Gitlab
 
       def project_to_overwrite
         strong_memoize(:project_to_overwrite) do
-          Project.find_by_full_path("#{@project.namespace.full_path}/#{original_path}")
+          Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}")
         end
       end
     end
   end
 end
+
+Gitlab::ImportExport::Importer.prepend_if_ee('EE::Gitlab::ImportExport::Importer')
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 2dd18616cd6..3fa5765fd4a 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -6,19 +6,21 @@ module Gitlab
       # Relations which cannot be saved at project level (and have a group assigned)
       GROUP_MODELS = [GroupLabel, Milestone].freeze
 
+      attr_reader :user
+      attr_reader :shared
+      attr_reader :project
+
       def initialize(user:, shared:, project:)
         @path = File.join(shared.export_path, 'project.json')
         @user = user
         @shared = shared
         @project = project
-        @project_id = project.id
         @saved = true
       end
 
       def restore
         begin
-          json = IO.read(@path)
-          @tree_hash = ActiveSupport::JSON.decode(json)
+          @tree_hash = read_tree_hash
         rescue => e
           Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger
           raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
@@ -30,26 +32,36 @@ module Gitlab
 
         ActiveRecord::Base.uncached do
           ActiveRecord::Base.no_touching do
+            update_project_params!
             create_relations
           end
         end
+
+        # ensure that we have latest version of the restore
+        @project.reload # rubocop:disable Cop/ActiveRecordAssociationReload
+
+        true
       rescue => e
         @shared.error(e)
         false
       end
 
-      def restored_project
-        return @project unless @tree_hash
+      private
 
-        @restored_project ||= restore_project
+      def read_tree_hash
+        json = IO.read(@path)
+        ActiveSupport::JSON.decode(json)
       end
 
-      private
-
       def members_mapper
         @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
                                                                     user: @user,
-                                                                    project: restored_project)
+                                                                    project: @project)
+      end
+
+      # A Hash of the imported merge request ID -> imported ID.
+      def merge_requests_mapping
+        @merge_requests_mapping ||= {}
       end
 
       # Loops through the tree of models defined in import_export.yml and
@@ -58,7 +70,7 @@ module Gitlab
       # the configuration yaml file too.
       # Finally, it updates each attribute in the newly imported project.
       def create_relations
-        project_relations_without_project_members.each do |relation_key, relation_definition|
+        project_relations.each do |relation_key, relation_definition|
           relation_key_s = relation_key.to_s
 
           if relation_definition.present?
@@ -78,10 +90,25 @@ module Gitlab
 
         remove_group_models(relation_hash) if relation_hash.is_a?(Array)
 
-        @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash)
+        @saved = false unless @project.append_or_update_attribute(relation_key, relation_hash)
 
-        # Restore the project again, extra query that skips holding the AR objects in memory
-        @restored_project = Project.find(@project_id)
+        save_id_mappings(relation_key, relation_hash_batch, relation_hash)
+
+        @project.reset
+      end
+
+      # Older, serialized CI pipeline exports may only have a
+      # merge_request_id and not the full hash of the merge request. To
+      # import these pipelines, we need to preserve the mapping between
+      # the old and new the merge request ID.
+      def save_id_mappings(relation_key, relation_hash_batch, relation_hash)
+        return unless relation_key == 'merge_requests'
+
+        relation_hash = Array(relation_hash)
+
+        Array(relation_hash_batch).each_with_index do |raw_data, index|
+          merge_requests_mapping[raw_data['id']] = relation_hash[index]['id']
+        end
       end
 
       # Remove project models that became group models as we found them at group level.
@@ -93,58 +120,44 @@ module Gitlab
         end
       end
 
-      def project_relations_without_project_members
-        # We remove `project_members` as they are deserialized separately
-        project_relations.except(:project_members)
+      def remove_feature_dependent_sub_relations!(_relation_item)
+        # no-op
       end
 
       def project_relations
-        reader.attributes_finder.find_relations_tree(:project)
+        @project_relations ||= reader.attributes_finder.find_relations_tree(:project)
       end
 
-      def restore_project
+      def update_project_params!
         Gitlab::Timeless.timeless(@project) do
-          @project.update(project_params)
-        end
-
-        @project
-      end
+          project_params = @tree_hash.reject do |key, value|
+            project_relations.include?(key.to_sym)
+          end
 
-      def project_params
-        @project_params ||= begin
-          attrs = json_params.merge(override_params).merge(visibility_level, external_label)
+          project_params = project_params.merge(present_project_override_params)
 
           # Cleaning all imported and overridden params
-          Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: attrs,
-                                                       relation_class: Project,
-                                                       excluded_keys: excluded_keys_for_relation(:project))
-        end
-      end
-
-      def override_params
-        @override_params ||= @project.import_data&.data&.fetch('override_params', nil) || {}
-      end
-
-      def json_params
-        @json_params ||= @tree_hash.reject do |key, value|
-          # return params that are not 1 to many or 1 to 1 relations
-          value.respond_to?(:each) && !Project.column_names.include?(key)
+          project_params = Gitlab::ImportExport::AttributeCleaner.clean(
+            relation_hash: project_params,
+            relation_class: Project,
+            excluded_keys: excluded_keys_for_relation(:project))
+
+          @project.assign_attributes(project_params)
+          @project.drop_visibility_level!
+          @project.save!
         end
       end
 
-      def visibility_level
-        level = override_params['visibility_level'] || json_params['visibility_level'] || @project.visibility_level
-        level = @project.group.visibility_level if @project.group && level.to_i > @project.group.visibility_level
-        level = Gitlab::VisibilityLevel::PRIVATE if level == Gitlab::VisibilityLevel::INTERNAL && Gitlab::CurrentSettings.restricted_visibility_levels.include?(level)
-
-        { 'visibility_level' => level }
+      def present_project_override_params
+        # we filter out the empty strings from the overrides
+        # keeping the default values configured
+        project_override_params.transform_values do |value|
+          value.is_a?(String) ? value.presence : value
+        end.compact
       end
 
-      def external_label
-        label = override_params['external_authorization_classification_label'].presence ||
-          json_params['external_authorization_classification_label'].presence
-
-        { 'external_authorization_classification_label' => label }
+      def project_override_params
+        @project_override_params ||= @project.import_data&.data&.fetch('override_params', nil) || {}
       end
 
       # Given a relation hash containing one or more models and its relationships,
@@ -159,17 +172,10 @@ module Gitlab
         return if tree_hash[relation_key].blank?
 
         tree_array = [tree_hash[relation_key]].flatten
-        null_iid_pipelines = []
 
         # Avoid keeping a possible heavy object in memory once we are done with it
-        while relation_item = (tree_array.shift || null_iid_pipelines.shift)
-          if nil_iid_pipeline?(relation_key, relation_item) && tree_array.any?
-            # Move pipelines with NULL IIDs to the end
-            # so they don't clash with existing IIDs.
-            null_iid_pipelines << relation_item
-
-            next
-          end
+        while relation_item = tree_array.shift
+          remove_feature_dependent_sub_relations!(relation_item)
 
           # The transaction at this level is less speedy than one single transaction
           # But we can't have it in the upper level or GC won't get rid of the AR objects
@@ -216,8 +222,9 @@ module Gitlab
             relation_sym: relation_key.to_sym,
             relation_hash: relation_hash,
             members_mapper: members_mapper,
+            merge_requests_mapping: merge_requests_mapping,
             user: @user,
-            project: @restored_project,
+            project: @project,
             excluded_keys: excluded_keys_for_relation(relation_key))
         end.compact
 
@@ -231,10 +238,8 @@ module Gitlab
       def excluded_keys_for_relation(relation)
         reader.attributes_finder.find_excluded_keys(relation)
       end
-
-      def nil_iid_pipeline?(relation_key, relation_item)
-        relation_key == 'ci_pipelines' && relation_item['iid'].nil?
-      end
     end
   end
 end
+
+Gitlab::ImportExport::ProjectTreeRestorer.prepend_if_ee('::EE::Gitlab::ImportExport::ProjectTreeRestorer')
diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb
index f75f69b2c75..63c71105efe 100644
--- a/lib/gitlab/import_export/project_tree_saver.rb
+++ b/lib/gitlab/import_export/project_tree_saver.rb
@@ -20,7 +20,8 @@ module Gitlab
 
         project_tree = serialize_project_tree
         fix_project_tree(project_tree)
-        File.write(full_path, project_tree.to_json)
+        project_tree_json = JSON.generate(project_tree)
+        File.write(full_path, project_tree_json)
 
         true
       rescue => e
@@ -30,6 +31,8 @@ module Gitlab
 
       private
 
+      # Aware that the resulting hash needs to be pure-hash and
+      # does not include any AR objects anymore, only objects that run `.to_json`
       def fix_project_tree(project_tree)
         if @params[:description].present?
           project_tree['description'] = @params[:description]
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index 1e9dff405c5..cb85af91f75 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -34,13 +34,13 @@ module Gitlab
 
       PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze
 
-      BUILD_MODELS = %w[Ci::Build commit_status].freeze
+      BUILD_MODELS = %i[Ci::Build commit_status].freeze
 
       IMPORTED_OBJECT_MAX_RETRIES = 5.freeze
 
-      EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature].freeze
+      EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature merge_request].freeze
 
-      TOKEN_RESET_MODELS = %w[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze
+      TOKEN_RESET_MODELS = %i[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze
 
       def self.create(*args)
         new(*args).create
@@ -55,10 +55,11 @@ module Gitlab
         relation_name.to_s.constantize
       end
 
-      def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:, excluded_keys: [])
-        @relation_name = self.class.overrides[relation_sym] || relation_sym
+      def initialize(relation_sym:, relation_hash:, members_mapper:, merge_requests_mapping:, user:, project:, excluded_keys: [])
+        @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym
         @relation_hash = relation_hash.except('noteable_id')
         @members_mapper = members_mapper
+        @merge_requests_mapping = merge_requests_mapping
         @user = user
         @project = project
         @imported_object_retries = 0
@@ -92,6 +93,10 @@ module Gitlab
         OVERRIDES
       end
 
+      def self.existing_object_check
+        EXISTING_OBJECT_CHECK
+      end
+
       private
 
       def setup_models
@@ -105,7 +110,10 @@ module Gitlab
         update_group_references
         remove_duplicate_assignees
 
-        setup_pipeline if @relation_name == 'Ci::Pipeline'
+        if @relation_name == :'Ci::Pipeline'
+          update_merge_request_references
+          setup_pipeline
+        end
 
         reset_tokens!
         remove_encrypted_attributes!
@@ -184,14 +192,36 @@ module Gitlab
       end
 
       def update_group_references
-        return unless EXISTING_OBJECT_CHECK.include?(@relation_name)
+        return unless self.class.existing_object_check.include?(@relation_name)
         return unless @relation_hash['group_id']
 
         @relation_hash['group_id'] = @project.namespace_id
       end
 
+      # This code is a workaround for broken project exports that don't
+      # export merge requests with CI pipelines (i.e. exports that were
+      # generated from
+      # https://gitlab.com/gitlab-org/gitlab/merge_requests/17844).
+      # This method can be removed in GitLab 12.6.
+      def update_merge_request_references
+        # If a merge request was properly created, we don't need to fix
+        # up this export.
+        return if @relation_hash['merge_request']
+
+        merge_request_id = @relation_hash['merge_request_id']
+
+        return unless merge_request_id
+
+        new_merge_request_id = @merge_requests_mapping[merge_request_id]
+
+        return unless new_merge_request_id
+
+        @relation_hash['merge_request_id'] = new_merge_request_id
+        parsed_relation_hash['merge_request_id'] = new_merge_request_id
+      end
+
       def reset_tokens!
-        return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name.to_s)
+        return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name)
 
         # If we import/export a project to the same instance, tokens will have to be reset.
         # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across.
@@ -255,14 +285,18 @@ module Gitlab
         # Only find existing records to avoid mapping tables such as milestones
         # Otherwise always create the record, skipping the extra SELECT clause.
         @existing_or_new_object ||= begin
-          if EXISTING_OBJECT_CHECK.include?(@relation_name)
+          if self.class.existing_object_check.include?(@relation_name)
             attribute_hash = attribute_hash_for(['events'])
 
             existing_object.assign_attributes(attribute_hash) if attribute_hash.any?
 
             existing_object
           else
-            relation_class.new(parsed_relation_hash)
+            object = relation_class.new
+
+            # Use #assign_attributes here to call object custom setters
+            object.assign_attributes(parsed_relation_hash)
+            object
           end
         end
       end
@@ -284,21 +318,27 @@ module Gitlab
       end
 
       def legacy_trigger?
-        @relation_name == 'Ci::Trigger' && @relation_hash['owner_id'].nil?
+        @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil?
       end
 
       def find_or_create_object!
         return relation_class.find_or_create_by(project_id: @project.id) if @relation_name == :project_feature
+        return find_or_create_merge_request! if @relation_name == :merge_request
 
         # Can't use IDs as validation exists calling `group` or `project` attributes
         finder_hash = parsed_relation_hash.tap do |hash|
           hash['group'] = @project.group if relation_class.attribute_method?('group_id')
-          hash['project'] = @project
+          hash['project'] = @project if relation_class.reflect_on_association(:project)
           hash.delete('project_id')
         end
 
         GroupProjectObjectBuilder.build(relation_class, finder_hash)
       end
+
+      def find_or_create_merge_request!
+        @project.merge_requests.find_by(iid: parsed_relation_hash['iid']) ||
+          relation_class.new(parsed_relation_hash)
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb
index 91167a9c4fb..3123687453f 100644
--- a/lib/gitlab/import_export/repo_restorer.rb
+++ b/lib/gitlab/import_export/repo_restorer.rb
@@ -6,19 +6,23 @@ module Gitlab
       include Gitlab::ImportExport::CommandLineUtil
 
       def initialize(project:, shared:, path_to_bundle:)
-        @project = project
+        @repository = project.repository
         @path_to_bundle = path_to_bundle
         @shared = shared
       end
 
       def restore
-        return true unless File.exist?(@path_to_bundle)
+        return true unless File.exist?(path_to_bundle)
 
-        @project.repository.create_from_bundle(@path_to_bundle)
+        repository.create_from_bundle(path_to_bundle)
       rescue => e
-        @shared.error(e)
+        shared.error(e)
         false
       end
+
+      private
+
+      attr_accessor :repository, :path_to_bundle, :shared
     end
   end
 end
diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb
index a60618dfcec..898cd7898ba 100644
--- a/lib/gitlab/import_export/repo_saver.rb
+++ b/lib/gitlab/import_export/repo_saver.rb
@@ -5,27 +5,35 @@ module Gitlab
     class RepoSaver
       include Gitlab::ImportExport::CommandLineUtil
 
-      attr_reader :full_path
+      attr_reader :project, :repository, :shared
 
       def initialize(project:, shared:)
         @project = project
         @shared = shared
+        @repository = @project.repository
       end
 
       def save
-        return true if @project.empty_repo? # it's ok to have no repo
+        return true unless repository_exists? # it's ok to have no repo
 
-        @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename)
         bundle_to_disk
       end
 
       private
 
+      def repository_exists?
+        repository.exists? && !repository.empty?
+      end
+
+      def bundle_full_path
+        File.join(shared.export_path, ImportExport.project_bundle_filename)
+      end
+
       def bundle_to_disk
-        mkdir_p(@shared.export_path)
-        @project.repository.bundle_to_disk(@full_path)
+        mkdir_p(shared.export_path)
+        repository.bundle_to_disk(bundle_full_path)
       rescue => e
-        @shared.error(e)
+        shared.error(e)
         false
       end
     end
diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb
index 725c1101d70..02d46a1f498 100644
--- a/lib/gitlab/import_export/shared.rb
+++ b/lib/gitlab/import_export/shared.rb
@@ -1,10 +1,32 @@
 # frozen_string_literal: true
-
+#
+# This class encapsulates the directories used by project import/export:
+#
+# 1. The project export job first generates the project metadata tree
+#    (e.g. `project.json) and repository bundle (e.g. `project.bundle`)
+#    inside a temporary `export_path`
+#    (e.g. /path/to/shared/tmp/project_exports/namespace/project/:randomA/:randomB).
+#
+# 2. The job then creates a tarball (e.g. `project.tar.gz`) in
+#    `archive_path` (e.g. /path/to/shared/tmp/project_exports/namespace/project/:randomA).
+#    CarrierWave moves this tarball files into its permanent location.
+#
+# 3. Lock files are used to indicate whether a project is in the
+#    `after_export` state.  These are stored in a directory
+#    (e.g. /path/to/shared/tmp/project_exports/namespace/project/locks. The
+#    number of lock files present signifies how many concurrent project
+#    exports are running. Note that this assumes the temporary directory
+#    is a shared mount:
+#    https://gitlab.com/gitlab-org/gitlab/issues/32203
+#
+# NOTE: Stale files should be cleaned up via ImportExportCleanupService.
 module Gitlab
   module ImportExport
     class Shared
       attr_reader :errors, :project
 
+      LOCKS_DIRECTORY = 'locks'
+
       def initialize(project)
         @project = project
         @errors = []
@@ -12,20 +34,31 @@ module Gitlab
       end
 
       def active_export_count
-        Dir[File.join(archive_path, '*')].count { |name| File.directory?(name) }
+        Dir[File.join(base_path, '*')].count { |name| File.basename(name) != LOCKS_DIRECTORY && File.directory?(name) }
       end
 
+      # The path where the project metadata and repository bundle is saved
       def export_path
         @export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path)
       end
 
+      # The path where the tarball is saved
       def archive_path
         @archive_path ||= Gitlab::ImportExport.export_path(relative_path: relative_archive_path)
       end
 
+      def base_path
+        @base_path ||= Gitlab::ImportExport.export_path(relative_path: relative_base_path)
+      end
+
+      def lock_files_path
+        @locks_files_path ||= File.join(base_path, LOCKS_DIRECTORY)
+      end
+
       def error(error)
-        log_error(message: error.message, caller: caller[0].dup)
-        log_debug(backtrace: error.backtrace&.join("\n"))
+        error_payload = { message: error.message }
+        error_payload[:error_backtrace] = Gitlab::Profiler.clean_backtrace(error.backtrace) if error.backtrace
+        log_error(error_payload)
 
         Gitlab::Sentry.track_acceptable_exception(error, extra: log_base_data)
 
@@ -37,16 +70,24 @@ module Gitlab
       end
 
       def after_export_in_progress?
-        File.exist?(after_export_lock_file)
+        locks_present?
+      end
+
+      def locks_present?
+        Dir.exist?(lock_files_path) && !Dir.empty?(lock_files_path)
       end
 
       private
 
       def relative_path
-        File.join(relative_archive_path, SecureRandom.hex)
+        @relative_path ||= File.join(relative_archive_path, SecureRandom.hex)
       end
 
       def relative_archive_path
+        @relative_archive_path ||= File.join(@project.disk_path, SecureRandom.hex)
+      end
+
+      def relative_base_path
         @project.disk_path
       end
 
@@ -70,10 +111,6 @@ module Gitlab
       def filtered_error_message(message)
         Projects::ImportErrorFilter.filter_message(message)
       end
-
-      def after_export_lock_file
-        AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project)
-      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb
index e232198150a..dca8e3a7449 100644
--- a/lib/gitlab/import_export/uploads_manager.rb
+++ b/lib/gitlab/import_export/uploads_manager.rb
@@ -68,7 +68,7 @@ module Gitlab
           yield(@project.avatar)
         else
           project_uploads_except_avatar(avatar_path).find_each(batch_size: UPLOADS_BATCH_SIZE) do |upload|
-            yield(upload.build_uploader)
+            yield(upload.retrieve_uploader)
           end
         end
       end
diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb
index 7303bcf61a4..93ae6f6b02a 100644
--- a/lib/gitlab/import_export/wiki_repo_saver.rb
+++ b/lib/gitlab/import_export/wiki_repo_saver.rb
@@ -4,28 +4,16 @@ module Gitlab
   module ImportExport
     class WikiRepoSaver < RepoSaver
       def save
-        @wiki = ProjectWiki.new(@project)
-        return true unless wiki_repository_exists? # it's okay to have no Wiki
+        wiki = ProjectWiki.new(project)
+        @repository = wiki.repository
 
-        bundle_to_disk(File.join(@shared.export_path, project_filename))
-      end
-
-      def bundle_to_disk(full_path)
-        mkdir_p(@shared.export_path)
-        @wiki.repository.bundle_to_disk(full_path)
-      rescue => e
-        @shared.error(e)
-        false
+        super
       end
 
       private
 
-      def project_filename
-        "project.wiki.bundle"
-      end
-
-      def wiki_repository_exists?
-        @wiki.repository.exists? && !@wiki.repository.empty?
+      def bundle_full_path
+        File.join(shared.export_path, ImportExport.wiki_repo_bundle_filename)
       end
     end
   end
diff --git a/lib/gitlab/import_export/wiki_restorer.rb b/lib/gitlab/import_export/wiki_restorer.rb
index 28b5e7449cd..359ba8ba769 100644
--- a/lib/gitlab/import_export/wiki_restorer.rb
+++ b/lib/gitlab/import_export/wiki_restorer.rb
@@ -6,19 +6,22 @@ module Gitlab
       def initialize(project:, shared:, path_to_bundle:, wiki_enabled:)
         super(project: project, shared: shared, path_to_bundle: path_to_bundle)
 
+        @project = project
         @wiki_enabled = wiki_enabled
       end
 
       def restore
-        @project.wiki if create_empty_wiki?
+        project.wiki if create_empty_wiki?
 
         super
       end
 
       private
 
+      attr_accessor :project, :wiki_enabled
+
       def create_empty_wiki?
-        !File.exist?(@path_to_bundle) && @wiki_enabled
+        !File.exist?(path_to_bundle) && wiki_enabled
       end
     end
   end
diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb
index 11a33a7b358..0c8b509740c 100644
--- a/lib/gitlab/jira/http_client.rb
+++ b/lib/gitlab/jira/http_client.rb
@@ -4,7 +4,7 @@ module Gitlab
   module Jira
     # Gitlab JIRA HTTP client to be used with jira-ruby gem, this subclasses JIRA::HTTPClient.
     # Uses Gitlab::HTTP to make requests to JIRA REST API.
-    # The parent class implementation can be found at: https://github.com/sumoheavy/jira-ruby/blob/v1.4.0/lib/jira/http_client.rb
+    # The parent class implementation can be found at: https://github.com/sumoheavy/jira-ruby/blob/v1.7.0/lib/jira/http_client.rb
     class HttpClient < JIRA::HttpClient
       extend ::Gitlab::Utils::Override
 
@@ -24,7 +24,7 @@ module Gitlab
           password: @options.delete(:password)
         }.to_json
 
-        make_request(:post, @options[:context_path] + '/rest/auth/1/session', body, { 'Content-Type' => 'application/json' })
+        make_request(:post, @options[:context_path] + '/rest/auth/1/session', body, 'Content-Type' => 'application/json')
       end
 
       override :make_request
diff --git a/lib/gitlab/kubernetes/helm/client_command.rb b/lib/gitlab/kubernetes/helm/client_command.rb
index 6ae68306a9b..a3f732e1283 100644
--- a/lib/gitlab/kubernetes/helm/client_command.rb
+++ b/lib/gitlab/kubernetes/helm/client_command.rb
@@ -17,7 +17,8 @@ module Gitlab
           # This is necessary to give Tiller time to restart after upgrade.
           # Ideally we'd be able to use --wait but cannot because of
           # https://github.com/helm/helm/issues/4855
-          "for i in $(seq 1 30); do #{helm_check} && break; sleep 1s; echo \"Retrying ($i)...\"; done"
+
+          "for i in $(seq 1 30); do #{helm_check} && s=0 && break || s=$?; sleep 1s; echo \"Retrying ($i)...\"; done; (exit $s)"
         end
 
         def repository_command
diff --git a/lib/gitlab/kubernetes/helm/reset_command.rb b/lib/gitlab/kubernetes/helm/reset_command.rb
index c8349639ec3..13176360227 100644
--- a/lib/gitlab/kubernetes/helm/reset_command.rb
+++ b/lib/gitlab/kubernetes/helm/reset_command.rb
@@ -18,7 +18,8 @@ module Gitlab
         def generate_script
           super + [
             reset_helm_command,
-            delete_tiller_replicaset
+            delete_tiller_replicaset,
+            delete_tiller_clusterrolebinding
           ].join("\n")
         end
 
@@ -43,6 +44,12 @@ module Gitlab
           Gitlab::Kubernetes::KubectlCmd.delete(*delete_args)
         end
 
+        def delete_tiller_clusterrolebinding
+          delete_args = %w[clusterrolebinding tiller-admin]
+
+          Gitlab::Kubernetes::KubectlCmd.delete(*delete_args)
+        end
+
         def reset_helm_command
           command = %w[helm reset] + optional_tls_flags
 
diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb
index 64317225ec6..66c28a9b702 100644
--- a/lib/gitlab/kubernetes/kube_client.rb
+++ b/lib/gitlab/kubernetes/kube_client.rb
@@ -39,7 +39,9 @@ module Gitlab
         :get_secret,
         :get_service,
         :get_service_account,
+        :delete_namespace,
         :delete_pod,
+        :delete_service_account,
         :create_config_map,
         :create_namespace,
         :create_pod,
diff --git a/lib/gitlab/legacy_github_import/release_formatter.rb b/lib/gitlab/legacy_github_import/release_formatter.rb
index fdab6b512ea..a083ae60726 100644
--- a/lib/gitlab/legacy_github_import/release_formatter.rb
+++ b/lib/gitlab/legacy_github_import/release_formatter.rb
@@ -10,7 +10,8 @@ module Gitlab
           name: raw_data.name,
           description: raw_data.body,
           created_at: raw_data.created_at,
-          released_at: raw_data.published_at,
+          # Draft releases will have a null published_at
+          released_at: raw_data.published_at || Time.current,
           updated_at: raw_data.created_at
         }
       end
diff --git a/lib/gitlab/lets_encrypt.rb b/lib/gitlab/lets_encrypt.rb
index 08ad2ab91b0..9d14b151f7d 100644
--- a/lib/gitlab/lets_encrypt.rb
+++ b/lib/gitlab/lets_encrypt.rb
@@ -5,5 +5,9 @@ module Gitlab
     def self.enabled?
       Gitlab::CurrentSettings.lets_encrypt_terms_of_service_accepted
     end
+
+    def self.terms_of_service_url
+      ::Gitlab::LetsEncrypt::Client.new.terms_of_service_url
+    end
   end
 end
diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb
index 124e34562c1..e90f3f05a33 100644
--- a/lib/gitlab/lfs_token.rb
+++ b/lib/gitlab/lfs_token.rb
@@ -34,8 +34,11 @@ module Gitlab
       HMACToken.new(actor).token(DEFAULT_EXPIRE_TIME)
     end
 
+    # When the token is an lfs one and the actor
+    # is blocked or the password has been changed,
+    # the token is no longer valid
     def token_valid?(token_to_check)
-      HMACToken.new(actor).token_valid?(token_to_check)
+      HMACToken.new(actor).token_valid?(token_to_check) && valid_user?
     end
 
     def deploy_key_pushable?(project)
@@ -46,6 +49,12 @@ module Gitlab
       user? ? :lfs_token : :lfs_deploy_token
     end
 
+    def valid_user?
+      return true unless user?
+
+      !actor.blocked? && (!actor.allow_password_authentication? || !actor.password_expired?)
+    end
+
     def authentication_payload(repository_http_path)
       {
         username: actor_name,
@@ -55,6 +64,10 @@ module Gitlab
       }
     end
 
+    def basic_encoding
+      ActionController::HttpAuthentication::Basic.encode_credentials(actor_name, token)
+    end
+
     private # rubocop:disable Lint/UselessAccessModifier
 
     class HMACToken
diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb
index c3169418371..297f109ff81 100644
--- a/lib/gitlab/metrics/dashboard/finder.rb
+++ b/lib/gitlab/metrics/dashboard/finder.rb
@@ -20,13 +20,17 @@ module Gitlab
           # @param options - dashboard_path [String] Path at which the
           #         dashboard can be found. Nil values will
           #         default to the system dashboard.
-          # @param options - group [String] Title of the group
+          # @param options - group [String, Group] Title of the group
           #         to which a panel might belong. Used by
-          #         embedded dashboards.
+          #         embedded dashboards. If cluster dashboard,
+          #         refers to the Group corresponding to the cluster.
           # @param options - title [String] Title of the panel.
           #         Used by embedded dashboards.
           # @param options - y_label [String] Y-Axis label of
           #         a panel. Used by embedded dashboards.
+          # @param options - cluster [Cluster]
+          # @param options - cluster_type [Symbol] The level of
+          #         cluster, one of [:admin, :project, :group]
           # @return [Hash]
           def find(project, user, options = {})
             service_for(options)
diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb
new file mode 100644
index 00000000000..7111835c85a
--- /dev/null
+++ b/lib/gitlab/metrics/exporter/base_exporter.rb
@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Exporter
+      class BaseExporter < Daemon
+        attr_reader :server
+
+        attr_accessor :readiness_checks
+
+        def enabled?
+          settings.enabled
+        end
+
+        def settings
+          raise NotImplementedError
+        end
+
+        def log_filename
+          raise NotImplementedError
+        end
+
+        private
+
+        def start_working
+          logger = WEBrick::Log.new(log_filename)
+          logger.time_format = "[%Y-%m-%dT%H:%M:%S.%L%z]"
+
+          access_log = [
+            [logger, WEBrick::AccessLog::COMBINED_LOG_FORMAT]
+          ]
+
+          @server = ::WEBrick::HTTPServer.new(
+            Port: settings.port, BindAddress: settings.address,
+            Logger: logger, AccessLog: access_log)
+          server.mount_proc '/readiness' do |req, res|
+            render_probe(readiness_probe, req, res)
+          end
+          server.mount_proc '/liveness' do |req, res|
+            render_probe(liveness_probe, req, res)
+          end
+          server.mount '/', Rack::Handler::WEBrick, rack_app
+
+          true
+        end
+
+        def run_thread
+          server&.start
+        rescue IOError
+          # ignore forcibily closed servers
+        end
+
+        def stop_working
+          if server
+            # we close sockets if thread is not longer running
+            # this happens, when the process forks
+            if thread.alive?
+              server.shutdown
+            else
+              server.listeners.each(&:close)
+            end
+          end
+
+          @server = nil
+        end
+
+        def rack_app
+          Rack::Builder.app do
+            use Rack::Deflater
+            use ::Prometheus::Client::Rack::Exporter if ::Gitlab::Metrics.metrics_folder_present?
+            run -> (env) { [404, {}, ['']] }
+          end
+        end
+
+        def readiness_probe
+          ::Gitlab::HealthChecks::Probes::Collection.new(*readiness_checks)
+        end
+
+        def liveness_probe
+          ::Gitlab::HealthChecks::Probes::Collection.new
+        end
+
+        def render_probe(probe, req, res)
+          result = probe.execute
+
+          res.status = result.http_status
+          res.content_type = 'application/json; charset=utf-8'
+          res.body = result.json.to_json
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb
new file mode 100644
index 00000000000..5ba7b29734b
--- /dev/null
+++ b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'webrick'
+require 'prometheus/client/rack/exporter'
+
+module Gitlab
+  module Metrics
+    module Exporter
+      class SidekiqExporter < BaseExporter
+        def settings
+          Settings.monitoring.sidekiq_exporter
+        end
+
+        def log_filename
+          File.join(Rails.root, 'log', 'sidekiq_exporter.log')
+        end
+
+        private
+
+        # Sidekiq Exporter does not work properly in sidekiq-cluster
+        # mode. It tries to start the service on the same port for
+        # each of the cluster workers, this results in failure
+        # due to duplicate binding.
+        #
+        # For now we ignore this error, as metrics are still "kind of"
+        # valid as they are rendered from shared directory.
+        #
+        # Issue: https://gitlab.com/gitlab-org/gitlab/issues/5714
+        def start_working
+          super
+        rescue Errno::EADDRINUSE => e
+          Sidekiq.logger.error(
+            class: self.class.to_s,
+            message: 'Cannot start sidekiq_exporter',
+            exception: e.message
+          )
+
+          false
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb
new file mode 100644
index 00000000000..3940f6fa155
--- /dev/null
+++ b/lib/gitlab/metrics/exporter/web_exporter.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'webrick'
+require 'prometheus/client/rack/exporter'
+
+module Gitlab
+  module Metrics
+    module Exporter
+      class WebExporter < BaseExporter
+        ExporterCheck = Struct.new(:exporter) do
+          def readiness
+            Gitlab::HealthChecks::Result.new(
+              'web_exporter', exporter.running)
+          end
+        end
+
+        attr_reader :running
+
+        # This exporter is always run on master process
+        def initialize
+          super
+
+          self.readiness_checks = [
+            WebExporter::ExporterCheck.new(self),
+            Gitlab::HealthChecks::PumaCheck,
+            Gitlab::HealthChecks::UnicornCheck
+          ]
+        end
+
+        def settings
+          Gitlab.config.monitoring.web_exporter
+        end
+
+        def log_filename
+          File.join(Rails.root, 'log', 'web_exporter.log')
+        end
+
+        private
+
+        def start_working
+          @running = true
+          super
+        end
+
+        def stop_working
+          @running = false
+          wait_in_blackout_period if server && thread.alive?
+          super
+        end
+
+        def wait_in_blackout_period
+          return unless blackout_seconds > 0
+
+          @server.logger.info(
+            message: 'starting blackout...',
+            duration_s: blackout_seconds)
+
+          sleep(blackout_seconds)
+        end
+
+        def blackout_seconds
+          settings['blackout_seconds'].to_i
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb
index 26aa0910047..46477587934 100644
--- a/lib/gitlab/metrics/requests_rack_middleware.rb
+++ b/lib/gitlab/metrics/requests_rack_middleware.rb
@@ -3,6 +3,18 @@
 module Gitlab
   module Metrics
     class RequestsRackMiddleware
+      HTTP_METHODS = {
+        "delete" => %w(200 202 204 303 400 401 403 404 410 422 500 503),
+        "get" => %w(200 204 301 302 303 304 307 400 401 403 404 410 412 422 429 500 503),
+        "head" => %w(200 204 301 302 303 304 400 401 403 404 410 429 500 503),
+        "options" => %w(200 404),
+        "patch" => %w(200 202 204 400 403 404 409 416 422 500),
+        "post" => %w(200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 413 415 422 429 500 503),
+        "propfind" => %w(404),
+        "put" => %w(200 202 204 400 401 403 404 405 406 409 410 415 422 500),
+        "report" =>  %w(404)
+      }.freeze
+
       def initialize(app)
         @app = app
       end
@@ -20,6 +32,14 @@ module Gitlab
                                                            {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25])
       end
 
+      def self.initialize_http_request_duration_seconds
+        HTTP_METHODS.each do |method, statuses|
+          statuses.each do |status|
+            http_request_duration_seconds.get({ method: method, status: status })
+          end
+        end
+      end
+
       def call(env)
         method = env['REQUEST_METHOD'].downcase
         started = Time.now.to_f
diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb
index d7d848d2833..90051f85f31 100644
--- a/lib/gitlab/metrics/samplers/base_sampler.rb
+++ b/lib/gitlab/metrics/samplers/base_sampler.rb
@@ -50,6 +50,11 @@ module Gitlab
 
         def start_working
           @running = true
+
+          true
+        end
+
+        def run_thread
           sleep(sleep_interval)
           while running
             safe_sample
diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb
index 8a24d4f3663..f788f51b1ce 100644
--- a/lib/gitlab/metrics/samplers/puma_sampler.rb
+++ b/lib/gitlab/metrics/samplers/puma_sampler.rb
@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'puma/state_file'
-
 module Gitlab
   module Metrics
     module Samplers
diff --git a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb
deleted file mode 100644
index 71a5406815f..00000000000
--- a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb
+++ /dev/null
@@ -1,49 +0,0 @@
-# frozen_string_literal: true
-
-require 'webrick'
-require 'prometheus/client/rack/exporter'
-
-module Gitlab
-  module Metrics
-    class SidekiqMetricsExporter < Daemon
-      LOG_FILENAME = File.join(Rails.root, 'log', 'sidekiq_exporter.log')
-
-      def enabled?
-        ::Gitlab::Metrics.metrics_folder_present? && settings.enabled
-      end
-
-      def settings
-        Settings.monitoring.sidekiq_exporter
-      end
-
-      private
-
-      attr_reader :server
-
-      def start_working
-        logger = WEBrick::Log.new(LOG_FILENAME)
-        access_log = [
-          [logger, WEBrick::AccessLog::COMBINED_LOG_FORMAT]
-        ]
-
-        @server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address,
-                                            Logger: logger, AccessLog: access_log)
-        server.mount "/", Rack::Handler::WEBrick, rack_app
-        server.start
-      end
-
-      def stop_working
-        server.shutdown if server
-        @server = nil
-      end
-
-      def rack_app
-        Rack::Builder.app do
-          use Rack::Deflater
-          use ::Prometheus::Client::Rack::Exporter
-          run -> (env) { [404, {}, ['']] }
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb
index 51f48095cb5..2a61b3de405 100644
--- a/lib/gitlab/metrics/system.rb
+++ b/lib/gitlab/metrics/system.rb
@@ -63,6 +63,21 @@ module Gitlab
       def self.monotonic_time
         Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second)
       end
+
+      def self.thread_cpu_time
+        # Not all OS kernels are supporting `Process::CLOCK_THREAD_CPUTIME_ID`
+        # Refer: https://gitlab.com/gitlab-org/gitlab/issues/30567#note_221765627
+        return unless defined?(Process::CLOCK_THREAD_CPUTIME_ID)
+
+        Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :float_second)
+      end
+
+      def self.thread_cpu_duration(start_time)
+        end_time = thread_cpu_time
+        return unless start_time && end_time
+
+        end_time - start_time
+      end
     end
   end
 end
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index ba2a0b2ecf8..115368c8bc6 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -44,6 +44,10 @@ module Gitlab
         duration.in_milliseconds.to_i
       end
 
+      def thread_cpu_duration
+        System.thread_cpu_duration(@thread_cputime_start)
+      end
+
       def allocated_memory
         @memory_after - @memory_before
       end
@@ -53,12 +57,14 @@ module Gitlab
 
         @memory_before = System.memory_usage
         @started_at = System.monotonic_time
+        @thread_cputime_start = System.thread_cpu_time
 
         yield
       ensure
         @memory_after = System.memory_usage
         @finished_at = System.monotonic_time
 
+        self.class.gitlab_transaction_cputime_seconds.observe(labels, thread_cpu_duration)
         self.class.gitlab_transaction_duration_seconds.observe(labels, duration)
         self.class.gitlab_transaction_allocated_memory_bytes.observe(labels, allocated_memory * 1024.0)
 
@@ -142,6 +148,12 @@ module Gitlab
         "#{labels[:controller]}##{labels[:action]}" if labels && !labels.empty?
       end
 
+      define_histogram :gitlab_transaction_cputime_seconds do
+        docstring 'Transaction thread cputime'
+        base_labels BASE_LABELS
+        buckets [0.1, 0.25, 0.5, 1.0, 2.5, 5.0]
+      end
+
       define_histogram :gitlab_transaction_duration_seconds do
         docstring 'Transaction duration'
         base_labels BASE_LABELS
diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb
index a29dc5395f3..b18f0eed1fa 100644
--- a/lib/gitlab/middleware/read_only/controller.rb
+++ b/lib/gitlab/middleware/read_only/controller.rb
@@ -20,6 +20,12 @@ module Gitlab
           'projects/lfs_locks_api' => %w{verify create unlock}
         }.freeze
 
+        WHITELISTED_GIT_REVISION_ROUTES = {
+          'projects/compare' => %w{create}
+        }.freeze
+
+        GRAPHQL_URL = '/api/graphql'
+
         def initialize(app, env)
           @app = app
           @env = env
@@ -79,7 +85,7 @@ module Gitlab
 
         # Overridden in EE module
         def whitelisted_routes
-          grack_route? || internal_route? || lfs_route? || sidekiq_route?
+          grack_route? || internal_route? || lfs_route? || compare_git_revisions_route? || sidekiq_route? || graphql_query?
         end
 
         def grack_route?
@@ -94,6 +100,13 @@ module Gitlab
           ReadOnly.internal_routes.any? { |path| request.path.include?(path) }
         end
 
+        def compare_git_revisions_route?
+          # Calling route_hash may be expensive. Only do it if we think there's a possible match
+          return false unless request.post? && request.path.end_with?('compare')
+
+          WHITELISTED_GIT_REVISION_ROUTES[route_hash[:controller]]&.include?(route_hash[:action])
+        end
+
         def lfs_route?
           # Calling route_hash may be expensive. Only do it if we think there's a possible match
           unless request.path.end_with?('/info/lfs/objects/batch',
@@ -108,6 +121,10 @@ module Gitlab
         def sidekiq_route?
           request.path.start_with?("#{relative_url}/admin/sidekiq")
         end
+
+        def graphql_query?
+          request.post? && request.path.start_with?(GRAPHQL_URL)
+        end
       end
     end
   end
diff --git a/lib/gitlab/pages_client.rb b/lib/gitlab/pages_client.rb
deleted file mode 100644
index 30a1f9ede25..00000000000
--- a/lib/gitlab/pages_client.rb
+++ /dev/null
@@ -1,119 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  class PagesClient
-    class << self
-      attr_reader :certificate, :token
-
-      def call(service, rpc, request, timeout: nil)
-        kwargs = request_kwargs(timeout)
-        stub(service).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend
-      end
-
-      # This function is not thread-safe. Call it from an initializer only.
-      def read_or_create_token
-        @token = read_token
-      rescue Errno::ENOENT
-        # TODO: uncomment this when omnibus knows how to write the token file for us
-        # https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests/2466
-        #
-        # write_token(SecureRandom.random_bytes(64))
-        #
-        # # Read from disk in case someone else won the race and wrote the file
-        # # before us. If this fails again let the exception bubble up.
-        # @token = read_token
-      end
-
-      # This function is not thread-safe. Call it from an initializer only.
-      def load_certificate
-        cert_path = config.certificate
-        return unless cert_path.present?
-
-        @certificate = File.read(cert_path)
-      end
-
-      def ping
-        request = Grpc::Health::V1::HealthCheckRequest.new
-        call(:health_check, :check, request, timeout: 5.seconds)
-      end
-
-      private
-
-      def request_kwargs(timeout)
-        encoded_token = Base64.strict_encode64(token.to_s)
-        metadata = {
-          'authorization' => "Bearer #{encoded_token}"
-        }
-
-        result = { metadata: metadata }
-
-        return result unless timeout
-
-        # Do not use `Time.now` for deadline calculation, since it
-        # will be affected by Timecop in some tests, but grpc's c-core
-        # uses system time instead of timecop's time, so tests will fail
-        # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will
-        # circumvent timecop
-        deadline = Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + timeout
-        result[:deadline] = deadline
-
-        result
-      end
-
-      def stub(name)
-        stub_class(name).new(address, grpc_creds)
-      end
-
-      def stub_class(name)
-        if name == :health_check
-          Grpc::Health::V1::Health::Stub
-        else
-          # TODO use pages namespace
-          Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub)
-        end
-      end
-
-      def address
-        addr = config.address
-        addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp'
-        addr
-      end
-
-      def grpc_creds
-        if address.start_with?('unix:')
-          :this_channel_is_insecure
-        elsif @certificate
-          GRPC::Core::ChannelCredentials.new(@certificate)
-        else
-          # Use system certificate pool
-          GRPC::Core::ChannelCredentials.new
-        end
-      end
-
-      def config
-        Gitlab.config.pages.admin
-      end
-
-      def read_token
-        File.read(token_path)
-      end
-
-      def token_path
-        Rails.root.join('.gitlab_pages_secret').to_s
-      end
-
-      def write_token(new_token)
-        Tempfile.open(File.basename(token_path), File.dirname(token_path), encoding: 'ascii-8bit') do |f|
-          f.write(new_token)
-          f.close
-          File.link(f.path, token_path)
-        end
-      rescue Errno::EACCES => ex
-        # TODO stop rescuing this exception in GitLab 11.0 https://gitlab.com/gitlab-org/gitlab-foss/issues/45672
-        Rails.logger.error("Could not write pages admin token file: #{ex}") # rubocop:disable Gitlab/RailsLogger
-      rescue Errno::EEXIST
-        # Another process wrote the token file concurrently with us. Use their token, not ours.
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/patch/prependable.rb b/lib/gitlab/patch/prependable.rb
index a9f6cfb19cb..22ece0a6a8b 100644
--- a/lib/gitlab/patch/prependable.rb
+++ b/lib/gitlab/patch/prependable.rb
@@ -24,7 +24,7 @@ module Gitlab
         super
 
         if const_defined?(:ClassMethods)
-          klass_methods = const_get(:ClassMethods)
+          klass_methods = const_get(:ClassMethods, false)
           base.singleton_class.prepend klass_methods
           base.instance_variable_set(:@_prepended_class_methods, klass_methods)
         end
@@ -40,7 +40,7 @@ module Gitlab
         super
 
         if instance_variable_defined?(:@_prepended_class_methods)
-          const_get(:ClassMethods).prepend @_prepended_class_methods
+          const_get(:ClassMethods, false).prepend @_prepended_class_methods
         end
       end
 
diff --git a/lib/gitlab/phabricator_import/base_worker.rb b/lib/gitlab/phabricator_import/base_worker.rb
index b69c65e78f8..d2c2ef8db48 100644
--- a/lib/gitlab/phabricator_import/base_worker.rb
+++ b/lib/gitlab/phabricator_import/base_worker.rb
@@ -23,6 +23,8 @@ module Gitlab
       include ProjectImportOptions # This marks the project as failed after too many tries
       include Gitlab::ExclusiveLeaseHelpers
 
+      feature_category :importers
+
       class << self
         def schedule(project_id, *args)
           perform_async(project_id, *args)
diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb
index 275151f7fc1..560618bb486 100644
--- a/lib/gitlab/profiler.rb
+++ b/lib/gitlab/profiler.rb
@@ -37,8 +37,7 @@ module Gitlab
     # - post_data: a string of raw POST data to use. Changes the HTTP verb to
     #   POST.
     #
-    # - user: a user to authenticate as. Only works if the user has a valid
-    #   personal access token.
+    # - user: a user to authenticate as.
     #
     # - private_token: instead of providing a user instance, the token can be
     #   given as a string. Takes precedence over the user option.
diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb
index ff9bb293b47..e04d6f250b1 100644
--- a/lib/gitlab/quick_actions/extractor.rb
+++ b/lib/gitlab/quick_actions/extractor.rb
@@ -50,7 +50,7 @@ module Gitlab
 
         content, commands = perform_substitutions(content, commands)
 
-        [content.strip, commands]
+        [content.rstrip, commands]
       end
 
       private
@@ -109,7 +109,7 @@ module Gitlab
                 [ ]
                 (?<arg>[^\n]*)
               )?
-              (?:\n|$)
+              (?:\s*\n|$)
             )
         }mix
       end
diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb
index 7e64fe2a1f4..404e0c31871 100644
--- a/lib/gitlab/quick_actions/issue_actions.rb
+++ b/lib/gitlab/quick_actions/issue_actions.rb
@@ -135,7 +135,8 @@ module Gitlab
         end
         types Issue
         condition do
-          current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target)
+          !quick_action_target.confidential? &&
+            current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target)
         end
         command :confidential do
           @updates[:confidential] = true
diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb
index 00f817c2399..ea2b03b42c1 100644
--- a/lib/gitlab/reference_extractor.rb
+++ b/lib/gitlab/reference_extractor.rb
@@ -3,7 +3,8 @@
 module Gitlab
   # Extract possible GFM references from an arbitrary String for further processing.
   class ReferenceExtractor < Banzai::ReferenceExtractor
-    REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range directly_addressed_user epic).freeze
+    REFERABLES = %i(user issue label milestone
+                    merge_request snippet commit commit_range directly_addressed_user epic).freeze
     attr_accessor :project, :current_user, :author
 
     def initialize(project, current_user = nil)
@@ -54,9 +55,9 @@ module Gitlab
     def self.references_pattern
       return @pattern if @pattern
 
-      patterns = REFERABLES.map do |ref|
-        ref.to_s.classify.constantize.try(:reference_pattern)
-      end
+      patterns = REFERABLES.map do |type|
+        Banzai::ReferenceParser[type].reference_type.to_s.classify.constantize.try(:reference_pattern)
+      end.uniq
 
       @pattern = Regexp.union(patterns.compact)
     end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 4bfa6f7e9a5..3d1f15c72ae 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -119,6 +119,15 @@ module Gitlab
     def breakline_regex
       @breakline_regex ||= /\r\n|\r|\n/
     end
+
+    # https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html
+    def aws_arn_regex
+      /\Aarn:\S+\z/
+    end
+
+    def aws_arn_regex_message
+      "must be a valid Amazon Resource Name"
+    end
   end
 end
 
diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb
index ab2549d5e68..13187836e02 100644
--- a/lib/gitlab/request_context.rb
+++ b/lib/gitlab/request_context.rb
@@ -6,6 +6,10 @@ module Gitlab
       def client_ip
         Gitlab::SafeRequestStore[:client_ip]
       end
+
+      def start_thread_cpu_time
+        Gitlab::SafeRequestStore[:start_thread_cpu_time]
+      end
     end
 
     def initialize(app)
@@ -23,6 +27,8 @@ module Gitlab
 
       Gitlab::SafeRequestStore[:client_ip] = req.ip
 
+      Gitlab::SafeRequestStore[:start_thread_cpu_time] = Gitlab::Metrics::System.thread_cpu_time
+
       @app.call(env)
     end
   end
diff --git a/lib/gitlab/sanitizers/exif.rb b/lib/gitlab/sanitizers/exif.rb
index 2f3d14ecebd..5eeb8b00ff3 100644
--- a/lib/gitlab/sanitizers/exif.rb
+++ b/lib/gitlab/sanitizers/exif.rb
@@ -68,7 +68,7 @@ module Gitlab
         }
 
         relation.find_each(find_params) do |upload|
-          clean(upload.build_uploader, dry_run: dry_run)
+          clean(upload.retrieve_uploader, dry_run: dry_run)
           sleep sleep_time if sleep_time
         rescue => err
           logger.error "failed to sanitize #{upload_ref(upload)}: #{err.message}"
diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb
index 93e172299b9..782ac534a7b 100644
--- a/lib/gitlab/search_results.rb
+++ b/lib/gitlab/search_results.rb
@@ -2,7 +2,7 @@
 
 module Gitlab
   class SearchResults
-    COUNT_LIMIT = 101
+    COUNT_LIMIT = 100
     COUNT_LIMIT_MESSAGE = "#{COUNT_LIMIT - 1}+"
 
     attr_reader :current_user, :query, :per_page
diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb
index 7dbed591b84..125d0d1cfbb 100644
--- a/lib/gitlab/shell.rb
+++ b/lib/gitlab/shell.rb
@@ -113,10 +113,6 @@ module Gitlab
       success
     end
 
-    # Move repository reroutes to mv_directory which is an alias for
-    # mv_namespace. Given the underlying implementation is a move action,
-    # indescriminate of what the folders might be.
-    #
     # storage - project's storage path
     # path - project disk path
     # new_path - new project disk path
@@ -126,7 +122,13 @@ module Gitlab
     def mv_repository(storage, path, new_path)
       return false if path.empty? || new_path.empty?
 
-      !!mv_directory(storage, "#{path}.git", "#{new_path}.git")
+      Gitlab::Git::Repository.new(storage, "#{path}.git", nil, nil).rename("#{new_path}.git")
+
+      true
+    rescue => e
+      Gitlab::Sentry.track_acceptable_exception(e, extra: { path: path, new_path: new_path, storage: storage })
+
+      false
     end
 
     # Fork repository to new path
@@ -151,9 +153,13 @@ module Gitlab
     def remove_repository(storage, name)
       return false if name.empty?
 
-      !!rm_directory(storage, "#{name}.git")
-    rescue ArgumentError => e
+      Gitlab::Git::Repository.new(storage, "#{name}.git", nil, nil).remove
+
+      true
+    rescue => e
       Rails.logger.warn("Repository does not exist: #{e} at: #{name}.git") # rubocop:disable Gitlab/RailsLogger
+      Gitlab::Sentry.track_acceptable_exception(e, extra: { path: name, storage: storage })
+
       false
     end
 
@@ -265,7 +271,6 @@ module Gitlab
 
       false
     end
-    alias_method :mv_directory, :mv_namespace # Note: ShellWorker uses this alias
 
     def url_to_repo(path)
       Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git"
@@ -292,6 +297,12 @@ module Gitlab
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
+    def repository_exists?(storage, dir_name)
+      Gitlab::Git::Repository.new(storage, dir_name, nil, nil).exists?
+    rescue GRPC::Internal
+      false
+    end
+
     def hooks_path
       File.join(gitlab_shell_path, 'hooks')
     end
diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb
index c102fa14cfc..ffceeb68f20 100644
--- a/lib/gitlab/sidekiq_config.rb
+++ b/lib/gitlab/sidekiq_config.rb
@@ -5,7 +5,11 @@ require 'set'
 
 module Gitlab
   module SidekiqConfig
-    QUEUE_CONFIG_PATHS = %w[app/workers/all_queues.yml ee/app/workers/all_queues.yml].freeze
+    QUEUE_CONFIG_PATHS = begin
+      result = %w[app/workers/all_queues.yml]
+      result << 'ee/app/workers/all_queues.yml' if Gitlab.ee?
+      result
+    end.freeze
 
     # This method is called by `ee/bin/sidekiq-cluster` in EE, which runs outside
     # of bundler/Rails context, so we cannot use any gem or Rails methods.
@@ -48,9 +52,11 @@ module Gitlab
     end
 
     def self.workers
-      @workers ||=
-        find_workers(Rails.root.join('app', 'workers')) +
-        find_workers(Rails.root.join('ee', 'app', 'workers'))
+      @workers ||= begin
+        result = find_workers(Rails.root.join('app', 'workers'))
+        result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'))) if Gitlab.ee?
+        result
+      end
     end
 
     def self.find_workers(root)
diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb
new file mode 100644
index 00000000000..9d0d67a488f
--- /dev/null
+++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb
@@ -0,0 +1,263 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqDaemon
+    class MemoryKiller < Daemon
+      include ::Gitlab::Utils::StrongMemoize
+
+      # Today 64-bit CPU support max 256T memory. It is big enough.
+      MAX_MEMORY_KB = 256 * 1024 * 1024 * 1024
+      # RSS below `soft_limit_rss` is considered safe
+      SOFT_LIMIT_RSS_KB = ENV.fetch('SIDEKIQ_MEMORY_KILLER_MAX_RSS', 2000000).to_i
+      # RSS above `hard_limit_rss` will be stopped
+      HARD_LIMIT_RSS_KB = ENV.fetch('SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS', MAX_MEMORY_KB).to_i
+      # RSS in range (soft_limit_rss, hard_limit_rss) is allowed for GRACE_BALLOON_SECONDS
+      GRACE_BALLOON_SECONDS = ENV.fetch('SIDEKIQ_MEMORY_KILLER_GRACE_TIME', 15 * 60).to_i
+      # Check RSS every CHECK_INTERVAL_SECONDS, minimum 2 seconds
+      CHECK_INTERVAL_SECONDS = [ENV.fetch('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', 3).to_i, 2].max
+      # Give Sidekiq up to 30 seconds to allow existing jobs to finish after exceeding the limit
+      SHUTDOWN_TIMEOUT_SECONDS = ENV.fetch('SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT', 30).to_i
+      # Developer/admin should always set `memory_killer_max_memory_growth_kb` explicitly
+      # In case not set, default to 300M. This is for extra-safe.
+      DEFAULT_MAX_MEMORY_GROWTH_KB = 300_000
+
+      # Phases of memory killer
+      PHASE = {
+        running: 1,
+        above_soft_limit: 2,
+        stop_fetching_new_jobs: 3,
+        shutting_down: 4,
+        killing_sidekiq: 5
+      }.freeze
+
+      def initialize
+        super
+
+        @enabled = true
+        @metrics = init_metrics
+      end
+
+      private
+
+      def init_metrics
+        {
+          sidekiq_current_rss:                  ::Gitlab::Metrics.gauge(:sidekiq_current_rss, 'Current RSS of Sidekiq Worker'),
+          sidekiq_memory_killer_soft_limit_rss: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_soft_limit_rss, 'Current soft_limit_rss of Sidekiq Worker'),
+          sidekiq_memory_killer_hard_limit_rss: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_hard_limit_rss, 'Current hard_limit_rss of Sidekiq Worker'),
+          sidekiq_memory_killer_phase:          ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_phase, 'Current phase of Sidekiq Worker')
+        }
+      end
+
+      def refresh_state(phase)
+        @phase = PHASE.fetch(phase)
+        @current_rss = get_rss
+        @soft_limit_rss = get_soft_limit_rss
+        @hard_limit_rss = get_hard_limit_rss
+
+        # track the current state as prometheus gauges
+        @metrics[:sidekiq_memory_killer_phase].set({}, @phase)
+        @metrics[:sidekiq_current_rss].set({}, @current_rss)
+        @metrics[:sidekiq_memory_killer_soft_limit_rss].set({}, @soft_limit_rss)
+        @metrics[:sidekiq_memory_killer_hard_limit_rss].set({}, @hard_limit_rss)
+      end
+
+      def run_thread
+        Sidekiq.logger.info(
+          class: self.class.to_s,
+          action: 'start',
+          pid: pid,
+          message: 'Starting Gitlab::SidekiqDaemon::MemoryKiller Daemon'
+        )
+
+        while enabled?
+          begin
+            sleep(CHECK_INTERVAL_SECONDS)
+            restart_sidekiq unless rss_within_range?
+          rescue => e
+            log_exception(e, __method__)
+          rescue Exception => e # rubocop:disable Lint/RescueException
+            log_exception(e, __method__ )
+            raise e
+          end
+        end
+      ensure
+        Sidekiq.logger.warn(
+          class: self.class.to_s,
+          action: 'stop',
+          pid: pid,
+          message: 'Stopping Gitlab::SidekiqDaemon::MemoryKiller Daemon'
+        )
+      end
+
+      def log_exception(exception, method)
+        Sidekiq.logger.warn(
+          class: self.class.to_s,
+          pid: pid,
+          message: "Exception from #{method}: #{exception.message}"
+        )
+      end
+
+      def stop_working
+        @enabled = false
+      end
+
+      def enabled?
+        @enabled
+      end
+
+      def restart_sidekiq
+        # Tell Sidekiq to stop fetching new jobs
+        # We first SIGNAL and then wait given time
+        # We also monitor a number of running jobs and allow to restart early
+        refresh_state(:stop_fetching_new_jobs)
+        signal_and_wait(SHUTDOWN_TIMEOUT_SECONDS, 'SIGTSTP', 'stop fetching new jobs')
+        return unless enabled?
+
+        # Tell sidekiq to restart itself
+        # Keep extra safe to wait `Sidekiq.options[:timeout] + 2` seconds before SIGKILL
+        refresh_state(:shutting_down)
+        signal_and_wait(Sidekiq.options[:timeout] + 2, 'SIGTERM', 'gracefully shut down')
+        return unless enabled?
+
+        # Ideally we should never reach this condition
+        # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't
+        # Kill the whole pgroup, so we can be sure no children are left behind
+        refresh_state(:killing_sidekiq)
+        signal_pgroup('SIGKILL', 'die')
+      end
+
+      def rss_within_range?
+        refresh_state(:running)
+
+        deadline = Gitlab::Metrics::System.monotonic_time + GRACE_BALLOON_SECONDS.seconds
+        loop do
+          return true unless enabled?
+
+          # RSS go above hard limit should trigger forcible shutdown right away
+          break if @current_rss > @hard_limit_rss
+
+          # RSS go below the soft limit
+          return true if @current_rss < @soft_limit_rss
+
+          # RSS did not go below the soft limit within deadline, restart
+          break if Gitlab::Metrics::System.monotonic_time > deadline
+
+          sleep(CHECK_INTERVAL_SECONDS)
+
+          refresh_state(:above_soft_limit)
+        end
+
+        # There are two chances to break from loop:
+        #   - above hard limit, or
+        #   - above soft limit after deadline
+        # When `above hard limit`, it immediately go to `stop_fetching_new_jobs`
+        # So ignore `above hard limit` and always set `above_soft_limit` here
+        refresh_state(:above_soft_limit)
+        log_rss_out_of_range(@current_rss, @hard_limit_rss, @soft_limit_rss)
+
+        false
+      end
+
+      def log_rss_out_of_range(current_rss, hard_limit_rss, soft_limit_rss)
+        Sidekiq.logger.warn(
+          class: self.class.to_s,
+          pid: pid,
+          message: 'Sidekiq worker RSS out of range',
+          current_rss: current_rss,
+          hard_limit_rss: hard_limit_rss,
+          soft_limit_rss: soft_limit_rss,
+          reason: out_of_range_description(current_rss, hard_limit_rss, soft_limit_rss)
+        )
+      end
+
+      def out_of_range_description(rss, hard_limit, soft_limit)
+        if rss > hard_limit
+          "current_rss(#{rss}) > hard_limit_rss(#{hard_limit})"
+        else
+          "current_rss(#{rss}) > soft_limit_rss(#{soft_limit}) longer than GRACE_BALLOON_SECONDS(#{GRACE_BALLOON_SECONDS})"
+        end
+      end
+
+      def get_rss
+        output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s)
+        return 0 unless status&.zero?
+
+        output.to_i
+      end
+
+      def get_soft_limit_rss
+        SOFT_LIMIT_RSS_KB + rss_increase_by_jobs
+      end
+
+      def get_hard_limit_rss
+        HARD_LIMIT_RSS_KB
+      end
+
+      def signal_and_wait(time, signal, explanation)
+        Sidekiq.logger.warn(
+          class: self.class.to_s,
+          pid: pid,
+          signal: signal,
+          explanation: explanation,
+          wait_time: time,
+          message: "Sending signal and waiting"
+        )
+        Process.kill(signal, pid)
+
+        deadline = Gitlab::Metrics::System.monotonic_time + time
+
+        # we try to finish as early as all jobs finished
+        # so we retest that in loop
+        sleep(CHECK_INTERVAL_SECONDS) while enabled? && any_jobs? && Gitlab::Metrics::System.monotonic_time < deadline
+      end
+
+      def signal_pgroup(signal, explanation)
+        if Process.getpgrp == pid
+          pid_or_pgrp_str = 'PGRP'
+          pid_to_signal = 0
+        else
+          pid_or_pgrp_str = 'PID'
+          pid_to_signal = pid
+        end
+
+        Sidekiq.logger.warn(
+          class: self.class.to_s,
+          signal: signal,
+          pid: pid,
+          message: "sending Sidekiq worker #{pid_or_pgrp_str}-#{pid} #{signal} (#{explanation})"
+        )
+        Process.kill(signal, pid_to_signal)
+      end
+
+      def rss_increase_by_jobs
+        Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job| # rubocop:disable CodeReuse/ActiveRecord
+          rss_increase_by_job(job)
+        end
+      end
+
+      def rss_increase_by_job(job)
+        memory_growth_kb = get_job_options(job, 'memory_killer_memory_growth_kb', 0).to_i
+        max_memory_growth_kb = get_job_options(job, 'memory_killer_max_memory_growth_kb', DEFAULT_MAX_MEMORY_GROWTH_KB).to_i
+
+        return 0 if memory_growth_kb.zero?
+
+        time_elapsed = [Gitlab::Metrics::System.monotonic_time - job[:started_at], 0].max
+        [memory_growth_kb * time_elapsed, max_memory_growth_kb].min
+      end
+
+      def get_job_options(job, key, default)
+        job[:worker_class].sidekiq_options.fetch(key, default)
+      rescue
+        default
+      end
+
+      def pid
+        Process.pid
+      end
+
+      def any_jobs?
+        Gitlab::SidekiqDaemon::Monitor.instance.jobs.any?
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_daemon/monitor.rb b/lib/gitlab/sidekiq_daemon/monitor.rb
index bbfca130425..a3d61c69ae1 100644
--- a/lib/gitlab/sidekiq_daemon/monitor.rb
+++ b/lib/gitlab/sidekiq_daemon/monitor.rb
@@ -14,19 +14,19 @@ module Gitlab
       # that should not be caught by application
       CancelledError = Class.new(Exception) # rubocop:disable Lint/InheritException
 
-      attr_reader :jobs_thread
+      attr_reader :jobs
       attr_reader :jobs_mutex
 
       def initialize
         super
 
-        @jobs_thread = {}
+        @jobs = {}
         @jobs_mutex = Mutex.new
       end
 
-      def within_job(jid, queue)
+      def within_job(worker_class, jid, queue)
         jobs_mutex.synchronize do
-          jobs_thread[jid] = Thread.current
+          jobs[jid] = { worker_class: worker_class, thread: Thread.current, started_at: Gitlab::Metrics::System.monotonic_time }
         end
 
         if cancelled?(jid)
@@ -43,7 +43,7 @@ module Gitlab
         yield
       ensure
         jobs_mutex.synchronize do
-          jobs_thread.delete(jid)
+          jobs.delete(jid)
         end
       end
 
@@ -61,24 +61,28 @@ module Gitlab
 
       private
 
-      def start_working
-        Sidekiq.logger.info(
-          class: self.class.to_s,
-          action: 'start',
-          message: 'Starting Monitor Daemon'
-        )
+      def run_thread
+        return unless notification_channel_enabled?
 
-        while enabled?
-          process_messages
-          sleep(RECONNECT_TIME)
-        end
+        begin
+          Sidekiq.logger.info(
+            class: self.class.to_s,
+            action: 'start',
+            message: 'Starting Monitor Daemon'
+          )
 
-      ensure
-        Sidekiq.logger.warn(
-          class: self.class.to_s,
-          action: 'stop',
-          message: 'Stopping Monitor Daemon'
-        )
+          while enabled?
+            process_messages
+            sleep(RECONNECT_TIME)
+          end
+
+        ensure
+          Sidekiq.logger.warn(
+            class: self.class.to_s,
+            action: 'stop',
+            message: 'Stopping Monitor Daemon'
+          )
+        end
       end
 
       def stop_working
@@ -156,7 +160,7 @@ module Gitlab
       # This is why it passes thread in block,
       # to ensure that we do process this thread
       def find_thread_unsafe(jid)
-        jobs_thread[jid]
+        jobs.dig(jid, :thread)
       end
 
       def find_thread_with_lock(jid)
@@ -179,6 +183,10 @@ module Gitlab
       def self.cancel_job_key(jid)
         "sidekiq:cancel:#{jid}"
       end
+
+      def notification_channel_enabled?
+        ENV.fetch("SIDEKIQ_MONITOR_WORKER", 0).to_i.nonzero?
+      end
     end
   end
 end
diff --git a/lib/gitlab/sidekiq_logging/exception_handler.rb b/lib/gitlab/sidekiq_logging/exception_handler.rb
new file mode 100644
index 00000000000..fba74b6c9ed
--- /dev/null
+++ b/lib/gitlab/sidekiq_logging/exception_handler.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqLogging
+    class ExceptionHandler
+      def call(job_exception, context)
+        data = {
+          error_class: job_exception.class.name,
+          error_message: job_exception.message
+        }
+
+        if context.is_a?(Hash)
+          data.merge!(context)
+          # correlation_id, jid, and class are available inside the job
+          # Hash, so promote these arguments to the root tree so that
+          # can be searched alongside other Sidekiq log messages.
+          job_data = data.delete(:job)
+          data.merge!(job_data) if job_data.present?
+        end
+
+        data[:error_backtrace] = Gitlab::Profiler.clean_backtrace(job_exception.backtrace) if job_exception.backtrace.present?
+
+        Sidekiq.logger.warn(data)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb
index 48b1524f9c7..853fb2777c3 100644
--- a/lib/gitlab/sidekiq_logging/structured_logger.rb
+++ b/lib/gitlab/sidekiq_logging/structured_logger.rb
@@ -58,8 +58,7 @@ module Gitlab
           payload['message'] = "#{message}: fail: #{payload['duration']} sec"
           payload['job_status'] = 'fail'
           payload['error_message'] = job_exception.message
-          payload['error'] = job_exception.class
-          payload['error_backtrace'] = backtrace_cleaner.clean(job_exception.backtrace)
+          payload['error_class'] = job_exception.class.name
         else
           payload['message'] = "#{message}: done: #{payload['duration']} sec"
           payload['job_status'] = 'done'
@@ -71,10 +70,11 @@ module Gitlab
       end
 
       def add_time_keys!(time, payload)
-        payload['duration'] = time[:duration].round(3)
-        payload['system_s'] = time[:stime].round(3)
-        payload['user_s'] = time[:utime].round(3)
-        payload['child_s'] = time[:ctime].round(3) if time[:ctime] > 0
+        payload['duration'] = time[:duration].round(6)
+
+        # ignore `cpu_s` if the platform does not support Process::CLOCK_THREAD_CPUTIME_ID (time[:cputime] == 0)
+        # supported OS version can be found at: https://www.rubydoc.info/stdlib/core/2.1.6/Process:clock_gettime
+        payload['cpu_s'] = time[:cputime].round(6) if time[:cputime] > 0
         payload['completed_at'] = Time.now.utc
       end
 
@@ -99,42 +99,32 @@ module Gitlab
       end
 
       def elapsed_by_absolute_time(start)
-        (Time.now.utc - start).to_f.round(3)
+        (Time.now.utc - start).to_f.round(6)
       end
 
       def elapsed(t0)
         t1 = get_time
         {
           duration: t1[:now] - t0[:now],
-          stime: t1[:times][:stime] - t0[:times][:stime],
-          utime: t1[:times][:utime] - t0[:times][:utime],
-          ctime: ctime(t1[:times]) - ctime(t0[:times])
+          cputime: t1[:thread_cputime] - t0[:thread_cputime]
         }
       end
 
       def get_time
         {
           now: current_time,
-          times: Process.times
+          thread_cputime: defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0
         }
       end
 
-      def ctime(times)
-        times[:cstime] + times[:cutime]
-      end
-
       def current_time
         Gitlab::Metrics::System.monotonic_time
       end
 
-      def backtrace_cleaner
-        @backtrace_cleaner ||= ActiveSupport::BacktraceCleaner.new
-      end
-
       def format_time(timestamp)
         return timestamp if timestamp.is_a?(String)
 
-        Time.at(timestamp).utc.iso8601(3)
+        Time.at(timestamp).utc.iso8601(6)
       end
 
       def limited_job_args(args)
diff --git a/lib/gitlab/sidekiq_middleware/metrics.rb b/lib/gitlab/sidekiq_middleware/metrics.rb
index 368f37a5d8c..8af353d8674 100644
--- a/lib/gitlab/sidekiq_middleware/metrics.rb
+++ b/lib/gitlab/sidekiq_middleware/metrics.rb
@@ -19,10 +19,16 @@ module Gitlab
           @metrics[:sidekiq_jobs_retried_total].increment(labels, 1)
         end
 
+        job_thread_cputime_start = get_thread_cputime
+
         realtime = Benchmark.realtime do
           yield
         end
 
+        job_thread_cputime_end = get_thread_cputime
+        job_thread_cputime = job_thread_cputime_end - job_thread_cputime_start
+        @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime)
+
         @metrics[:sidekiq_jobs_completion_seconds].observe(labels, realtime)
       rescue Exception # rubocop: disable Lint/RescueException
         @metrics[:sidekiq_jobs_failed_total].increment(labels, 1)
@@ -35,6 +41,7 @@ module Gitlab
 
       def init_metrics
         {
+          sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
           sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
           sidekiq_jobs_failed_total:       ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'),
           sidekiq_jobs_retried_total:      ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'),
@@ -47,6 +54,10 @@ module Gitlab
           queue: queue
         }
       end
+
+      def get_thread_cputime
+        defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0
+      end
     end
   end
 end
diff --git a/lib/gitlab/sidekiq_middleware/monitor.rb b/lib/gitlab/sidekiq_middleware/monitor.rb
index 00965bf5506..ed825dbfd60 100644
--- a/lib/gitlab/sidekiq_middleware/monitor.rb
+++ b/lib/gitlab/sidekiq_middleware/monitor.rb
@@ -4,7 +4,7 @@ module Gitlab
   module SidekiqMiddleware
     class Monitor
       def call(worker, job, queue)
-        Gitlab::SidekiqDaemon::Monitor.instance.within_job(job['jid'], queue) do
+        Gitlab::SidekiqDaemon::Monitor.instance.within_job(worker.class, job['jid'], queue) do
           yield
         end
       rescue Gitlab::SidekiqDaemon::Monitor::CancelledError
diff --git a/lib/gitlab/slash_commands/presenters/access.rb b/lib/gitlab/slash_commands/presenters/access.rb
index b1bfaa6cb59..9ce1bcfb37c 100644
--- a/lib/gitlab/slash_commands/presenters/access.rb
+++ b/lib/gitlab/slash_commands/presenters/access.rb
@@ -15,6 +15,15 @@ module Gitlab
           MESSAGE
         end
 
+        def deactivated
+          ephemeral_response(text: <<~MESSAGE)
+            You are not allowed to perform the given chatops command since
+            your account has been deactivated by your administrator.
+
+            Please log back in from a web browser to reactivate your account at #{Gitlab.config.gitlab.url}
+          MESSAGE
+        end
+
         def not_found
           ephemeral_response(text: "404 not found! GitLab couldn't find what you were looking for! :boom:")
         end
diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb
index ac3b219e0c7..e955ccd35da 100644
--- a/lib/gitlab/snippet_search_results.rb
+++ b/lib/gitlab/snippet_search_results.rb
@@ -4,19 +4,19 @@ module Gitlab
   class SnippetSearchResults < SearchResults
     include SnippetsHelper
 
-    attr_reader :limit_snippets
+    attr_reader :current_user
 
-    def initialize(limit_snippets, query)
-      @limit_snippets = limit_snippets
+    def initialize(current_user, query)
+      @current_user = current_user
       @query = query
     end
 
     def objects(scope, page = nil)
       case scope
       when 'snippet_titles'
-        snippet_titles.page(page).per(per_page)
+        paginated_objects(snippet_titles, page)
       when 'snippet_blobs'
-        snippet_blobs.page(page).per(per_page)
+        paginated_objects(snippet_blobs, page)
       else
         super(scope, nil, false)
       end
@@ -25,38 +25,53 @@ module Gitlab
     def formatted_count(scope)
       case scope
       when 'snippet_titles'
-        snippet_titles_count.to_s
+        formatted_limited_count(limited_snippet_titles_count)
       when 'snippet_blobs'
-        snippet_blobs_count.to_s
+        formatted_limited_count(limited_snippet_blobs_count)
       else
         super
       end
     end
 
-    def snippet_titles_count
-      @snippet_titles_count ||= snippet_titles.count
+    def limited_snippet_titles_count
+      @limited_snippet_titles_count ||= limited_count(snippet_titles)
     end
 
-    def snippet_blobs_count
-      @snippet_blobs_count ||= snippet_blobs.count
+    def limited_snippet_blobs_count
+      @limited_snippet_blobs_count ||= limited_count(snippet_blobs)
     end
 
     private
 
     # rubocop: disable CodeReuse/ActiveRecord
-    def snippet_titles
-      limit_snippets.search(query).order('updated_at DESC').includes(:author)
+    def snippets
+      SnippetsFinder.new(current_user, finder_params)
+        .execute
+        .includes(:author)
+        .reorder(updated_at: :desc)
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
-    # rubocop: disable CodeReuse/ActiveRecord
+    def snippet_titles
+      snippets.search(query)
+    end
+
     def snippet_blobs
-      limit_snippets.search_code(query).order('updated_at DESC').includes(:author)
+      snippets.search_code(query)
     end
-    # rubocop: enable CodeReuse/ActiveRecord
 
     def default_scope
       'snippet_blobs'
     end
+
+    def paginated_objects(relation, page)
+      relation.page(page).per(per_page)
+    end
+
+    def finder_params
+      {}
+    end
   end
 end
+
+Gitlab::SnippetSearchResults.prepend_if_ee('::EE::Gitlab::SnippetSearchResults')
diff --git a/lib/gitlab/submodule_links.rb b/lib/gitlab/submodule_links.rb
index 18fd604a3b0..b0ee0877f30 100644
--- a/lib/gitlab/submodule_links.rb
+++ b/lib/gitlab/submodule_links.rb
@@ -6,6 +6,7 @@ module Gitlab
 
     def initialize(repository)
       @repository = repository
+      @cache_store = {}
     end
 
     def for(submodule, sha)
@@ -18,8 +19,9 @@ module Gitlab
     attr_reader :repository
 
     def submodule_urls_for(sha)
-      strong_memoize(:"submodule_urls_for_#{sha}") do
-        repository.submodule_urls_for(sha)
+      @cache_store.fetch(sha) do
+        submodule_urls = repository.submodule_urls_for(sha)
+        @cache_store[sha] = submodule_urls
       end
     end
 
diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb
index 78177c6d306..2470685bc00 100644
--- a/lib/gitlab/tracking.rb
+++ b/lib/gitlab/tracking.rb
@@ -6,6 +6,21 @@ module Gitlab
   module Tracking
     SNOWPLOW_NAMESPACE = 'gl'
 
+    module ControllerConcern
+      extend ActiveSupport::Concern
+
+      protected
+
+      def track_event(action = action_name, **args)
+        category = args.delete(:category) || self.class.name
+        Gitlab::Tracking.event(category, action.to_s, **args)
+      end
+
+      def track_self_describing_event(schema_url, event_data_json, **args)
+        Gitlab::Tracking.self_describing_event(schema_url, event_data_json, **args)
+      end
+    end
+
     class << self
       def enabled?
         Gitlab::CurrentSettings.snowplow_enabled?
@@ -17,6 +32,13 @@ module Gitlab
         snowplow.track_struct_event(category, action, label, property, value, context, Time.now.to_i)
       end
 
+      def self_describing_event(schema_url, event_data_json, context: nil)
+        return unless enabled?
+
+        event_json = SnowplowTracker::SelfDescribingJson.new(schema_url, event_data_json)
+        snowplow.track_self_describing_event(event_json, context, Time.now.to_i)
+      end
+
       def snowplow_options(group)
         additional_features = Feature.enabled?(:additional_snowplow_tracking, group)
         {
@@ -33,7 +55,7 @@ module Gitlab
 
       def snowplow
         @snowplow ||= SnowplowTracker::Tracker.new(
-          SnowplowTracker::Emitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname),
+          SnowplowTracker::AsyncEmitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname, protocol: 'https'),
           SnowplowTracker::Subject.new,
           SNOWPLOW_NAMESPACE,
           Gitlab::CurrentSettings.snowplow_site_id
diff --git a/lib/gitlab/tracking/incident_management.rb b/lib/gitlab/tracking/incident_management.rb
new file mode 100644
index 00000000000..bd8d1669dd3
--- /dev/null
+++ b/lib/gitlab/tracking/incident_management.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Tracking
+    module IncidentManagement
+      class << self
+        def track_from_params(incident_params)
+          return if incident_params.blank?
+
+          incident_params.each do |k, v|
+            prefix = ['', '0'].include?(v.to_s) ? 'disabled' : 'enabled'
+
+            key = tracking_keys.dig(k, :name)
+            label = tracking_keys.dig(k, :label)
+
+            next if key.blank?
+
+            details = label ? { label: label, property: v } : {}
+
+            ::Gitlab::Tracking.event('IncidentManagement::Settings', "#{prefix}_#{key}", **details )
+          end
+        end
+
+        def tracking_keys
+          {
+            create_issue: {
+              name: 'issue_auto_creation_on_alerts'
+            },
+            issue_template_key: {
+              name: 'issue_template_on_alerts',
+              label: 'Template name'
+            },
+            send_email: {
+              name: 'sending_emails'
+            }
+          }.with_indifferent_access.freeze
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/uploads/migration_helper.rb b/lib/gitlab/uploads/migration_helper.rb
new file mode 100644
index 00000000000..4ff064007f1
--- /dev/null
+++ b/lib/gitlab/uploads/migration_helper.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Uploads
+    class MigrationHelper
+      attr_reader :logger
+
+      CATEGORIES = [%w(AvatarUploader Project :avatar),
+                    %w(AvatarUploader Group :avatar),
+                    %w(AvatarUploader User :avatar),
+                    %w(AttachmentUploader Note :attachment),
+                    %w(AttachmentUploader Appearance :logo),
+                    %w(AttachmentUploader Appearance :header_logo),
+                    %w(FaviconUploader Appearance :favicon),
+                    %w(FileUploader Project),
+                    %w(PersonalFileUploader Snippet),
+                    %w(NamespaceFileUploader Snippet),
+                    %w(FileUploader MergeRequest)].freeze
+
+      def initialize(args, logger)
+        prepare_variables(args, logger)
+      end
+
+      def migrate_to_remote_storage
+        @to_store = ObjectStorage::Store::REMOTE
+
+        uploads.each_batch(of: batch_size, &method(:enqueue_batch))
+      end
+
+      def migrate_to_local_storage
+        @to_store = ObjectStorage::Store::LOCAL
+
+        uploads(ObjectStorage::Store::REMOTE).each_batch(of: batch_size, &method(:enqueue_batch))
+      end
+
+      private
+
+      def batch_size
+        ENV.fetch('MIGRATION_BATCH_SIZE', 200).to_i
+      end
+
+      def prepare_variables(args, logger)
+        @mounted_as     = args.mounted_as&.gsub(':', '')&.to_sym
+        @uploader_class = args.uploader_class.constantize
+        @model_class    = args.model_class.constantize
+        @logger         = logger
+      end
+
+      def enqueue_batch(batch, index)
+        job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch,
+                                                           @model_class,
+                                                           @mounted_as,
+                                                           @to_store)
+        logger.info(message: "[Uploads migration] Enqueued upload migration job", index: index, job_id: job)
+      rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e
+        # continue for the next batch
+        logger.warn(message: "[Uploads migration] Could not enqueue batch", ids: batch.ids, reason: e.message) # rubocop:disable CodeReuse/ActiveRecord
+      end
+
+      # rubocop:disable CodeReuse/ActiveRecord
+      def uploads(store_type = [nil, ObjectStorage::Store::LOCAL])
+        Upload.class_eval { include EachBatch } unless Upload < EachBatch
+
+        Upload
+          .where(store: store_type,
+                 uploader: @uploader_class.to_s,
+                 model_type: @model_class.base_class.sti_name)
+      end
+      # rubocop:enable CodeReuse/ActiveRecord
+    end
+  end
+end
diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb
index 4285b2675c5..0adca34440c 100644
--- a/lib/gitlab/url_blocker.rb
+++ b/lib/gitlab/url_blocker.rb
@@ -125,6 +125,11 @@ module Gitlab
         # If the addr can't be resolved or the url is invalid (i.e http://1.1.1.1.1)
         # we block the url
         raise BlockedUrlError, "Host cannot be resolved or invalid"
+      rescue ArgumentError => error
+        # Addrinfo.getaddrinfo errors if the domain exceeds 1024 characters.
+        raise unless error.message.include?('hostname too long')
+
+        raise BlockedUrlError, "Host is too long (maximum is 1024 characters)"
       end
 
       def validate_local_request(
diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb
index 42cf1ec1f0e..038067eeae4 100644
--- a/lib/gitlab/url_builder.rb
+++ b/lib/gitlab/url_builder.rb
@@ -81,3 +81,5 @@ module Gitlab
     end
   end
 end
+
+::Gitlab::UrlBuilder.prepend_if_ee('EE::Gitlab::UrlBuilder')
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index c6c2876033d..cb492b69fec 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -17,7 +17,6 @@ module Gitlab
           .merge(features_usage_data)
           .merge(components_usage_data)
           .merge(cycle_analytics_usage_data)
-          .merge(usage_counters)
       end
 
       def to_json(force_refresh: false)
@@ -38,7 +37,7 @@ module Gitlab
         usage_data
       end
 
-      # rubocop:disable Metrics/AbcSize
+      # rubocop: disable Metrics/AbcSize
       # rubocop: disable CodeReuse/ActiveRecord
       def system_usage_data
         {
@@ -97,13 +96,16 @@ module Gitlab
             todos: count(Todo),
             uploads: count(Upload),
             web_hooks: count(WebHook)
-          }.merge(services_usage)
-            .merge(approximate_counts)
-        }.tap do |data|
-          data[:counts][:user_preferences] = user_preferences_usage
-        end
+          }.merge(
+            services_usage,
+            approximate_counts,
+            usage_counters,
+            user_preferences_usage
+          )
+        }
       end
       # rubocop: enable CodeReuse/ActiveRecord
+      # rubocop: enable Metrics/AbcSize
 
       def cycle_analytics_usage_data
         Gitlab::CycleAnalytics::UsageData.new.to_json
@@ -116,6 +118,7 @@ module Gitlab
       def features_usage_data_ce
         {
           container_registry_enabled: Gitlab.config.registry.enabled,
+          dependency_proxy_enabled: Gitlab.config.try(:dependency_proxy)&.enabled,
           gitlab_shared_runners_enabled: Gitlab.config.gitlab_ci.shared_runners_enabled,
           gravatar_enabled: Gitlab::CurrentSettings.gravatar_enabled?,
           influxdb_metrics_enabled: Gitlab::Metrics.influx_metrics_enabled?,
@@ -136,15 +139,15 @@ module Gitlab
       # @return [Array<#totals>] An array of objects that respond to `#totals`
       def usage_data_counters
         [
-         Gitlab::UsageDataCounters::WikiPageCounter,
-         Gitlab::UsageDataCounters::WebIdeCounter,
-         Gitlab::UsageDataCounters::NoteCounter,
-         Gitlab::UsageDataCounters::SnippetCounter,
-         Gitlab::UsageDataCounters::SearchCounter,
-         Gitlab::UsageDataCounters::CycleAnalyticsCounter,
-         Gitlab::UsageDataCounters::ProductivityAnalyticsCounter,
-         Gitlab::UsageDataCounters::SourceCodeCounter,
-         Gitlab::UsageDataCounters::MergeRequestCounter
+          Gitlab::UsageDataCounters::WikiPageCounter,
+          Gitlab::UsageDataCounters::WebIdeCounter,
+          Gitlab::UsageDataCounters::NoteCounter,
+          Gitlab::UsageDataCounters::SnippetCounter,
+          Gitlab::UsageDataCounters::SearchCounter,
+          Gitlab::UsageDataCounters::CycleAnalyticsCounter,
+          Gitlab::UsageDataCounters::ProductivityAnalyticsCounter,
+          Gitlab::UsageDataCounters::SourceCodeCounter,
+          Gitlab::UsageDataCounters::MergeRequestCounter
         ]
       end
 
@@ -186,7 +189,7 @@ module Gitlab
           .find_in_batches(batch_size: BATCH_SIZE) do |services|
 
           counts = services.group_by do |service|
-            # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab-ce/issues/63084
+            # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404
             service_url = service.data_fields&.url || (service.properties && service.properties['url'])
             service_url&.include?('.atlassian.net') ? :cloud : :server
           end
diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb
index c66ce0434a4..7fbfc4c45c4 100644
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@@ -13,14 +13,6 @@ module Gitlab
       path
     end
 
-    # Run system command without outputting to stdout.
-    #
-    # @param  cmd [Array<String>]
-    # @return [Boolean]
-    def system_silent(cmd)
-      Popen.popen(cmd).last.zero?
-    end
-
     def force_utf8(str)
       str.dup.force_encoding(Encoding::UTF_8)
     end
diff --git a/lib/gitlab/utils/inline_hash.rb b/lib/gitlab/utils/inline_hash.rb
new file mode 100644
index 00000000000..41e5f3ee4c3
--- /dev/null
+++ b/lib/gitlab/utils/inline_hash.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Utils
+    module InlineHash
+      extend self
+
+      # Transforms a Hash into an inline Hash by merging its nested keys.
+      #
+      # Input
+      #
+      #  {
+      #    'root_param' => 'Root',
+      #    12 => 'number',
+      #    symbol: 'symbol',
+      #    nested_param: {
+      #      key: 'Value'
+      #    },
+      #    'very' => {
+      #      'deep' => {
+      #        'nested' => {
+      #          12 => 'Deep nested value'
+      #        }
+      #      }
+      #    }
+      #  }
+      #
+      #
+      # Result
+      #
+      #  {
+      #    'root_param' => 'Root',
+      #     12 => 'number',
+      #     symbol: symbol,
+      #    'nested_param.key' => 'Value',
+      #    'very.deep.nested.12' => 'Deep nested value'
+      #  }
+      #
+      def merge_keys(hash, prefix: nil, connector: '.')
+        result = {}
+        pairs =
+          if prefix
+            base_prefix = "#{prefix}#{connector}"
+            hash.map { |key, value| ["#{base_prefix}#{key}", value] }
+          else
+            hash.to_a
+          end
+
+        until pairs.empty?
+          key, value = pairs.shift
+
+          if value.is_a?(Hash)
+            value.each { |k, v| pairs.unshift ["#{key}#{connector}#{k}", v] }
+          else
+            result[key] = value
+          end
+        end
+
+        result
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/utils/safe_inline_hash.rb b/lib/gitlab/utils/safe_inline_hash.rb
new file mode 100644
index 00000000000..644d87c6876
--- /dev/null
+++ b/lib/gitlab/utils/safe_inline_hash.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Utils
+    class SafeInlineHash
+      # Validates the hash size using `Gitlab::Utils::DeepSize` before merging keys using `Gitlab::Utils::InlineHash`
+      def initialize(hash, prefix: nil, connector: '.')
+        @hash = hash
+      end
+
+      def self.merge_keys!(hash, prefix: nil, connector: '.')
+        new(hash).merge_keys!(prefix: prefix, connector: connector)
+      end
+
+      def merge_keys!(prefix:, connector:)
+        raise ArgumentError, 'The Hash is too big' unless valid?
+
+        Gitlab::Utils::InlineHash.merge_keys(hash, prefix: prefix, connector: connector)
+      end
+
+      private
+
+      attr_reader :hash
+
+      def valid?
+        Gitlab::Utils::DeepSize.new(hash).valid?
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb
index 875e8a120e9..afcdbd087d2 100644
--- a/lib/gitlab/verify/uploads.rb
+++ b/lib/gitlab/verify/uploads.rb
@@ -32,7 +32,7 @@ module Gitlab
       end
 
       def remote_object_exists?(upload)
-        upload.build_uploader.file.exists?
+        upload.retrieve_uploader.file.exists?
       end
     end
   end
diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb
index 9f01a3f97ce..9085835dee6 100644
--- a/lib/google_api/cloud_platform/client.rb
+++ b/lib/google_api/cloud_platform/client.rb
@@ -2,6 +2,7 @@
 
 require 'google/apis/compute_v1'
 require 'google/apis/container_v1'
+require 'google/apis/container_v1beta1'
 require 'google/apis/cloudbilling_v1'
 require 'google/apis/cloudresourcemanager_v1'
 
@@ -53,30 +54,13 @@ module GoogleApi
         service.get_zone_cluster(project_id, zone, cluster_id, options: user_agent_header)
       end
 
-      def projects_zones_clusters_create(project_id, zone, cluster_name, cluster_size, machine_type:, legacy_abac:)
-        service = Google::Apis::ContainerV1::ContainerService.new
+      def projects_zones_clusters_create(project_id, zone, cluster_name, cluster_size, machine_type:, legacy_abac:, enable_addons: [])
+        service = Google::Apis::ContainerV1beta1::ContainerService.new
         service.authorization = access_token
 
-        request_body = Google::Apis::ContainerV1::CreateClusterRequest.new(
-          {
-            "cluster": {
-              "name": cluster_name,
-              "initial_node_count": cluster_size,
-              "node_config": {
-                "machine_type": machine_type
-              },
-              "master_auth": {
-                "username": CLUSTER_MASTER_AUTH_USERNAME,
-                "client_certificate_config": {
-                  issue_client_certificate: true
-                }
-              },
-              "legacy_abac": {
-                "enabled": legacy_abac
-              }
-            }
-          }
-        )
+        cluster_options = make_cluster_options(cluster_name, cluster_size, machine_type, legacy_abac, enable_addons)
+
+        request_body = Google::Apis::ContainerV1beta1::CreateClusterRequest.new(cluster_options)
 
         service.create_cluster(project_id, zone, request_body, options: user_agent_header)
       end
@@ -95,6 +79,33 @@ module GoogleApi
 
       private
 
+      def make_cluster_options(cluster_name, cluster_size, machine_type, legacy_abac, enable_addons)
+        {
+          cluster: {
+            name: cluster_name,
+            initial_node_count: cluster_size,
+            node_config: {
+              machine_type: machine_type
+            },
+            master_auth: {
+              username: CLUSTER_MASTER_AUTH_USERNAME,
+              client_certificate_config: {
+                issue_client_certificate: true
+              }
+            },
+            legacy_abac: {
+              enabled: legacy_abac
+            },
+            ip_allocation_policy: {
+              use_ip_aliases: true
+            },
+            addons_config: enable_addons.each_with_object({}) do |addon, hash|
+              hash[addon] = { disabled: false }
+            end
+          }
+        }
+      end
+
       def token_life_time(expires_at)
         DateTime.strptime(expires_at, '%s').to_time.utc - Time.now.utc
       end
diff --git a/lib/grafana/client.rb b/lib/grafana/client.rb
new file mode 100644
index 00000000000..0765630f9bb
--- /dev/null
+++ b/lib/grafana/client.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Grafana
+  class Client
+    Error = Class.new(StandardError)
+
+    # @param api_url [String] Base URL of the Grafana instance
+    # @param token [String] Admin-level API token for instance
+    def initialize(api_url:, token:)
+      @api_url = api_url
+      @token = token
+    end
+
+    # @param datasource_id [String] Grafana ID for the datasource
+    # @param proxy_path [String] Path to proxy - ex) 'api/v1/query_range'
+    def proxy_datasource(datasource_id:, proxy_path:, query: {})
+      http_get("#{@api_url}/api/datasources/proxy/#{datasource_id}/#{proxy_path}", query: query)
+    end
+
+    private
+
+    def http_get(url, params = {})
+      response = handle_request_exceptions do
+        Gitlab::HTTP.get(url, **request_params.merge(params))
+      end
+
+      handle_response(response)
+    end
+
+    def request_params
+      {
+        headers: {
+          'Authorization' => "Bearer #{@token}",
+          'Accept' => 'application/json',
+          'Content-Type' => 'application/json'
+        },
+        follow_redirects: false
+      }
+    end
+
+    def handle_request_exceptions
+      yield
+    rescue Gitlab::HTTP::Error
+      raise_error 'Error when connecting to Grafana'
+    rescue Net::OpenTimeout
+      raise_error 'Connection to Grafana timed out'
+    rescue SocketError
+      raise_error 'Received SocketError when trying to connect to Grafana'
+    rescue OpenSSL::SSL::SSLError
+      raise_error 'Grafana returned invalid SSL data'
+    rescue Errno::ECONNREFUSED
+      raise_error 'Connection refused'
+    rescue => e
+      raise_error "Grafana request failed due to #{e.class}"
+    end
+
+    def handle_response(response)
+      return response if response.code == 200
+
+      raise_error "Grafana response status code: #{response.code}"
+    end
+
+    def raise_error(message)
+      raise Client::Error, message
+    end
+  end
+end
diff --git a/lib/quality/test_level.rb b/lib/quality/test_level.rb
index a65657dadd0..b7822adf6ed 100644
--- a/lib/quality/test_level.rb
+++ b/lib/quality/test_level.rb
@@ -53,11 +53,11 @@ module Quality
     end
 
     def pattern(level)
-      @patterns[level] ||= "#{prefix}spec/{#{TEST_LEVEL_FOLDERS.fetch(level).join(',')}}{,/**/}*_spec.rb"
+      @patterns[level] ||= "#{prefix}spec/#{folders_pattern(level)}{,/**/}*_spec.rb"
     end
 
     def regexp(level)
-      @regexps[level] ||= Regexp.new("#{prefix}spec/(#{TEST_LEVEL_FOLDERS.fetch(level).join('|')})").freeze
+      @regexps[level] ||= Regexp.new("#{prefix}spec/#{folders_regex(level)}").freeze
     end
 
     def level_for(file_path)
@@ -72,5 +72,27 @@ module Quality
         raise UnknownTestLevelError, "Test level for #{file_path} couldn't be set. Please rename the file properly or change the test level detection regexes in #{__FILE__}."
       end
     end
+
+    private
+
+    def folders_pattern(level)
+      case level
+      # Geo specs aren't in a specific folder, but they all have the :geo tag, so we must search for them globally
+      when :all, :geo
+        '**'
+      else
+        "{#{TEST_LEVEL_FOLDERS.fetch(level).join(',')}}"
+      end
+    end
+
+    def folders_regex(level)
+      case level
+      # Geo specs aren't in a specific folder, but they all have the :geo tag, so we must search for them globally
+      when :all, :geo
+        ''
+      else
+        "(#{TEST_LEVEL_FOLDERS.fetch(level).join('|')})"
+      end
+    end
   end
 end
diff --git a/lib/tasks/frontend.rake b/lib/tasks/frontend.rake
index 1cac7520227..6e90229830d 100644
--- a/lib/tasks/frontend.rake
+++ b/lib/tasks/frontend.rake
@@ -2,7 +2,10 @@ unless Rails.env.production?
   namespace :frontend do
     desc 'GitLab | Frontend | Generate fixtures for JavaScript tests'
     RSpec::Core::RakeTask.new(:fixtures, [:pattern]) do |t, args|
-      args.with_defaults(pattern: '{spec,ee/spec}/frontend/fixtures/*.rb')
+      directories = %w[spec]
+      directories << 'ee/spec' if Gitlab.ee?
+      directory_glob = "{#{directories.join(',')}}"
+      args.with_defaults(pattern: "#{directory_glob}/frontend/fixtures/*.rb")
       ENV['NO_KNAPSACK'] = 'true'
       t.pattern = args[:pattern]
       t.rspec_opts = '--format documentation'
diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake
index 9012e55a70c..0d09fd0a4e3 100644
--- a/lib/tasks/gitlab/artifacts/migrate.rake
+++ b/lib/tasks/gitlab/artifacts/migrate.rake
@@ -6,18 +6,31 @@ namespace :gitlab do
   namespace :artifacts do
     task migrate: :environment do
       logger = Logger.new(STDOUT)
-      logger.info('Starting transfer of artifacts')
+      logger.info('Starting transfer of artifacts to remote storage')
 
-      Ci::Build.joins(:project)
-        .with_artifacts_stored_locally
-        .find_each(batch_size: 10) do |build|
+      helper = Gitlab::Artifacts::MigrationHelper.new
 
-        build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE)
-        build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE)
+      begin
+        helper.migrate_to_remote_storage do |artifact|
+          logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to object storage")
+        end
+      rescue => e
+        logger.error(e.message)
+      end
+    end
+
+    task migrate_to_local: :environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Starting transfer of artifacts to local storage')
+
+      helper = Gitlab::Artifacts::MigrationHelper.new
 
-        logger.info("Transferred artifact ID #{build.id} with size #{build.artifacts_size} to object storage")
+      begin
+        helper.migrate_to_local_storage do |artifact|
+          logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to local storage")
+        end
       rescue => e
-        logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}")
+        logger.error(e.message)
       end
     end
   end
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index 4d854cd178d..0a0ee7b4bfa 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -3,69 +3,6 @@ require 'set'
 
 namespace :gitlab do
   namespace :cleanup do
-    desc "GitLab | Cleanup | Clean namespaces"
-    task dirs: :gitlab_environment do
-      namespaces = Set.new(Namespace.pluck(:path))
-      namespaces << Storage::HashedProject::REPOSITORY_PATH_PREFIX
-
-      Gitaly::Server.all.each do |server|
-        all_dirs = Gitlab::GitalyClient::StorageService
-          .new(server.storage)
-          .list_directories(depth: 0)
-          .reject { |dir| dir.ends_with?('.git') || namespaces.include?(File.basename(dir)) }
-
-        puts "Looking for directories to remove... "
-        all_dirs.each do |dir_path|
-          if remove?
-            begin
-              Gitlab::GitalyClient::NamespaceService.new(server.storage)
-                .remove(dir_path)
-
-              puts "Removed...#{dir_path}"
-            rescue StandardError => e
-              puts "Cannot remove #{dir_path}: #{e.message}".color(:red)
-            end
-          else
-            puts "Can be removed: #{dir_path}".color(:red)
-          end
-        end
-      end
-
-      unless remove?
-        puts "To cleanup this directories run this command with REMOVE=true".color(:yellow)
-      end
-    end
-
-    desc "GitLab | Cleanup | Clean repositories"
-    task repos: :gitlab_environment do
-      move_suffix = "+orphaned+#{Time.now.to_i}"
-
-      Gitaly::Server.all.each do |server|
-        Gitlab::GitalyClient::StorageService
-          .new(server.storage)
-          .list_directories
-          .each do |path|
-          repo_with_namespace = path.chomp('.git').chomp('.wiki')
-
-          # TODO ignoring hashed repositories for now.  But revisit to fully support
-          # possible orphaned hashed repos
-          next if repo_with_namespace.start_with?(Storage::HashedProject::REPOSITORY_PATH_PREFIX)
-          next if Project.find_by_full_path(repo_with_namespace)
-
-          new_path = path + move_suffix
-          puts path.inspect + ' -> ' + new_path.inspect
-
-          begin
-            Gitlab::GitalyClient::NamespaceService
-              .new(server.storage)
-              .rename(path, new_path)
-          rescue StandardError => e
-            puts "Error occurred while moving the repository: #{e.message}".color(:red)
-          end
-        end
-      end
-    end
-
     desc "GitLab | Cleanup | Block users that have been removed in LDAP"
     task block_removed_ldap_users: :gitlab_environment do
       warn_user_is_not_gitlab
diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake
index fd8df015903..902f22684ee 100644
--- a/lib/tasks/gitlab/graphql.rake
+++ b/lib/tasks/gitlab/graphql.rake
@@ -11,10 +11,28 @@ namespace :gitlab do
     task compile_docs: :environment do
       renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options)
 
-      renderer.render
+      renderer.write
 
       puts "Documentation compiled."
     end
+
+    desc 'GitLab | Check if GraphQL docs are up to date'
+    task check_docs: :environment do
+      renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options)
+
+      doc = File.read(Rails.root.join(OUTPUT_DIR, 'index.md'))
+
+      if doc == renderer.contents
+        puts "GraphQL documentation is up to date"
+      else
+        puts '#' * 10
+        puts '#'
+        puts '# GraphQL documentation is outdated! Please update it by running `bundle exec rake gitlab:graphql:compile_docs`.'
+        puts '#'
+        puts '#' * 10
+        abort
+      end
+    end
   end
 end
 
diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake
index 97c15175a23..4142903d9c3 100644
--- a/lib/tasks/gitlab/lfs/migrate.rake
+++ b/lib/tasks/gitlab/lfs/migrate.rake
@@ -17,5 +17,20 @@ namespace :gitlab do
           logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}")
         end
     end
+
+    task migrate_to_local: :environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Starting transfer of LFS files to local storage')
+
+      LfsObject.with_files_stored_remotely
+        .find_each(batch_size: 10) do |lfs_object|
+
+        lfs_object.file.migrate!(LfsObjectUploader::Store::LOCAL)
+
+        logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to local storage")
+      rescue => e
+        logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}")
+      end
+    end
   end
 end
diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake
deleted file mode 100644
index 100e480bd66..00000000000
--- a/lib/tasks/gitlab/pages.rake
+++ /dev/null
@@ -1,9 +0,0 @@
-namespace :gitlab do
-  namespace :pages do
-    desc 'Ping the pages admin API'
-    task admin_ping: :gitlab_environment do
-      Gitlab::PagesClient.ping
-      puts "OK: gitlab-pages admin API is reachable"
-    end
-  end
-end
diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake
index 5d86d6e466c..50774de77c9 100644
--- a/lib/tasks/gitlab/setup.rake
+++ b/lib/tasks/gitlab/setup.rake
@@ -31,7 +31,6 @@ namespace :gitlab do
     terminate_all_connections unless Rails.env.production?
 
     Rake::Task["db:reset"].invoke
-    Rake::Task["setup_postgresql"].invoke
     Rake::Task["db:seed_fu"].invoke
   rescue Gitlab::TaskAbortedByUserError
     puts "Quitting...".color(:red)
diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake
deleted file mode 100644
index 5e1ec481ece..00000000000
--- a/lib/tasks/gitlab/traces.rake
+++ /dev/null
@@ -1,38 +0,0 @@
-require 'logger'
-require 'resolv-replace'
-
-desc "GitLab | Archive legacy traces to trace artifacts"
-namespace :gitlab do
-  namespace :traces do
-    task archive: :environment do
-      logger = Logger.new(STDOUT)
-      logger.info('Archiving legacy traces')
-
-      Ci::Build.finished.without_archived_trace
-        .order(id: :asc)
-        .find_in_batches(batch_size: 1000) do |jobs|
-        job_ids = jobs.map { |job| [job.id] }
-
-        ArchiveTraceWorker.bulk_perform_async(job_ids)
-
-        logger.info("Scheduled #{job_ids.count} jobs. From #{job_ids.min} to #{job_ids.max}")
-      end
-    end
-
-    task migrate: :environment do
-      logger = Logger.new(STDOUT)
-      logger.info('Starting transfer of job traces')
-
-      Ci::Build.joins(:project)
-        .with_archived_trace_stored_locally
-        .find_each(batch_size: 10) do |build|
-
-        build.job_artifacts_trace.file.migrate!(ObjectStorage::Store::REMOTE)
-
-        logger.info("Transferred job trace of #{build.id} to object storage")
-      rescue => e
-        logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}")
-      end
-    end
-  end
-end
diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake
index 1c93609a006..44536a447c7 100644
--- a/lib/tasks/gitlab/uploads/migrate.rake
+++ b/lib/tasks/gitlab/uploads/migrate.rake
@@ -3,19 +3,7 @@ namespace :gitlab do
     namespace :migrate do
       desc "GitLab | Uploads | Migrate all uploaded files to object storage"
       task all: :environment do
-        categories = [%w(AvatarUploader Project :avatar),
-                      %w(AvatarUploader Group :avatar),
-                      %w(AvatarUploader User :avatar),
-                      %w(AttachmentUploader Note :attachment),
-                      %w(AttachmentUploader Appearance :logo),
-                      %w(AttachmentUploader Appearance :header_logo),
-                      %w(FaviconUploader Appearance :favicon),
-                      %w(FileUploader Project),
-                      %w(PersonalFileUploader Snippet),
-                      %w(NamespaceFileUploader Snippet),
-                      %w(FileUploader MergeRequest)]
-
-        categories.each do |args|
+        Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args|
           Rake::Task["gitlab:uploads:migrate"].invoke(*args)
           Rake::Task["gitlab:uploads:migrate"].reenable
         end
@@ -25,34 +13,23 @@ namespace :gitlab do
     # The following is the actual rake task that migrates uploads of specified
     # category to object storage
     desc 'GitLab | Uploads | Migrate the uploaded files of specified type to object storage'
-    task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args|
-      batch_size     = ENV.fetch('BATCH', 200).to_i
-      @to_store      = ObjectStorage::Store::REMOTE
-      @mounted_as    = args.mounted_as&.gsub(':', '')&.to_sym
-      @uploader_class = args.uploader_class.constantize
-      @model_class    = args.model_class.constantize
-
-      uploads.each_batch(of: batch_size, &method(:enqueue_batch))
+    task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args|
+      Gitlab::Uploads::MigrationHelper.new(args, Logger.new(STDOUT)).migrate_to_remote_storage
     end
 
-    def enqueue_batch(batch, index)
-      job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch,
-                                                         @model_class,
-                                                         @mounted_as,
-                                                         @to_store)
-      puts "Enqueued job ##{index}: #{job}"
-    rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e
-      # continue for the next batch
-      puts "Could not enqueue batch (#{batch.ids}) #{e.message}".color(:red)
+    namespace :migrate_to_local do
+      desc "GitLab | Uploads | Migrate all uploaded files to local storage"
+      task all: :environment do
+        Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args|
+          Rake::Task["gitlab:uploads:migrate_to_local"].invoke(*args)
+          Rake::Task["gitlab:uploads:migrate_to_local"].reenable
+        end
+      end
     end
 
-    def uploads
-      Upload.class_eval { include EachBatch } unless Upload < EachBatch
-
-      Upload
-        .where(store: [nil, ObjectStorage::Store::LOCAL],
-               uploader: @uploader_class.to_s,
-               model_type: @model_class.base_class.sti_name)
+    desc 'GitLab | Uploads | Migrate the uploaded files of specified type to local storage'
+    task :migrate_to_local, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args|
+      Gitlab::Uploads::MigrationHelper.new(args, Logger.new(STDOUT)).migrate_to_local_storage
     end
   end
 end
diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake
index cda88c130bb..4c8f13b63a4 100644
--- a/lib/tasks/migrate/setup_postgresql.rake
+++ b/lib/tasks/migrate/setup_postgresql.rake
@@ -1,14 +1,3 @@
-desc 'GitLab | Sets up PostgreSQL'
-task setup_postgresql: :environment do
-  require Rails.root.join('db/migrate/20180215181245_users_name_lower_index.rb')
-  require Rails.root.join('db/migrate/20180504195842_project_name_lower_index.rb')
-  require Rails.root.join('db/post_migrate/20180306164012_add_path_index_to_redirect_routes.rb')
-
-  UsersNameLowerIndex.new.up
-  ProjectNameLowerIndex.new.up
-  AddPathIndexToRedirectRoutes.new.up
-end
-
 desc 'GitLab | Generate PostgreSQL Password Hash'
 task :postgresql_md5_hash do
   require 'digest'
diff --git a/lib/tasks/services.rake b/lib/tasks/services.rake
deleted file mode 100644
index 4ec4fdd281f..00000000000
--- a/lib/tasks/services.rake
+++ /dev/null
@@ -1,98 +0,0 @@
-services_template = <<-ERB
-# Services
-
-<% services.each do |service| %>
-## <%= service[:title] %>
-
-
-<% unless service[:description].blank? %>
-<%= service[:description] %>
-<% end %>
-
-
-### Create/Edit <%= service[:title] %> service
-
-Set <%= service[:title] %> service for a project.
-<% unless service[:help].blank? %>
-
-> <%= service[:help].gsub("\n", ' ') %>
-
-<% end %>
-
-```
-PUT /projects/:id/services/<%= service[:dashed_name] %>
-
-```
-
-Parameters:
-
-<% service[:params].each do |param| %>
-- `<%= param[:name] %>` <%= param[:required] ? "(**required**)" : "(optional)"  %><%= [" -",  param[:description]].join(" ").gsub("\n", '') unless param[:description].blank? %>
-
-<% end %>
-
-### Delete <%= service[:title] %> service
-
-Delete <%= service[:title] %> service for a project.
-
-```
-DELETE /projects/:id/services/<%= service[:dashed_name] %>
-
-```
-
-### Get <%= service[:title] %> service settings
-
-Get <%= service[:title] %> service settings for a project.
-
-```
-GET /projects/:id/services/<%= service[:dashed_name] %>
-
-```
-
-<% end %>
-ERB
-
-namespace :services do
-  task doc: :environment do
-    services = Service.available_services_names.map do |s|
-      service_start = Time.now
-      klass = "#{s}_service".classify.constantize
-
-      service = klass.new
-
-      service_hash = {}
-
-      service_hash[:title] = service.title
-      service_hash[:dashed_name] = s.dasherize
-      service_hash[:description] = service.description
-      service_hash[:help] = service.help
-      service_hash[:params] = service.fields.map do |p|
-        param_hash = {}
-
-        param_hash[:name] = p[:name]
-        param_hash[:description] = p[:placeholder] || p[:title]
-        param_hash[:required] = klass.validators_on(p[:name].to_sym).any? do |v|
-          v.class == ActiveRecord::Validations::PresenceValidator
-        end
-
-        param_hash
-      end
-      service_hash[:params].sort_by! { |p| p[:required] ? 0 : 1 }
-
-      puts "Collected data for: #{service.title}, #{Time.now - service_start}"
-      service_hash
-    end
-
-    doc_start = Time.now
-    doc_path = File.join(Rails.root, 'doc', 'api', 'services.md')
-
-    result = ERB.new(services_template, trim_mode: '>')
-      .result(OpenStruct.new(services: services).instance_eval { binding })
-
-    File.open(doc_path, 'w') do |f|
-      f.write result
-    end
-
-    puts "write a new service.md to: #{doc_path}, #{Time.now - doc_start}"
-  end
-end
diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb
index aae542f02ac..424db653fb8 100644
--- a/lib/uploaded_file.rb
+++ b/lib/uploaded_file.rb
@@ -6,6 +6,7 @@ require "fileutils"
 
 class UploadedFile
   InvalidPathError = Class.new(StandardError)
+  UnknownSizeError = Class.new(StandardError)
 
   # The filename, *not* including the path, of the "uploaded" file
   attr_reader :original_filename
@@ -18,37 +19,50 @@ class UploadedFile
 
   attr_reader :remote_id
   attr_reader :sha256
-
-  def initialize(path, filename: nil, content_type: "application/octet-stream", sha256: nil, remote_id: nil)
-    raise InvalidPathError, "#{path} file does not exist" unless ::File.exist?(path)
+  attr_reader :size
+
+  def initialize(path, filename: nil, content_type: "application/octet-stream", sha256: nil, remote_id: nil, size: nil)
+    if path.present?
+      raise InvalidPathError, "#{path} file does not exist" unless ::File.exist?(path)
+
+      @tempfile = File.new(path, 'rb')
+      @size = @tempfile.size
+    else
+      begin
+        @size = Integer(size)
+      rescue ArgumentError, TypeError
+        raise UnknownSizeError, 'Unable to determine file size'
+      end
+    end
 
     @content_type = content_type
-    @original_filename = sanitize_filename(filename || path)
+    @original_filename = sanitize_filename(filename || path || '')
     @content_type = content_type
     @sha256 = sha256
     @remote_id = remote_id
-    @tempfile = File.new(path, 'rb')
   end
 
   def self.from_params(params, field, upload_paths)
-    unless params["#{field}.path"]
-      raise InvalidPathError, "file is invalid" if params["#{field}.remote_id"]
-
-      return
-    end
-
-    file_path = File.realpath(params["#{field}.path"])
-
-    paths = Array(upload_paths) << Dir.tmpdir
-    unless self.allowed_path?(file_path, paths.compact)
-      raise InvalidPathError, "insecure path used '#{file_path}'"
+    path = params["#{field}.path"]
+    remote_id = params["#{field}.remote_id"]
+    return if path.blank? && remote_id.blank?
+
+    file_path = nil
+    if path
+      file_path = File.realpath(path)
+
+      paths = Array(upload_paths) << Dir.tmpdir
+      unless self.allowed_path?(file_path, paths.compact)
+        raise InvalidPathError, "insecure path used '#{file_path}'"
+      end
     end
 
     UploadedFile.new(file_path,
       filename: params["#{field}.name"],
       content_type: params["#{field}.type"] || 'application/octet-stream',
       sha256: params["#{field}.sha256"],
-      remote_id: params["#{field}.remote_id"])
+      remote_id: remote_id,
+      size: params["#{field}.size"])
   end
 
   def self.allowed_path?(file_path, paths)
@@ -68,7 +82,11 @@ class UploadedFile
   end
 
   def path
-    @tempfile.path
+    @tempfile&.path
+  end
+
+  def close
+    @tempfile&.close
   end
 
   alias_method :local_path, :path