Merge branch 'master' into '46448-add-timestamps-for-each-stage-of-gitlab-rake-gitlab-backup-restore'

# Conflicts:
#   spec/tasks/gitlab/backup_rake_spec.rb

1400 files changed, 27187 insertions, 10579 deletions

diff --git a/lib/additional_email_headers_interceptor.rb b/lib/additional_email_headers_interceptor.rb
deleted file mode 100644
index 3cb1694b9f1..00000000000
--- a/lib/additional_email_headers_interceptor.rb
+++ /dev/null
@@ -1,6 +0,0 @@
-class AdditionalEmailHeadersInterceptor
-  def self.delivering_email(message)
-    message.header['Auto-Submitted'] ||= 'auto-generated'
-    message.header['X-Auto-Response-Suppress'] ||= 'All'
-  end
-end
diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb
index a4d8507960e..6fb7985f955 100644
--- a/lib/after_commit_queue.rb
+++ b/lib/after_commit_queue.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module AfterCommitQueue
   extend ActiveSupport::Concern
 
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb
index ae13c248171..ee8dc822098 100644
--- a/lib/api/access_requests.rb
+++ b/lib/api/access_requests.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class AccessRequests < Grape::API
     include PaginationParams
@@ -10,7 +12,7 @@ module API
       params do
         requires :id, type: String, desc: "The #{source_type} ID"
       end
-      resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+      resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
         desc "Gets a list of access requests for a #{source_type}." do
           detail 'This feature was introduced in GitLab 8.11.'
           success Entities::AccessRequester
@@ -18,6 +20,7 @@ module API
         params do
           use :pagination
         end
+        # rubocop: disable CodeReuse/ActiveRecord
         get ":id/access_requests" do
           source = find_source(source_type, params[:id])
 
@@ -26,6 +29,7 @@ module API
 
           present access_requesters, with: Entities::AccessRequester
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         desc "Requests access for the authenticated user to a #{source_type}." do
           detail 'This feature was introduced in GitLab 8.11.'
@@ -50,6 +54,7 @@ module API
           requires :user_id, type: Integer, desc: 'The user ID of the access requester'
           optional :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)'
         end
+        # rubocop: disable CodeReuse/ActiveRecord
         put ':id/access_requests/:user_id/approve' do
           source = find_source(source_type, params[:id])
 
@@ -61,6 +66,7 @@ module API
           status :created
           present member, with: Entities::Member
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         desc 'Denies an access request for the given user.' do
           detail 'This feature was introduced in GitLab 8.11.'
@@ -68,6 +74,7 @@ module API
         params do
           requires :user_id, type: Integer, desc: 'The user ID of the access requester'
         end
+        # rubocop: disable CodeReuse/ActiveRecord
         delete ":id/access_requests/:user_id" do
           source = find_source(source_type, params[:id])
           member = source.requesters.find_by!(user_id: params[:user_id])
@@ -76,6 +83,7 @@ module API
             ::Members::DestroyService.new(current_user).execute(member)
           end
         end
+        # rubocop: enable CodeReuse/ActiveRecord
       end
     end
   end
diff --git a/lib/api/api.rb b/lib/api/api.rb
index 7ea575a9661..9cbfc0e35ff 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class API < Grape::API
     include APIGuard
@@ -5,8 +7,9 @@ module API
     LOG_FILENAME = Rails.root.join("log", "api_json.log")
 
     NO_SLASH_URL_PART_REGEX = %r{[^/]+}
-    PROJECT_ENDPOINT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze
-    COMMIT_ENDPOINT_REQUIREMENTS = PROJECT_ENDPOINT_REQUIREMENTS.merge(sha: NO_SLASH_URL_PART_REGEX).freeze
+    NAMESPACE_OR_PROJECT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze
+    COMMIT_ENDPOINT_REQUIREMENTS = NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(sha: NO_SLASH_URL_PART_REGEX).freeze
+    USER_REQUIREMENTS = { user_id: NO_SLASH_URL_PART_REGEX }.freeze
 
     insert_before Grape::Middleware::Error,
                   GrapeLogging::Middleware::RequestLogger,
@@ -15,8 +18,11 @@ module API
                   include: [
                     GrapeLogging::Loggers::FilterParameters.new,
                     GrapeLogging::Loggers::ClientEnv.new,
+                    Gitlab::GrapeLogging::Loggers::RouteLogger.new,
                     Gitlab::GrapeLogging::Loggers::UserLogger.new,
-                    Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new
+                    Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new,
+                    Gitlab::GrapeLogging::Loggers::PerfLogger.new,
+                    Gitlab::GrapeLogging::Loggers::CorrelationIdLogger.new
                   ]
 
     allow_access_with_scope :api
@@ -46,6 +52,10 @@ module API
       rack_response({ 'message' => '404 Not found' }.to_json, 404)
     end
 
+    rescue_from ::Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError do
+      rack_response({ 'message' => '409 Conflict: Resource lock' }.to_json, 409)
+    end
+
     rescue_from UploadedFile::InvalidPathError do |e|
       rack_response({ 'message' => e.message }.to_json, 400)
     end
@@ -76,13 +86,13 @@ module API
     content_type :txt, "text/plain"
 
     # Ensure the namespace is right, otherwise we might load Grape::API::Helpers
-    helpers ::SentryHelper
     helpers ::API::Helpers
     helpers ::API::Helpers::CommonHelpers
 
     # Keep in alphabetical order
     mount ::API::AccessRequests
     mount ::API::Applications
+    mount ::API::Avatar
     mount ::API::AwardEmoji
     mount ::API::Badges
     mount ::API::Boards
@@ -91,6 +101,7 @@ module API
     mount ::API::CircuitBreakers
     mount ::API::Commits
     mount ::API::CommitStatuses
+    mount ::API::ContainerRegistry
     mount ::API::DeployKeys
     mount ::API::Deployments
     mount ::API::Environments
@@ -98,12 +109,14 @@ module API
     mount ::API::Features
     mount ::API::Files
     mount ::API::GroupBoards
-    mount ::API::Groups
     mount ::API::GroupMilestones
+    mount ::API::Groups
+    mount ::API::GroupVariables
+    mount ::API::ImportGithub
     mount ::API::Internal
     mount ::API::Issues
-    mount ::API::Jobs
     mount ::API::JobArtifacts
+    mount ::API::Jobs
     mount ::API::Keys
     mount ::API::Labels
     mount ::API::Lint
@@ -114,18 +127,24 @@ module API
     mount ::API::Namespaces
     mount ::API::Notes
     mount ::API::Discussions
+    mount ::API::ResourceLabelEvents
     mount ::API::NotificationSettings
     mount ::API::PagesDomains
     mount ::API::Pipelines
     mount ::API::PipelineSchedules
+    mount ::API::ProjectClusters
     mount ::API::ProjectExport
     mount ::API::ProjectImport
     mount ::API::ProjectHooks
-    mount ::API::Projects
     mount ::API::ProjectMilestones
+    mount ::API::Projects
     mount ::API::ProjectSnapshots
     mount ::API::ProjectSnippets
+    mount ::API::ProjectTemplates
     mount ::API::ProtectedBranches
+    mount ::API::ProtectedTags
+    mount ::API::Releases
+    mount ::API::Release::Links
     mount ::API::Repositories
     mount ::API::Runner
     mount ::API::Runners
@@ -134,7 +153,9 @@ module API
     mount ::API::Settings
     mount ::API::SidekiqMetrics
     mount ::API::Snippets
+    mount ::API::Submodules
     mount ::API::Subscriptions
+    mount ::API::Suggestions
     mount ::API::SystemHooks
     mount ::API::Tags
     mount ::API::Templates
@@ -142,7 +163,6 @@ module API
     mount ::API::Triggers
     mount ::API::Users
     mount ::API::Variables
-    mount ::API::GroupVariables
     mount ::API::Version
     mount ::API::Wikis
 
diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb
index c17089759de..af9b519ed9e 100644
--- a/lib/api/api_guard.rb
+++ b/lib/api/api_guard.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Guard API with OAuth 2.0 Access Token
 
 require 'rack/oauth2'
@@ -84,7 +86,7 @@ module API
       end
     end
 
-    module ClassMethods
+    class_methods do
       private
 
       def install_error_responders(base)
@@ -92,6 +94,7 @@ module API
                          Gitlab::Auth::TokenNotFoundError,
                          Gitlab::Auth::ExpiredError,
                          Gitlab::Auth::RevokedError,
+                         Gitlab::Auth::ImpersonationDisabled,
                          Gitlab::Auth::InsufficientScopeError]
 
         base.__send__(:rescue_from, *error_classes, oauth2_bearer_token_error_handler) # rubocop:disable GitlabSecurity/PublicSend
@@ -119,6 +122,11 @@ module API
                 :invalid_token,
                 "Token was revoked. You have to re-authorize from the user.")
 
+            when Gitlab::Auth::ImpersonationDisabled
+              Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
+                :invalid_token,
+                "Token is an impersonation token but impersonation was disabled.")
+
             when Gitlab::Auth::InsufficientScopeError
               # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2)
               # does not include WWW-Authenticate header, which breaks the standard.
diff --git a/lib/api/applications.rb b/lib/api/applications.rb
index b122cdefe4e..92717e04543 100644
--- a/lib/api/applications.rb
+++ b/lib/api/applications.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   # External applications API
   class Applications < Grape::API
@@ -22,6 +24,22 @@ module API
           render_validation_error! application
         end
       end
+
+      desc 'Get applications' do
+        success Entities::Application
+      end
+      get do
+        applications = ApplicationsFinder.new.execute
+        present applications, with: Entities::Application
+      end
+
+      desc 'Delete an application'
+      delete ':id' do
+        application = ApplicationsFinder.new(params).execute
+        application.destroy
+
+        status 204
+      end
     end
   end
 end
diff --git a/lib/api/avatar.rb b/lib/api/avatar.rb
new file mode 100644
index 00000000000..0f14d003065
--- /dev/null
+++ b/lib/api/avatar.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module API
+  class Avatar < Grape::API
+    resource :avatar do
+      desc 'Return avatar url for a user' do
+        success Entities::Avatar
+      end
+      params do
+        requires :email, type: String, desc: 'Public email address of the user'
+        optional :size, type: Integer, desc: 'Single pixel dimension for Gravatar images'
+      end
+      get do
+        forbidden!('Unauthorized access') unless can?(current_user, :read_users_list)
+
+        user = User.find_by_public_email(params[:email])
+        user ||= User.new(email: params[:email])
+
+        present user, with: Entities::Avatar, size: params[:size]
+      end
+    end
+  end
+end
diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb
index c3d93996816..a1851ba3627 100644
--- a/lib/api/award_emoji.rb
+++ b/lib/api/award_emoji.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class AwardEmoji < Grape::API
     include PaginationParams
@@ -12,7 +14,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       AWARDABLES.each do |awardable_params|
         awardable_string = awardable_params[:type].pluralize
         awardable_id_string = "#{awardable_params[:type]}_#{awardable_params[:find_by]}"
@@ -100,9 +102,10 @@ module API
       end
 
       def can_award_awardable?
-        awardable.user_can_award?(current_user, params[:name])
+        awardable.user_can_award?(current_user)
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def awardable
         @awardable ||=
           begin
@@ -119,6 +122,7 @@ module API
             end
           end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def read_ability(awardable)
         case awardable
diff --git a/lib/api/badges.rb b/lib/api/badges.rb
index 8ceffe9c5ef..ba554e00a16 100644
--- a/lib/api/badges.rb
+++ b/lib/api/badges.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Badges < Grape::API
     include PaginationParams
@@ -20,7 +22,7 @@ module API
       params do
         requires :id, type: String, desc: "The ID of a #{source_type}"
       end
-      resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+      resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
         desc "Gets a list of #{source_type} badges viewable by the authenticated user." do
           detail 'This feature was introduced in GitLab 10.6.'
           success Entities::Badge
diff --git a/lib/api/boards.rb b/lib/api/boards.rb
index 6c706b2b4e1..b7c77730afb 100644
--- a/lib/api/boards.rb
+++ b/lib/api/boards.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Boards < Grape::API
     include BoardsResponses
@@ -14,7 +16,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       segment ':id/boards' do
         desc 'Get all project boards' do
           detail 'This feature was introduced in 8.13'
@@ -33,6 +35,7 @@ module API
           success Entities::Board
         end
         get '/:board_id' do
+          authorize!(:read_board, user_project)
           present board, with: Entities::Board
         end
       end
@@ -70,12 +73,10 @@ module API
           success Entities::List
         end
         params do
-          requires :label_id, type: Integer, desc: 'The ID of an existing label'
+          use :list_creation_params
         end
         post '/lists' do
-          unless available_labels_for(user_project).exists?(params[:label_id])
-            render_api_error!({ error: 'Label not found!' }, 400)
-          end
+          authorize_list_type_resource!
 
           authorize!(:admin_list, user_project)
 
diff --git a/lib/api/boards_responses.rb b/lib/api/boards_responses.rb
index ead0943a74d..86d9b24802f 100644
--- a/lib/api/boards_responses.rb
+++ b/lib/api/boards_responses.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   module BoardsResponses
     extend ActiveSupport::Concern
@@ -14,7 +16,7 @@ module API
 
         def create_list
           create_list_service =
-            ::Boards::Lists::CreateService.new(board_parent, current_user, { label_id: params[:label_id] })
+            ::Boards::Lists::CreateService.new(board_parent, current_user, create_list_params)
 
           list = create_list_service.execute(board)
 
@@ -25,6 +27,10 @@ module API
           end
         end
 
+        def create_list_params
+          params.slice(:label_id)
+        end
+
         def move_list(list)
           move_list_service =
             ::Boards::Lists::MoveService.new(board_parent, current_user, { position: params[:position].to_i })
@@ -44,6 +50,18 @@ module API
             end
           end
         end
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def authorize_list_type_resource!
+          unless available_labels_for(board_parent).exists?(params[:label_id])
+            render_api_error!({ error: 'Label not found!' }, 400)
+          end
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        params :list_creation_params do
+          requires :label_id, type: Integer, desc: 'The ID of an existing label'
+        end
       end
     end
   end
diff --git a/lib/api/branches.rb b/lib/api/branches.rb
index 13cfba728fa..07f529b01bb 100644
--- a/lib/api/branches.rb
+++ b/lib/api/branches.rb
@@ -1,31 +1,26 @@
+# frozen_string_literal: true
+
 require 'mime/types'
 
 module API
   class Branches < Grape::API
     include PaginationParams
 
-    BRANCH_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(branch: API::NO_SLASH_URL_PART_REGEX)
+    BRANCH_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(branch: API::NO_SLASH_URL_PART_REGEX)
 
     before { authorize! :download_code, user_project }
 
     helpers do
-      def find_branch!(branch_name)
-        begin
-          user_project.repository.find_branch(branch_name) || not_found!('Branch')
-        rescue Gitlab::Git::CommandError
-          render_api_error!('The branch refname is invalid', 400)
-        end
-      end
-
       params :filter_params do
         optional :search, type: String, desc: 'Return list of branches matching the search criteria'
+        optional :sort, type: String, desc: 'Return list of branches sorted by the given field'
       end
     end
 
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Get a project repository branches' do
         success Entities::Branch
       end
@@ -39,12 +34,13 @@ module API
         repository = user_project.repository
 
         branches = BranchesFinder.new(repository, declared_params(include_missing: false)).execute
-
+        branches = ::Kaminari.paginate_array(branches)
         merged_branch_names = repository.merged_branch_names(branches.map(&:name))
 
         present(
-          paginate(::Kaminari.paginate_array(branches)),
+          paginate(branches),
           with: Entities::Branch,
+          current_user: current_user,
           project: user_project,
           merged_branch_names: merged_branch_names
         )
@@ -63,7 +59,7 @@ module API
         get do
           branch = find_branch!(params[:branch])
 
-          present branch, with: Entities::Branch, project: user_project
+          present branch, with: Entities::Branch, current_user: current_user, project: user_project
         end
       end
 
@@ -75,10 +71,11 @@ module API
         success Entities::Branch
       end
       params do
-        requires :branch, type: String, desc: 'The name of the branch'
+        requires :branch, type: String, desc: 'The name of the branch', allow_blank: false
         optional :developers_can_push, type: Boolean, desc: 'Flag if developers can push to that branch'
         optional :developers_can_merge, type: Boolean, desc: 'Flag if developers can merge to that branch'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       put ':id/repository/branches/:branch/protect', requirements: BRANCH_ENDPOINT_REQUIREMENTS do
         authorize_admin_project
 
@@ -101,19 +98,21 @@ module API
                            end
 
         if protected_branch.valid?
-          present branch, with: Entities::Branch, project: user_project
+          present branch, with: Entities::Branch, current_user: current_user, project: user_project
         else
           render_api_error!(protected_branch.errors.full_messages, 422)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       # Note: This API will be deprecated in favor of the protected branches API.
       desc 'Unprotect a single branch' do
         success Entities::Branch
       end
       params do
-        requires :branch, type: String, desc: 'The name of the branch'
+        requires :branch, type: String, desc: 'The name of the branch', allow_blank: false
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       put ':id/repository/branches/:branch/unprotect', requirements: BRANCH_ENDPOINT_REQUIREMENTS do
         authorize_admin_project
 
@@ -121,15 +120,16 @@ module API
         protected_branch = user_project.protected_branches.find_by(name: branch.name)
         protected_branch&.destroy
 
-        present branch, with: Entities::Branch, project: user_project
+        present branch, with: Entities::Branch, current_user: current_user, project: user_project
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Create branch' do
         success Entities::Branch
       end
       params do
-        requires :branch, type: String, desc: 'The name of the branch'
-        requires :ref, type: String, desc: 'Create branch from commit sha or existing branch'
+        requires :branch, type: String, desc: 'The name of the branch', allow_blank: false
+        requires :ref, type: String, desc: 'Create branch from commit sha or existing branch', allow_blank: false
       end
       post ':id/repository/branches' do
         authorize_push_project
@@ -140,6 +140,7 @@ module API
         if result[:status] == :success
           present result[:branch],
                   with: Entities::Branch,
+                  current_user: current_user,
                   project: user_project
         else
           render_api_error!(result[:message], 400)
@@ -148,7 +149,7 @@ module API
 
       desc 'Delete a branch'
       params do
-        requires :branch, type: String, desc: 'The name of the branch'
+        requires :branch, type: String, desc: 'The name of the branch', allow_blank: false
       end
       delete ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do
         authorize_push_project
diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb
index d7138b2f2fe..19148758fc5 100644
--- a/lib/api/broadcast_messages.rb
+++ b/lib/api/broadcast_messages.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class BroadcastMessages < Grape::API
     include PaginationParams
diff --git a/lib/api/circuit_breakers.rb b/lib/api/circuit_breakers.rb
index c13154dc0ec..da756daadcc 100644
--- a/lib/api/circuit_breakers.rb
+++ b/lib/api/circuit_breakers.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class CircuitBreakers < Grape::API
     before { authenticated_as_admin! }
@@ -11,37 +13,24 @@ module API
       end
       resource ':type' do
         namespace '', requirements: { type: 'repository_storage' } do
-          helpers do
-            def failing_storage_health
-              @failing_storage_health ||= Gitlab::Git::Storage::Health.for_failing_storages
-            end
-
-            def storage_health
-              @storage_health ||= Gitlab::Git::Storage::Health.for_all_storages
-            end
-          end
-
           desc 'Get all git storages' do
             detail 'This feature was introduced in GitLab 9.5'
-            success Entities::RepositoryStorageHealth
           end
           get do
-            present storage_health, with: Entities::RepositoryStorageHealth
+            present []
           end
 
           desc 'Get all failing git storages' do
             detail 'This feature was introduced in GitLab 9.5'
-            success Entities::RepositoryStorageHealth
           end
           get 'failing' do
-            present failing_storage_health, with: Entities::RepositoryStorageHealth
+            present []
           end
 
           desc 'Reset all storage failures and open circuitbreaker' do
             detail 'This feature was introduced in GitLab 9.5'
           end
           delete do
-            Gitlab::Git::Storage::FailureInfo.reset_all!
           end
         end
       end
diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb
index 829eef18795..08b4f8db8b0 100644
--- a/lib/api/commit_statuses.rb
+++ b/lib/api/commit_statuses.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'mime/types'
 
 module API
@@ -5,7 +7,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       include PaginationParams
 
       before { authenticate! }
@@ -21,12 +23,13 @@ module API
         optional :all,   type: String, desc: 'Show all statuses, default: false'
         use :pagination
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ':id/repository/commits/:sha/statuses' do
         authorize!(:read_commit_status, user_project)
 
         not_found!('Commit') unless user_project.commit(params[:sha])
 
-        pipelines = user_project.pipelines.where(sha: params[:sha])
+        pipelines = user_project.ci_pipelines.where(sha: params[:sha])
         statuses = ::CommitStatus.where(pipeline: pipelines)
         statuses = statuses.latest unless to_boolean(params[:all])
         statuses = statuses.where(ref: params[:ref]) if params[:ref].present?
@@ -34,6 +37,7 @@ module API
         statuses = statuses.where(name: params[:name]) if params[:name].present?
         present paginate(statuses), with: Entities::CommitStatus
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Post status to a commit' do
         success Entities::CommitStatus
@@ -49,6 +53,7 @@ module API
         optional :context,     type: String,  desc: 'A string label to differentiate this status from the status of other systems. Default: "default"'
         optional :coverage,    type: Float,   desc: 'The total code coverage'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       post ':id/statuses/:sha' do
         authorize! :create_commit_status, user_project
 
@@ -70,7 +75,7 @@ module API
 
         pipeline = @project.pipeline_for(ref, commit.sha)
         unless pipeline
-          pipeline = @project.pipelines.create!(
+          pipeline = @project.ci_pipelines.create!(
             source: :external,
             sha: commit.sha,
             ref: ref,
@@ -111,13 +116,14 @@ module API
           end
 
           MergeRequest.where(source_project: @project, source_branch: ref)
-            .update_all(head_pipeline_id: pipeline) if pipeline.latest?
+            .update_all(head_pipeline_id: pipeline.id) if pipeline.latest?
 
           present status, with: Entities::CommitStatus
         rescue StateMachines::InvalidTransition => e
           render_api_error!(e.message, 400)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index 684955a1b24..9d23daafe95 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'mime/types'
 
 module API
@@ -6,28 +8,42 @@ module API
 
     before { authorize! :download_code, user_project }
 
+    helpers do
+      def user_access
+        @user_access ||= Gitlab::UserAccess.new(current_user, project: user_project)
+      end
+
+      def authorize_push_to_branch!(branch)
+        unless user_access.can_push_to_branch?(branch)
+          forbidden!("You are not allowed to push into this branch")
+        end
+      end
+    end
+
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Get a project repository commits' do
         success Entities::Commit
       end
       params do
         optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used'
-        optional :since,    type: DateTime, desc: 'Only commits after or on this date will be returned'
-        optional :until,    type: DateTime, desc: 'Only commits before or on this date will be returned'
-        optional :path,     type: String, desc: 'The file path'
-        optional :all,      type: Boolean, desc: 'Every commit will be returned'
+        optional :since, type: DateTime, desc: 'Only commits after or on this date will be returned'
+        optional :until, type: DateTime, desc: 'Only commits before or on this date will be returned'
+        optional :path, type: String, desc: 'The file path'
+        optional :all, type: Boolean, desc: 'Every commit will be returned'
+        optional :with_stats, type: Boolean, desc: 'Stats about each commit will be added to the response'
         use :pagination
       end
       get ':id/repository/commits' do
-        path   = params[:path]
+        path = params[:path]
         before = params[:until]
-        after  = params[:since]
-        ref    = params[:ref_name] || user_project.try(:default_branch) || 'master' unless params[:all]
+        after = params[:since]
+        ref = params[:ref_name] || user_project.try(:default_branch) || 'master' unless params[:all]
         offset = (params[:page] - 1) * params[:per_page]
-        all    = params[:all]
+        all = params[:all]
+        with_stats = params[:with_stats]
 
         commits = user_project.repository.commits(ref,
                                                   path: path,
@@ -47,7 +63,9 @@ module API
 
         paginated_commits = Kaminari.paginate_array(commits, total_count: commit_count)
 
-        present paginate(paginated_commits), with: Entities::Commit
+        serializer = with_stats ? Entities::CommitWithStats : Entities::Commit
+
+        present paginate(paginated_commits), with: serializer
       end
 
       desc 'Commit multiple file changes as one commit' do
@@ -55,15 +73,35 @@ module API
         detail 'This feature was introduced in GitLab 8.13'
       end
       params do
-        requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.'
+        requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.', allow_blank: false
         requires :commit_message, type: String, desc: 'Commit message'
-        requires :actions, type: Array[Hash], desc: 'Actions to perform in commit'
+        requires :actions, type: Array, desc: 'Actions to perform in commit' do
+          requires :action, type: String, desc: 'The action to perform, `create`, `delete`, `move`, `update`, `chmod`', values: %w[create update move delete chmod].freeze
+          requires :file_path, type: String, desc: 'Full path to the file. Ex. `lib/class.rb`'
+          given action: ->(action) { action == 'move' } do
+            requires :previous_path, type: String, desc: 'Original full path to the file being moved. Ex. `lib/class1.rb`'
+          end
+          given action: ->(action) { %w[create move].include? action } do
+            optional :content, type: String, desc: 'File content'
+          end
+          given action: ->(action) { action == 'update' } do
+            requires :content, type: String, desc: 'File content'
+          end
+          optional :encoding, type: String, desc: '`text` or `base64`', default: 'text', values: %w[text base64]
+          given action: ->(action) { %w[update move delete].include? action } do
+            optional :last_commit_id, type: String, desc: 'Last known file commit id'
+          end
+          given action: ->(action) { action == 'chmod' } do
+            requires :execute_filemode, type: Boolean, desc: 'When `true/false` enables/disables the execute flag on the file.'
+          end
+        end
         optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from'
         optional :author_email, type: String, desc: 'Author email for commit'
         optional :author_name, type: String, desc: 'Author name for commit'
+        optional :stats, type: Boolean, default: true, desc: 'Include commit stats'
       end
       post ':id/repository/commits' do
-        authorize! :push_code, user_project
+        authorize_push_to_branch!(params[:branch])
 
         attrs = declared_params
         attrs[:branch_name] = attrs.delete(:branch)
@@ -73,7 +111,10 @@ module API
 
         if result[:status] == :success
           commit_detail = user_project.repository.commit(result[:result])
-          present commit_detail, with: Entities::CommitDetail
+
+          Gitlab::WebIdeCommitsCounter.increment if find_user_from_warden
+
+          present commit_detail, with: Entities::CommitDetail, stats: params[:stats]
         else
           render_api_error!(result[:message], 400)
         end
@@ -120,6 +161,7 @@ module API
         use :pagination
         requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ':id/repository/commits/:sha/comments', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do
         commit = user_project.commit(params[:sha])
 
@@ -128,6 +170,7 @@ module API
 
         present paginate(notes), with: Entities::CommitNote
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Cherry pick commit into a branch' do
         detail 'This feature was introduced in GitLab 8.15'
@@ -135,16 +178,49 @@ module API
       end
       params do
         requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag to be cherry picked'
-        requires :branch, type: String, desc: 'The name of the branch'
+        requires :branch, type: String, desc: 'The name of the branch', allow_blank: false
       end
       post ':id/repository/commits/:sha/cherry_pick', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do
-        authorize! :push_code, user_project
+        authorize_push_to_branch!(params[:branch])
+
+        commit = user_project.commit(params[:sha])
+        not_found!('Commit') unless commit
+
+        find_branch!(params[:branch])
+
+        commit_params = {
+          commit: commit,
+          start_branch: params[:branch],
+          branch_name: params[:branch]
+        }
+
+        result = ::Commits::CherryPickService
+          .new(user_project, current_user, commit_params)
+          .execute
+
+        if result[:status] == :success
+          present user_project.repository.commit(result[:result]),
+            with: Entities::Commit
+        else
+          render_api_error!(result[:message], 400)
+        end
+      end
+
+      desc 'Revert a commit in a branch' do
+        detail 'This feature was introduced in GitLab 11.5'
+        success Entities::Commit
+      end
+      params do
+        requires :sha, type: String, desc: 'Commit SHA to revert'
+        requires :branch, type: String, desc: 'Target branch name', allow_blank: false
+      end
+      post ':id/repository/commits/:sha/revert', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do
+        authorize_push_to_branch!(params[:branch])
 
         commit = user_project.commit(params[:sha])
         not_found!('Commit') unless commit
 
-        branch = user_project.repository.find_branch(params[:branch])
-        not_found!('Branch') unless branch
+        find_branch!(params[:branch])
 
         commit_params = {
           commit: commit,
@@ -152,11 +228,13 @@ module API
           branch_name: params[:branch]
         }
 
-        result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute
+        result = ::Commits::RevertService
+          .new(user_project, current_user, commit_params)
+          .execute
 
         if result[:status] == :success
-          branch = user_project.repository.find_branch(params[:branch])
-          present user_project.repository.commit(branch.dereferenced_target), with: Entities::Commit
+          present user_project.repository.commit(result[:result]),
+            with: Entities::Commit
         else
           render_api_error!(result[:message], 400)
         end
diff --git a/lib/api/container_registry.rb b/lib/api/container_registry.rb
new file mode 100644
index 00000000000..e4493910196
--- /dev/null
+++ b/lib/api/container_registry.rb
@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module API
+  class ContainerRegistry < Grape::API
+    include PaginationParams
+
+    REGISTRY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(
+      tag_name: API::NO_SLASH_URL_PART_REGEX)
+
+    before { error!('404 Not Found', 404) unless Feature.enabled?(:container_registry_api, user_project, default_enabled: true) }
+    before { authorize_read_container_images! }
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      desc 'Get a project container repositories' do
+        detail 'This feature was introduced in GitLab 11.8.'
+        success Entities::ContainerRegistry::Repository
+      end
+      params do
+        use :pagination
+      end
+      get ':id/registry/repositories' do
+        repositories = user_project.container_repositories.ordered
+
+        present paginate(repositories), with: Entities::ContainerRegistry::Repository
+      end
+
+      desc 'Delete repository' do
+        detail 'This feature was introduced in GitLab 11.8.'
+      end
+      params do
+        requires :repository_id, type: Integer, desc: 'The ID of the repository'
+      end
+      delete ':id/registry/repositories/:repository_id', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
+        authorize_admin_container_image!
+
+        DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id)
+
+        status :accepted
+      end
+
+      desc 'Get a list of repositories tags' do
+        detail 'This feature was introduced in GitLab 11.8.'
+        success Entities::ContainerRegistry::Tag
+      end
+      params do
+        requires :repository_id, type: Integer, desc: 'The ID of the repository'
+        use :pagination
+      end
+      get ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
+        authorize_read_container_image!
+
+        tags = Kaminari.paginate_array(repository.tags)
+        present paginate(tags), with: Entities::ContainerRegistry::Tag
+      end
+
+      desc 'Delete repository tags (in bulk)' do
+        detail 'This feature was introduced in GitLab 11.8.'
+      end
+      params do
+        requires :repository_id, type: Integer, desc: 'The ID of the repository'
+        requires :name_regex, type: String, desc: 'The tag name regexp to delete, specify .* to delete all'
+        optional :keep_n, type: Integer, desc: 'Keep n of latest tags with matching name'
+        optional :older_than, type: String, desc: 'Delete older than: 1h, 1d, 1month'
+      end
+      delete ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
+        authorize_admin_container_image!
+
+        CleanupContainerRepositoryWorker.perform_async(current_user.id, repository.id,
+          declared_params.except(:repository_id)) # rubocop: disable CodeReuse/ActiveRecord
+
+        status :accepted
+      end
+
+      desc 'Get a details about repository tag' do
+        detail 'This feature was introduced in GitLab 11.8.'
+        success Entities::ContainerRegistry::TagDetails
+      end
+      params do
+        requires :repository_id, type: Integer, desc: 'The ID of the repository'
+        requires :tag_name, type: String, desc: 'The name of the tag'
+      end
+      get ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
+        authorize_read_container_image!
+        validate_tag!
+
+        present tag, with: Entities::ContainerRegistry::TagDetails
+      end
+
+      desc 'Delete repository tag' do
+        detail 'This feature was introduced in GitLab 11.8.'
+      end
+      params do
+        requires :repository_id, type: Integer, desc: 'The ID of the repository'
+        requires :tag_name, type: String, desc: 'The name of the tag'
+      end
+      delete ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
+        authorize_destroy_container_image!
+        validate_tag!
+
+        tag.delete
+
+        status :ok
+      end
+    end
+
+    helpers do
+      def authorize_read_container_images!
+        authorize! :read_container_image, user_project
+      end
+
+      def authorize_read_container_image!
+        authorize! :read_container_image, repository
+      end
+
+      def authorize_update_container_image!
+        authorize! :update_container_image, repository
+      end
+
+      def authorize_destroy_container_image!
+        authorize! :admin_container_image, repository
+      end
+
+      def authorize_admin_container_image!
+        authorize! :admin_container_image, repository
+      end
+
+      def repository
+        @repository ||= user_project.container_repositories.find(params[:repository_id])
+      end
+
+      def tag
+        @tag ||= repository.tag(params[:tag_name])
+      end
+
+      def validate_tag!
+        not_found!('Tag') unless tag.valid?
+      end
+    end
+  end
+end
diff --git a/lib/api/custom_attributes_endpoints.rb b/lib/api/custom_attributes_endpoints.rb
index 5000aa0d9ac..2149e04451e 100644
--- a/lib/api/custom_attributes_endpoints.rb
+++ b/lib/api/custom_attributes_endpoints.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   module CustomAttributesEndpoints
     extend ActiveSupport::Concern
@@ -30,6 +32,7 @@ module API
       params do
         use :custom_attributes_key
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ':id/custom_attributes/:key' do
         resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend
         authorize! :read_custom_attribute
@@ -38,12 +41,14 @@ module API
 
         present custom_attribute, with: Entities::CustomAttribute
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc "Set a custom attribute on a #{attributable_name}"
       params do
         use :custom_attributes_key
         requires :value, type: String, desc: 'The value of the custom attribute'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       put ':id/custom_attributes/:key' do
         resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend
         authorize! :update_custom_attribute
@@ -59,11 +64,13 @@ module API
           render_validation_error!(custom_attribute)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc "Delete a custom attribute on a #{attributable_name}"
       params do
         use :custom_attributes_key
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete ':id/custom_attributes/:key' do
         resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend
         authorize! :update_custom_attribute
@@ -72,6 +79,7 @@ module API
 
         status 204
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb
index b7aadc27e71..df6d2721977 100644
--- a/lib/api/deploy_keys.rb
+++ b/lib/api/deploy_keys.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class DeployKeys < Grape::API
     include PaginationParams
@@ -9,9 +11,11 @@ module API
         project.deploy_keys_projects.create(attrs)
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def find_by_deploy_key(project, key_id)
         project.deploy_keys_projects.find_by!(deploy_key: key_id)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
 
     desc 'Return all deploy keys'
@@ -27,7 +31,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of the project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       before { authorize_admin_project }
 
       desc "Get a specific project's deploy keys" do
@@ -36,11 +40,13 @@ module API
       params do
         use :pagination
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ":id/deploy_keys" do
         keys = user_project.deploy_keys_projects.preload(:deploy_key)
 
         present paginate(keys), with: Entities::DeployKeysProject
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Get single deploy key' do
         success Entities::DeployKeysProject
@@ -62,6 +68,7 @@ module API
         requires :title, type: String, desc: 'The name of the deploy key'
         optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository"
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       post ":id/deploy_keys" do
         params[:key].strip!
 
@@ -94,6 +101,7 @@ module API
           render_validation_error!(deploy_key_project)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Update an existing deploy key for a project' do
         success Entities::SSHKey
@@ -112,9 +120,9 @@ module API
         can_push = params[:can_push].nil? ? deploy_keys_project.can_push : params[:can_push]
         title = params[:title] || deploy_keys_project.deploy_key.title
 
-        result = deploy_keys_project.update_attributes(can_push: can_push,
-                                                       deploy_key_attributes: { id: params[:key_id],
-                                                                                title: title })
+        result = deploy_keys_project.update(can_push: can_push,
+                                            deploy_key_attributes: { id: params[:key_id],
+                                                                     title: title })
 
         if result
           present deploy_keys_project, with: Entities::DeployKeysProject
@@ -147,12 +155,14 @@ module API
       params do
         requires :key_id, type: Integer, desc: 'The ID of the deploy key'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete ":id/deploy_keys/:key_id" do
         deploy_key_project = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id])
         not_found!('Deploy Key') unless deploy_key_project
 
         destroy_conditionally!(deploy_key_project)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb
index 184fae0eb76..eb45df31ff9 100644
--- a/lib/api/deployments.rb
+++ b/lib/api/deployments.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   # Deployments RESTful API endpoints
   class Deployments < Grape::API
@@ -8,7 +10,7 @@ module API
     params do
       requires :id, type: String, desc: 'The project ID'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Get all deployments of the project' do
         detail 'This feature was introduced in GitLab 8.11.'
         success Entities::Deployment
@@ -18,18 +20,20 @@ module API
         optional :order_by, type: String, values: %w[id iid created_at ref], default: 'id', desc: 'Return deployments ordered by `id` or `iid` or `created_at` or `ref`'
         optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ':id/deployments' do
         authorize! :read_deployment, user_project
 
         present paginate(user_project.deployments.order(params[:order_by] => params[:sort])), with: Entities::Deployment
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Gets a specific deployment' do
         detail 'This feature was introduced in GitLab 8.11.'
         success Entities::Deployment
       end
       params do
-        requires :deployment_id, type: Integer,  desc: 'The deployment ID'
+        requires :deployment_id, type: Integer, desc: 'The deployment ID'
       end
       get ':id/deployments/:deployment_id' do
         authorize! :read_deployment, user_project
diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb
index 13c34e3473a..91eb6a23701 100644
--- a/lib/api/discussions.rb
+++ b/lib/api/discussions.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Discussions < Grape::API
     include PaginationParams
@@ -15,7 +17,7 @@ module API
       params do
         requires :id, type: String, desc: "The ID of a #{parent_type}"
       end
-      resource parent_type.pluralize.to_sym, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+      resource parent_type.pluralize.to_sym, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
         desc "Get a list of #{noteable_type.to_s.downcase} discussions" do
           success Entities::Discussion
         end
@@ -23,6 +25,7 @@ module API
           requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable'
           use :pagination
         end
+        # rubocop: disable CodeReuse/ActiveRecord
         get ":id/#{noteables_path}/:noteable_id/discussions" do
           noteable = find_noteable(parent_type, noteables_str, params[:noteable_id])
 
@@ -36,6 +39,7 @@ module API
 
           present paginate(discussions), with: Entities::Discussion
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         desc "Get a single #{noteable_type.to_s.downcase} discussion" do
           success Entities::Discussion
@@ -219,6 +223,7 @@ module API
     end
 
     helpers do
+      # rubocop: disable CodeReuse/ActiveRecord
       def readable_discussion_notes(noteable, discussion_id)
         notes = noteable.notes
           .where(discussion_id: discussion_id)
@@ -228,6 +233,7 @@ module API
 
         notes.reject { |n| n.cross_reference_not_visible_for?(current_user) }
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index c4537036a3a..9f1394571d8 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   module Entities
     class WikiPageBasic < Grape::Entity
@@ -10,6 +12,28 @@ module API
       expose :content
     end
 
+    class WikiAttachment < Grape::Entity
+      include Gitlab::FileMarkdownLinkBuilder
+
+      expose :file_name
+      expose :file_path
+      expose :branch
+      expose :link do
+        expose :file_path, as: :url
+        expose :markdown do |_entity|
+          self.markdown_link
+        end
+      end
+
+      def filename
+        object.file_name
+      end
+
+      def secure_url
+        object.file_path
+      end
+    end
+
     class UserSafe < Grape::Entity
       expose :id, :name, :username
     end
@@ -30,8 +54,8 @@ module API
     end
 
     class User < UserBasic
-      expose :created_at
-      expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization
+      expose :created_at, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) }
+      expose :bio, :location, :public_email, :skype, :linkedin, :twitter, :website_url, :organization
     end
 
     class UserActivity < Grape::Entity
@@ -55,12 +79,21 @@ module API
       expose :can_create_project?, as: :can_create_project
       expose :two_factor_enabled?, as: :two_factor_enabled
       expose :external
+      expose :private_profile
     end
 
     class UserWithAdmin < UserPublic
       expose :admin?, as: :is_admin
     end
 
+    class UserStatus < Grape::Entity
+      expose :emoji
+      expose :message
+      expose :message_html do |entity|
+        MarkupHelper.markdown_field(entity, :message)
+      end
+    end
+
     class Email < Grape::Entity
       expose :id, :email
     end
@@ -74,6 +107,7 @@ module API
       expose :project_id, :issues_events, :confidential_issues_events
       expose :note_events, :confidential_note_events, :pipeline_events, :wiki_page_events
       expose :job_events
+      expose :push_events_branch_filter
     end
 
     class SharedGroup < Grape::Entity
@@ -81,7 +115,11 @@ module API
       expose :group_name do |group_link, options|
         group_link.group.name
       end
+      expose :group_full_path do |group_link, options|
+        group_link.group.full_path
+      end
       expose :group_access, as: :group_access_level
+      expose :expires_at
     end
 
     class ProjectIdentity < Grape::Entity
@@ -110,7 +148,9 @@ module API
       expose :import_status
 
       # TODO: Use `expose_nil` once we upgrade the grape-entity gem
-      expose :import_error, if: lambda { |status, _ops| status.import_error }
+      expose :import_error, if: lambda { |project, _ops| project.import_state&.last_error } do |project|
+        project.import_state.last_error
+      end
     end
 
     class BasicProjectDetails < ProjectIdentity
@@ -125,21 +165,41 @@ module API
         # (fixed in https://github.com/rails/rails/pull/25976).
         project.tags.map(&:name).sort
       end
+
       expose :ssh_url_to_repo, :http_url_to_repo, :web_url, :readme_url
+
+      expose :license_url, if: :license do |project|
+        license = project.repository.license_blob
+
+        if license
+          Gitlab::Routing.url_helpers.project_blob_url(project, File.join(project.default_branch, license.path))
+        end
+      end
+
+      expose :license, with: 'API::Entities::LicenseBasic', if: :license do |project|
+        project.repository.license
+      end
+
       expose :avatar_url do |project, options|
         project.avatar_url(only_path: false)
       end
+
       expose :star_count, :forks_count
       expose :last_activity_at
-
+      expose :namespace, using: 'API::Entities::NamespaceBasic'
       expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes
 
-      def self.preload_relation(projects_relation, options =  {})
+      # rubocop: disable CodeReuse/ActiveRecord
+      def self.preload_relation(projects_relation, options = {})
+        # Preloading tags, should be done with using only `:tags`,
+        # as `:tags` are defined as: `has_many :tags, through: :taggings`
+        # N+1 is solved then by using `subject.tags.map(&:name)`
+        # MR describing the solution: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/20555
         projects_relation.preload(:project_feature, :route)
-                         .preload(:import_state)
-                         .preload(namespace: [:route, :owner],
-                                  tags: :taggings)
+                         .preload(:import_state, :tags)
+                         .preload(namespace: [:route, :owner])
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
 
     class Project < BasicProjectDetails
@@ -191,10 +251,12 @@ module API
       expose :shared_runners_enabled
       expose :lfs_enabled?, as: :lfs_enabled
       expose :creator_id
-      expose :namespace, using: 'API::Entities::NamespaceBasic'
       expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? }
       expose :import_status
-      expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] }
+
+      expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } do |project|
+        project.import_state&.last_error
+      end
 
       expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) }
       expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] }
@@ -211,13 +273,19 @@ module API
 
       expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics
 
-      def self.preload_relation(projects_relation, options =  {})
+      # rubocop: disable CodeReuse/ActiveRecord
+      def self.preload_relation(projects_relation, options = {})
+        # Preloading tags, should be done with using only `:tags`,
+        # as `:tags` are defined as: `has_many :tags, through: :taggings`
+        # N+1 is solved then by using `subject.tags.map(&:name)`
+        # MR describing the solution: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/20555
         super(projects_relation).preload(:group)
-                                .preload(project_group_links: :group,
+                                .preload(project_group_links: { group: :route },
                                          fork_network: :root_project,
-                                         forked_project_link: :forked_from_project,
-                                         forked_from_project: [:route, :forks, namespace: :route, tags: :taggings])
+                                         fork_network_member: :forked_from_project,
+                                         forked_from_project: [:route, :forks, :tags, namespace: :route])
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def self.forks_counting_projects(projects_relation)
         projects_relation + projects_relation.map(&:forked_from_project).compact
@@ -258,7 +326,7 @@ module API
       expose :request_access_enabled
       expose :full_name, :full_path
 
-      if ::Group.supports_nested_groups?
+      if ::Group.supports_nested_objects?
         expose :parent_id
       end
 
@@ -276,19 +344,23 @@ module API
 
     class GroupDetail < Group
       expose :projects, using: Entities::Project do |group, options|
-        GroupProjectsFinder.new(
+        projects = GroupProjectsFinder.new(
           group: group,
           current_user: options[:current_user],
           options: { only_owned: true }
         ).execute
+
+        Entities::Project.prepare_relation(projects)
       end
 
       expose :shared_projects, using: Entities::Project do |group, options|
-        GroupProjectsFinder.new(
+        projects = GroupProjectsFinder.new(
           group: group,
           current_user: options[:current_user],
           options: { only_shared: true }
         ).execute
+
+        Entities::Project.prepare_relation(projects)
       end
     end
 
@@ -308,6 +380,10 @@ module API
       expose :additions, :deletions, :total
     end
 
+    class CommitWithStats < Commit
+      expose :stats, using: Entities::CommitStats
+    end
+
     class CommitDetail < Commit
       expose :stats, using: Entities::CommitStats, if: :stats
       expose :status
@@ -345,6 +421,14 @@ module API
       expose :developers_can_merge do |repo_branch, options|
         options[:project].protected_branches.developers_can?(:merge, repo_branch.name)
       end
+
+      expose :can_push do |repo_branch, options|
+        Gitlab::UserAccess.new(options[:current_user], project: options[:project]).can_push_to_branch?(repo_branch.name)
+      end
+
+      expose :default do |repo_branch, options|
+        options[:project].default_branch == repo_branch.name
+      end
     end
 
     class TreeObject < Grape::Entity
@@ -358,7 +442,7 @@ module API
     end
 
     class Snippet < Grape::Entity
-      expose :id, :title, :file_name, :description
+      expose :id, :title, :file_name, :description, :visibility
       expose :author, using: Entities::UserBasic
       expose :updated_at, :created_at
       expose :project_id
@@ -404,6 +488,11 @@ module API
       expose :merge_access_levels, using: Entities::ProtectedRefAccess
     end
 
+    class ProtectedTag < Grape::Entity
+      expose :name
+      expose :create_access_levels, using: Entities::ProtectedRefAccess
+    end
+
     class Milestone < Grape::Entity
       expose :id, :iid
       expose :project_id, if: -> (entity, options) { entity&.project_id }
@@ -412,6 +501,10 @@ module API
       expose :state, :created_at, :updated_at
       expose :due_date
       expose :start_date
+
+      expose :web_url do |milestone, _options|
+        Gitlab::UrlBuilder.build(milestone)
+      end
     end
 
     class IssueBasic < ProjectEntity
@@ -497,10 +590,12 @@ module API
         expose :total_time_spent, as: :human_total_time_spent
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def total_time_spent
         # Avoids an N+1 query since timelogs are preloaded
         object.timelogs.map(&:time_spent).sum
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
 
     class ExternalIssue < Grape::Entity
@@ -510,6 +605,10 @@ module API
 
     class PipelineBasic < Grape::Entity
       expose :id, :sha, :ref, :status
+
+      expose :web_url do |pipeline, _options|
+        Gitlab::Routing.url_helpers.project_pipeline_url(pipeline.project, pipeline)
+      end
     end
 
     class MergeRequestSimple < ProjectEntity
@@ -520,6 +619,28 @@ module API
     end
 
     class MergeRequestBasic < ProjectEntity
+      expose :merged_by, using: Entities::UserBasic do |merge_request, _options|
+        merge_request.metrics&.merged_by
+      end
+
+      expose :merged_at do |merge_request, _options|
+        merge_request.metrics&.merged_at
+      end
+
+      expose :closed_by, using: Entities::UserBasic do |merge_request, _options|
+        merge_request.metrics&.latest_closed_by
+      end
+
+      expose :closed_at do |merge_request, _options|
+        merge_request.metrics&.latest_closed_at
+      end
+
+      expose :title_html, if: -> (_, options) { options[:render_html] } do |entity|
+        MarkupHelper.markdown_field(entity, :title)
+      end
+      expose :description_html, if: -> (_, options) { options[:render_html] } do |entity|
+        MarkupHelper.markdown_field(entity, :description)
+      end
       expose :target_branch, :source_branch
       expose :upvotes do |merge_request, options|
         if options[:issuable_metadata]
@@ -559,7 +680,9 @@ module API
       expose :discussion_locked
       expose :should_remove_source_branch?, as: :should_remove_source_branch
       expose :force_remove_source_branch?, as: :force_remove_source_branch
-      expose :allow_maintainer_to_push, if: -> (merge_request, _) { merge_request.for_fork? }
+      expose :allow_collaboration, if: -> (merge_request, _) { merge_request.for_fork? }
+      # Deprecated
+      expose :allow_collaboration, as: :allow_maintainer_to_push, if: -> (merge_request, _) { merge_request.for_fork? }
 
       expose :web_url do |merge_request, options|
         Gitlab::UrlBuilder.build(merge_request)
@@ -581,22 +704,6 @@ module API
         merge_request.merge_request_diff.real_size
       end
 
-      expose :merged_by, using: Entities::UserBasic do |merge_request, _options|
-        merge_request.metrics&.merged_by
-      end
-
-      expose :merged_at do |merge_request, _options|
-        merge_request.metrics&.merged_at
-      end
-
-      expose :closed_by, using: Entities::UserBasic do |merge_request, _options|
-        merge_request.metrics&.latest_closed_by
-      end
-
-      expose :closed_at do |merge_request, _options|
-        merge_request.metrics&.latest_closed_at
-      end
-
       expose :latest_build_started_at, if: -> (_, options) { build_available?(options) } do |merge_request, _options|
         merge_request.metrics&.latest_build_started_at
       end
@@ -615,6 +722,12 @@ module API
 
       expose :diff_refs, using: Entities::DiffRefs
 
+      # Allow the status of a rebase to be determined
+      expose :merge_error
+      expose :rebase_in_progress?, as: :rebase_in_progress, if: -> (_, options) { options[:include_rebase_in_progress] }
+
+      expose :diverged_commits_count, as: :diverged_commits_count, if: -> (_, options) { options[:include_diverged_commits_count] }
+
       def build_available?(options)
         options[:project]&.feature_available?(:builds, options[:current_user])
       end
@@ -674,7 +787,7 @@ module API
       expose :system?, as: :system
       expose :noteable_id, :noteable_type
 
-      expose :position, if: ->(note, options) { note.diff_note? } do |note|
+      expose :position, if: ->(note, options) { note.is_a?(DiffNote) } do |note|
         note.position.to_h
       end
 
@@ -692,6 +805,12 @@ module API
       expose :notes, using: Entities::Note
     end
 
+    class Avatar < Grape::Entity
+      expose :avatar_url do |avatarable, options|
+        avatarable.avatar_url(only_path: false, size: options[:size])
+      end
+    end
+
     class AwardEmoji < Grape::Entity
       expose :id
       expose :name
@@ -749,28 +868,33 @@ module API
 
     class Todo < Grape::Entity
       expose :id
-      expose :project, using: Entities::BasicProjectDetails
+      expose :project, using: Entities::ProjectIdentity, if: -> (todo, _) { todo.project_id }
+      expose :group, using: 'API::Entities::NamespaceBasic', if: -> (todo, _) { todo.group_id }
       expose :author, using: Entities::UserBasic
       expose :action_name
       expose :target_type
 
       expose :target do |todo, options|
-        Entities.const_get(todo.target_type).represent(todo.target, options)
+        todo_target_class(todo.target_type).represent(todo.target, options)
       end
 
       expose :target_url do |todo, options|
         target_type   = todo.target_type.underscore
-        target_url    = "namespace_project_#{target_type}_url"
+        target_url    = "#{todo.parent.class.to_s.underscore}_#{target_type}_url"
         target_anchor = "note_#{todo.note_id}" if todo.note_id?
 
         Gitlab::Routing
           .url_helpers
-          .public_send(target_url, todo.project.namespace, todo.project, todo.target, anchor: target_anchor) # rubocop:disable GitlabSecurity/PublicSend
+          .public_send(target_url, todo.parent, todo.target, anchor: target_anchor) # rubocop:disable GitlabSecurity/PublicSend
       end
 
       expose :body
       expose :state
       expose :created_at
+
+      def todo_target_class(target_type)
+        ::API::Entities.const_get(target_type)
+      end
     end
 
     class NamespaceBasic < Grape::Entity
@@ -805,7 +929,7 @@ module API
     class NotificationSetting < Grape::Entity
       expose :level
       expose :events, if: ->(notification_setting, _) { notification_setting.custom? } do
-        ::NotificationSetting::EMAIL_EVENTS.each do |event|
+        ::NotificationSetting.email_events.each do |event|
           expose event
         end
       end
@@ -844,12 +968,13 @@ module API
             if options[:group_members]
               options[:group_members].find { |member| member.source_id == project.namespace_id }
             else
-              project.group.group_member(options[:current_user])
+              project.group.highest_group_member(options[:current_user])
             end
           end
         end
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def self.preload_relation(projects_relation, options = {})
         relation = super(projects_relation, options)
 
@@ -874,6 +999,7 @@ module API
 
         relation
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
 
     class LabelBasic < Grape::Entity
@@ -968,11 +1094,42 @@ module API
       expose :password_authentication_enabled_for_web, as: :signin_enabled
     end
 
-    class Release < Grape::Entity
+    # deprecated old Release representation
+    class TagRelease < Grape::Entity
       expose :tag, as: :tag_name
       expose :description
     end
 
+    module Releases
+      class Link < Grape::Entity
+        expose :id
+        expose :name
+        expose :url
+        expose :external?, as: :external
+      end
+
+      class Source < Grape::Entity
+        expose :format
+        expose :url
+      end
+    end
+
+    class Release < TagRelease
+      expose :name
+      expose :description_html
+      expose :created_at
+      expose :author, using: Entities::UserBasic, if: -> (release, _) { release.author.present? }
+      expose :commit, using: Entities::Commit
+
+      expose :assets do
+        expose :assets_count, as: :count
+        expose :sources, using: Entities::Releases::Source
+        expose :links, using: Entities::Releases::Link do |release, options|
+          release.links.sorted
+        end
+      end
+    end
+
     class Tag < Grape::Entity
       expose :name, :message, :target
 
@@ -980,9 +1137,11 @@ module API
         options[:project].repository.commit(repo_tag.dereferenced_target)
       end
 
-      expose :release, using: Entities::Release do |repo_tag, options|
+      # rubocop: disable CodeReuse/ActiveRecord
+      expose :release, using: Entities::TagRelease do |repo_tag, options|
         options[:project].releases.find_by(tag: repo_tag.name)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
 
     class Runner < Grape::Entity
@@ -990,7 +1149,7 @@ module API
       expose :description
       expose :ip_address
       expose :active
-      expose :is_shared
+      expose :instance_type?, as: :is_shared
       expose :name
       expose :online?, as: :online
       expose :status
@@ -1004,7 +1163,8 @@ module API
       expose :access_level
       expose :version, :revision, :platform, :architecture
       expose :contacted_at
-      expose :token, if: lambda { |runner, options| options[:current_user].admin? || !runner.is_shared? }
+      expose :token, if: lambda { |runner, options| options[:current_user].admin? || !runner.instance_type? }
+      # rubocop: disable CodeReuse/ActiveRecord
       expose :projects, with: Entities::BasicProjectDetails do |runner, options|
         if options[:current_user].admin?
           runner.projects
@@ -1012,6 +1172,8 @@ module API
           options[:current_user].authorized_projects.where(id: runner.projects)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
+      # rubocop: disable CodeReuse/ActiveRecord
       expose :groups, with: Entities::BasicGroupDetails do |runner, options|
         if options[:current_user].admin?
           runner.groups
@@ -1019,6 +1181,7 @@ module API
           options[:current_user].authorized_groups.where(id: runner.groups)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
 
     class RunnerRegistrationDetails < Grape::Entity
@@ -1026,7 +1189,12 @@ module API
     end
 
     class JobArtifactFile < Grape::Entity
-      expose :filename, :size
+      expose :filename
+      expose :cached_size, as: :size
+    end
+
+    class JobArtifact < Grape::Entity
+      expose :file_type, :size, :filename, :file_format
     end
 
     class JobBasic < Grape::Entity
@@ -1036,10 +1204,16 @@ module API
       expose :user, with: User
       expose :commit, with: Commit
       expose :pipeline, with: PipelineBasic
+
+      expose :web_url do |job, _options|
+        Gitlab::Routing.url_helpers.project_job_url(job.project, job)
+      end
     end
 
     class Job < JobBasic
+      # artifacts_file is included in job_artifacts, but kept for backward compatibility (remove in api/v5)
       expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? }
+      expose :job_artifacts, as: :artifacts, using: JobArtifact
       expose :runner, with: Runner
       expose :artifacts_expire_at
     end
@@ -1049,8 +1223,11 @@ module API
     end
 
     class Trigger < Grape::Entity
+      include ::API::Helpers::Presentable
+
       expose :id
-      expose :token, :description
+      expose :token
+      expose :description
       expose :created_at, :updated_at, :last_used
       expose :owner, using: Entities::UserBasic
     end
@@ -1096,11 +1273,14 @@ module API
       expose :deployable,  using: Entities::Job
     end
 
-    class License < Grape::Entity
+    class LicenseBasic < Grape::Entity
       expose :key, :name, :nickname
-      expose :featured, as: :popular
       expose :url, as: :html_url
       expose(:source_url) { |license| license.meta['source'] }
+    end
+
+    class License < LicenseBasic
+      expose :popular?, as: :popular
       expose(:description) { |license| license.meta['description'] }
       expose(:conditions) { |license| license.meta['conditions'] }
       expose(:permissions) { |license| license.meta['permissions'] }
@@ -1109,6 +1289,7 @@ module API
     end
 
     class TemplatesList < Grape::Entity
+      expose :key
       expose :name
     end
 
@@ -1133,7 +1314,11 @@ module API
       expose :token
     end
 
-    class ImpersonationToken < PersonalAccessTokenWithToken
+    class ImpersonationToken < PersonalAccessToken
+      expose :impersonation
+    end
+
+    class ImpersonationTokenWithToken < PersonalAccessTokenWithToken
       expose :impersonation
     end
 
@@ -1178,6 +1363,7 @@ module API
 
       class RunnerInfo < Grape::Entity
         expose :metadata_timeout, as: :timeout
+        expose :runner_session_url
       end
 
       class Step < Grape::Entity
@@ -1193,7 +1379,13 @@ module API
       end
 
       class Artifacts < Grape::Entity
-        expose :name, :untracked, :paths, :when, :expire_in
+        expose :name
+        expose :untracked
+        expose :paths
+        expose :when
+        expose :expire_in
+        expose :artifact_type
+        expose :artifact_format
       end
 
       class Cache < Grape::Entity
@@ -1244,12 +1436,6 @@ module API
       expose :submitted, as: :akismet_submitted
     end
 
-    class RepositoryStorageHealth < Grape::Entity
-      expose :storage_name
-      expose :failing_on_hosts
-      expose :total_failures
-    end
-
     class CustomAttribute < Grape::Entity
       expose :key
       expose :value
@@ -1298,7 +1484,9 @@ module API
     end
 
     class Application < Grape::Entity
+      expose :id
       expose :uid, as: :application_id
+      expose :name, as: :application_name
       expose :redirect_uri, as: :callback_url
     end
 
@@ -1334,5 +1522,64 @@ module API
         badge.type == 'ProjectBadge' ? 'project' : 'group'
       end
     end
+
+    class ResourceLabelEvent < Grape::Entity
+      expose :id
+      expose :user, using: Entities::UserBasic
+      expose :created_at
+      expose :resource_type do |event, options|
+        event.issuable.class.name
+      end
+      expose :resource_id do |event, options|
+        event.issuable.id
+      end
+      expose :label, using: Entities::LabelBasic
+      expose :action
+    end
+
+    class Suggestion < Grape::Entity
+      expose :id
+      expose :from_original_line
+      expose :to_original_line
+      expose :from_line
+      expose :to_line
+      expose :appliable?, as: :appliable
+      expose :applied
+      expose :from_content
+      expose :to_content
+    end
+
+    module Platform
+      class Kubernetes < Grape::Entity
+        expose :api_url
+        expose :namespace
+        expose :authorization_type
+        expose :ca_cert
+      end
+    end
+
+    module Provider
+      class Gcp < Grape::Entity
+        expose :cluster_id
+        expose :status_name
+        expose :gcp_project_id
+        expose :zone
+        expose :machine_type
+        expose :num_nodes
+        expose :endpoint
+      end
+    end
+
+    class Cluster < Grape::Entity
+      expose :id, :name, :created_at
+      expose :provider_type, :platform_type, :environment_scope, :cluster_type
+      expose :user, using: Entities::UserBasic
+      expose :platform_kubernetes, using: Entities::Platform::Kubernetes
+      expose :provider_gcp, using: Entities::Provider::Gcp
+    end
+
+    class ClusterProject < Cluster
+      expose :project, using: Entities::BasicProjectDetails
+    end
   end
 end
diff --git a/lib/api/entities/container_registry.rb b/lib/api/entities/container_registry.rb
new file mode 100644
index 00000000000..00833ca7480
--- /dev/null
+++ b/lib/api/entities/container_registry.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module API
+  module Entities
+    module ContainerRegistry
+      class Repository < Grape::Entity
+        expose :id
+        expose :name
+        expose :path
+        expose :location
+        expose :created_at
+      end
+
+      class Tag < Grape::Entity
+        expose :name
+        expose :path
+        expose :location
+      end
+
+      class TagDetails < Tag
+        expose :revision
+        expose :short_revision
+        expose :digest
+        expose :created_at
+        expose :total_size
+      end
+    end
+  end
+end
diff --git a/lib/api/environments.rb b/lib/api/environments.rb
index 5c63ec028d9..0278c6c54a5 100644
--- a/lib/api/environments.rb
+++ b/lib/api/environments.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   # Environments RESTfull API endpoints
   class Environments < Grape::API
@@ -9,7 +11,7 @@ module API
     params do
       requires :id, type: String, desc: 'The project ID'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Get all environments of the project' do
         detail 'This feature was introduced in GitLab 8.11.'
         success Entities::Environment
@@ -72,7 +74,7 @@ module API
         success Entities::Environment
       end
       params do
-        requires :environment_id, type: Integer,  desc: 'The environment ID'
+        requires :environment_id, type: Integer, desc: 'The environment ID'
       end
       delete ':id/environments/:environment_id' do
         authorize! :update_environment, user_project
@@ -86,12 +88,13 @@ module API
         success Entities::Environment
       end
       params do
-        requires :environment_id, type: Integer,  desc: 'The environment ID'
+        requires :environment_id, type: Integer, desc: 'The environment ID'
       end
       post ':id/environments/:environment_id/stop' do
-        authorize! :create_deployment, user_project
+        authorize! :read_environment, user_project
 
         environment = user_project.environments.find(params[:environment_id])
+        authorize! :stop_environment, environment
 
         environment.stop_with_action!(current_user)
 
diff --git a/lib/api/events.rb b/lib/api/events.rb
index b0713ff1d54..b98aa9f31e1 100644
--- a/lib/api/events.rb
+++ b/lib/api/events.rb
@@ -1,6 +1,9 @@
+# frozen_string_literal: true
+
 module API
   class Events < Grape::API
     include PaginationParams
+    include APIGuard
 
     helpers do
       params :event_filter_params do
@@ -16,13 +19,19 @@ module API
       end
 
       def present_events(events)
-        events = events.reorder(created_at: params[:sort])
+        events = paginate(events)
+
+        present events, with: Entities::Event
+      end
 
-        present paginate(events), with: Entities::Event
+      def find_events(source)
+        EventsFinder.new(params.merge(source: source, current_user: current_user, with_associations: true)).execute
       end
     end
 
     resource :events do
+      allow_access_with_scope :read_user, if: -> (request) { request.get? }
+
       desc "List currently authenticated user's events" do
         detail 'This feature was introduced in GitLab 9.3.'
         success Entities::Event
@@ -32,10 +41,11 @@ module API
         use :event_filter_params
         use :sort_params
       end
+
       get do
         authenticate!
 
-        events = EventsFinder.new(params.merge(source: current_user, current_user: current_user)).execute.preload(:author, :target)
+        events = find_events(current_user)
 
         present_events(events)
       end
@@ -45,6 +55,8 @@ module API
       requires :id, type: String, desc: 'The ID or Username of the user'
     end
     resource :users do
+      allow_access_with_scope :read_user, if: -> (request) { request.get? }
+
       desc 'Get the contribution events of a specified user' do
         detail 'This feature was introduced in GitLab 8.13.'
         success Entities::Event
@@ -54,11 +66,12 @@ module API
         use :event_filter_params
         use :sort_params
       end
+
       get ':id/events' do
         user = find_user(params[:id])
         not_found!('User') unless user
 
-        events = EventsFinder.new(params.merge(source: user, current_user: current_user)).execute.preload(:author, :target)
+        events = find_events(user)
 
         present_events(events)
       end
@@ -67,7 +80,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc "List a Project's visible events" do
         success Entities::Event
       end
@@ -76,8 +89,9 @@ module API
         use :event_filter_params
         use :sort_params
       end
+
       get ":id/events" do
-        events = EventsFinder.new(params.merge(source: user_project, current_user: current_user)).execute.preload(:author, :target)
+        events = find_events(user_project)
 
         present_events(events)
       end
diff --git a/lib/api/features.rb b/lib/api/features.rb
index 11d848584d9..835aac05905 100644
--- a/lib/api/features.rb
+++ b/lib/api/features.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Features < Grape::API
     before { authenticated_as_admin! }
@@ -15,11 +17,11 @@ module API
       end
 
       def gate_targets(params)
-        targets = []
-        targets << Feature.group(params[:feature_group]) if params[:feature_group]
-        targets << User.find_by_username(params[:user]) if params[:user]
+        Feature::Target.new(params).targets
+      end
 
-        targets
+      def gate_specified?(params)
+        Feature::Target.new(params).gate_specified?
       end
     end
 
@@ -40,6 +42,7 @@ module API
         requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time'
         optional :feature_group, type: String, desc: 'A Feature group name'
         optional :user, type: String, desc: 'A GitLab username'
+        optional :project, type: String, desc: 'A projects path, like gitlab-org/gitlab-ce'
       end
       post ':name' do
         feature = Feature.get(params[:name])
@@ -48,13 +51,13 @@ module API
 
         case value
         when true
-          if targets.present?
+          if gate_specified?(params)
             targets.each { |target| feature.enable(target) }
           else
             feature.enable
           end
         when false
-          if targets.present?
+          if gate_specified?(params)
             targets.each { |target| feature.disable(target) }
           else
             feature.disable
diff --git a/lib/api/files.rb b/lib/api/files.rb
index 1598d3c00b8..ca59d330e1c 100644
--- a/lib/api/files.rb
+++ b/lib/api/files.rb
@@ -1,10 +1,16 @@
+# frozen_string_literal: true
+
 module API
   class Files < Grape::API
-    FILE_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(file_path: API::NO_SLASH_URL_PART_REGEX)
+    include APIGuard
+
+    FILE_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(file_path: API::NO_SLASH_URL_PART_REGEX)
 
     # Prevents returning plain/text responses for files with .txt extension
     after_validation { content_type "application/json" }
 
+    helpers ::API::Helpers::HeadersHelpers
+
     helpers do
       def commit_params(attrs)
         {
@@ -40,10 +46,24 @@ module API
         }
       end
 
+      def blob_data
+        {
+          file_name: @blob.name,
+          file_path: @blob.path,
+          size: @blob.size,
+          encoding: "base64",
+          content_sha256: Digest::SHA256.hexdigest(@blob.data),
+          ref: params[:ref],
+          blob_id: @blob.id,
+          commit_id: @commit.id,
+          last_commit_id: @repo.last_commit_id_for_path(@commit.sha, params[:file_path])
+        }
+      end
+
       params :simple_file_params do
         requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb'
-        requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.'
-        requires :commit_message, type: String, desc: 'Commit message'
+        requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.', allow_blank: false
+        requires :commit_message, type: String, allow_blank: false, desc: 'Commit message'
         optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from'
         optional :author_email, type: String, desc: 'The email of the author'
         optional :author_name, type: String, desc: 'The name of the author'
@@ -61,36 +81,56 @@ module API
       requires :id, type: String, desc: 'The project ID'
     end
     resource :projects, requirements: FILE_ENDPOINT_REQUIREMENTS do
+      allow_access_with_scope :read_repository, if: -> (request) { request.get? || request.head? }
+
+      desc 'Get raw file metadata from repository'
+      params do
+        requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb'
+        requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false
+      end
+      head ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do
+        assign_file_vars!
+
+        set_http_headers(blob_data)
+      end
+
       desc 'Get raw file contents from the repository'
       params do
         requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb'
-        requires :ref, type: String, desc: 'The name of branch, tag commit'
+        requires :ref, type: String, desc: 'The name of branch, tag commit', allow_blank: false
       end
       get ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do
         assign_file_vars!
 
+        set_http_headers(blob_data)
+
         send_git_blob @repo, @blob
       end
 
+      desc 'Get file metadata from repository'
+      params do
+        requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb'
+        requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false
+      end
+      head ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do
+        assign_file_vars!
+
+        set_http_headers(blob_data)
+      end
+
       desc 'Get a file from the repository'
       params do
         requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb'
-        requires :ref, type: String, desc: 'The name of branch, tag or commit'
+        requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false
       end
       get ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do
         assign_file_vars!
 
-        {
-          file_name: @blob.name,
-          file_path: @blob.path,
-          size: @blob.size,
-          encoding: "base64",
-          content: Base64.strict_encode64(@blob.data),
-          ref: params[:ref],
-          blob_id: @blob.id,
-          commit_id: @commit.id,
-          last_commit_id: @repo.last_commit_id_for_path(@commit.sha, params[:file_path])
-        }
+        data = blob_data
+
+        set_http_headers(data)
+
+        data.merge(content: Base64.strict_encode64(@blob.data))
       end
 
       desc 'Create new file in repository'
diff --git a/lib/api/group_boards.rb b/lib/api/group_boards.rb
index aa9fff25fc8..9a20ee8c8b9 100644
--- a/lib/api/group_boards.rb
+++ b/lib/api/group_boards.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class GroupBoards < Grape::API
     include BoardsResponses
@@ -17,7 +19,7 @@ module API
       requires :id, type: String, desc: 'The ID of a group'
     end
 
-    resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       segment ':id/boards' do
         desc 'Find a group board' do
           detail 'This feature was introduced in 10.6'
@@ -70,12 +72,10 @@ module API
           success Entities::List
         end
         params do
-          requires :label_id, type: Integer, desc: 'The ID of an existing label'
+          use :list_creation_params
         end
         post '/lists' do
-          unless available_labels_for(board_parent).exists?(params[:label_id])
-            render_api_error!({ error: 'Label not found!' }, 400)
-          end
+          authorize_list_type_resource!
 
           authorize!(:admin_list, user_group)
 
diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb
index 93fa0b95857..d4287e4a7c4 100644
--- a/lib/api/group_milestones.rb
+++ b/lib/api/group_milestones.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class GroupMilestones < Grape::API
     include MilestoneResponses
@@ -10,7 +12,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a group'
     end
-    resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Get a list of group milestones' do
         success Entities::Milestone
       end
@@ -41,7 +43,7 @@ module API
         use :optional_params
       end
       post ":id/milestones" do
-        authorize! :admin_milestones, user_group
+        authorize! :admin_milestone, user_group
 
         create_milestone_for(user_group)
       end
@@ -53,11 +55,21 @@ module API
         use :update_params
       end
       put ":id/milestones/:milestone_id" do
-        authorize! :admin_milestones, user_group
+        authorize! :admin_milestone, user_group
 
         update_milestone_for(user_group)
       end
 
+      desc 'Remove a project milestone'
+      delete ":id/milestones/:milestone_id" do
+        authorize! :admin_milestone, user_group
+
+        milestone = user_group.milestones.find(params[:milestone_id])
+        Milestones::DestroyService.new(user_group, current_user).execute(milestone)
+
+        status(204)
+      end
+
       desc 'Get all issues for a single group milestone' do
         success Entities::IssueBasic
       end
diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb
index 55d5c7f1606..3f048e0dc56 100644
--- a/lib/api/group_variables.rb
+++ b/lib/api/group_variables.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class GroupVariables < Grape::API
     include PaginationParams
@@ -9,7 +11,7 @@ module API
       requires :id, type: String, desc: 'The ID of a group'
     end
 
-    resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Get group-level variables' do
         success Entities::Variable
       end
@@ -27,6 +29,7 @@ module API
       params do
         requires :key, type: String, desc: 'The key of the variable'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ':id/variables/:key' do
         key = params[:key]
         variable = user_group.variables.find_by(key: key)
@@ -35,6 +38,7 @@ module API
 
         present variable, with: Entities::Variable
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Create a new variable in a group' do
         success Entities::Variable
@@ -64,6 +68,7 @@ module API
         optional :value, type: String, desc: 'The value of the variable'
         optional :protected, type: String, desc: 'Whether the variable is protected'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       put ':id/variables/:key' do
         variable = user_group.variables.find_by(key: params[:key])
 
@@ -77,6 +82,7 @@ module API
           render_validation_error!(variable)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Delete an existing variable from a group' do
         success Entities::Variable
@@ -84,12 +90,14 @@ module API
       params do
         requires :key, type: String, desc: 'The key of the variable'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete ':id/variables/:key' do
         variable = user_group.variables.find_by(key: params[:key])
         not_found!('GroupVariable') unless variable
 
         destroy_conditionally!(variable)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 03b6b30a0d8..64958ff982a 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Groups < Grape::API
     include PaginationParams
@@ -32,13 +34,15 @@ module API
         optional :all_available, type: Boolean, desc: 'Show all group that you have access to'
         optional :search, type: String, desc: 'Search for a specific group'
         optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user'
-        optional :order_by, type: String, values: %w[name path], default: 'name', desc: 'Order by name or path'
+        optional :order_by, type: String, values: %w[name path id], default: 'name', desc: 'Order by name, path or id'
         optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)'
+        optional :min_access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'Minimum access level of authenticated user'
         use :pagination
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def find_groups(params, parent_id = nil)
-        find_params = params.slice(:all_available, :custom_attributes, :owned)
+        find_params = params.slice(:all_available, :custom_attributes, :owned, :min_access_level)
         find_params[:parent] = find_group!(parent_id) if parent_id
         find_params[:all_available] =
           find_params.fetch(:all_available, current_user&.full_private_access?)
@@ -46,14 +50,29 @@ module API
         groups = GroupsFinder.new(current_user, find_params).execute
         groups = groups.search(params[:search]) if params[:search].present?
         groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?
-        groups = groups.reorder(params[:order_by] => params[:sort])
+        order_options = { params[:order_by] => params[:sort] }
+        order_options["id"] ||= "asc"
+        groups = groups.reorder(order_options)
 
         groups
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def find_group_projects(params)
         group = find_group!(params[:id])
-        projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute
+        options = {
+          only_owned: !params[:with_shared],
+          include_subgroups: params[:include_subgroups]
+        }
+
+        projects = GroupProjectsFinder.new(
+          group: group,
+          current_user: current_user,
+          params: project_finder_params,
+          options: options
+        ).execute
+        projects = projects.with_issues_available_for_user(current_user) if params[:with_issues_enabled]
+        projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled]
         projects = reorder_projects(projects)
         paginate(projects)
       end
@@ -94,7 +113,7 @@ module API
         requires :name, type: String, desc: 'The name of the group'
         requires :path, type: String, desc: 'The path of the group'
 
-        if ::Group.supports_nested_groups?
+        if ::Group.supports_nested_objects?
           optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group'
         end
 
@@ -121,7 +140,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a group'
     end
-    resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Update a group. Available only for users who can administrate groups.' do
         success Entities::Group
       end
@@ -146,12 +165,13 @@ module API
       end
       params do
         use :with_custom_attributes
+        optional :with_projects, type: Boolean, default: true, desc: 'Omit project details'
       end
       get ":id" do
         group = find_group!(params[:id])
 
         options = {
-          with: Entities::GroupDetail,
+          with: params[:with_projects] ? Entities::GroupDetail : Entities::Group,
           current_user: current_user
         }
 
@@ -189,6 +209,10 @@ module API
                           desc: 'Return only the ID, URL, name, and path of each project'
         optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user'
         optional :starred, type: Boolean, default: false, desc: 'Limit by starred status'
+        optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature'
+        optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature'
+        optional :with_shared, type: Boolean, default: true, desc: 'Include projects shared to this group'
+        optional :include_subgroups, type: Boolean, default: false, desc: 'Includes projects in subgroups of this group'
 
         use :pagination
         use :with_custom_attributes
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 2ed331d4fd2..fa6c9777824 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   module Helpers
     include Gitlab::Utils
@@ -95,20 +97,20 @@ module API
     end
 
     def find_user(id)
-      if id =~ /^\d+$/
-        User.find_by(id: id)
-      else
-        User.find_by(username: id)
-      end
+      UserFinder.new(id).find_by_id_or_username
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def find_project(id)
+      projects = Project.without_deleted
+
       if id.is_a?(Integer) || id =~ /^\d+$/
-        Project.find_by(id: id)
+        projects.find_by(id: id)
       elsif id.include?("/")
-        Project.find_by_full_path(id)
+        projects.find_by_full_path(id)
       end
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     def find_project!(id)
       project = find_project(id)
@@ -120,6 +122,7 @@ module API
       end
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def find_group(id)
       if id.to_s =~ /^\d+$/
         Group.find_by(id: id)
@@ -127,6 +130,7 @@ module API
         Group.find_by_full_path(id)
       end
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     def find_group!(id)
       group = find_group(id)
@@ -138,6 +142,7 @@ module API
       end
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def find_namespace(id)
       if id.to_s =~ /^\d+$/
         Namespace.find_by(id: id)
@@ -145,6 +150,7 @@ module API
         Namespace.find_by_full_path(id)
       end
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     def find_namespace!(id)
       namespace = find_namespace(id)
@@ -156,6 +162,14 @@ module API
       end
     end
 
+    def find_branch!(branch_name)
+      if Gitlab::GitRefValidator.validate(branch_name)
+        user_project.repository.find_branch(branch_name) || not_found!('Branch')
+      else
+        render_api_error!('The branch refname is invalid', 400)
+      end
+    end
+
     def find_project_label(id)
       labels = available_labels_for(user_project)
       label = labels.find_by_id(id) || labels.find_by_title(id)
@@ -163,13 +177,17 @@ module API
       label || not_found!('Label')
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def find_project_issue(iid)
       IssuesFinder.new(current_user, project_id: user_project.id).find_by!(iid: iid)
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def find_project_merge_request(iid)
       MergeRequestsFinder.new(current_user, project_id: user_project.id).find_by!(iid: iid)
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     def find_project_commit(id)
       user_project.commit_by(oid: id)
@@ -180,11 +198,13 @@ module API
       SnippetsFinder.new(current_user, finder_params).find(id)
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def find_merge_request_with_access(iid, access_level = :read_merge_request)
       merge_request = user_project.merge_requests.find_by!(iid: iid)
       authorize! access_level, merge_request
       merge_request
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     def find_build!(id)
       user_project.builds.find(id.to_i)
@@ -215,8 +235,8 @@ module API
       forbidden! unless current_user.admin?
     end
 
-    def authorize!(action, subject = :global)
-      forbidden! unless can?(current_user, action, subject)
+    def authorize!(action, subject = :global, reason = nil)
+      forbidden!(reason) unless can?(current_user, action, subject)
     end
 
     def authorize_push_project
@@ -272,12 +292,15 @@ module API
           attrs[key] = params_hash[key]
         end
       end
-      ActionController::Parameters.new(attrs).permit!
+      permitted_attrs = ActionController::Parameters.new(attrs).permit!
+      permitted_attrs.to_h
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def filter_by_iid(items, iid)
       items.where(iid: iid)
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     def filter_by_search(items, text)
       items.search(text)
@@ -347,18 +370,19 @@ module API
     end
 
     def handle_api_exception(exception)
-      if sentry_enabled? && report_exception?(exception)
+      if report_exception?(exception)
         define_params_for_grape_middleware
-        sentry_context
-        Raven.capture_exception(exception, extra: params)
+        Gitlab::Sentry.context(current_user)
+        Gitlab::Sentry.track_acceptable_exception(exception, extra: params)
       end
 
       # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60
       trace = exception.backtrace
 
-      message = "\n#{exception.class} (#{exception.message}):\n"
+      message = ["\n#{exception.class} (#{exception.message}):\n"]
       message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code)
       message << "  " << trace.join("\n  ")
+      message = message.join
 
       API.logger.add Logger::FATAL, message
 
@@ -374,20 +398,23 @@ module API
 
     # project helpers
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def reorder_projects(projects)
       projects.reorder(params[:order_by] => params[:sort])
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     def project_finder_params
-      finder_params = {}
+      finder_params = { without_deleted: true }
       finder_params[:owned] = true if params[:owned].present?
       finder_params[:non_public] = true if params[:membership].present?
       finder_params[:starred] = true if params[:starred].present?
       finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility]
-      finder_params[:archived] = params[:archived]
+      finder_params[:archived] = archived_param unless params[:archived].nil?
       finder_params[:search] = params[:search] if params[:search]
       finder_params[:user] = params.delete(:user) if params[:user]
       finder_params[:custom_attributes] = params[:custom_attributes] if params[:custom_attributes]
+      finder_params[:min_access_level] = params[:min_access_level] if params[:min_access_level]
       finder_params
     end
 
@@ -469,6 +496,11 @@ module API
     def send_git_blob(repository, blob)
       env['api.format'] = :txt
       content_type 'text/plain'
+      header['Content-Disposition'] = content_disposition('inline', blob.name)
+
+      # Let Workhorse examine the content and determine the better content disposition
+      header[Gitlab::Workhorse::DETECT_HEADER] = "true"
+
       header(*Gitlab::Workhorse.send_git_blob(repository, blob))
     end
 
@@ -484,7 +516,7 @@ module API
     # `request`. We workaround this by defining methods that returns the right
     # values.
     def define_params_for_grape_middleware
-      self.define_singleton_method(:request) { Rack::Request.new(env) }
+      self.define_singleton_method(:request) { ActionDispatch::Request.new(env) }
       self.define_singleton_method(:params) { request.params.symbolize_keys }
     end
 
@@ -495,5 +527,17 @@ module API
 
       exception.status == 500
     end
+
+    def archived_param
+      return 'only' if params[:archived]
+
+      params[:archived]
+    end
+
+    def content_disposition(disposition, filename)
+      disposition += %(; filename=#{filename.inspect}) if filename.present?
+
+      disposition
+    end
   end
 end
diff --git a/lib/api/helpers/badges_helpers.rb b/lib/api/helpers/badges_helpers.rb
index 1f8afbf3c90..46ce5b4e7b5 100644
--- a/lib/api/helpers/badges_helpers.rb
+++ b/lib/api/helpers/badges_helpers.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   module Helpers
     module BadgesHelpers
diff --git a/lib/api/helpers/common_helpers.rb b/lib/api/helpers/common_helpers.rb
index 9993caa5249..7551ca50a7f 100644
--- a/lib/api/helpers/common_helpers.rb
+++ b/lib/api/helpers/common_helpers.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   module Helpers
     module CommonHelpers
diff --git a/lib/api/helpers/custom_attributes.rb b/lib/api/helpers/custom_attributes.rb
index 10d652e33f5..88208226c40 100644
--- a/lib/api/helpers/custom_attributes.rb
+++ b/lib/api/helpers/custom_attributes.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   module Helpers
     module CustomAttributes
@@ -12,6 +14,7 @@ module API
                                          desc: 'Filter with custom attributes'
           end
 
+          # rubocop: disable CodeReuse/ActiveRecord
           def with_custom_attributes(collection_or_resource, options = {})
             options = options.merge(
               with_custom_attributes: params[:with_custom_attributes] &&
@@ -24,6 +27,7 @@ module API
 
             [collection_or_resource, options]
           end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb
index dd4f6c41131..1058f4e8a5e 100644
--- a/lib/api/helpers/custom_validators.rb
+++ b/lib/api/helpers/custom_validators.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   module Helpers
     module CustomValidators
@@ -8,8 +10,21 @@ module API
           raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], message: message(:absence)
         end
       end
+
+      class IntegerNoneAny < Grape::Validations::Base
+        def validate_param!(attr_name, params)
+          value = params[attr_name]
+
+          return if value.is_a?(Integer) ||
+              [IssuableFinder::FILTER_NONE, IssuableFinder::FILTER_ANY].include?(value.to_s.downcase)
+
+          raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)],
+                                               message: "should be an integer, 'None' or 'Any'"
+        end
+      end
     end
   end
 end
 
 Grape::Validations.register_validator(:absence, ::API::Helpers::CustomValidators::Absence)
+Grape::Validations.register_validator(:integer_none_any, ::API::Helpers::CustomValidators::IntegerNoneAny)
diff --git a/lib/api/helpers/headers_helpers.rb b/lib/api/helpers/headers_helpers.rb
new file mode 100644
index 00000000000..7553af9d156
--- /dev/null
+++ b/lib/api/helpers/headers_helpers.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module API
+  module Helpers
+    module HeadersHelpers
+      def set_http_headers(header_data)
+        header_data.each do |key, value|
+          if value.is_a?(Enumerable)
+            raise ArgumentError.new("Header value should be a string")
+          end
+
+          header "X-Gitlab-#{key.to_s.split('_').collect(&:capitalize).join('-')}", value.to_s
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb
index 83151be82ad..4eaaca96b49 100644
--- a/lib/api/helpers/internal_helpers.rb
+++ b/lib/api/helpers/internal_helpers.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   module Helpers
     module InternalHelpers
diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb
index a50ea0b52aa..73d58ee7f37 100644
--- a/lib/api/helpers/members_helpers.rb
+++ b/lib/api/helpers/members_helpers.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # rubocop:disable GitlabSecurity/PublicSend
 
 module API
@@ -10,6 +12,35 @@ module API
       def authorize_admin_source!(source_type, source)
         authorize! :"admin_#{source_type}", source
       end
+
+      def find_all_members(source_type, source)
+        members = source_type == 'project' ? find_all_members_for_project(source) : find_all_members_for_group(source)
+        members.non_invite
+          .non_request
+      end
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def find_all_members_for_project(project)
+        shared_group_ids = project.project_group_links.pluck(:group_id)
+        project_group_ids = project.group&.self_and_ancestors&.pluck(:id)
+        source_ids = [project.id, project_group_ids, shared_group_ids]
+          .flatten
+          .compact
+        Member.includes(:user)
+          .joins(user: :project_authorizations)
+          .where(project_authorizations: { project_id: project.id })
+          .where(source_id: source_ids)
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def find_all_members_for_group(group)
+        source_ids = group.self_and_ancestors.pluck(:id)
+        Member.includes(:user)
+          .where(source_id: source_ids)
+          .where(source_type: 'Namespace')
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb
index b4bfb677d72..216b2c45741 100644
--- a/lib/api/helpers/notes_helpers.rb
+++ b/lib/api/helpers/notes_helpers.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   module Helpers
     module NotesHelpers
@@ -92,10 +94,9 @@ module API
 
         parent = noteable_parent(noteable)
 
-        if opts[:created_at]
-          opts.delete(:created_at) unless
-            current_user.admin? || parent.owned_by?(current_user)
-        end
+        opts.delete(:created_at) unless current_user.can?(:set_note_created_at, policy_object)
+
+        opts[:updated_at] = opts[:created_at] if opts[:created_at]
 
         project = parent if parent.is_a?(Project)
         ::Notes::CreateService.new(project, current_user, opts).execute
diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb
index 3308212216e..de59c915d66 100644
--- a/lib/api/helpers/pagination.rb
+++ b/lib/api/helpers/pagination.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   module Helpers
     module Pagination
@@ -91,6 +93,7 @@ module API
           @request_context = request_context
         end
 
+        # rubocop: disable CodeReuse/ActiveRecord
         def paginate(relation)
           pagination = KeysetPaginationInfo.new(relation, request_context)
 
@@ -112,6 +115,7 @@ module API
 
           paged_relation
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         private
 
@@ -142,7 +146,7 @@ module API
         end
 
         def add_default_pagination_headers
-          header 'X-Per-Page',    per_page.to_s
+          header 'X-Per-Page', per_page.to_s
         end
 
         def add_navigation_links(next_page_params)
@@ -174,15 +178,27 @@ module API
         end
 
         def paginate(relation)
-          relation = add_default_order(relation)
-
-          relation.page(params[:page]).per(params[:per_page]).tap do |data|
+          paginate_with_limit_optimization(add_default_order(relation)).tap do |data|
             add_pagination_headers(data)
           end
         end
 
         private
 
+        def paginate_with_limit_optimization(relation)
+          pagination_data = relation.page(params[:page]).per(params[:per_page])
+          return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation)
+          return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit)
+
+          limited_total_count = pagination_data.total_count_with_limit
+          if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT
+            pagination_data.without_count
+          else
+            pagination_data
+          end
+        end
+
+        # rubocop: disable CodeReuse/ActiveRecord
         def add_default_order(relation)
           if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty?
             relation = relation.order(:id)
@@ -190,6 +206,7 @@ module API
 
           relation
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         def add_pagination_headers(paginated_data)
           header 'X-Per-Page',    paginated_data.limit_value.to_s
diff --git a/lib/api/helpers/presentable.rb b/lib/api/helpers/presentable.rb
new file mode 100644
index 00000000000..973c2132efe
--- /dev/null
+++ b/lib/api/helpers/presentable.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module API
+  module Helpers
+    ##
+    # This module makes it possible to use `app/presenters` with
+    # Grape Entities. It instantiates model presenter and passes
+    # options defined in the API endpoint to the presenter itself.
+    #
+    #   present object, with: Entities::Something,
+    #                   current_user: current_user,
+    #                   another_option: 'my options'
+    #
+    # Example above will make `current_user` and `another_option`
+    # values available in the subclass of `Gitlab::View::Presenter`
+    # thorough a separate method in the presenter.
+    #
+    # The model class needs to have `::Presentable` module mixed in
+    # if you want to use `API::Helpers::Presentable`.
+    #
+    module Presentable
+      extend ActiveSupport::Concern
+
+      def initialize(object, options = {})
+        super(object.present(options), options)
+      end
+    end
+  end
+end
diff --git a/lib/api/helpers/project_snapshots_helpers.rb b/lib/api/helpers/project_snapshots_helpers.rb
index 94798a8cb51..1b5dc281e38 100644
--- a/lib/api/helpers/project_snapshots_helpers.rb
+++ b/lib/api/helpers/project_snapshots_helpers.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   module Helpers
     module ProjectSnapshotsHelpers
diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb
index 381d5e8968c..e6a72b949f9 100644
--- a/lib/api/helpers/projects_helpers.rb
+++ b/lib/api/helpers/projects_helpers.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   module Helpers
     module ProjectsHelpers
@@ -26,6 +28,7 @@ module API
             optional :avatar, type: File, desc: 'Avatar image for project'
             optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line'
             optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests'
+            optional :initialize_with_readme, type: Boolean, desc: "Initialize a project with a README.md"
           end
 
           params :optional_project_params do
diff --git a/lib/api/helpers/related_resources_helpers.rb b/lib/api/helpers/related_resources_helpers.rb
index bc7333ca4b3..793ae11b41d 100644
--- a/lib/api/helpers/related_resources_helpers.rb
+++ b/lib/api/helpers/related_resources_helpers.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   module Helpers
     module RelatedResourcesHelpers
diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb
index 35ac0b4cbca..16df8e830e1 100644
--- a/lib/api/helpers/runner.rb
+++ b/lib/api/helpers/runner.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   module Helpers
     module Runner
@@ -24,7 +26,7 @@ module API
       end
 
       def get_runner_ip
-        { ip_address: request.ip }
+        { ip_address: request.env["HTTP_X_FORWARDED_FOR"] || request.ip }
       end
 
       def current_runner
@@ -59,6 +61,11 @@ module API
       def max_artifacts_size
         Gitlab::CurrentSettings.max_artifacts_size.megabytes.to_i
       end
+
+      def job_forbidden!(job, reason)
+        header 'Job-Status', job.status
+        forbidden!(reason)
+      end
     end
   end
 end
diff --git a/lib/api/helpers/version.rb b/lib/api/helpers/version.rb
new file mode 100644
index 00000000000..7f53094e90c
--- /dev/null
+++ b/lib/api/helpers/version.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module API
+  module Helpers
+    class Version
+      include Helpers::RelatedResourcesHelpers
+
+      def initialize(version)
+        @version = version.to_s
+
+        unless API.versions.include?(version)
+          raise ArgumentError, 'Unknown API version!'
+        end
+      end
+
+      def root_path
+        File.join('/', API.prefix.to_s, @version)
+      end
+
+      def root_url
+        @root_url ||= expose_url(root_path)
+      end
+
+      def to_s
+        @version
+      end
+    end
+  end
+end
diff --git a/lib/api/import_github.rb b/lib/api/import_github.rb
new file mode 100644
index 00000000000..bb4e536cf57
--- /dev/null
+++ b/lib/api/import_github.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module API
+  class ImportGithub < Grape::API
+    rescue_from Octokit::Unauthorized, with: :provider_unauthorized
+
+    helpers do
+      def client
+        @client ||= Gitlab::LegacyGithubImport::Client.new(params[:personal_access_token], client_options)
+      end
+
+      def access_params
+        { github_access_token: params[:personal_access_token] }
+      end
+
+      def client_options
+        {}
+      end
+
+      def provider
+        :github
+      end
+    end
+
+    desc 'Import a GitHub project' do
+      detail 'This feature was introduced in GitLab 11.3.4.'
+      success Entities::ProjectEntity
+    end
+    params do
+      requires :personal_access_token, type: String, desc: 'GitHub personal access token'
+      requires :repo_id, type: Integer, desc: 'GitHub repository ID'
+      optional :new_name, type: String, desc: 'New repo name'
+      requires :target_namespace, type: String, desc: 'Namespace to import repo into'
+    end
+    post 'import/github' do
+      result = Import::GithubService.new(client, current_user, params).execute(access_params, provider)
+
+      if result[:status] == :success
+        present ProjectSerializer.new.represent(result[:project])
+      else
+        status result[:http_status]
+        { errors: result[:message] }
+      end
+    end
+  end
+end
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index a9803be9f69..9488b3469d9 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   # Internal access API
   class Internal < Grape::API
@@ -6,19 +8,28 @@ module API
     helpers ::API::Helpers::InternalHelpers
     helpers ::Gitlab::Identifier
 
+    UNKNOWN_CHECK_RESULT_ERROR = 'Unknown check result'.freeze
+
+    helpers do
+      def response_with_status(code: 200, success: true, message: nil, **extra_options)
+        status code
+        { status: success, message: message }.merge(extra_options).compact
+      end
+    end
+
     namespace 'internal' do
-      # Check if git command is allowed to project
+      # Check if git command is allowed for project
       #
       # Params:
       #   key_id - ssh key id for Git over SSH
-      #   user_id - user id for Git over HTTP
+      #   user_id - user id for Git over HTTP or over SSH in keyless SSH CERT mode
+      #   username - user name for Git over SSH in keyless SSH cert mode
       #   protocol - Git access protocol being used, e.g. HTTP or SSH
       #   project - project full_path (not path on disk)
       #   action - git action (git-upload-pack or git-receive-pack)
       #   changes - changes as "oldrev newrev ref", see Gitlab::ChangesList
+      # rubocop: disable CodeReuse/ActiveRecord
       post "/allowed" do
-        status 200
-
         # Stores some Git-specific env thread-safely
         env = parse_env
         Gitlab::Git::HookEnv.set(gl_repository, env) if project
@@ -28,6 +39,8 @@ module API
             Key.find_by(id: params[:key_id])
           elsif params[:user_id]
             User.find_by(id: params[:user_id])
+          elsif params[:username]
+            UserFinder.new(params[:username]).find_by_username
           end
 
         protocol = params[:protocol]
@@ -46,35 +59,65 @@ module API
                     namespace_path: namespace_path, project_path: project_path,
                     redirected_path: redirected_path)
 
-        begin
-          access_checker.check(params[:action], params[:changes])
-          @project ||= access_checker.project
-        rescue Gitlab::GitAccess::UnauthorizedError, Gitlab::GitAccess::NotFoundError => e
-          break { status: false, message: e.message }
-        end
+        check_result = begin
+                         result = access_checker.check(params[:action], params[:changes])
+                         @project ||= access_checker.project
+                         result
+                       rescue Gitlab::GitAccess::UnauthorizedError => e
+                         break response_with_status(code: 401, success: false, message: e.message)
+                       rescue Gitlab::GitAccess::TimeoutError => e
+                         break response_with_status(code: 503, success: false, message: e.message)
+                       rescue Gitlab::GitAccess::NotFoundError => e
+                         break response_with_status(code: 404, success: false, message: e.message)
+                       end
 
         log_user_activity(actor)
 
-        {
-          status: true,
-          gl_repository: gl_repository,
-          gl_username: user&.username,
-
-          # This repository_path is a bogus value but gitlab-shell still requires
-          # its presence. https://gitlab.com/gitlab-org/gitlab-shell/issues/135
-          repository_path: '/',
+        case check_result
+        when ::Gitlab::GitAccessResult::Success
+          payload = {
+            gl_repository: gl_repository,
+            gl_id: Gitlab::GlId.gl_id(user),
+            gl_username: user&.username,
+            git_config_options: [],
+
+            # This repository_path is a bogus value but gitlab-shell still requires
+            # its presence. https://gitlab.com/gitlab-org/gitlab-shell/issues/135
+            repository_path: '/',
+
+            gitaly: gitaly_payload(params[:action])
+          }
+
+          # Custom option for git-receive-pack command
+          receive_max_input_size = Gitlab::CurrentSettings.receive_max_input_size.to_i
+          if receive_max_input_size > 0
+            payload[:git_config_options] << "receive.maxInputSize=#{receive_max_input_size.megabytes}"
+          end
 
-          gitaly: gitaly_payload(params[:action])
-        }
+          response_with_status(**payload)
+        when ::Gitlab::GitAccessResult::CustomAction
+          response_with_status(code: 300, message: check_result.message, payload: check_result.payload)
+        else
+          response_with_status(code: 500, success: false, message: UNKNOWN_CHECK_RESULT_ERROR)
+        end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
+      # rubocop: disable CodeReuse/ActiveRecord
       post "/lfs_authenticate" do
         status 200
 
-        key = Key.find(params[:key_id])
-        key.update_last_used_at
+        if params[:key_id]
+          actor = Key.find(params[:key_id])
+          actor.update_last_used_at
+        elsif params[:user_id]
+          actor = User.find_by(id: params[:user_id])
+          raise ActiveRecord::RecordNotFound.new("No such user id!") unless actor
+        else
+          raise ActiveRecord::RecordNotFound.new("No key_id or user_id passed!")
+        end
 
-        token_handler = Gitlab::LfsToken.new(key)
+        token_handler = Gitlab::LfsToken.new(actor)
 
         {
           username: token_handler.actor_name,
@@ -82,6 +125,7 @@ module API
           repository_http_path: project.http_url_to_repo
         }
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       get "/merge_request_urls" do
         merge_request_urls
@@ -90,6 +134,7 @@ module API
       #
       # Get a ssh key using the fingerprint
       #
+      # rubocop: disable CodeReuse/ActiveRecord
       get "/authorized_keys" do
         fingerprint = params.fetch(:fingerprint) do
           Gitlab::InsecureKeyFingerprint.new(params.fetch(:key)).fingerprint
@@ -98,20 +143,25 @@ module API
         not_found!("Key") if key.nil?
         present key, with: Entities::SSHKey
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       #
-      # Discover user by ssh key or user id
+      # Discover user by ssh key, user id or username
       #
+      # rubocop: disable CodeReuse/ActiveRecord
       get "/discover" do
         if params[:key_id]
           key = Key.find(params[:key_id])
           user = key.user
         elsif params[:user_id]
           user = User.find_by(id: params[:user_id])
+        elsif params[:username]
+          user = UserFinder.new(params[:username]).find_by_username
         end
 
         present user, with: Entities::UserSafe
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       get "/check" do
         {
@@ -138,25 +188,34 @@ module API
         end
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       post '/two_factor_recovery_codes' do
         status 200
 
-        key = Key.find_by(id: params[:key_id])
+        if params[:key_id]
+          key = Key.find_by(id: params[:key_id])
 
-        if key
-          key.update_last_used_at
-        else
-          break { 'success' => false, 'message' => 'Could not find the given key' }
-        end
+          if key
+            key.update_last_used_at
+          else
+            break { 'success' => false, 'message' => 'Could not find the given key' }
+          end
 
-        if key.is_a?(DeployKey)
-          break { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' }
-        end
+          if key.is_a?(DeployKey)
+            break { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' }
+          end
+
+          user = key.user
 
-        user = key.user
+          unless user
+            break { success: false, message: 'Could not find a user for the given key' }
+          end
+        elsif params[:user_id]
+          user = User.find_by(id: params[:user_id])
 
-        unless user
-          break { success: false, message: 'Could not find a user for the given key' }
+          unless user
+            break { success: false, message: 'Could not find the given user' }
+          end
         end
 
         unless user.two_factor_enabled?
@@ -171,6 +230,7 @@ module API
 
         { success: true, recovery_codes: codes }
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       post '/pre_receive' do
         status 200
@@ -196,8 +256,9 @@ module API
 
       post '/post_receive' do
         status 200
+
         PostReceive.perform_async(params[:gl_repository], params[:identifier],
-          params[:changes])
+          params[:changes], params[:push_options].to_a)
         broadcast_message = BroadcastMessage.current&.last&.message
         reference_counter_decreased = Gitlab::ReferenceCounter.new(params[:gl_repository]).decrease
 
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index b64f465ce56..afa3ac80121 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Issues < Grape::API
     include PaginationParams
@@ -6,7 +8,17 @@ module API
 
     helpers ::Gitlab::IssuableMetadata
 
+    # EE::API::Issues would override the following helpers
+    helpers do
+      params :issues_params_ee do
+      end
+
+      params :issue_params_ee do
+      end
+    end
+
     helpers do
+      # rubocop: disable CodeReuse/ActiveRecord
       def find_issues(args = {})
         args = declared_params.merge(args)
 
@@ -16,10 +28,11 @@ module API
         args[:scope] = args[:scope].underscore if args[:scope]
 
         issues = IssuesFinder.new(current_user, args).execute
-          .preload(:assignees, :labels, :notes, :timelogs, :project)
+          .preload(:assignees, :labels, :notes, :timelogs, :project, :author, :closed_by)
 
         issues.reorder(args[:order_by] => args[:sort])
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       params :issues_params do
         optional :labels, type: String, desc: 'Comma-separated list of label names'
@@ -36,14 +49,17 @@ module API
         optional :updated_after, type: DateTime, desc: 'Return issues updated after the specified time'
         optional :updated_before, type: DateTime, desc: 'Return issues updated before the specified time'
         optional :author_id, type: Integer, desc: 'Return issues which are authored by the user with the given ID'
-        optional :assignee_id, type: Integer, desc: 'Return issues which are assigned to the user with the given ID'
+        optional :assignee_id, types: [Integer, String], integer_none_any: true,
+                               desc: 'Return issues which are assigned to the user with the given ID'
         optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all],
                          desc: 'Return issues for the given scope: `created_by_me`, `assigned_to_me` or `all`'
         optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji'
         use :pagination
+
+        use :issues_params_ee
       end
 
-      params :issue_params_ce do
+      params :issue_params do
         optional :description, type: String, desc: 'The description of an issue'
         optional :assignee_ids, type: Array[Integer], desc: 'The array of user IDs to assign issue'
         optional :assignee_id,  type: Integer, desc: '[Deprecated] The ID of a user to assign issue'
@@ -52,10 +68,8 @@ module API
         optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY'
         optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential'
         optional :discussion_locked, type: Boolean, desc: " Boolean parameter indicating if the issue's discussion is locked"
-      end
 
-      params :issue_params do
-        use :issue_params_ce
+        use :issue_params_ee
       end
     end
 
@@ -87,7 +101,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a group'
     end
-    resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Get a list of group issues' do
         success Entities::IssueBasic
       end
@@ -114,7 +128,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       include TimeTrackingEndpoints
 
       desc 'Get a list of project issues' do
@@ -162,6 +176,9 @@ module API
                                                            desc: 'The IID of a merge request for which to resolve discussions'
         optional :discussion_to_resolve, type: String,
                                          desc: 'The ID of a discussion to resolve, also pass `merge_request_to_resolve_discussions_of`'
+        optional :iid, type: Integer,
+                       desc: 'The internal ID of a project issue. Available only for admins and project owners.'
+
         use :issue_params
       end
       post ':id/issues' do
@@ -169,10 +186,8 @@ module API
 
         authorize! :create_issue, user_project
 
-        # Setting created_at time only allowed for admins and project owners
-        unless current_user.admin? || user_project.owner == current_user
-          params.delete(:created_at)
-        end
+        params.delete(:created_at) unless current_user.can?(:set_issue_created_at, user_project)
+        params.delete(:iid) unless current_user.can?(:set_issue_iid, user_project)
 
         issue_params = declared_params(include_missing: false)
 
@@ -206,14 +221,15 @@ module API
         at_least_one_of :title, :description, :assignee_ids, :assignee_id, :milestone_id, :discussion_locked,
                         :labels, :created_at, :due_date, :confidential, :state_event
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       put ':id/issues/:issue_iid' do
         Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42322')
 
         issue = user_project.issues.find_by!(iid: params.delete(:issue_iid))
         authorize! :update_issue, issue
 
-        # Setting created_at time only allowed for admins and project owners
-        unless current_user.admin? || user_project.owner == current_user
+        # Setting created_at time only allowed for admins and project/group owners
+        unless current_user.admin? || user_project.owner == current_user || current_user.owned_groups.include?(user_project.owner)
           params.delete(:updated_at)
         end
 
@@ -233,6 +249,7 @@ module API
           render_validation_error!(issue)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Move an existing issue' do
         success Entities::Issue
@@ -241,6 +258,7 @@ module API
         requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue'
         requires :to_project_id, type: Integer, desc: 'The ID of the new project'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       post ':id/issues/:issue_iid/move' do
         Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42323')
 
@@ -257,11 +275,13 @@ module API
           render_api_error!(error.message, 400)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Delete a project issue'
       params do
         requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete ":id/issues/:issue_iid" do
         issue = user_project.issues.find_by(iid: params[:issue_iid])
         not_found!('Issue') unless issue
@@ -272,13 +292,39 @@ module API
           Issuable::DestroyService.new(user_project, current_user).execute(issue)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      desc 'List merge requests that are related to the issue' do
+        success Entities::MergeRequestBasic
+      end
+      params do
+        requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue'
+      end
+      get ':id/issues/:issue_iid/related_merge_requests' do
+        issue = find_project_issue(params[:issue_iid])
+
+        merge_request_iids = ::Issues::ReferencedMergeRequestsService.new(user_project, current_user)
+          .execute(issue)
+          .flatten
+          .map(&:iid)
+
+        merge_requests =
+          if merge_request_iids.present?
+            MergeRequestsFinder.new(current_user, project_id: user_project.id, iids: merge_request_iids).execute
+          else
+            MergeRequest.none
+          end
+
+        present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project
+      end
 
-      desc 'List merge requests closing issue'  do
+      desc 'List merge requests closing issue' do
         success Entities::MergeRequestBasic
       end
       params do
         requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ':id/issues/:issue_iid/closed_by' do
         issue = find_project_issue(params[:issue_iid])
 
@@ -287,8 +333,9 @@ module API
 
         present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
-      desc 'List participants for an issue'  do
+      desc 'List participants for an issue' do
         success Entities::UserBasic
       end
       params do
diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb
index 32379d7c8ab..933bd067e26 100644
--- a/lib/api/job_artifacts.rb
+++ b/lib/api/job_artifacts.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class JobArtifacts < Grape::API
     before { authenticate_non_get! }
@@ -12,7 +14,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Download the artifacts archive from a job' do
         detail 'This feature was introduced in GitLab 8.10'
       end
@@ -25,12 +27,34 @@ module API
         requirements: { ref_name: /.+/ } do
         authorize_download_artifacts!
 
-        builds = user_project.latest_successful_builds_for(params[:ref_name])
-        latest_build = builds.find_by!(name: params[:job])
+        latest_build = user_project.latest_successful_build_for!(params[:job], params[:ref_name])
 
         present_carrierwave_file!(latest_build.artifacts_file)
       end
 
+      desc 'Download a specific file from artifacts archive from a ref' do
+        detail 'This feature was introduced in GitLab 11.5'
+      end
+      params do
+        requires :ref_name, type: String, desc: 'The ref from repository'
+        requires :job, type: String, desc: 'The name for the job'
+        requires :artifact_path, type: String, desc: 'Artifact path'
+      end
+      get ':id/jobs/artifacts/:ref_name/raw/*artifact_path',
+          format: false,
+          requirements: { ref_name: /.+/ } do
+        authorize_download_artifacts!
+
+        build = user_project.latest_successful_build_for!(params[:job], params[:ref_name])
+
+        path = Gitlab::Ci::Build::Artifacts::Path
+                 .new(params[:artifact_path])
+
+        bad_request! unless path.valid?
+
+        send_artifacts_entry(build, path)
+      end
+
       desc 'Download the artifacts archive from a job' do
         detail 'This feature was introduced in GitLab 8.5'
       end
@@ -61,6 +85,7 @@ module API
 
         path = Gitlab::Ci::Build::Artifacts::Path
           .new(params[:artifact_path])
+
         bad_request! unless path.valid?
 
         send_artifacts_entry(build, path)
diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb
index 54d1acbd412..59f0dbe8a9b 100644
--- a/lib/api/jobs.rb
+++ b/lib/api/jobs.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Jobs < Grape::API
     include PaginationParams
@@ -7,7 +9,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       helpers do
         params :optional_scope do
           optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show',
@@ -34,29 +36,39 @@ module API
         use :optional_scope
         use :pagination
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ':id/jobs' do
+        authorize_read_builds!
+
         builds = user_project.builds.order('id DESC')
         builds = filter_builds(builds, params[:scope])
 
-        builds = builds.preload(:user, :job_artifacts_archive, :runner, pipeline: :project)
+        builds = builds.preload(:user, :job_artifacts_archive, :job_artifacts, :runner, pipeline: :project)
         present paginate(builds), with: Entities::Job
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Get pipeline jobs' do
         success Entities::Job
       end
       params do
-        requires :pipeline_id, type: Integer,  desc: 'The pipeline ID'
+        requires :pipeline_id, type: Integer, desc: 'The pipeline ID'
         use :optional_scope
         use :pagination
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ':id/pipelines/:pipeline_id/jobs' do
-        pipeline = user_project.pipelines.find(params[:pipeline_id])
+        authorize!(:read_pipeline, user_project)
+        pipeline = user_project.ci_pipelines.find(params[:pipeline_id])
+        authorize!(:read_build, pipeline)
+
         builds = pipeline.builds
         builds = filter_builds(builds, params[:scope])
+        builds = builds.preload(:job_artifacts_archive, :job_artifacts, project: [:namespace])
 
         present paginate(builds), with: Entities::Job
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Get a specific job of a project' do
         success Entities::Job
@@ -144,7 +156,7 @@ module API
         present build, with: Entities::Job
       end
 
-      desc 'Trigger a manual job' do
+      desc 'Trigger a actionable job (manual, delayed, etc)' do
         success Entities::Job
         detail 'This feature was added in GitLab 8.11'
       end
@@ -167,6 +179,7 @@ module API
     end
 
     helpers do
+      # rubocop: disable CodeReuse/ActiveRecord
       def filter_builds(builds, scope)
         return builds if scope.nil? || scope.empty?
 
@@ -177,6 +190,7 @@ module API
 
         builds.where(status: available_statuses && scope)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/api/keys.rb b/lib/api/keys.rb
index 767f27ef334..d5280a0035d 100644
--- a/lib/api/keys.rb
+++ b/lib/api/keys.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   # Keys API
   class Keys < Grape::API
@@ -12,7 +14,7 @@ module API
 
         key = Key.find(params[:id])
 
-        present key, with: Entities::SSHKeyWithUser
+        present key, with: Entities::SSHKeyWithUser, current_user: current_user
       end
     end
   end
diff --git a/lib/api/labels.rb b/lib/api/labels.rb
index 81eaf56e48e..d5eb2b94669 100644
--- a/lib/api/labels.rb
+++ b/lib/api/labels.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Labels < Grape::API
     include PaginationParams
@@ -7,7 +9,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Get all labels of the project' do
         success Entities::Label
       end
@@ -27,6 +29,7 @@ module API
         optional :description, type: String, desc: 'The description of label to be created'
         optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       post ':id/labels' do
         authorize! :admin_label, user_project
 
@@ -43,6 +46,7 @@ module API
           render_validation_error!(label)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Delete an existing label' do
         success Entities::Label
@@ -50,6 +54,7 @@ module API
       params do
         requires :name, type: String, desc: 'The name of the label to be deleted'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete ':id/labels' do
         authorize! :admin_label, user_project
 
@@ -58,18 +63,20 @@ module API
 
         destroy_conditionally!(label)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Update an existing label. At least one optional parameter is required.' do
         success Entities::Label
       end
       params do
-        requires :name,  type: String, desc: 'The name of the label to be updated'
+        requires :name, type: String, desc: 'The name of the label to be updated'
         optional :new_name, type: String, desc: 'The new name of the label'
         optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names"
         optional :description, type: String, desc: 'The new description of label'
         optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true
         at_least_one_of :new_name, :color, :description, :priority
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       put ':id/labels' do
         authorize! :admin_label, user_project
 
@@ -95,6 +102,7 @@ module API
 
         present label, with: Entities::Label, current_user: current_user, project: user_project
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/api/lint.rb b/lib/api/lint.rb
index d202eaa4c49..a7672021db0 100644
--- a/lib/api/lint.rb
+++ b/lib/api/lint.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Lint < Grape::API
     namespace :ci do
@@ -6,7 +8,8 @@ module API
         requires :content, type: String, desc: 'Content of .gitlab-ci.yml'
       end
       post '/lint' do
-        error = Gitlab::Ci::YamlProcessor.validation_message(params[:content])
+        error = Gitlab::Ci::YamlProcessor.validation_message(params[:content],
+          user: current_user)
 
         status 200
 
diff --git a/lib/api/markdown.rb b/lib/api/markdown.rb
index b9ed68aa584..de77bef43ce 100644
--- a/lib/api/markdown.rb
+++ b/lib/api/markdown.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Markdown < Grape::API
     params do
@@ -10,9 +12,8 @@ module API
         detail "This feature was introduced in GitLab 11.0."
       end
       post do
-        # Explicitly set CommonMark as markdown engine to use.
-        # Remove this set when https://gitlab.com/gitlab-org/gitlab-ce/issues/43011 is done.
-        context = { markdown_engine: :common_mark, only_path: false }
+        context = { only_path: false, current_user: current_user }
+        context[:pipeline] = params[:gfm] ? :full : :plain_markdown
 
         if params[:project]
           project = Project.find_by_full_path(params[:project])
@@ -24,9 +25,7 @@ module API
           context[:skip_project_check] = true
         end
 
-        context[:pipeline] = params[:gfm] ? :full : :plain_markdown
-
-        { html: Banzai.render(params[:text], context) }
+        { html: Banzai.render_and_post_process(params[:text], context) }
       end
     end
   end
diff --git a/lib/api/members.rb b/lib/api/members.rb
index 8b12986d09e..461ffe71a62 100644
--- a/lib/api/members.rb
+++ b/lib/api/members.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Members < Grape::API
     include PaginationParams
@@ -10,7 +12,7 @@ module API
       params do
         requires :id, type: String, desc: "The #{source_type} ID"
       end
-      resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+      resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
         desc 'Gets a list of group or project members viewable by the authenticated user.' do
           success Entities::Member
         end
@@ -18,6 +20,7 @@ module API
           optional :query, type: String, desc: 'A query string to search for members'
           use :pagination
         end
+        # rubocop: disable CodeReuse/ActiveRecord
         get ":id/members" do
           source = find_source(source_type, params[:id])
 
@@ -27,6 +30,26 @@ module API
 
           present members, with: Entities::Member
         end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        desc 'Gets a list of group or project members viewable by the authenticated user, including those who gained membership through ancestor group.' do
+          success Entities::Member
+        end
+        params do
+          optional :query, type: String, desc: 'A query string to search for members'
+          use :pagination
+        end
+        # rubocop: disable CodeReuse/ActiveRecord
+        get ":id/members/all" do
+          source = find_source(source_type, params[:id])
+
+          members = find_all_members(source_type, source)
+          members = members.includes(:user).references(:user).merge(User.search(params[:query])) if params[:query].present?
+          members = paginate(members)
+
+          present members, with: Entities::Member
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         desc 'Gets a member of a group or project.' do
           success Entities::Member
@@ -34,6 +57,7 @@ module API
         params do
           requires :user_id, type: Integer, desc: 'The user ID of the member'
         end
+        # rubocop: disable CodeReuse/ActiveRecord
         get ":id/members/:user_id" do
           source = find_source(source_type, params[:id])
 
@@ -42,6 +66,7 @@ module API
 
           present member, with: Entities::Member
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         desc 'Adds a member to a group or project.' do
           success Entities::Member
@@ -51,6 +76,7 @@ module API
           requires :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)'
           optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY'
         end
+        # rubocop: disable CodeReuse/ActiveRecord
         post ":id/members" do
           source = find_source(source_type, params[:id])
           authorize_admin_source!(source_type, source)
@@ -58,7 +84,10 @@ module API
           member = source.members.find_by(user_id: params[:user_id])
           conflict!('Member already exists') if member
 
-          member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at])
+          user = User.find_by_id(params[:user_id])
+          not_found!('User') unless user
+
+          member = source.add_user(user, params[:access_level], current_user: current_user, expires_at: params[:expires_at])
 
           if !member
             not_allowed! # This currently can only be reached in EE
@@ -68,6 +97,7 @@ module API
             render_validation_error!(member)
           end
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         desc 'Updates a member of a group or project.' do
           success Entities::Member
@@ -77,6 +107,7 @@ module API
           requires :access_level, type: Integer, desc: 'A valid access level'
           optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY'
         end
+        # rubocop: disable CodeReuse/ActiveRecord
         put ":id/members/:user_id" do
           source = find_source(source_type, params.delete(:id))
           authorize_admin_source!(source_type, source)
@@ -93,11 +124,13 @@ module API
             render_validation_error!(updated_member)
           end
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         desc 'Removes a user from a group or project.'
         params do
           requires :user_id, type: Integer, desc: 'The user ID of the member'
         end
+        # rubocop: disable CodeReuse/ActiveRecord
         delete ":id/members/:user_id" do
           source = find_source(source_type, params[:id])
           member = source.members.find_by!(user_id: params[:user_id])
@@ -106,6 +139,7 @@ module API
             ::Members::DestroyService.new(current_user).execute(member)
           end
         end
+        # rubocop: enable CodeReuse/ActiveRecord
       end
     end
   end
diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb
index 95ef8f42954..6ad30aa56e0 100644
--- a/lib/api/merge_request_diffs.rb
+++ b/lib/api/merge_request_diffs.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   # MergeRequestDiff API
   class MergeRequestDiffs < Grape::API
@@ -8,7 +10,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Get a list of merge request diff versions' do
         detail 'This feature was introduced in GitLab 8.12.'
         success Entities::MergeRequestDiff
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index b1e510d72de..132b19164d0 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class MergeRequests < Grape::API
     include PaginationParams
@@ -28,9 +30,9 @@ module API
     end
 
     helpers do
+      # rubocop: disable CodeReuse/ActiveRecord
       def find_merge_requests(args = {})
         args = declared_params.merge(args)
-
         args[:milestone_title] = args.delete(:milestone)
         args[:label_name] = args.delete(:labels)
         args[:scope] = args[:scope].underscore if args[:scope]
@@ -38,13 +40,14 @@ module API
         merge_requests = MergeRequestsFinder.new(current_user, args).execute
                            .reorder(args[:order_by] => args[:sort])
         merge_requests = paginate(merge_requests)
-                           .preload(:target_project)
+                           .preload(:source_project, :target_project)
 
         return merge_requests if args[:view] == 'simple'
 
         merge_requests
-          .preload(:notes, :author, :assignee, :milestone, :latest_merge_request_diff, :labels, :timelogs)
+          .preload(:notes, :author, :assignee, :milestone, :latest_merge_request_diff, :labels, :timelogs, metrics: [:latest_closed_by, :merged_by])
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def merge_request_pipelines_with_access
         authorize! :read_pipeline, user_project
@@ -71,9 +74,22 @@ module API
         options
       end
 
+      def authorize_push_to_merge_request!(merge_request)
+        forbidden!('Source branch does not exist') unless
+          merge_request.source_branch_exists?
+
+        user_access = Gitlab::UserAccess.new(
+          current_user,
+          project: merge_request.source_project
+        )
+
+        forbidden!('Cannot push to source branch') unless
+          user_access.can_push_to_branch?(merge_request.source_branch)
+      end
+
       params :merge_requests_params do
-        optional :state, type: String, values: %w[opened closed merged all], default: 'all',
-                         desc: 'Return opened, closed, merged, or all merge requests'
+        optional :state, type: String, values: %w[opened closed locked merged all], default: 'all',
+                         desc: 'Return opened, closed, locked, merged, or all merge requests'
         optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at',
                             desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.'
         optional :sort, type: String, values: %w[asc desc], default: 'desc',
@@ -86,13 +102,15 @@ module API
         optional :updated_before, type: DateTime, desc: 'Return merge requests updated before the specified time'
         optional :view, type: String, values: %w[simple], desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request'
         optional :author_id, type: Integer, desc: 'Return merge requests which are authored by the user with the given ID'
-        optional :assignee_id, type: Integer, desc: 'Return merge requests which are assigned to the user with the given ID'
+        optional :assignee_id, types: [Integer, String], integer_none_any: true,
+                               desc: 'Return merge requests which are assigned to the user with the given ID'
         optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all],
                          desc: 'Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`'
         optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji'
         optional :source_branch, type: String, desc: 'Return merge requests with the given source branch'
         optional :target_branch, type: String, desc: 'Return merge requests with the given target branch'
         optional :search, type: String, desc: 'Search merge requests for text present in the title or description'
+        optional :wip, type: String, values: %w[yes no], desc: 'Search merge requests for WIP in the title'
         use :pagination
       end
     end
@@ -117,7 +135,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a group'
     end
-    resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Get a list of group merge requests' do
         success Entities::MergeRequestBasic
       end
@@ -136,7 +154,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       include TimeTrackingEndpoints
 
       helpers do
@@ -161,8 +179,9 @@ module API
           optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request'
           optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request'
           optional :labels, type: String, desc: 'Comma-separated list of label names'
-          optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging'
-          optional :allow_maintainer_to_push, type: Boolean, desc: 'Whether a maintainer of the target project can push to the source project'
+          optional :remove_source_branch, type: Boolean, desc: 'Delete source branch when merging'
+          optional :allow_collaboration, type: Boolean, desc: 'Allow commits from members who can merge to the target branch'
+          optional :allow_maintainer_to_push, type: Boolean, as: :allow_collaboration, desc: '[deprecated] See allow_collaboration'
           optional :squash, type: Grape::API::Boolean, desc: 'When true, the commits will be squashed into a single commit on merge'
 
           use :optional_params_ee
@@ -231,6 +250,9 @@ module API
 
       params do
         requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request'
+        optional :render_html, type: Boolean, desc: 'Returns the description and title rendered HTML'
+        optional :include_diverged_commits_count, type: Boolean, desc: 'Returns the commits count behind the target branch'
+        optional :include_rebase_in_progress, type: Boolean, desc: 'Returns whether a rebase operation is ongoing '
       end
       desc 'Get a single merge request' do
         success Entities::MergeRequest
@@ -238,7 +260,13 @@ module API
       get ':id/merge_requests/:merge_request_iid' do
         merge_request = find_merge_request_with_access(params[:merge_request_iid])
 
-        present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project
+        present merge_request,
+          with: Entities::MergeRequest,
+          current_user: current_user,
+          project: user_project,
+          render_html: params[:render_html],
+          include_diverged_commits_count: params[:include_diverged_commits_count],
+          include_rebase_in_progress: params[:include_rebase_in_progress]
       end
 
       desc 'Get the participants of a merge request' do
@@ -370,6 +398,19 @@ module API
           .cancel(merge_request)
       end
 
+      desc 'Rebase the merge request against its target branch' do
+        detail 'This feature was added in GitLab 11.6'
+      end
+      put ':id/merge_requests/:merge_request_iid/rebase' do
+        merge_request = find_project_merge_request(params[:merge_request_iid])
+
+        authorize_push_to_merge_request!(merge_request)
+
+        RebaseWorker.perform_async(merge_request.id, current_user.id)
+
+        status :accepted
+      end
+
       desc 'List issues that will be closed on merge' do
         success Entities::MRNote
       end
@@ -378,7 +419,7 @@ module API
       end
       get ':id/merge_requests/:merge_request_iid/closes_issues' do
         merge_request = find_merge_request_with_access(params[:merge_request_iid])
-        issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user))
+        issues = ::Kaminari.paginate_array(merge_request.visible_closing_issues_for(current_user))
         issues = paginate(issues)
 
         external_issues, internal_issues = issues.partition { |issue| issue.is_a?(ExternalIssue) }
diff --git a/lib/api/milestone_responses.rb b/lib/api/milestone_responses.rb
index a8eb137e46a..a0ca39b69d4 100644
--- a/lib/api/milestone_responses.rb
+++ b/lib/api/milestone_responses.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   module MilestoneResponses
     extend ActiveSupport::Concern
diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb
index 32b77aedba8..3cc09f6ac3f 100644
--- a/lib/api/namespaces.rb
+++ b/lib/api/namespaces.rb
@@ -1,23 +1,40 @@
+# frozen_string_literal: true
+
 module API
   class Namespaces < Grape::API
     include PaginationParams
 
     before { authenticate! }
 
+    helpers do
+      params :optional_list_params_ee do
+        # EE::API::Namespaces would override this helper
+      end
+
+      # EE::API::Namespaces would override this method
+      def custom_namespace_present_options
+        {}
+      end
+    end
+
     resource :namespaces do
       desc 'Get a namespaces list' do
         success Entities::Namespace
       end
       params do
         optional :search, type: String, desc: "Search query for namespaces"
+
         use :pagination
+        use :optional_list_params_ee
       end
       get do
         namespaces = current_user.admin ? Namespace.all : current_user.namespaces
 
         namespaces = namespaces.search(params[:search]) if params[:search].present?
 
-        present paginate(namespaces), with: Entities::Namespace, current_user: current_user
+        options = { with: Entities::Namespace, current_user: current_user }
+
+        present paginate(namespaces), options.reverse_merge(custom_namespace_present_options)
       end
 
       desc 'Get a namespace by ID' do
@@ -26,7 +43,7 @@ module API
       params do
         requires :id, type: String, desc: "Namespace's ID or path"
       end
-      get ':id' do
+      get ':id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
         present user_namespace, with: Entities::Namespace, current_user: current_user
       end
     end
diff --git a/lib/api/notes.rb b/lib/api/notes.rb
index 39923e6d5b5..1bdf7aeb119 100644
--- a/lib/api/notes.rb
+++ b/lib/api/notes.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Notes < Grape::API
     include PaginationParams
@@ -14,7 +16,7 @@ module API
       params do
         requires :id, type: String, desc: "The ID of a #{parent_type}"
       end
-      resource parent_type.pluralize.to_sym, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+      resource parent_type.pluralize.to_sym, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
         noteables_str = noteable_type.to_s.underscore.pluralize
 
         desc "Get a list of #{noteable_type.to_s.downcase} notes" do
@@ -28,6 +30,7 @@ module API
                           desc: 'Return notes sorted in `asc` or `desc` order.'
           use :pagination
         end
+        # rubocop: disable CodeReuse/ActiveRecord
         get ":id/#{noteables_str}/:noteable_id/notes" do
           noteable = find_noteable(parent_type, noteables_str, params[:noteable_id])
 
@@ -45,6 +48,7 @@ module API
             .reject { |n| n.cross_reference_not_visible_for?(current_user) }
           present notes, with: Entities::Note
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         desc "Get a single #{noteable_type.to_s.downcase} note" do
           success Entities::Note
diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb
index 0266bf2f717..8cb46bd3ad6 100644
--- a/lib/api/notification_settings.rb
+++ b/lib/api/notification_settings.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   # notification_settings API
   class NotificationSettings < Grape::API
@@ -23,7 +25,7 @@ module API
       params do
         optional :level, type: String, desc: 'The global notification level'
         optional :notification_email, type: String, desc: 'The email address to send notifications'
-        NotificationSetting::EMAIL_EVENTS.each do |event|
+        NotificationSetting.email_events.each do |event|
           optional event, type: Boolean, desc: 'Enable/disable this notification'
         end
       end
@@ -50,11 +52,13 @@ module API
       end
     end
 
-    %w[group project].each do |source_type|
+    [Group, Project].each do |source_class|
+      source_type = source_class.name.underscore
+
       params do
         requires :id, type: String, desc: "The #{source_type} ID"
       end
-      resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+      resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
         desc "Get #{source_type} level notification level settings, defaults to Global" do
           detail 'This feature was introduced in GitLab 8.12'
           success Entities::NotificationSetting
@@ -73,7 +77,7 @@ module API
         end
         params do
           optional :level, type: String, desc: "The #{source_type} notification level"
-          NotificationSetting::EMAIL_EVENTS.each do |event|
+          NotificationSetting.email_events(source_class).each do |event|
             optional event, type: Boolean, desc: 'Enable/disable this notification'
           end
         end
diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb
index ba33993d852..78442f465bd 100644
--- a/lib/api/pages_domains.rb
+++ b/lib/api/pages_domains.rb
@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 module API
   class PagesDomains < Grape::API
     include PaginationParams
 
-    PAGES_DOMAINS_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(domain: API::NO_SLASH_URL_PART_REGEX)
+    PAGES_DOMAINS_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(domain: API::NO_SLASH_URL_PART_REGEX)
 
     before do
       authenticate!
@@ -13,9 +15,11 @@ module API
     end
 
     helpers do
+      # rubocop: disable CodeReuse/ActiveRecord
       def find_pages_domain!
         user_project.pages_domains.find_by(domain: params[:domain]) || not_found!('PagesDomain')
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def pages_domain
         @pages_domain ||= find_pages_domain!
@@ -50,7 +54,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       before do
         require_pages_enabled!
       end
@@ -61,11 +65,13 @@ module API
       params do
         use :pagination
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ":id/pages/domains" do
         authorize! :read_pages, user_project
 
         present paginate(user_project.pages_domains.order(:domain)), with: Entities::PagesDomain
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Get a single pages domain' do
         success Entities::PagesDomain
diff --git a/lib/api/pagination_params.rb b/lib/api/pagination_params.rb
index f566eb3ed2b..ae03595eb25 100644
--- a/lib/api/pagination_params.rb
+++ b/lib/api/pagination_params.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   # Concern for declare pagination params.
   #
diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb
index 37f32411296..c86b50d3736 100644
--- a/lib/api/pipeline_schedules.rb
+++ b/lib/api/pipeline_schedules.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class PipelineSchedules < Grape::API
     include PaginationParams
@@ -7,7 +9,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Get all pipeline schedules' do
         success Entities::PipelineSchedule
       end
@@ -16,6 +18,7 @@ module API
         optional :scope,    type: String, values: %w[active inactive],
                             desc: 'The scope of pipeline schedules'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ':id/pipeline_schedules' do
         authorize! :read_pipeline_schedule, user_project
 
@@ -23,12 +26,13 @@ module API
           .preload([:owner, :last_pipeline])
         present paginate(schedules), with: Entities::PipelineSchedule
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Get a single pipeline schedule' do
         success Entities::PipelineScheduleDetails
       end
       params do
-        requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
+        requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id'
       end
       get ':id/pipeline_schedules/:pipeline_schedule_id' do
         present pipeline_schedule, with: Entities::PipelineScheduleDetails
@@ -39,7 +43,7 @@ module API
       end
       params do
         requires :description, type: String, desc: 'The description of pipeline schedule'
-        requires :ref, type: String, desc: 'The branch/tag name will be triggered'
+        requires :ref, type: String, desc: 'The branch/tag name will be triggered', allow_blank: false
         requires :cron, type: String, desc: 'The cron'
         optional :cron_timezone, type: String, default: 'UTC', desc: 'The timezone'
         optional :active, type: Boolean, default: true, desc: 'The activation of pipeline schedule'
@@ -83,7 +87,7 @@ module API
         success Entities::PipelineScheduleDetails
       end
       params do
-        requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
+        requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id'
       end
       post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do
         authorize! :update_pipeline_schedule, pipeline_schedule
@@ -99,7 +103,7 @@ module API
         success Entities::PipelineScheduleDetails
       end
       params do
-        requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
+        requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id'
       end
       delete ':id/pipeline_schedules/:pipeline_schedule_id' do
         authorize! :admin_pipeline_schedule, pipeline_schedule
@@ -161,6 +165,7 @@ module API
     end
 
     helpers do
+      # rubocop: disable CodeReuse/ActiveRecord
       def pipeline_schedule
         @pipeline_schedule ||=
           user_project
@@ -172,7 +177,9 @@ module API
               end
             end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def pipeline_schedule_variable
         @pipeline_schedule_variable ||=
           pipeline_schedule.variables.find_by(key: params[:key]).tap do |pipeline_schedule_variable|
@@ -181,6 +188,7 @@ module API
             end
           end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb
index 735591fedd5..ac8fe98e55e 100644
--- a/lib/api/pipelines.rb
+++ b/lib/api/pipelines.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Pipelines < Grape::API
     include PaginationParams
@@ -7,7 +9,7 @@ module API
     params do
       requires :id, type: String, desc: 'The project ID'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Get all Pipelines of the project' do
         detail 'This feature was introduced in GitLab 8.11.'
         success Entities::PipelineBasic
@@ -31,7 +33,7 @@ module API
       get ':id/pipelines' do
         authorize! :read_pipeline, user_project
 
-        pipelines = PipelinesFinder.new(user_project, params).execute
+        pipelines = PipelinesFinder.new(user_project, current_user, params).execute
         present paginate(pipelines), with: Entities::PipelineBasic
       end
 
@@ -40,16 +42,22 @@ module API
         success Entities::Pipeline
       end
       params do
-        requires :ref, type: String,  desc: 'Reference'
+        requires :ref, type: String, desc: 'Reference'
+        optional :variables, Array, desc: 'Array of variables available in the pipeline'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       post ':id/pipeline' do
         Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42124')
 
         authorize! :create_pipeline, user_project
 
+        pipeline_params = declared_params(include_missing: false)
+          .merge(variables_attributes: params[:variables])
+          .except(:variables)
+
         new_pipeline = Ci::CreatePipelineService.new(user_project,
                                                      current_user,
-                                                     declared_params(include_missing: false))
+                                                     pipeline_params)
                            .execute(:api, ignore_skip_ci: true, save_on_errors: false)
 
         if new_pipeline.persisted?
@@ -58,6 +66,7 @@ module API
           render_validation_error!(new_pipeline)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Gets a specific pipeline for the project' do
         detail 'This feature was introduced in GitLab 8.11'
@@ -67,20 +76,35 @@ module API
         requires :pipeline_id, type: Integer, desc: 'The pipeline ID'
       end
       get ':id/pipelines/:pipeline_id' do
-        authorize! :read_pipeline, user_project
+        authorize! :read_pipeline, pipeline
 
         present pipeline, with: Entities::Pipeline
       end
 
+      desc 'Deletes a pipeline' do
+        detail 'This feature was introduced in GitLab 11.6'
+        http_codes [[204, 'Pipeline was deleted'], [403, 'Forbidden']]
+      end
+      params do
+        requires :pipeline_id, type: Integer, desc: 'The pipeline ID'
+      end
+      delete ':id/pipelines/:pipeline_id' do
+        authorize! :destroy_pipeline, pipeline
+
+        destroy_conditionally!(pipeline) do
+          ::Ci::DestroyPipelineService.new(user_project, current_user).execute(pipeline)
+        end
+      end
+
       desc 'Retry builds in the pipeline' do
         detail 'This feature was introduced in GitLab 8.11.'
         success Entities::Pipeline
       end
       params do
-        requires :pipeline_id, type: Integer,  desc: 'The pipeline ID'
+        requires :pipeline_id, type: Integer, desc: 'The pipeline ID'
       end
       post ':id/pipelines/:pipeline_id/retry' do
-        authorize! :update_pipeline, user_project
+        authorize! :update_pipeline, pipeline
 
         pipeline.retry_failed(current_user)
 
@@ -92,10 +116,10 @@ module API
         success Entities::Pipeline
       end
       params do
-        requires :pipeline_id, type: Integer,  desc: 'The pipeline ID'
+        requires :pipeline_id, type: Integer, desc: 'The pipeline ID'
       end
       post ':id/pipelines/:pipeline_id/cancel' do
-        authorize! :update_pipeline, user_project
+        authorize! :update_pipeline, pipeline
 
         pipeline.cancel_running
 
@@ -106,7 +130,7 @@ module API
 
     helpers do
       def pipeline
-        @pipeline ||= user_project.pipelines.find(params[:pipeline_id])
+        @pipeline ||= user_project.ci_pipelines.find(params[:pipeline_id])
       end
     end
   end
diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb
new file mode 100644
index 00000000000..c96261a7b57
--- /dev/null
+++ b/lib/api/project_clusters.rb
@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module API
+  class ProjectClusters < Grape::API
+    include PaginationParams
+
+    before { authenticate! }
+
+    # EE::API::ProjectClusters will
+    # override these methods
+    helpers do
+      params :create_params_ee do
+      end
+
+      params :update_params_ee do
+      end
+    end
+
+    params do
+      requires :id, type: String, desc: 'The ID of the project'
+    end
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      desc 'Get all clusters from the project' do
+        detail 'This feature was introduced in GitLab 11.7.'
+        success Entities::Cluster
+      end
+      params do
+        use :pagination
+      end
+      get ':id/clusters' do
+        authorize! :read_cluster, user_project
+
+        present paginate(clusters_for_current_user), with: Entities::Cluster
+      end
+
+      desc 'Get specific cluster for the project' do
+        detail 'This feature was introduced in GitLab 11.7.'
+        success Entities::ClusterProject
+      end
+      params do
+        requires :cluster_id, type: Integer, desc: 'The cluster ID'
+      end
+      get ':id/clusters/:cluster_id' do
+        authorize! :read_cluster, cluster
+
+        present cluster, with: Entities::ClusterProject
+      end
+
+      desc 'Adds an existing cluster' do
+        detail 'This feature was introduced in GitLab 11.7.'
+        success Entities::ClusterProject
+      end
+      params do
+        requires :name, type: String, desc: 'Cluster name'
+        optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true'
+        requires :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do
+          requires :api_url, type: String, allow_blank: false, desc: 'URL to access the Kubernetes API'
+          requires :token, type: String, desc: 'Token to authenticate against Kubernetes'
+          optional :ca_cert, type: String, desc: 'TLS certificate (needed if API is using a self-signed TLS certificate)'
+          optional :namespace, type: String, desc: 'Unique namespace related to Project'
+          optional :authorization_type, type: String, values: Clusters::Platforms::Kubernetes.authorization_types.keys, default: 'rbac', desc: 'Cluster authorization type, defaults to RBAC'
+        end
+        use :create_params_ee
+      end
+      post ':id/clusters/user' do
+        authorize! :add_cluster, user_project, 'Instance does not support multiple Kubernetes clusters'
+
+        user_cluster = ::Clusters::CreateService
+          .new(current_user, create_cluster_user_params)
+          .execute
+
+        if user_cluster.persisted?
+          present user_cluster, with: Entities::ClusterProject
+        else
+          render_validation_error!(user_cluster)
+        end
+      end
+
+      desc 'Update an existing cluster' do
+        detail 'This feature was introduced in GitLab 11.7.'
+        success Entities::ClusterProject
+      end
+      params do
+        requires :cluster_id, type: Integer, desc: 'The cluster ID'
+        optional :name, type: String, desc: 'Cluster name'
+        optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do
+          optional :api_url, type: String, desc: 'URL to access the Kubernetes API'
+          optional :token, type: String, desc: 'Token to authenticate against Kubernetes'
+          optional :ca_cert, type: String, desc: 'TLS certificate (needed if API is using a self-signed TLS certificate)'
+          optional :namespace, type: String, desc: 'Unique namespace related to Project'
+        end
+        use :update_params_ee
+      end
+      put ':id/clusters/:cluster_id' do
+        authorize! :update_cluster, cluster
+
+        update_service = Clusters::UpdateService.new(current_user, update_cluster_params)
+
+        if update_service.execute(cluster)
+          present cluster, with: Entities::ClusterProject
+        else
+          render_validation_error!(cluster)
+        end
+      end
+
+      desc 'Remove a cluster' do
+        detail 'This feature was introduced in GitLab 11.7.'
+        success Entities::ClusterProject
+      end
+      params do
+        requires :cluster_id, type: Integer, desc: 'The Cluster ID'
+      end
+      delete ':id/clusters/:cluster_id' do
+        authorize! :admin_cluster, cluster
+
+        destroy_conditionally!(cluster)
+      end
+    end
+
+    helpers do
+      def clusters_for_current_user
+        @clusters_for_current_user ||= ClustersFinder.new(user_project, current_user, :all).execute
+      end
+
+      def cluster
+        @cluster ||= clusters_for_current_user.find(params[:cluster_id])
+      end
+
+      def create_cluster_user_params
+        declared_params.merge({
+          provider_type: :user,
+          platform_type: :kubernetes,
+          clusterable: user_project
+        })
+      end
+
+      def update_cluster_params
+        declared_params(include_missing: false).without(:cluster_id)
+      end
+    end
+  end
+end
diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb
index 5ef4e9d530c..e34ed0bdb44 100644
--- a/lib/api/project_export.rb
+++ b/lib/api/project_export.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class ProjectExport < Grape::API
     before do
@@ -21,11 +23,11 @@ module API
         detail 'This feature was introduced in GitLab 10.6.'
       end
       get ':id/export/download' do
-        path = user_project.export_project_path
-
-        render_api_error!('404 Not found or has expired', 404) unless path
-
-        present_disk_file!(path, File.basename(path), 'application/gzip')
+        if user_project.export_file_exists?
+          present_carrierwave_file!(user_project.export_file)
+        else
+          render_api_error!('404 Not found or has expired', 404)
+        end
       end
 
       desc 'Start export' do
diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb
index 68921ae439b..0e7576c9243 100644
--- a/lib/api/project_hooks.rb
+++ b/lib/api/project_hooks.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class ProjectHooks < Grape::API
     include PaginationParams
@@ -20,13 +22,14 @@ module API
         optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events"
         optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook"
         optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response"
+        optional :push_events_branch_filter, type: String, desc: "Trigger hook on specified branch only"
       end
     end
 
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Get project hooks' do
         success Entities::ProjectHook
       end
@@ -63,6 +66,7 @@ module API
           present hook, with: Entities::ProjectHook
         else
           error!("Invalid url given", 422) if hook.errors[:url].present?
+          error!("Invalid branch filter given", 422) if hook.errors[:push_events_branch_filter].present?
 
           not_found!("Project hook #{hook.errors.messages}")
         end
@@ -80,10 +84,11 @@ module API
 
         update_params = declared_params(include_missing: false)
 
-        if hook.update_attributes(update_params)
+        if hook.update(update_params)
           present hook, with: Entities::ProjectHook
         else
           error!("Invalid url given", 422) if hook.errors[:url].present?
+          error!("Invalid branch filter given", 422) if hook.errors[:push_events_branch_filter].present?
 
           not_found!("Project hook #{hook.errors.messages}")
         end
diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb
index bc5152e539f..c64ec2fcc95 100644
--- a/lib/api/project_import.rb
+++ b/lib/api/project_import.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class ProjectImport < Grape::API
     include PaginationParams
@@ -21,7 +23,7 @@ module API
       forbidden! unless Gitlab::CurrentSettings.import_sources.include?('gitlab_project')
     end
 
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       params do
         requires :path, type: String, desc: 'The new project path and name'
         requires :file, type: File, desc: 'The project export file to be imported'
diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb
index 306dc0e63d7..da31bcb8dac 100644
--- a/lib/api/project_milestones.rb
+++ b/lib/api/project_milestones.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class ProjectMilestones < Grape::API
     include PaginationParams
@@ -10,7 +12,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Get a list of project milestones' do
         success Entities::Milestone
       end
@@ -64,7 +66,8 @@ module API
       delete ":id/milestones/:milestone_id" do
         authorize! :admin_milestone, user_project
 
-        user_project.milestones.find(params[:milestone_id]).destroy
+        milestone = user_project.milestones.find(params[:milestone_id])
+        Milestones::DestroyService.new(user_project, current_user).execute(milestone)
 
         status(204)
       end
diff --git a/lib/api/project_snapshots.rb b/lib/api/project_snapshots.rb
index 71005acc587..175fbb2ce92 100644
--- a/lib/api/project_snapshots.rb
+++ b/lib/api/project_snapshots.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class ProjectSnapshots < Grape::API
     helpers ::API::Helpers::ProjectSnapshotsHelpers
diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb
index 1de5551fee9..a607df411a6 100644
--- a/lib/api/project_snippets.rb
+++ b/lib/api/project_snippets.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class ProjectSnippets < Grape::API
     include PaginationParams
@@ -7,7 +9,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       helpers do
         def handle_project_member_errors(errors)
           if errors[:project_access].any?
@@ -49,7 +51,7 @@ module API
       params do
         requires :title, type: String, desc: 'The title of the snippet'
         requires :file_name, type: String, desc: 'The file name of the snippet'
-        requires :code, type: String, desc: 'The content of the snippet'
+        requires :code, type: String, allow_blank: false, desc: 'The content of the snippet'
         optional :description, type: String, desc: 'The description of a snippet'
         requires :visibility, type: String,
                               values: Gitlab::VisibilityLevel.string_values,
@@ -78,13 +80,14 @@ module API
         requires :snippet_id, type: Integer, desc: 'The ID of a project snippet'
         optional :title, type: String, desc: 'The title of the snippet'
         optional :file_name, type: String, desc: 'The file name of the snippet'
-        optional :code, type: String, desc: 'The content of the snippet'
+        optional :code, type: String, allow_blank: false, desc: 'The content of the snippet'
         optional :description, type: String, desc: 'The description of a snippet'
         optional :visibility, type: String,
                               values: Gitlab::VisibilityLevel.string_values,
                               desc: 'The visibility of the snippet'
         at_least_one_of :title, :file_name, :code, :visibility_level
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       put ":id/snippets/:snippet_id" do
         snippet = snippets_for_current_user.find_by(id: params.delete(:snippet_id))
         not_found!('Snippet') unless snippet
@@ -107,11 +110,13 @@ module API
           render_validation_error!(snippet)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Delete a project snippet'
       params do
         requires :snippet_id, type: Integer, desc: 'The ID of a project snippet'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete ":id/snippets/:snippet_id" do
         snippet = snippets_for_current_user.find_by(id: params[:snippet_id])
         not_found!('Snippet') unless snippet
@@ -120,11 +125,13 @@ module API
 
         destroy_conditionally!(snippet)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Get a raw project snippet'
       params do
         requires :snippet_id, type: Integer, desc: 'The ID of a project snippet'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ":id/snippets/:snippet_id/raw" do
         snippet = snippets_for_current_user.find_by(id: params[:snippet_id])
         not_found!('Snippet') unless snippet
@@ -133,6 +140,7 @@ module API
         content_type 'text/plain'
         present snippet.content
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Get the user agent details for a project snippet' do
         success Entities::UserAgentDetail
@@ -140,6 +148,7 @@ module API
       params do
         requires :snippet_id, type: Integer, desc: 'The ID of a project snippet'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ":id/snippets/:snippet_id/user_agent_detail" do
         authenticated_as_admin!
 
@@ -149,6 +158,7 @@ module API
 
         present snippet.user_agent_detail, with: Entities::UserAgentDetail
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb
new file mode 100644
index 00000000000..d05ddad7466
--- /dev/null
+++ b/lib/api/project_templates.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module API
+  class ProjectTemplates < Grape::API
+    include PaginationParams
+
+    TEMPLATE_TYPES = %w[dockerfiles gitignores gitlab_ci_ymls licenses].freeze
+
+    before { authenticate_non_get! }
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+      requires :type, type: String, values: TEMPLATE_TYPES, desc: 'The type (dockerfiles|gitignores|gitlab_ci_ymls|licenses) of the template'
+    end
+    resource :projects do
+      desc 'Get a list of templates available to this project' do
+        detail 'This endpoint was introduced in GitLab 11.4'
+      end
+      params do
+        use :pagination
+      end
+      get ':id/templates/:type' do
+        templates = TemplateFinder
+          .build(params[:type], user_project)
+          .execute
+
+        present paginate(::Kaminari.paginate_array(templates)), with: Entities::TemplatesList
+      end
+
+      desc 'Download a template available to this project' do
+        detail 'This endpoint was introduced in GitLab 11.4'
+      end
+      params do
+        requires :name, type: String, desc: 'The name of the template'
+
+        optional :project, type: String, desc: 'The project name to use when expanding placeholders in the template. Only affects licenses'
+        optional :fullname, type: String, desc: 'The full name of the copyright holder to use when expanding placeholders in the template. Only affects licenses'
+      end
+      get ':id/templates/:type/:name', requirements: { name: /[\w\.-]+/ } do
+        template = TemplateFinder
+          .build(params[:type], user_project, name: params[:name])
+          .execute
+
+        not_found!('Template') unless template.present?
+
+        template.resolve!(
+          project_name: params[:project].presence,
+          fullname: params[:fullname].presence || current_user&.name
+        )
+
+        if template.is_a?(::LicenseTemplate)
+          present template, with: Entities::License
+        else
+          present template, with: Entities::Template
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 3ef3680c5d9..3afa2d8a6b0 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_dependency 'declarative_policy'
 
 module API
@@ -9,6 +11,56 @@ module API
     before { authenticate_non_get! }
 
     helpers do
+      params :optional_filter_params_ee do
+        # EE::API::Projects would override this helper
+      end
+
+      params :optional_update_params_ee do
+        # EE::API::Projects would override this helper
+      end
+
+      # EE::API::Projects would override this method
+      def apply_filters(projects)
+        projects = projects.with_issues_available_for_user(current_user) if params[:with_issues_enabled]
+        projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled]
+        projects = projects.with_statistics if params[:statistics]
+
+        projects
+      end
+
+      def verify_update_project_attrs!(project, attrs)
+      end
+    end
+
+    def self.update_params_at_least_one_of
+      [
+        :jobs_enabled,
+        :resolve_outdated_diff_discussions,
+        :ci_config_path,
+        :container_registry_enabled,
+        :default_branch,
+        :description,
+        :issues_enabled,
+        :lfs_enabled,
+        :merge_requests_enabled,
+        :merge_method,
+        :name,
+        :only_allow_merge_if_all_discussions_are_resolved,
+        :only_allow_merge_if_pipeline_succeeds,
+        :path,
+        :printing_merge_request_link_enabled,
+        :public_builds,
+        :request_access_enabled,
+        :shared_runners_enabled,
+        :snippets_enabled,
+        :tag_list,
+        :visibility,
+        :wiki_enabled,
+        :avatar
+      ]
+    end
+
+    helpers do
       params :statistics_params do
         optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
       end
@@ -30,7 +82,7 @@ module API
       end
 
       params :filter_params do
-        optional :archived, type: Boolean, default: false, desc: 'Limit by archived status'
+        optional :archived, type: Boolean, desc: 'Limit by archived status'
         optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values,
                               desc: 'Limit by visibility'
         optional :search, type: String, desc: 'Return list of projects matching the search criteria'
@@ -39,6 +91,9 @@ module API
         optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of'
         optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature'
         optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature'
+        optional :min_access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'Limit by minimum access level of authenticated user'
+
+        use :optional_filter_params_ee
       end
 
       params :create_params do
@@ -52,16 +107,15 @@ module API
 
       def present_projects(projects, options = {})
         projects = reorder_projects(projects)
-        projects = projects.with_issues_available_for_user(current_user) if params[:with_issues_enabled]
-        projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled]
-        projects = projects.with_statistics if params[:statistics]
+        projects = apply_filters(projects)
         projects = paginate(projects)
         projects, options = with_custom_attributes(projects, options)
 
         options = options.reverse_merge(
           with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails,
           statistics: params[:statistics],
-          current_user: current_user
+          current_user: current_user,
+          license: false
         )
         options[:with] = Entities::BasicProjectDetails if params[:simple]
 
@@ -74,7 +128,7 @@ module API
       end
     end
 
-    resource :users, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :users, requirements: API::USER_REQUIREMENTS do
       desc 'Get a user projects' do
         success Entities::BasicProjectDetails
       end
@@ -147,6 +201,7 @@ module API
         use :optional_project_params
         use :create_params
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       post "user/:user_id" do
         authenticated_as_admin!
         user = User.find_by(id: params.delete(:user_id))
@@ -163,25 +218,30 @@ module API
           render_validation_error!(project)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
 
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Get a single project' do
         success Entities::ProjectWithAccess
       end
       params do
         use :statistics_params
         use :with_custom_attributes
+
+        optional :license, type: Boolean, default: false,
+                           desc: 'Include project license data'
       end
       get ":id" do
         options = {
           with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails,
           current_user: current_user,
           user_can_admin_project: can?(current_user, :admin_project, user_project),
-          statistics: params[:statistics]
+          statistics: params[:statistics],
+          license: params[:license]
         }
 
         project, options = with_custom_attributes(user_project, options)
@@ -232,42 +292,23 @@ module API
         present_projects forks
       end
 
+      desc 'Check pages access of this project'
+      get ':id/pages_access' do
+        authorize! :read_pages_content, user_project unless user_project.public_pages?
+        status 200
+      end
+
       desc 'Update an existing project' do
         success Entities::Project
       end
       params do
-        # CE
-        at_least_one_of_ce =
-          [
-            :jobs_enabled,
-            :resolve_outdated_diff_discussions,
-            :ci_config_path,
-            :container_registry_enabled,
-            :default_branch,
-            :description,
-            :issues_enabled,
-            :lfs_enabled,
-            :merge_requests_enabled,
-            :merge_method,
-            :name,
-            :only_allow_merge_if_all_discussions_are_resolved,
-            :only_allow_merge_if_pipeline_succeeds,
-            :path,
-            :printing_merge_request_link_enabled,
-            :public_builds,
-            :request_access_enabled,
-            :shared_runners_enabled,
-            :snippets_enabled,
-            :tag_list,
-            :visibility,
-            :wiki_enabled
-          ]
         optional :name, type: String, desc: 'The name of the project'
         optional :default_branch, type: String, desc: 'The default branch of the project'
         optional :path, type: String, desc: 'The path of the repository'
 
         use :optional_project_params
-        at_least_one_of(*at_least_one_of_ce)
+
+        at_least_one_of(*::API::Projects.update_params_at_least_one_of)
       end
       put ':id' do
         authorize_admin_project
@@ -277,6 +318,8 @@ module API
 
         attrs = translate_params_for_compatibility(attrs)
 
+        verify_update_project_attrs!(user_project, attrs)
+
         result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute
 
         if result[:status] == :success
@@ -293,7 +336,7 @@ module API
       post ':id/archive' do
         authorize!(:archive_project, user_project)
 
-        user_project.archive!
+        ::Projects::UpdateService.new(user_project, current_user, archived: true).execute
 
         present user_project, with: Entities::Project
       end
@@ -304,7 +347,7 @@ module API
       post ':id/unarchive' do
         authorize!(:archive_project, user_project)
 
-        user_project.unarchive!
+        ::Projects::UpdateService.new(@project, current_user, archived: false).execute
 
         present user_project, with: Entities::Project
       end
@@ -358,7 +401,7 @@ module API
         requires :forked_from_id, type: String, desc: 'The ID of the project it was forked from'
       end
       post ":id/fork/:forked_from_id" do
-        authenticated_as_admin!
+        authorize! :admin_project, user_project
 
         fork_from_project = find_project!(params[:forked_from_id])
 
@@ -416,6 +459,7 @@ module API
       params do
         requires :group_id, type: Integer, desc: 'The ID of the group'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete ":id/share/:group_id" do
         authorize! :admin_project, user_project
 
@@ -424,13 +468,14 @@ module API
 
         destroy_conditionally!(link)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Upload a file'
       params do
         requires :file, type: File, desc: 'The file to be uploaded'
       end
       post ":id/uploads" do
-        UploadService.new(user_project, params[:file]).execute
+        UploadService.new(user_project, params[:file]).execute.to_h
       end
 
       desc 'Get the users list of a project' do
@@ -459,6 +504,23 @@ module API
           conflict!(error.message)
         end
       end
+
+      desc 'Transfer a project to a new namespace'
+      params do
+        requires :namespace, type: String, desc: 'The ID or path of the new namespace'
+      end
+      put ":id/transfer" do
+        authorize! :change_namespace, user_project
+
+        namespace = find_namespace!(params[:namespace])
+        result = ::Projects::TransferService.new(user_project, current_user).execute(namespace)
+
+        if result
+          present user_project, with: Entities::Project
+        else
+          render_api_error!("Failed to transfer project #{user_project.errors.messages}", 400)
+        end
+      end
     end
   end
 end
diff --git a/lib/api/projects_relation_builder.rb b/lib/api/projects_relation_builder.rb
index 6482fd94ab8..263468c9aa6 100644
--- a/lib/api/projects_relation_builder.rb
+++ b/lib/api/projects_relation_builder.rb
@@ -1,15 +1,17 @@
+# frozen_string_literal: true
+
 module API
   module ProjectsRelationBuilder
     extend ActiveSupport::Concern
 
-    module ClassMethods
+    class_methods do
       def prepare_relation(projects_relation, options = {})
         projects_relation = preload_relation(projects_relation, options)
         execute_batch_counting(projects_relation)
         projects_relation
       end
 
-      def preload_relation(projects_relation, options =  {})
+      def preload_relation(projects_relation, options = {})
         projects_relation
       end
 
diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb
index aa7cab4a741..5af43448727 100644
--- a/lib/api/protected_branches.rb
+++ b/lib/api/protected_branches.rb
@@ -1,26 +1,30 @@
+# frozen_string_literal: true
+
 module API
   class ProtectedBranches < Grape::API
     include PaginationParams
 
-    BRANCH_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(name: API::NO_SLASH_URL_PART_REGEX)
+    BRANCH_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(name: API::NO_SLASH_URL_PART_REGEX)
 
     before { authorize_admin_project }
 
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc "Get a project's protected branches" do
         success Entities::ProtectedBranch
       end
       params do
         use :pagination
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ':id/protected_branches' do
         protected_branches = user_project.protected_branches.preload(:push_access_levels, :merge_access_levels)
 
         present paginate(protected_branches), with: Entities::ProtectedBranch, project: user_project
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Get a single protected branch' do
         success Entities::ProtectedBranch
@@ -28,11 +32,13 @@ module API
       params do
         requires :name, type: String, desc: 'The name of the branch or wildcard'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do
         protected_branch = user_project.protected_branches.find_by!(name: params[:name])
 
         present protected_branch, with: Entities::ProtectedBranch, project: user_project
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Protect a single branch or wildcard' do
         success Entities::ProtectedBranch
@@ -40,12 +46,13 @@ module API
       params do
         requires :name, type: String, desc: 'The name of the protected branch'
         optional :push_access_level, type: Integer,
-                                     values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS,
-                                     desc: 'Access levels allowed to push (defaults: `40`, master access level)'
+                                     values: ProtectedBranch::PushAccessLevel.allowed_access_levels,
+                                     desc: 'Access levels allowed to push (defaults: `40`, maintainer access level)'
         optional :merge_access_level, type: Integer,
-                                      values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS,
-                                      desc: 'Access levels allowed to merge (defaults: `40`, master access level)'
+                                      values: ProtectedBranch::MergeAccessLevel.allowed_access_levels,
+                                      desc: 'Access levels allowed to merge (defaults: `40`, maintainer access level)'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       post ':id/protected_branches' do
         protected_branch = user_project.protected_branches.find_by(name: params[:name])
         if protected_branch
@@ -62,11 +69,13 @@ module API
           render_api_error!(protected_branch.errors.full_messages, 422)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Unprotect a single branch'
       params do
         requires :name, type: String, desc: 'The name of the protected branch'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do
         protected_branch = user_project.protected_branches.find_by!(name: params[:name])
 
@@ -75,6 +84,7 @@ module API
           destroy_service.execute(protected_branch)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/api/protected_tags.rb b/lib/api/protected_tags.rb
new file mode 100644
index 00000000000..ee13473c848
--- /dev/null
+++ b/lib/api/protected_tags.rb
@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module API
+  class ProtectedTags < Grape::API
+    include PaginationParams
+
+    TAG_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(name: API::NO_SLASH_URL_PART_REGEX)
+
+    before { authorize_admin_project }
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      desc "Get a project's protected tags" do
+        detail 'This feature was introduced in GitLab 11.3.'
+        success Entities::ProtectedTag
+      end
+      params do
+        use :pagination
+      end
+      # rubocop: disable CodeReuse/ActiveRecord
+      get ':id/protected_tags' do
+        protected_tags = user_project.protected_tags.preload(:create_access_levels)
+
+        present paginate(protected_tags), with: Entities::ProtectedTag, project: user_project
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      desc 'Get a single protected tag' do
+        detail 'This feature was introduced in GitLab 11.3.'
+        success Entities::ProtectedTag
+      end
+      params do
+        requires :name, type: String, desc: 'The name of the tag or wildcard'
+      end
+      # rubocop: disable CodeReuse/ActiveRecord
+      get ':id/protected_tags/:name', requirements: TAG_ENDPOINT_REQUIREMENTS do
+        protected_tag = user_project.protected_tags.find_by!(name: params[:name])
+
+        present protected_tag, with: Entities::ProtectedTag, project: user_project
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      desc 'Protect a single tag or wildcard' do
+        detail 'This feature was introduced in GitLab 11.3.'
+        success Entities::ProtectedTag
+      end
+      params do
+        requires :name, type: String, desc: 'The name of the protected tag'
+        optional :create_access_level, type: Integer, default: Gitlab::Access::MAINTAINER,
+                                       values: ProtectedTag::CreateAccessLevel.allowed_access_levels,
+                                       desc: 'Access levels allowed to create (defaults: `40`, maintainer access level)'
+      end
+      post ':id/protected_tags' do
+        protected_tags_params = {
+          name: params[:name],
+          create_access_levels_attributes: [{ access_level: params[:create_access_level] }]
+        }
+
+        protected_tag = ::ProtectedTags::CreateService.new(user_project,
+                                                           current_user,
+                                                           protected_tags_params).execute
+
+        if protected_tag.persisted?
+          present protected_tag, with: Entities::ProtectedTag, project: user_project
+        else
+          render_api_error!(protected_tag.errors.full_messages, 422)
+        end
+      end
+
+      desc 'Unprotect a single tag' do
+        detail 'This feature was introduced in GitLab 11.3.'
+      end
+      params do
+        requires :name, type: String, desc: 'The name of the protected tag'
+      end
+      # rubocop: disable CodeReuse/ActiveRecord
+      delete ':id/protected_tags/:name', requirements: TAG_ENDPOINT_REQUIREMENTS do
+        protected_tag = user_project.protected_tags.find_by!(name: params[:name])
+
+        destroy_conditionally!(protected_tag)
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+    end
+  end
+end
diff --git a/lib/api/release/links.rb b/lib/api/release/links.rb
new file mode 100644
index 00000000000..e3072684ef7
--- /dev/null
+++ b/lib/api/release/links.rb
@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module API
+  module Release
+    class Links < Grape::API
+      include PaginationParams
+
+      RELEASE_ENDPOINT_REQUIREMETS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS
+        .merge(tag_name: API::NO_SLASH_URL_PART_REGEX)
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource 'projects/:id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+        params do
+          requires :tag_name, type: String, desc: 'The name of the tag', as: :tag
+        end
+        resource 'releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do
+          resource :assets do
+            desc 'Get a list of links of a release' do
+              detail 'This feature was introduced in GitLab 11.7.'
+              success Entities::Releases::Link
+            end
+            params do
+              use :pagination
+            end
+            get 'links' do
+              authorize! :read_release, release
+
+              present paginate(release.links.sorted), with: Entities::Releases::Link
+            end
+
+            desc 'Create a link of a release' do
+              detail 'This feature was introduced in GitLab 11.7.'
+              success Entities::Releases::Link
+            end
+            params do
+              requires :name, type: String, desc: 'The name of the link'
+              requires :url, type: String, desc: 'The URL of the link'
+            end
+            post 'links' do
+              authorize! :create_release, release
+
+              new_link = release.links.create(declared_params(include_missing: false))
+
+              if new_link.persisted?
+                present new_link, with: Entities::Releases::Link
+              else
+                render_api_error!(new_link.errors.messages, 400)
+              end
+            end
+
+            params do
+              requires :link_id, type: String, desc: 'The id of the link'
+            end
+            resource 'links/:link_id' do
+              desc 'Get a link detail of a release' do
+                detail 'This feature was introduced in GitLab 11.7.'
+                success Entities::Releases::Link
+              end
+              get do
+                authorize! :read_release, release
+
+                present link, with: Entities::Releases::Link
+              end
+
+              desc 'Update a link of a release' do
+                detail 'This feature was introduced in GitLab 11.7.'
+                success Entities::Releases::Link
+              end
+              params do
+                optional :name, type: String, desc: 'The name of the link'
+                optional :url, type: String, desc: 'The URL of the link'
+                at_least_one_of :name, :url
+              end
+              put do
+                authorize! :update_release, release
+
+                if link.update(declared_params(include_missing: false))
+                  present link, with: Entities::Releases::Link
+                else
+                  render_api_error!(link.errors.messages, 400)
+                end
+              end
+
+              desc 'Delete a link of a release' do
+                detail 'This feature was introduced in GitLab 11.7.'
+                success Entities::Releases::Link
+              end
+              delete do
+                authorize! :destroy_release, release
+
+                if link.destroy
+                  present link, with: Entities::Releases::Link
+                else
+                  render_api_error!(link.errors.messages, 400)
+                end
+              end
+            end
+          end
+        end
+      end
+
+      helpers do
+        def release
+          @release ||= user_project.releases.find_by_tag!(params[:tag])
+        end
+
+        def link
+          @link ||= release.links.find(params[:link_id])
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/releases.rb b/lib/api/releases.rb
new file mode 100644
index 00000000000..cb85028f22c
--- /dev/null
+++ b/lib/api/releases.rb
@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module API
+  class Releases < Grape::API
+    include PaginationParams
+
+    RELEASE_ENDPOINT_REQUIREMETS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS
+      .merge(tag_name: API::NO_SLASH_URL_PART_REGEX)
+
+    before { authorize_read_releases! }
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      desc 'Get a project releases' do
+        detail 'This feature was introduced in GitLab 11.7.'
+        success Entities::Release
+      end
+      params do
+        use :pagination
+      end
+      get ':id/releases' do
+        releases = ::ReleasesFinder.new(user_project, current_user).execute
+
+        present paginate(releases), with: Entities::Release
+      end
+
+      desc 'Get a single project release' do
+        detail 'This feature was introduced in GitLab 11.7.'
+        success Entities::Release
+      end
+      params do
+        requires :tag_name, type: String, desc: 'The name of the tag', as: :tag
+      end
+      get ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do
+        authorize_read_release!
+
+        present release, with: Entities::Release
+      end
+
+      desc 'Create a new release' do
+        detail 'This feature was introduced in GitLab 11.7.'
+        success Entities::Release
+      end
+      params do
+        requires :tag_name,    type: String, desc: 'The name of the tag', as: :tag
+        requires :name,        type: String, desc: 'The name of the release'
+        requires :description, type: String, desc: 'The release notes'
+        optional :ref,         type: String, desc: 'The commit sha or branch name'
+        optional :assets, type: Hash do
+          optional :links, type: Array do
+            requires :name, type: String
+            requires :url, type: String
+          end
+        end
+      end
+      post ':id/releases' do
+        authorize_create_release!
+
+        result = ::Releases::CreateService
+          .new(user_project, current_user, declared_params(include_missing: false))
+          .execute
+
+        if result[:status] == :success
+          present result[:release], with: Entities::Release
+        else
+          render_api_error!(result[:message], result[:http_status])
+        end
+      end
+
+      desc 'Update a release' do
+        detail 'This feature was introduced in GitLab 11.7.'
+        success Entities::Release
+      end
+      params do
+        requires :tag_name,    type: String, desc: 'The name of the tag', as: :tag
+        optional :name,        type: String, desc: 'The name of the release'
+        optional :description, type: String, desc: 'Release notes with markdown support'
+      end
+      put ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do
+        authorize_update_release!
+
+        result = ::Releases::UpdateService
+          .new(user_project, current_user, declared_params(include_missing: false))
+          .execute
+
+        if result[:status] == :success
+          present result[:release], with: Entities::Release
+        else
+          render_api_error!(result[:message], result[:http_status])
+        end
+      end
+
+      desc 'Delete a release' do
+        detail 'This feature was introduced in GitLab 11.7.'
+        success Entities::Release
+      end
+      params do
+        requires :tag_name, type: String, desc: 'The name of the tag', as: :tag
+      end
+      delete ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do
+        authorize_destroy_release!
+
+        result = ::Releases::DestroyService
+          .new(user_project, current_user, declared_params(include_missing: false))
+          .execute
+
+        if result[:status] == :success
+          present result[:release], with: Entities::Release
+        else
+          render_api_error!(result[:message], result[:http_status])
+        end
+      end
+    end
+
+    helpers do
+      def authorize_create_release!
+        authorize! :create_release, user_project
+      end
+
+      def authorize_read_releases!
+        authorize! :read_release, user_project
+      end
+
+      def authorize_read_release!
+        authorize! :read_release, release
+      end
+
+      def authorize_update_release!
+        authorize! :update_release, release
+      end
+
+      def authorize_destroy_release!
+        authorize! :destroy_release, release
+      end
+
+      def release
+        @release ||= user_project.releases.find_by_tag(params[:tag])
+      end
+    end
+  end
+end
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index bb3fa99af38..32e05d84491 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'mime/types'
 
 module API
@@ -9,7 +11,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       helpers do
         def handle_project_member_errors(errors)
           if errors[:project_access].any?
@@ -100,9 +102,10 @@ module API
       params do
         requires :from, type: String, desc: 'The commit, branch name, or tag name to start comparison'
         requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison'
+        optional :straight, type: Boolean, desc: 'Comparison method, `true` for direct comparison between `from` and `to` (`from`..`to`), `false` to compare using merge base (`from`...`to`)', default: false
       end
       get ':id/repository/compare' do
-        compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to])
+        compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to], straight: params[:straight])
         present compare, with: Entities::Compare
       end
 
@@ -122,6 +125,34 @@ module API
           not_found!
         end
       end
+
+      desc 'Get the common ancestor between commits' do
+        success Entities::Commit
+      end
+      params do
+        requires :refs, type: Array[String]
+      end
+      get ':id/repository/merge_base' do
+        refs = params[:refs]
+
+        if refs.size < 2
+          render_api_error!('Provide at least 2 refs', 400)
+        end
+
+        merge_base = Gitlab::Git::MergeBase.new(user_project.repository, refs)
+
+        if merge_base.unknown_refs.any?
+          ref_noun = 'ref'.pluralize(merge_base.unknown_refs.size)
+          message = "Could not find #{ref_noun}: #{merge_base.unknown_refs.join(', ')}"
+          render_api_error!(message, 400)
+        end
+
+        if merge_base.commit
+          present merge_base.commit, with: Entities::Commit
+        else
+          not_found!("Merge Base")
+        end
+      end
     end
   end
 end
diff --git a/lib/api/resource_label_events.rb b/lib/api/resource_label_events.rb
new file mode 100644
index 00000000000..0c328f7268e
--- /dev/null
+++ b/lib/api/resource_label_events.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module API
+  class ResourceLabelEvents < Grape::API
+    include PaginationParams
+    helpers ::API::Helpers::NotesHelpers
+
+    before { authenticate! }
+
+    EVENTABLE_TYPES = [Issue, MergeRequest].freeze
+
+    EVENTABLE_TYPES.each do |eventable_type|
+      parent_type = eventable_type.parent_class.to_s.underscore
+      eventables_str = eventable_type.to_s.underscore.pluralize
+
+      params do
+        requires :id, type: String, desc: "The ID of a #{parent_type}"
+      end
+      resource parent_type.pluralize.to_sym, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+        desc "Get a list of #{eventable_type.to_s.downcase} resource label events" do
+          success Entities::ResourceLabelEvent
+          detail 'This feature was introduced in 11.3'
+        end
+        params do
+          requires :eventable_id, types: [Integer, String], desc: 'The ID of the eventable'
+          use :pagination
+        end
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        get ":id/#{eventables_str}/:eventable_id/resource_label_events" do
+          eventable = find_noteable(parent_type, eventables_str, params[:eventable_id])
+          events = eventable.resource_label_events.includes(:label, :user)
+
+          present paginate(events), with: Entities::ResourceLabelEvent
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        desc "Get a single #{eventable_type.to_s.downcase} resource label event" do
+          success Entities::ResourceLabelEvent
+          detail 'This feature was introduced in 11.3'
+        end
+        params do
+          requires :event_id, type: String, desc: 'The ID of a resource label event'
+          requires :eventable_id, types: [Integer, String], desc: 'The ID of the eventable'
+        end
+        get ":id/#{eventables_str}/:eventable_id/resource_label_events/:event_id" do
+          eventable = find_noteable(parent_type, eventables_str, params[:eventable_id])
+          event = eventable.resource_label_events.find(params[:event_id])
+
+          present event, with: Entities::ResourceLabelEvent
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/runner.rb b/lib/api/runner.rb
index e9886c76870..c60d25b88cb 100644
--- a/lib/api/runner.rb
+++ b/lib/api/runner.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Runner < Grape::API
     helpers ::API::Helpers::Runner
@@ -24,13 +26,13 @@ module API
         attributes =
           if runner_registration_token_valid?
             # Create shared runner. Requires admin access
-            attributes.merge(is_shared: true, runner_type: :instance_type)
-          elsif project = Project.find_by(runners_token: params[:token])
+            attributes.merge(runner_type: :instance_type)
+          elsif project = Project.find_by_runners_token(params[:token])
             # Create a specific runner for the project
-            attributes.merge(is_shared: false, runner_type: :project_type, projects: [project])
-          elsif group = Group.find_by(runners_token: params[:token])
+            attributes.merge(runner_type: :project_type, projects: [project])
+          elsif group = Group.find_by_runners_token(params[:token])
             # Create a specific runner for the group
-            attributes.merge(is_shared: false, runner_type: :group_type, groups: [group])
+            attributes.merge(runner_type: :group_type, groups: [group])
           else
             forbidden!
           end
@@ -80,26 +82,44 @@ module API
       params do
         requires :token, type: String, desc: %q(Runner's authentication token)
         optional :last_update, type: String, desc: %q(Runner's queue last_update token)
-        optional :info, type: Hash, desc: %q(Runner's metadata)
+        optional :info, type: Hash, desc: %q(Runner's metadata) do
+          optional :name, type: String, desc: %q(Runner's name)
+          optional :version, type: String, desc: %q(Runner's version)
+          optional :revision, type: String, desc: %q(Runner's revision)
+          optional :platform, type: String, desc: %q(Runner's platform)
+          optional :architecture, type: String, desc: %q(Runner's architecture)
+          optional :executor, type: String, desc: %q(Runner's executor)
+          optional :features, type: Hash, desc: %q(Runner's features)
+        end
+        optional :session, type: Hash, desc: %q(Runner's session data) do
+          optional :url, type: String, desc: %q(Session's url)
+          optional :certificate, type: String, desc: %q(Session's certificate)
+          optional :authorization, type: String, desc: %q(Session's authorization)
+        end
       end
       post '/request' do
         authenticate_runner!
-        no_content! unless current_runner.active?
 
-        if current_runner.runner_queue_value_latest?(params[:last_update])
-          header 'X-GitLab-Last-Update', params[:last_update]
+        unless current_runner.active?
+          header 'X-GitLab-Last-Update', current_runner.ensure_runner_queue_value
+          break no_content!
+        end
+
+        runner_params = declared_params(include_missing: false)
+
+        if current_runner.runner_queue_value_latest?(runner_params[:last_update])
+          header 'X-GitLab-Last-Update', runner_params[:last_update]
           Gitlab::Metrics.add_event(:build_not_found_cached)
           break no_content!
         end
 
         new_update = current_runner.ensure_runner_queue_value
-        result = ::Ci::RegisterJobService.new(current_runner).execute
+        result = ::Ci::RegisterJobService.new(current_runner).execute(runner_params)
 
         if result.valid?
           if result.build
-            Gitlab::Metrics.add_event(:build_found,
-                                      project: result.build.project.full_path)
-            present result.build, with: Entities::JobRequest::Response
+            Gitlab::Metrics.add_event(:build_found)
+            present Ci::BuildRunnerPresenter.new(result.build), with: Entities::JobRequest::Response
           else
             Gitlab::Metrics.add_event(:build_not_found)
             header 'X-GitLab-Last-Update', new_update
@@ -120,19 +140,19 @@ module API
         requires :id, type: Integer, desc: %q(Job's ID)
         optional :trace, type: String, desc: %q(Job's full trace)
         optional :state, type: String, desc: %q(Job's status: success, failed)
-        optional :failure_reason, type: String, values: CommitStatus.failure_reasons.keys,
-                                  desc: %q(Job's failure_reason)
+        optional :failure_reason, type: String, desc: %q(Job's failure_reason)
       end
       put '/:id' do
         job = authenticate_job!
-        forbidden!('Job is not running') unless job.running?
+        job_forbidden!(job, 'Job is not running') unless job.running?
 
         job.trace.set(params[:trace]) if params[:trace]
 
-        Gitlab::Metrics.add_event(:update_build,
-                                  project: job.project.full_path)
+        Gitlab::Metrics.add_event(:update_build)
 
         case params[:state].to_s
+        when 'running'
+          job.touch if job.needs_touch?
         when 'success'
           job.success!
         when 'failed'
@@ -152,7 +172,7 @@ module API
       end
       patch '/:id/trace' do
         job = authenticate_job!
-        forbidden!('Job is not running') unless job.running?
+        job_forbidden!(job, 'Job is not running') unless job.running?
 
         error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range')
         content_range = request.headers['Content-Range']
@@ -205,7 +225,7 @@ module API
 
         status 200
         content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
-        JobArtifactUploader.workhorse_authorize
+        JobArtifactUploader.workhorse_authorize(has_length: false, maximum_size: max_artifacts_size)
       end
 
       desc 'Upload artifacts for job' do
@@ -220,6 +240,10 @@ module API
         requires :id, type: Integer, desc: %q(Job's ID)
         optional :token, type: String, desc: %q(Job's authentication token)
         optional :expire_in, type: String, desc: %q(Specify when artifacts should expire)
+        optional :artifact_type, type: String, desc: %q(The type of artifact),
+                                 default: 'archive', values: Ci::JobArtifact.file_types.keys
+        optional :artifact_format, type: String, desc: %q(The format of artifact),
+                                   default: 'zip', values: Ci::JobArtifact.file_formats.keys
         optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
         optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse))
         optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse))
@@ -243,29 +267,29 @@ module API
         bad_request!('Missing artifacts file!') unless artifacts
         file_to_large! unless artifacts.size < max_artifacts_size
 
-        bad_request!("Already uploaded") if job.job_artifacts_archive
-
         expire_in = params['expire_in'] ||
           Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in
 
-        job.build_job_artifacts_archive(
+        job.job_artifacts.build(
           project: job.project,
           file: artifacts,
-          file_type: :archive,
+          file_type: params['artifact_type'],
+          file_format: params['artifact_format'],
           file_sha256: artifacts.sha256,
           expire_in: expire_in)
 
         if metadata
-          job.build_job_artifacts_metadata(
+          job.job_artifacts.build(
             project: job.project,
             file: metadata,
             file_type: :metadata,
+            file_format: :gzip,
             file_sha256: metadata.sha256,
             expire_in: expire_in)
         end
 
         if job.update(artifacts_expire_in: expire_in)
-          present job, with: Entities::JobRequest::Response
+          present Ci::BuildRunnerPresenter.new(job), with: Entities::JobRequest::Response
         else
           render_validation_error!(job)
         end
diff --git a/lib/api/runners.rb b/lib/api/runners.rb
index 2b78075ddbf..f72b33605a7 100644
--- a/lib/api/runners.rb
+++ b/lib/api/runners.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Runners < Grape::API
     include PaginationParams
@@ -9,12 +11,20 @@ module API
         success Entities::Runner
       end
       params do
-        optional :scope, type: String, values: %w[active paused online],
+        optional :scope, type: String, values: Ci::Runner::AVAILABLE_STATUSES,
                          desc: 'The scope of specific runners to show'
+        optional :type, type: String, values: Ci::Runner::AVAILABLE_TYPES,
+                        desc: 'The type of the runners to show'
+        optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES,
+                          desc: 'The status of the runners to show'
         use :pagination
       end
       get do
-        runners = filter_runners(current_user.ci_owned_runners, params[:scope], without: %w(specific shared))
+        runners = current_user.ci_owned_runners
+        runners = filter_runners(runners, params[:scope], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES)
+        runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES)
+        runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES)
+
         present paginate(runners), with: Entities::Runner
       end
 
@@ -22,13 +32,22 @@ module API
         success Entities::Runner
       end
       params do
-        optional :scope, type: String, values: %w[active paused online specific shared],
+        optional :scope, type: String, values: Ci::Runner::AVAILABLE_SCOPES,
                          desc: 'The scope of specific runners to show'
+        optional :type, type: String, values: Ci::Runner::AVAILABLE_TYPES,
+                        desc: 'The type of the runners to show'
+        optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES,
+                          desc: 'The status of the runners to show'
         use :pagination
       end
       get 'all' do
         authenticated_as_admin!
-        runners = filter_runners(Ci::Runner.all, params[:scope])
+
+        runners = Ci::Runner.all
+        runners = filter_runners(runners, params[:scope])
+        runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES)
+        runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES)
+
         present paginate(runners), with: Entities::Runner
       end
 
@@ -58,7 +77,7 @@ module API
         optional :access_level, type: String, values: Ci::Runner.access_levels.keys,
                                 desc: 'The access_level of the runner'
         optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this Runner will handle the job'
-        at_least_one_of :description, :active, :tag_list, :run_untagged, :locked, :access_level
+        at_least_one_of :description, :active, :tag_list, :run_untagged, :locked, :access_level, :maximum_timeout
       end
       put ':id' do
         runner = get_runner(params.delete(:id))
@@ -94,7 +113,7 @@ module API
         optional :status, type: String, desc: 'Status of the job', values: Ci::Build::AVAILABLE_STATUSES
         use :pagination
       end
-      get  ':id/jobs' do
+      get ':id/jobs' do
         runner = get_runner(params[:id])
         authenticate_list_runners_jobs!(runner)
 
@@ -107,19 +126,27 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       before { authorize_admin_project }
 
       desc 'Get runners available for project' do
         success Entities::Runner
       end
       params do
-        optional :scope, type: String, values: %w[active paused online specific shared],
+        optional :scope, type: String, values: Ci::Runner::AVAILABLE_SCOPES,
                          desc: 'The scope of specific runners to show'
+        optional :type, type: String, values: Ci::Runner::AVAILABLE_TYPES,
+                        desc: 'The type of the runners to show'
+        optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES,
+                          desc: 'The status of the runners to show'
         use :pagination
       end
       get ':id/runners' do
-        runners = filter_runners(Ci::Runner.owned_or_shared(user_project.id), params[:scope])
+        runners = Ci::Runner.owned_or_instance_wide(user_project.id)
+        runners = filter_runners(runners, params[:scope])
+        runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES)
+        runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES)
+
         present paginate(runners), with: Entities::Runner
       end
 
@@ -146,6 +173,7 @@ module API
       params do
         requires :runner_id, type: Integer, desc: 'The ID of the runner'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete ':id/runners/:runner_id' do
         runner_project = user_project.runner_projects.find_by(runner_id: params[:runner_id])
         not_found!('Runner') unless runner_project
@@ -155,19 +183,20 @@ module API
 
         destroy_conditionally!(runner_project)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
 
     helpers do
-      def filter_runners(runners, scope, options = {})
+      def filter_runners(runners, scope, allowed_scopes: ::Ci::Runner::AVAILABLE_SCOPES)
         return runners unless scope.present?
 
-        available_scopes = ::Ci::Runner::AVAILABLE_SCOPES
-        if options[:without]
-          available_scopes = available_scopes - options[:without]
+        unless allowed_scopes.include?(scope)
+          render_api_error!('Scope contains invalid value', 400)
         end
 
-        if (available_scopes & [scope]).empty?
-          render_api_error!('Scope contains invalid value', 400)
+        # Support deprecated scopes
+        if runners.respond_to?("deprecated_#{scope}")
+          scope = "deprecated_#{scope}"
         end
 
         runners.public_send(scope) # rubocop:disable GitlabSecurity/PublicSend
@@ -180,7 +209,7 @@ module API
       end
 
       def authenticate_show_runner!(runner)
-        return if runner.is_shared || current_user.admin?
+        return if runner.instance_type? || current_user.admin?
 
         forbidden!("No access granted") unless can?(current_user, :read_runner, runner)
       end
diff --git a/lib/api/scope.rb b/lib/api/scope.rb
index d5165b2e482..707775e5d15 100644
--- a/lib/api/scope.rb
+++ b/lib/api/scope.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Encapsulate a scope used for authorization, such as `api`, or `read_user`
 module API
   class Scope
diff --git a/lib/api/search.rb b/lib/api/search.rb
index 5d9ec617cb7..f5db692afe5 100644
--- a/lib/api/search.rb
+++ b/lib/api/search.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Search < Grape::API
     include PaginationParams
@@ -33,14 +35,7 @@ module API
       end
 
       def process_results(results)
-        case params[:scope]
-        when 'wiki_blobs'
-          paginate(results).map { |blob| Gitlab::ProjectSearchResults.parse_search_result(blob, user_project) }
-        when 'blobs'
-          paginate(results).map { |blob| blob[1] }
-        else
-          paginate(results)
-        end
+        paginate(results)
       end
 
       def snippets?
@@ -70,7 +65,7 @@ module API
       end
     end
 
-    resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Search on GitLab' do
         detail 'This feature was introduced in GitLab 10.5.'
       end
@@ -89,7 +84,7 @@ module API
       end
     end
 
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Search on GitLab' do
         detail 'This feature was introduced in GitLab 10.5.'
       end
diff --git a/lib/api/services.rb b/lib/api/services.rb
index 794fdab8f2b..637b5a8a89a 100644
--- a/lib/api/services.rb
+++ b/lib/api/services.rb
@@ -298,6 +298,14 @@ module API
           desc: 'Title'
         }
       ],
+      'discord' => [
+        {
+          required: true,
+          name: :webhook,
+          type: String,
+          desc: 'Discord webhook. e.g. https://discordapp.com/api/webhooks/…'
+        }
+      ],
       'drone-ci' => [
         {
           required: true,
@@ -354,18 +362,12 @@ module API
           desc: 'Flowdock token'
         }
       ],
-      'gemnasium' => [
+      'hangouts-chat' => [
         {
           required: true,
-          name: :api_key,
-          type: String,
-          desc: 'Your personal API key on gemnasium.com'
-        },
-        {
-          required: true,
-          name: :token,
+          name: :webhook,
           type: String,
-          desc: "The project's slug on gemnasium.com"
+          desc: 'The Hangouts Chat webhook. e.g. https://chat.googleapis.com/v1/spaces…'
         }
       ],
       'hipchat' => [
@@ -683,11 +685,12 @@ module API
       BuildkiteService,
       CampfireService,
       CustomIssueTrackerService,
+      DiscordService,
       DroneCiService,
       EmailsOnPushService,
       ExternalWikiService,
       FlowdockService,
-      GemnasiumService,
+      HangoutsChatService,
       HipchatService,
       IrkerService,
       JiraService,
@@ -760,7 +763,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS  do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       before { authenticate! }
       before { authorize_admin_project }
 
@@ -787,7 +790,7 @@ module API
           service = user_project.find_or_initialize_service(service_slug.underscore)
           service_params = declared_params(include_missing: false).merge(active: true)
 
-          if service.update_attributes(service_params)
+          if service.update(service_params)
             present service, with: Entities::ProjectService
           else
             render_api_error!('400 Bad Request', 400)
@@ -807,7 +810,7 @@ module API
             hash.merge!(key => nil)
           end
 
-          unless service.update_attributes(attrs.merge(active: false))
+          unless service.update(attrs.merge(active: false))
             render_api_error!('400 Bad Request', 400)
           end
         end
@@ -827,17 +830,19 @@ module API
 
     TRIGGER_SERVICES.each do |service_slug, settings|
       helpers do
+        # rubocop: disable CodeReuse/ActiveRecord
         def slash_command_service(project, service_slug, params)
           project.services.active.where(template: false).find do |service|
             service.try(:token) == params[:token] && service.to_param == service_slug.underscore
           end
         end
+        # rubocop: enable CodeReuse/ActiveRecord
       end
 
       params do
         requires :id, type: String, desc: 'The ID of a project'
       end
-      resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS  do
+      resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
         desc "Trigger a slash command for #{service_slug}" do
           detail 'Added in GitLab 8.13'
         end
diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index 02ef89f997f..95371961398 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Settings < Grape::API
     before { authenticated_as_admin! }
@@ -20,116 +22,105 @@ module API
       success Entities::ApplicationSetting
     end
     params do
-      optional :default_branch_protection, type: Integer, values: [0, 1, 2], desc: 'Determine if developers can push to master'
+      optional :admin_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.'
+      optional :after_sign_up_text, type: String, desc: 'Text shown after sign up'
+      optional :after_sign_out_path, type: String, desc: 'We will redirect users to this page after they sign out'
+      optional :akismet_enabled, type: Boolean, desc: 'Helps prevent bots from creating issues'
+      given akismet_enabled: ->(val) { val } do
+        requires :akismet_api_key, type: String, desc: 'Generate API key at http://www.akismet.com'
+      end
+      optional :clientside_sentry_enabled, type: Boolean, desc: 'Sentry can also be used for reporting and logging clientside exceptions. https://sentry.io/for/javascript/'
+      given clientside_sentry_enabled: ->(val) { val } do
+        requires :clientside_sentry_dsn, type: String, desc: 'Clientside Sentry Data Source Name'
+      end
+      optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)'
+      optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts"
+      optional :default_branch_protection, type: Integer, values: Gitlab::Access.protection_values, desc: 'Determine if developers can push to master'
+      optional :default_group_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default group visibility'
       optional :default_project_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default project visibility'
+      optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects'
       optional :default_snippet_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default snippet visibility'
-      optional :default_group_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default group visibility'
-      optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for groups, projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.'
-      optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project],
-                                desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com'
       optional :disabled_oauth_sign_in_sources, type: Array[String], desc: 'Disable certain OAuth sign-in sources'
-      optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.'
-      optional :project_export_enabled, type: Boolean, desc: 'Enable project export'
-      optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled'
-      optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects'
-      optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB'
-      optional :session_expire_delay, type: Integer, desc: 'Session duration in minutes. GitLab restart is required to apply changes.'
-      optional :user_oauth_applications, type: Boolean, desc: 'Allow users to register any application to use GitLab as an OAuth provider'
-      optional :user_default_external, type: Boolean, desc: 'Newly registered users will by default be external'
-      optional :signup_enabled, type: Boolean, desc: 'Flag indicating if sign up is enabled'
-      optional :send_user_confirmation_email, type: Boolean, desc: 'Send confirmation email on sign-up'
-      optional :domain_whitelist, type: String, desc: 'ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com'
       optional :domain_blacklist_enabled, type: Boolean, desc: 'Enable domain blacklist for sign ups'
       given domain_blacklist_enabled: ->(val) { val } do
         requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com'
       end
-      optional :after_sign_up_text, type: String, desc: 'Text shown after sign up'
-      optional :password_authentication_enabled_for_web, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface'
-      optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5
-      optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5
-      mutually_exclusive :password_authentication_enabled_for_web, :password_authentication_enabled, :signin_enabled
-      optional :password_authentication_enabled_for_git, type: Boolean, desc: 'Flag indicating if password authentication is enabled for Git over HTTP(S)'
-      optional :performance_bar_allowed_group_path, type: String, desc: 'Path of the group that is allowed to toggle the performance bar.'
-      optional :performance_bar_allowed_group_id, type: String, desc: 'Depreated: Use :performance_bar_allowed_group_path instead. Path of the group that is allowed to toggle the performance bar.' # support legacy names, can be removed in v6
-      optional :performance_bar_enabled, type: String, desc: 'Deprecated: Pass `performance_bar_allowed_group_path: nil` instead. Allow enabling the performance.' # support legacy names, can be removed in v6
-      optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication'
-      given require_two_factor_authentication: ->(val) { val } do
-        requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication'
-      end
-      optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page'
-      optional :after_sign_out_path, type: String, desc: 'We will redirect users to this page after they sign out'
-      optional :sign_in_text, type: String, desc: 'The sign in text of the GitLab application'
+      optional :domain_whitelist, type: String, desc: 'ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com'
+      optional :email_author_in_body, type: Boolean, desc: 'Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead.'
+      optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.'
+      optional :gitaly_timeout_default, type: Integer, desc: 'Default Gitaly timeout, in seconds. Set to 0 to disable timeouts.'
+      optional :gitaly_timeout_fast, type: Integer, desc: 'Gitaly fast operation timeout, in seconds. Set to 0 to disable timeouts.'
+      optional :gitaly_timeout_medium, type: Integer, desc: 'Medium Gitaly timeout, in seconds. Set to 0 to disable timeouts.'
+      optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled'
       optional :help_page_hide_commercial_content, type: Boolean, desc: 'Hide marketing-related entries from help'
-      optional :help_page_text, type: String, desc: 'Custom text displayed on the help page'
       optional :help_page_support_url, type: String, desc: 'Alternate support URL for help page'
-      optional :shared_runners_enabled, type: Boolean, desc: 'Enable shared runners for new projects'
-      given shared_runners_enabled: ->(val) { val } do
-        requires :shared_runners_text, type: String, desc: 'Shared runners text '
+      optional :help_page_text, type: String, desc: 'Custom text displayed on the help page'
+      optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page'
+      optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)'
+      given housekeeping_enabled: ->(val) { val } do
+        requires :housekeeping_bitmaps_enabled, type: Boolean, desc: "Creating pack file bitmaps makes housekeeping take a little longer but bitmaps should accelerate 'git clone' performance."
+        requires :housekeeping_full_repack_period, type: Integer, desc: "Number of Git pushes after which a full 'git repack' is run."
+        requires :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run."
+        requires :housekeeping_incremental_repack_period, type: Integer, desc: "Number of Git pushes after which an incremental 'git repack' is run."
       end
+      optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.'
+      optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project manifest],
+                                desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com'
       optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size for each job's artifacts"
-      optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts"
+      optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB'
       optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB'
-      optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)'
-      optional :prometheus_metrics_enabled, type: Boolean, desc: 'Enable Prometheus metrics'
       optional :metrics_enabled, type: Boolean, desc: 'Enable the InfluxDB metrics'
       given metrics_enabled: ->(val) { val } do
         requires :metrics_host, type: String, desc: 'The InfluxDB host'
-        requires :metrics_port, type: Integer, desc: 'The UDP port to use for connecting to InfluxDB'
-        requires :metrics_pool_size, type: Integer, desc: 'The amount of InfluxDB connections to open'
-        requires :metrics_timeout, type: Integer, desc: 'The amount of seconds after which an InfluxDB connection will time out'
         requires :metrics_method_call_threshold, type: Integer, desc: 'A method call is only tracked when it takes longer to complete than the given amount of milliseconds.'
-        requires :metrics_sample_interval, type: Integer, desc: 'The sampling interval in seconds'
         requires :metrics_packet_size, type: Integer, desc: 'The amount of points to store in a single UDP packet'
+        requires :metrics_pool_size, type: Integer, desc: 'The amount of InfluxDB connections to open'
+        requires :metrics_port, type: Integer, desc: 'The UDP port to use for connecting to InfluxDB'
+        requires :metrics_sample_interval, type: Integer, desc: 'The sampling interval in seconds'
+        requires :metrics_timeout, type: Integer, desc: 'The amount of seconds after which an InfluxDB connection will time out'
       end
-      optional :sidekiq_throttling_enabled, type: Boolean, desc: 'Enable Sidekiq Job Throttling'
-      given sidekiq_throttling_enabled: ->(val) { val } do
-        requires :sidekiq_throttling_queus, type: Array[String], desc: 'Choose which queues you wish to throttle'
-        requires :sidekiq_throttling_factor, type: Float, desc: 'The factor by which the queues should be throttled. A value between 0.0 and 1.0, exclusive.'
+      optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5
+      optional :password_authentication_enabled_for_web, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface'
+      mutually_exclusive :password_authentication_enabled_for_web, :password_authentication_enabled, :signin_enabled
+      optional :password_authentication_enabled_for_git, type: Boolean, desc: 'Flag indicating if password authentication is enabled for Git over HTTP(S)'
+      optional :performance_bar_allowed_group_id, type: String, desc: 'Deprecated: Use :performance_bar_allowed_group_path instead. Path of the group that is allowed to toggle the performance bar.' # support legacy names, can be removed in v6
+      optional :performance_bar_allowed_group_path, type: String, desc: 'Path of the group that is allowed to toggle the performance bar.'
+      optional :performance_bar_enabled, type: String, desc: 'Deprecated: Pass `performance_bar_allowed_group_path: nil` instead. Allow enabling the performance.' # support legacy names, can be removed in v6
+      optional :plantuml_enabled, type: Boolean, desc: 'Enable PlantUML'
+      given plantuml_enabled: ->(val) { val } do
+        requires :plantuml_url, type: String, desc: 'The PlantUML server URL'
       end
+      optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.'
+      optional :project_export_enabled, type: Boolean, desc: 'Enable project export'
+      optional :prometheus_metrics_enabled, type: Boolean, desc: 'Enable Prometheus metrics'
       optional :recaptcha_enabled, type: Boolean, desc: 'Helps prevent bots from creating accounts'
       given recaptcha_enabled: ->(val) { val } do
         requires :recaptcha_site_key, type: String, desc: 'Generate site key at http://www.google.com/recaptcha'
         requires :recaptcha_private_key, type: String, desc: 'Generate private key at http://www.google.com/recaptcha'
       end
-      optional :akismet_enabled, type: Boolean, desc: 'Helps prevent bots from creating issues'
-      given akismet_enabled: ->(val) { val } do
-        requires :akismet_api_key, type: String, desc: 'Generate API key at http://www.akismet.com'
+      optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues."
+      optional :repository_storages, type: Array[String], desc: 'Storage paths for new projects'
+      optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to set up Two-factor authentication'
+      given require_two_factor_authentication: ->(val) { val } do
+        requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication'
       end
-      optional :admin_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.'
+      optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for groups, projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.'
+      optional :send_user_confirmation_email, type: Boolean, desc: 'Send confirmation email on sign-up'
       optional :sentry_enabled, type: Boolean, desc: 'Sentry is an error reporting and logging tool which is currently not shipped with GitLab, get it here: https://getsentry.com'
       given sentry_enabled: ->(val) { val } do
         requires :sentry_dsn, type: String, desc: 'Sentry Data Source Name'
       end
-      optional :clientside_sentry_enabled, type: Boolean, desc: 'Sentry can also be used for reporting and logging clientside exceptions. https://sentry.io/for/javascript/'
-      given clientside_sentry_enabled: ->(val) { val } do
-        requires :clientside_sentry_dsn, type: String, desc: 'Clientside Sentry Data Source Name'
-      end
-      optional :repository_storages, type: Array[String], desc: 'Storage paths for new projects'
-      optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues."
-      optional :koding_enabled, type: Boolean, desc: 'Enable Koding'
-      given koding_enabled: ->(val) { val } do
-        requires :koding_url, type: String, desc: 'The Koding team URL'
-      end
-      optional :plantuml_enabled, type: Boolean, desc: 'Enable PlantUML'
-      given plantuml_enabled: ->(val) { val } do
-        requires :plantuml_url, type: String, desc: 'The PlantUML server URL'
-      end
-      optional :version_check_enabled, type: Boolean, desc: 'Let GitLab inform you when an update is available.'
-      optional :email_author_in_body, type: Boolean, desc: 'Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead.'
-      optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.'
-      optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)'
-      given housekeeping_enabled: ->(val) { val } do
-        requires :housekeeping_bitmaps_enabled, type: Boolean, desc: "Creating pack file bitmaps makes housekeeping take a little longer but bitmaps should accelerate 'git clone' performance."
-        requires :housekeeping_incremental_repack_period, type: Integer, desc: "Number of Git pushes after which an incremental 'git repack' is run."
-        requires :housekeeping_full_repack_period, type: Integer, desc: "Number of Git pushes after which a full 'git repack' is run."
-        requires :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run."
+      optional :session_expire_delay, type: Integer, desc: 'Session duration in minutes. GitLab restart is required to apply changes.'
+      optional :shared_runners_enabled, type: Boolean, desc: 'Enable shared runners for new projects'
+      given shared_runners_enabled: ->(val) { val } do
+        requires :shared_runners_text, type: String, desc: 'Shared runners text '
       end
+      optional :sign_in_text, type: String, desc: 'The sign in text of the GitLab application'
+      optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5
+      optional :signup_enabled, type: Boolean, desc: 'Flag indicating if sign up is enabled'
       optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.'
-      optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.'
-      optional :gitaly_timeout_default, type: Integer, desc: 'Default Gitaly timeout, in seconds. Set to 0 to disable timeouts.'
-      optional :gitaly_timeout_medium, type: Integer, desc: 'Medium Gitaly timeout, in seconds. Set to 0 to disable timeouts.'
-      optional :gitaly_timeout_fast, type: Integer, desc: 'Gitaly fast operation timeout, in seconds. Set to 0 to disable timeouts.'
       optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.'
+      optional :instance_statistics_visibility_private, type: Boolean, desc: 'When set to `true` Instance statistics will only be available to admins'
 
       ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type|
         optional :"#{type}_key_restriction",
diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb
index 11f2b40269a..daa9598a204 100644
--- a/lib/api/sidekiq_metrics.rb
+++ b/lib/api/sidekiq_metrics.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sidekiq/api'
 
 module API
diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb
index b30305b4bc9..326d55afd0e 100644
--- a/lib/api/snippets.rb
+++ b/lib/api/snippets.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   # Snippets API
   class Snippets < Grape::API
@@ -12,7 +14,7 @@ module API
         end
 
         def public_snippets
-          SnippetsFinder.new(current_user, visibility: Snippet::PUBLIC).execute
+          SnippetsFinder.new(current_user, scope: :are_public).execute
         end
       end
 
@@ -92,6 +94,7 @@ module API
                               desc: 'The visibility of the snippet'
         at_least_one_of :title, :file_name, :content, :visibility
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       put ':id' do
         snippet = snippets_for_current_user.find_by(id: params.delete(:id))
         break not_found!('Snippet') unless snippet
@@ -110,6 +113,7 @@ module API
           render_validation_error!(snippet)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Remove snippet' do
         detail 'This feature was introduced in GitLab 8.15.'
@@ -118,6 +122,7 @@ module API
       params do
         requires :id, type: Integer, desc: 'The ID of a snippet'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete ':id' do
         snippet = snippets_for_current_user.find_by(id: params.delete(:id))
         break not_found!('Snippet') unless snippet
@@ -126,6 +131,7 @@ module API
 
         destroy_conditionally!(snippet)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Get a raw snippet' do
         detail 'This feature was introduced in GitLab 8.15.'
@@ -133,14 +139,17 @@ module API
       params do
         requires :id, type: Integer, desc: 'The ID of a snippet'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ":id/raw" do
         snippet = snippets_for_current_user.find_by(id: params.delete(:id))
         break not_found!('Snippet') unless snippet
 
         env['api.format'] = :txt
         content_type 'text/plain'
+        header['Content-Disposition'] = 'attachment'
         present snippet.content
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Get the user agent details for a snippet' do
         success Entities::UserAgentDetail
@@ -148,6 +157,7 @@ module API
       params do
         requires :id, type: Integer, desc: 'The ID of a snippet'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ":id/user_agent_detail" do
         authenticated_as_admin!
 
@@ -157,6 +167,7 @@ module API
 
         present snippet.user_agent_detail, with: Entities::UserAgentDetail
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/api/submodules.rb b/lib/api/submodules.rb
new file mode 100644
index 00000000000..72d7d994102
--- /dev/null
+++ b/lib/api/submodules.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module API
+  class Submodules < Grape::API
+    before { authenticate! }
+
+    helpers do
+      def commit_params(attrs)
+        {
+          submodule: attrs[:submodule],
+          commit_sha: attrs[:commit_sha],
+          branch_name: attrs[:branch],
+          commit_message: attrs[:commit_message]
+        }
+      end
+    end
+
+    params do
+      requires :id, type: String, desc: 'The project ID'
+    end
+    resource :projects, requirements: Files::FILE_ENDPOINT_REQUIREMENTS do
+      desc 'Update existing submodule reference in repository' do
+        success Entities::Commit
+      end
+      params do
+        requires :submodule, type: String, desc: 'Url encoded full path to submodule.'
+        requires :commit_sha, type: String, desc: 'Commit sha to update the submodule to.'
+        requires :branch, type: String, desc: 'Name of the branch to commit into.'
+        optional :commit_message, type: String, desc: 'Commit message. If no message is provided a default one will be set.'
+      end
+      put ":id/repository/submodules/:submodule", requirements: Files::FILE_ENDPOINT_REQUIREMENTS do
+        authorize! :push_code, user_project
+
+        submodule_params = declared_params(include_missing: false)
+
+        result = ::Submodules::UpdateService.new(user_project, current_user, commit_params(submodule_params)).execute
+
+        if result[:status] == :success
+          commit_detail = user_project.repository.commit(result[:result])
+          present commit_detail, with: Entities::CommitDetail
+        else
+          render_api_error!(result[:message], result[:http_status] || 400)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb
index b3e1e23031a..74ad3c35a61 100644
--- a/lib/api/subscriptions.rb
+++ b/lib/api/subscriptions.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Subscriptions < Grape::API
     before { authenticate! }
@@ -12,7 +14,7 @@ module API
       requires :id, type: String, desc: 'The ID of a project'
       requires :subscribable_id, type: String, desc: 'The ID of a resource'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       subscribable_types.each do |type, finder|
         type_singularized = type.singularize
         entity_class = Entities.const_get(type_singularized.camelcase)
diff --git a/lib/api/suggestions.rb b/lib/api/suggestions.rb
new file mode 100644
index 00000000000..d008d1b9e97
--- /dev/null
+++ b/lib/api/suggestions.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module API
+  class Suggestions < Grape::API
+    before { authenticate! }
+
+    resource :suggestions do
+      desc 'Apply suggestion patch in the Merge Request it was created' do
+        success Entities::Suggestion
+      end
+      params do
+        requires :id, type: String, desc: 'The suggestion ID'
+      end
+      put ':id/apply' do
+        suggestion = Suggestion.find_by_id(params[:id])
+
+        not_found! unless suggestion
+        authorize! :apply_suggestion, suggestion
+
+        result = ::Suggestions::ApplyService.new(current_user).execute(suggestion)
+
+        if result[:status] == :success
+          present suggestion, with: Entities::Suggestion, current_user: current_user
+        else
+          http_status = result[:http_status] || 400
+          render_api_error!(result[:message], http_status)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb
index c7a460df46a..51fae0e54aa 100644
--- a/lib/api/system_hooks.rb
+++ b/lib/api/system_hooks.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class SystemHooks < Grape::API
     include PaginationParams
@@ -63,12 +65,14 @@ module API
       params do
         requires :id, type: Integer, desc: 'The ID of the system hook'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete ":id" do
         hook = SystemHook.find_by(id: params[:id])
         not_found!('System hook') unless hook
 
         destroy_conditionally!(hook)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/api/tags.rb b/lib/api/tags.rb
index 5e0afc6a7e4..f5359fd316c 100644
--- a/lib/api/tags.rb
+++ b/lib/api/tags.rb
@@ -1,15 +1,17 @@
+# frozen_string_literal: true
+
 module API
   class Tags < Grape::API
     include PaginationParams
 
-    TAG_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(tag_name: API::NO_SLASH_URL_PART_REGEX)
+    TAG_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(tag_name: API::NO_SLASH_URL_PART_REGEX)
 
     before { authorize! :download_code, user_project }
 
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Get a project repository tags' do
         success Entities::Tag
       end
@@ -18,12 +20,15 @@ module API
                         desc: 'Return tags sorted in updated by `asc` or `desc` order.'
         optional :order_by, type: String, values: %w[name updated], default: 'updated',
                             desc: 'Return tags ordered by `name` or `updated` fields.'
+        optional :search, type: String, desc: 'Return list of tags matching the search criteria'
         use :pagination
       end
       get ':id/repository/tags' do
-        tags = ::Kaminari.paginate_array(::TagsFinder.new(user_project.repository, sort: "#{params[:order_by]}_#{params[:sort]}").execute)
+        tags = ::TagsFinder.new(user_project.repository,
+                                sort: "#{params[:order_by]}_#{params[:sort]}",
+                                search: params[:search]).execute
 
-        present paginate(tags), with: Entities::Tag, project: user_project
+        present paginate(::Kaminari.paginate_array(tags)), with: Entities::Tag, project: user_project
       end
 
       desc 'Get a single repository tag' do
@@ -40,21 +45,35 @@ module API
       end
 
       desc 'Create a new repository tag' do
+        detail 'This optional release_description parameter was deprecated in GitLab 11.7.'
         success Entities::Tag
       end
       params do
         requires :tag_name,            type: String, desc: 'The name of the tag'
         requires :ref,                 type: String, desc: 'The commit sha or branch name'
         optional :message,             type: String, desc: 'Specifying a message creates an annotated tag'
-        optional :release_description, type: String, desc: 'Specifying release notes stored in the GitLab database'
+        optional :release_description, type: String, desc: 'Specifying release notes stored in the GitLab database (deprecated in GitLab 11.7)'
       end
       post ':id/repository/tags' do
         authorize_push_project
 
         result = ::Tags::CreateService.new(user_project, current_user)
-          .execute(params[:tag_name], params[:ref], params[:message], params[:release_description])
+          .execute(params[:tag_name], params[:ref], params[:message])
 
         if result[:status] == :success
+          # Release creation with Tags API was deprecated in GitLab 11.7
+          if params[:release_description].present?
+            release_create_params = {
+              tag: params[:tag_name],
+              name: params[:tag_name], # Name can be specified in new API
+              description: params[:release_description]
+            }
+
+            ::Releases::CreateService
+              .new(user_project, current_user, release_create_params)
+              .execute
+          end
+
           present result[:tag],
                   with: Entities::Tag,
                   project: user_project
@@ -86,44 +105,72 @@ module API
       end
 
       desc 'Add a release note to a tag' do
-        success Entities::Release
+        detail 'This feature was deprecated in GitLab 11.7.'
+        success Entities::TagRelease
       end
       params do
-        requires :tag_name,    type: String, desc: 'The name of the tag'
+        requires :tag_name,    type: String, desc: 'The name of the tag', as: :tag
         requires :description, type: String, desc: 'Release notes with markdown support'
       end
       post ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS do
-        authorize_push_project
+        authorize_create_release!
 
-        result = CreateReleaseService.new(user_project, current_user)
-          .execute(params[:tag_name], params[:description])
+        ##
+        # Legacy API does not support tag auto creation.
+        not_found!('Tag') unless user_project.repository.find_tag(params[:tag])
+
+        release_create_params = {
+          tag: params[:tag],
+          name: params[:tag], # Name can be specified in new API
+          description: params[:description]
+        }
+
+        result = ::Releases::CreateService
+          .new(user_project, current_user, release_create_params)
+          .execute
 
         if result[:status] == :success
-          present result[:release], with: Entities::Release
+          present result[:release], with: Entities::TagRelease
         else
           render_api_error!(result[:message], result[:http_status])
         end
       end
 
       desc "Update a tag's release note" do
-        success Entities::Release
+        detail 'This feature was deprecated in GitLab 11.7.'
+        success Entities::TagRelease
       end
       params do
-        requires :tag_name,    type: String, desc: 'The name of the tag'
+        requires :tag_name,    type: String, desc: 'The name of the tag', as: :tag
         requires :description, type: String, desc: 'Release notes with markdown support'
       end
       put ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS do
-        authorize_push_project
+        authorize_update_release!
 
-        result = UpdateReleaseService.new(user_project, current_user)
-          .execute(params[:tag_name], params[:description])
+        result = ::Releases::UpdateService
+          .new(user_project, current_user, declared_params(include_missing: false))
+          .execute
 
         if result[:status] == :success
-          present result[:release], with: Entities::Release
+          present result[:release], with: Entities::TagRelease
         else
           render_api_error!(result[:message], result[:http_status])
         end
       end
     end
+
+    helpers do
+      def authorize_create_release!
+        authorize! :create_release, user_project
+      end
+
+      def authorize_update_release!
+        authorize! :update_release, release
+      end
+
+      def release
+        @release ||= user_project.releases.find_by_tag(params[:tag])
+      end
+    end
   end
 end
diff --git a/lib/api/templates.rb b/lib/api/templates.rb
index 41862768a3f..51f357d9477 100644
--- a/lib/api/templates.rb
+++ b/lib/api/templates.rb
@@ -1,46 +1,22 @@
+# frozen_string_literal: true
+
 module API
   class Templates < Grape::API
     include PaginationParams
 
     GLOBAL_TEMPLATE_TYPES = {
       gitignores: {
-        klass: Gitlab::Template::GitignoreTemplate,
         gitlab_version: 8.8
       },
       gitlab_ci_ymls: {
-        klass: Gitlab::Template::GitlabCiYmlTemplate,
         gitlab_version: 8.9
       },
       dockerfiles: {
-        klass: Gitlab::Template::DockerfileTemplate,
         gitlab_version: 8.15
       }
     }.freeze
-    PROJECT_TEMPLATE_REGEX =
-      %r{[\<\{\[]
-        (project|description|
-        one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here
-      [\>\}\]]}xi.freeze
-    YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze
-    FULLNAME_TEMPLATE_REGEX =
-      %r{[\<\{\[]
-        (fullname|name\sof\s(author|copyright\sowner))
-      [\>\}\]]}xi.freeze
 
     helpers do
-      def parsed_license_template
-        # We create a fresh Licensee::License object since we'll modify its
-        # content in place below.
-        template = Licensee::License.new(params[:name])
-
-        template.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s)
-        template.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present?
-
-        fullname = params[:fullname].presence || current_user.try(:name)
-        template.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname
-        template
-      end
-
       def render_response(template_type, template)
         not_found!(template_type.to_s.singularize) unless template
         present template, with: Entities::Template
@@ -56,11 +32,12 @@ module API
       use :pagination
     end
     get "templates/licenses" do
-      options = {
-        featured: declared(params)[:popular].present? ? true : nil
-      }
-      licences = ::Kaminari.paginate_array(Licensee::License.all(options))
-      present paginate(licences), with: Entities::License
+      popular = declared(params)[:popular]
+      popular = to_boolean(popular) if popular.present?
+
+      templates = TemplateFinder.build(:licenses, nil, popular: popular).execute
+
+      present paginate(::Kaminari.paginate_array(templates)), with: ::API::Entities::License
     end
 
     desc 'Get the text for a specific license' do
@@ -71,15 +48,19 @@ module API
       requires :name, type: String, desc: 'The name of the template'
     end
     get "templates/licenses/:name", requirements: { name: /[\w\.-]+/ } do
-      not_found!('License') unless Licensee::License.find(declared(params)[:name])
+      template = TemplateFinder.build(:licenses, nil, name: params[:name]).execute
+
+      not_found!('License') unless template.present?
 
-      template = parsed_license_template
+      template.resolve!(
+        project_name: params[:project].presence,
+        fullname: params[:fullname].presence || current_user&.name
+      )
 
       present template, with: ::API::Entities::License
     end
 
     GLOBAL_TEMPLATE_TYPES.each do |template_type, properties|
-      klass = properties[:klass]
       gitlab_version = properties[:gitlab_version]
 
       desc 'Get the list of the available template' do
@@ -90,7 +71,7 @@ module API
         use :pagination
       end
       get "templates/#{template_type}" do
-        templates = ::Kaminari.paginate_array(klass.all)
+        templates = ::Kaminari.paginate_array(TemplateFinder.build(template_type, nil).execute)
         present paginate(templates), with: Entities::TemplatesList
       end
 
@@ -101,8 +82,9 @@ module API
       params do
         requires :name, type: String, desc: 'The name of the template'
       end
-      get "templates/#{template_type}/:name" do
-        new_template = klass.find(declared(params)[:name])
+      get "templates/#{template_type}/:name", requirements: { name: /[\w\.-]+/ } do
+        finder = TemplateFinder.build(template_type, nil, name: declared(params)[:name])
+        new_template = finder.execute
 
         render_response(template_type, new_template)
       end
diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb
index 2bb451dea89..93fe06bec27 100644
--- a/lib/api/time_tracking_endpoints.rb
+++ b/lib/api/time_tracking_endpoints.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   module TimeTrackingEndpoints
     extend ActiveSupport::Concern
diff --git a/lib/api/todos.rb b/lib/api/todos.rb
index c6dbcf84e3a..64ac8ece56c 100644
--- a/lib/api/todos.rb
+++ b/lib/api/todos.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Todos < Grape::API
     include PaginationParams
@@ -12,7 +14,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS  do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       ISSUABLE_TYPES.each do |type, finder|
         type_id_str = "#{type.singularize}_iid".to_sym
 
diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb
index b29e660c6e0..8fc7c7361e1 100644
--- a/lib/api/triggers.rb
+++ b/lib/api/triggers.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Triggers < Grape::API
     include PaginationParams
@@ -5,12 +7,12 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS  do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Trigger a GitLab project pipeline' do
         success Entities::Pipeline
       end
       params do
-        requires :ref, type: String, desc: 'The commit sha or name of a branch or tag'
+        requires :ref, type: String, desc: 'The commit sha or name of a branch or tag', allow_blank: false
         requires :token, type: String, desc: 'The unique token of trigger'
         optional :variables, type: Hash, desc: 'The list of variables to be injected into build'
       end
@@ -42,20 +44,22 @@ module API
       params do
         use :pagination
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ':id/triggers' do
         authenticate!
         authorize! :admin_build, user_project
 
         triggers = user_project.triggers.includes(:trigger_requests)
 
-        present paginate(triggers), with: Entities::Trigger
+        present paginate(triggers), with: Entities::Trigger, current_user: current_user
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Get specific trigger of a project' do
         success Entities::Trigger
       end
       params do
-        requires :trigger_id, type: Integer,  desc: 'The trigger ID'
+        requires :trigger_id, type: Integer, desc: 'The trigger ID'
       end
       get ':id/triggers/:trigger_id' do
         authenticate!
@@ -64,14 +68,14 @@ module API
         trigger = user_project.triggers.find(params.delete(:trigger_id))
         break not_found!('Trigger') unless trigger
 
-        present trigger, with: Entities::Trigger
+        present trigger, with: Entities::Trigger, current_user: current_user
       end
 
       desc 'Create a trigger' do
         success Entities::Trigger
       end
       params do
-        requires :description, type: String,  desc: 'The trigger description'
+        requires :description, type: String, desc: 'The trigger description'
       end
       post ':id/triggers' do
         authenticate!
@@ -81,7 +85,7 @@ module API
           declared_params(include_missing: false).merge(owner: current_user))
 
         if trigger.valid?
-          present trigger, with: Entities::Trigger
+          present trigger, with: Entities::Trigger, current_user: current_user
         else
           render_validation_error!(trigger)
         end
@@ -102,7 +106,7 @@ module API
         break not_found!('Trigger') unless trigger
 
         if trigger.update(declared_params(include_missing: false))
-          present trigger, with: Entities::Trigger
+          present trigger, with: Entities::Trigger, current_user: current_user
         else
           render_validation_error!(trigger)
         end
@@ -112,7 +116,7 @@ module API
         success Entities::Trigger
       end
       params do
-        requires :trigger_id, type: Integer,  desc: 'The trigger ID'
+        requires :trigger_id, type: Integer, desc: 'The trigger ID'
       end
       post ':id/triggers/:trigger_id/take_ownership' do
         authenticate!
@@ -123,7 +127,7 @@ module API
 
         if trigger.update(owner: current_user)
           status :ok
-          present trigger, with: Entities::Trigger
+          present trigger, with: Entities::Trigger, current_user: current_user
         else
           render_validation_error!(trigger)
         end
@@ -133,7 +137,7 @@ module API
         success Entities::Trigger
       end
       params do
-        requires :trigger_id, type: Integer,  desc: 'The trigger ID'
+        requires :trigger_id, type: Integer, desc: 'The trigger ID'
       end
       delete ':id/triggers/:trigger_id' do
         authenticate!
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 14b8a796c8e..8ce09a8881b 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Users < Grape::API
     include PaginationParams
@@ -14,11 +16,14 @@ module API
       end
 
       helpers do
+        # rubocop: disable CodeReuse/ActiveRecord
         def find_user_by_id(params)
           id = params[:user_id] || params[:id]
           User.find_by(id: id) || not_found!('User')
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
+        # rubocop: disable CodeReuse/ActiveRecord
         def reorder_users(users)
           if params[:order_by] && params[:sort]
             users.reorder(params[:order_by] => params[:sort])
@@ -26,6 +31,7 @@ module API
             users
           end
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         params :optional_attributes do
           optional :skype, type: String, desc: 'The Skype username'
@@ -38,10 +44,12 @@ module API
           optional :provider, type: String, desc: 'The external provider'
           optional :bio, type: String, desc: 'The biography of the user'
           optional :location, type: String, desc: 'The location of the user'
+          optional :public_email, type: String, desc: 'The public email of the user'
           optional :admin, type: Boolean, desc: 'Flag indicating the user is an administrator'
           optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups'
           optional :external, type: Boolean, desc: 'Flag indicating the user is an external user'
           optional :avatar, type: File, desc: 'Avatar image for user'
+          optional :private_profile, type: Boolean, desc: 'Flag indicating the user has a private profile'
           all_or_none_of :extern_uid, :provider
         end
 
@@ -73,6 +81,7 @@ module API
         use :pagination
         use :with_custom_attributes
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get do
         authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?)
 
@@ -96,10 +105,11 @@ module API
 
         entity = current_user&.admin? ? Entities::UserWithAdmin : Entities::UserBasic
         users = users.preload(:identities, :u2f_registrations) if entity == Entities::UserWithAdmin
-        users, options = with_custom_attributes(users, with: entity)
+        users, options = with_custom_attributes(users, { with: entity, current_user: current_user })
 
         present paginate(users), options
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Get a single user' do
         success Entities::User
@@ -109,15 +119,28 @@ module API
 
         use :with_custom_attributes
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ":id" do
         user = User.find_by(id: params[:id])
         not_found!('User') unless user && can?(current_user, :read_user, user)
 
-        opts = current_user&.admin? ? { with: Entities::UserWithAdmin } : { with: Entities::User }
+        opts = { with: current_user&.admin? ? Entities::UserWithAdmin : Entities::User, current_user: current_user }
         user, opts = with_custom_attributes(user, opts)
 
         present user, opts
       end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      desc "Get the status of a user"
+      params do
+        requires :user_id, type: String, desc: 'The ID or username of the user'
+      end
+      get ":user_id/status", requirements: API::USER_REQUIREMENTS do
+        user = find_user(params[:user_id])
+        not_found!('User') unless user && can?(current_user, :read_user, user)
+
+        present user.status || {}, with: Entities::UserStatus
+      end
 
       desc 'Create a user. Available only for admins.' do
         success Entities::UserPublic
@@ -139,15 +162,15 @@ module API
         user = ::Users::CreateService.new(current_user, params).execute(skip_authorization: true)
 
         if user.persisted?
-          present user, with: Entities::UserPublic
+          present user, with: Entities::UserPublic, current_user: current_user
         else
           conflict!('Email has already been taken') if User
-              .where(email: user.email)
-              .count > 0
+            .by_any_email(user.email.downcase)
+            .any?
 
           conflict!('Username has already been taken') if User
-              .where(username: user.username)
-              .count > 0
+            .by_username(user.username)
+            .any?
 
           render_validation_error!(user)
         end
@@ -165,6 +188,7 @@ module API
         optional :username, type: String, desc: 'The username of the user'
         use :optional_attributes
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       put ":id" do
         authenticated_as_admin!
 
@@ -172,11 +196,11 @@ module API
         not_found!('User') unless user
 
         conflict!('Email has already been taken') if params[:email] &&
-            User.where(email: params[:email])
+            User.by_any_email(params[:email].downcase)
                 .where.not(id: user.id).count > 0
 
         conflict!('Username has already been taken') if params[:username] &&
-            User.where(username: params[:username])
+            User.by_username(params[:username])
                 .where.not(id: user.id).count > 0
 
         user_params = declared_params(include_missing: false)
@@ -186,7 +210,7 @@ module API
           identity = user.identities.find_by(provider: identity_attrs[:provider])
 
           if identity
-            identity.update_attributes(identity_attrs)
+            identity.update(identity_attrs)
           else
             identity = user.identities.build(identity_attrs)
             identity.save
@@ -198,11 +222,12 @@ module API
         result = ::Users::UpdateService.new(current_user, user_params.except(:extern_uid, :provider).merge(user: user)).execute
 
         if result[:status] == :success
-          present user, with: Entities::UserPublic
+          present user, with: Entities::UserPublic, current_user: current_user
         else
           render_validation_error!(user)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Add an SSH key to a specified user. Available only for admins.' do
         success Entities::SSHKey
@@ -212,6 +237,7 @@ module API
         requires :key, type: String, desc: 'The new SSH key'
         requires :title, type: String, desc: 'The title of the new SSH key'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       post ":id/keys" do
         authenticated_as_admin!
 
@@ -226,22 +252,23 @@ module API
           render_validation_error!(key)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
-      desc 'Get the SSH keys of a specified user. Available only for admins.' do
+      desc 'Get the SSH keys of a specified user.' do
         success Entities::SSHKey
       end
       params do
         requires :id, type: Integer, desc: 'The ID of the user'
         use :pagination
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ':id/keys' do
-        authenticated_as_admin!
-
         user = User.find_by(id: params[:id])
-        not_found!('User') unless user
+        not_found!('User') unless user && can?(current_user, :read_user, user)
 
         present paginate(user.keys), with: Entities::SSHKey
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Delete an existing SSH key from a specified user. Available only for admins.' do
         success Entities::SSHKey
@@ -250,6 +277,7 @@ module API
         requires :id, type: Integer, desc: 'The ID of the user'
         requires :key_id, type: Integer, desc: 'The ID of the SSH key'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete ':id/keys/:key_id' do
         authenticated_as_admin!
 
@@ -261,6 +289,7 @@ module API
 
         destroy_conditionally!(key)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Add a GPG key to a specified user. Available only for admins.' do
         detail 'This feature was added in GitLab 10.0'
@@ -270,6 +299,7 @@ module API
         requires :id, type: Integer, desc: 'The ID of the user'
         requires :key, type: String, desc: 'The new GPG key'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       post ':id/gpg_keys' do
         authenticated_as_admin!
 
@@ -284,6 +314,7 @@ module API
           render_validation_error!(key)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Get the GPG keys of a specified user. Available only for admins.' do
         detail 'This feature was added in GitLab 10.0'
@@ -293,6 +324,7 @@ module API
         requires :id, type: Integer, desc: 'The ID of the user'
         use :pagination
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ':id/gpg_keys' do
         authenticated_as_admin!
 
@@ -301,6 +333,7 @@ module API
 
         present paginate(user.gpg_keys), with: Entities::GPGKey
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Delete an existing GPG key from a specified user. Available only for admins.' do
         detail 'This feature was added in GitLab 10.0'
@@ -309,6 +342,7 @@ module API
         requires :id, type: Integer, desc: 'The ID of the user'
         requires :key_id, type: Integer, desc: 'The ID of the GPG key'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete ':id/gpg_keys/:key_id' do
         authenticated_as_admin!
 
@@ -321,6 +355,7 @@ module API
         status 204
         key.destroy
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Revokes an existing GPG key from a specified user. Available only for admins.' do
         detail 'This feature was added in GitLab 10.0'
@@ -329,6 +364,7 @@ module API
         requires :id, type: Integer, desc: 'The ID of the user'
         requires :key_id, type: Integer, desc: 'The ID of the GPG key'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       post ':id/gpg_keys/:key_id/revoke' do
         authenticated_as_admin!
 
@@ -341,6 +377,7 @@ module API
         key.revoke
         status :accepted
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Add an email address to a specified user. Available only for admins.' do
         success Entities::Email
@@ -348,7 +385,9 @@ module API
       params do
         requires :id, type: Integer, desc: 'The ID of the user'
         requires :email, type: String, desc: 'The email of the user'
+        optional :skip_confirmation, type: Boolean, desc: 'Skip confirmation of email and assume it is verified'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       post ":id/emails" do
         authenticated_as_admin!
 
@@ -363,6 +402,7 @@ module API
           render_validation_error!(email)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Get the emails addresses of a specified user. Available only for admins.' do
         success Entities::Email
@@ -371,6 +411,7 @@ module API
         requires :id, type: Integer, desc: 'The ID of the user'
         use :pagination
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ':id/emails' do
         authenticated_as_admin!
         user = User.find_by(id: params[:id])
@@ -378,6 +419,7 @@ module API
 
         present paginate(user.emails), with: Entities::Email
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Delete an email address of a specified user. Available only for admins.' do
         success Entities::Email
@@ -386,6 +428,7 @@ module API
         requires :id, type: Integer, desc: 'The ID of the user'
         requires :email_id, type: Integer, desc: 'The ID of the email'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete ':id/emails/:email_id' do
         authenticated_as_admin!
         user = User.find_by(id: params[:id])
@@ -398,6 +441,7 @@ module API
           Emails::DestroyService.new(current_user, user: user).execute(email)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Delete a user. Available only for admins.' do
         success Entities::Email
@@ -406,6 +450,7 @@ module API
         requires :id, type: Integer, desc: 'The ID of the user'
         optional :hard_delete, type: Boolean, desc: "Whether to remove a user's contributions"
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete ":id" do
         Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42279')
 
@@ -418,11 +463,13 @@ module API
           user.delete_async(deleted_by: current_user, params: params)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Block a user. Available only for admins.'
       params do
         requires :id, type: Integer, desc: 'The ID of the user'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       post ':id/block' do
         authenticated_as_admin!
         user = User.find_by(id: params[:id])
@@ -434,11 +481,13 @@ module API
           forbidden!('LDAP blocked users cannot be modified by the API')
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Unblock a user. Available only for admins.'
       params do
         requires :id, type: Integer, desc: 'The ID of the user'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       post ':id/unblock' do
         authenticated_as_admin!
         user = User.find_by(id: params[:id])
@@ -450,6 +499,7 @@ module API
           user.activate
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       params do
         requires :user_id, type: Integer, desc: 'The ID of the user'
@@ -463,7 +513,7 @@ module API
             end
 
             def find_impersonation_token
-              finder.find_by(id: declared_params[:impersonation_token_id]) || not_found!('Impersonation Token')
+              finder.find_by_id(declared_params[:impersonation_token_id]) || not_found!('Impersonation Token')
             end
           end
 
@@ -481,7 +531,7 @@ module API
 
           desc 'Create a impersonation token. Available only for admins.' do
             detail 'This feature was introduced in GitLab 9.0'
-            success Entities::ImpersonationToken
+            success Entities::ImpersonationTokenWithToken
           end
           params do
             requires :name, type: String, desc: 'The name of the impersonation token'
@@ -492,7 +542,7 @@ module API
             impersonation_token = finder.build(declared_params(include_missing: false))
 
             if impersonation_token.save
-              present impersonation_token, with: Entities::ImpersonationToken
+              present impersonation_token, with: Entities::ImpersonationTokenWithToken
             else
               render_validation_error!(impersonation_token)
             end
@@ -531,18 +581,22 @@ module API
         authenticate!
       end
 
-      desc 'Get the currently authenticated user' do
-        success Entities::UserPublic
-      end
-      get do
-        entity =
-          if current_user.admin?
-            Entities::UserWithAdmin
-          else
-            Entities::UserPublic
-          end
+      # Enabling /user endpoint for the v3 version to allow oauth
+      # authentication through this endpoint.
+      version %w(v3 v4), using: :path do
+        desc 'Get the currently authenticated user' do
+          success Entities::UserPublic
+        end
+        get do
+          entity =
+            if current_user.admin?
+              Entities::UserWithAdmin
+            else
+              Entities::UserPublic
+            end
 
-        present current_user, with: entity
+          present current_user, with: entity, current_user: current_user
+        end
       end
 
       desc "Get the currently authenticated user's SSH keys" do
@@ -561,12 +615,14 @@ module API
       params do
         requires :key_id, type: Integer, desc: 'The ID of the SSH key'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get "keys/:key_id" do
         key = current_user.keys.find_by(id: params[:key_id])
         not_found!('Key') unless key
 
         present key, with: Entities::SSHKey
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Add a new SSH key to the currently authenticated user' do
         success Entities::SSHKey
@@ -591,12 +647,14 @@ module API
       params do
         requires :key_id, type: Integer, desc: 'The ID of the SSH key'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete "keys/:key_id" do
         key = current_user.keys.find_by(id: params[:key_id])
         not_found!('Key') unless key
 
         destroy_conditionally!(key)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc "Get the currently authenticated user's GPG keys" do
         detail 'This feature was added in GitLab 10.0'
@@ -616,12 +674,14 @@ module API
       params do
         requires :key_id, type: Integer, desc: 'The ID of the GPG key'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get 'gpg_keys/:key_id' do
         key = current_user.gpg_keys.find_by(id: params[:key_id])
         not_found!('GPG Key') unless key
 
         present key, with: Entities::GPGKey
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Add a new GPG key to the currently authenticated user' do
         detail 'This feature was added in GitLab 10.0'
@@ -646,6 +706,7 @@ module API
       params do
         requires :key_id, type: Integer, desc: 'The ID of the GPG key'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       post 'gpg_keys/:key_id/revoke' do
         key = current_user.gpg_keys.find_by(id: params[:key_id])
         not_found!('GPG Key') unless key
@@ -653,6 +714,7 @@ module API
         key.revoke
         status :accepted
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Delete a GPG key from the currently authenticated user' do
         detail 'This feature was added in GitLab 10.0'
@@ -660,6 +722,7 @@ module API
       params do
         requires :key_id, type: Integer, desc: 'The ID of the SSH key'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete 'gpg_keys/:key_id' do
         key = current_user.gpg_keys.find_by(id: params[:key_id])
         not_found!('GPG Key') unless key
@@ -667,6 +730,7 @@ module API
         status 204
         key.destroy
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc "Get the currently authenticated user's email addresses" do
         success Entities::Email
@@ -684,12 +748,14 @@ module API
       params do
         requires :email_id, type: Integer, desc: 'The ID of the email'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get "emails/:email_id" do
         email = current_user.emails.find_by(id: params[:email_id])
         not_found!('Email') unless email
 
         present email, with: Entities::Email
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Add new email address to the currently authenticated user' do
         success Entities::Email
@@ -711,6 +777,7 @@ module API
       params do
         requires :email_id, type: Integer, desc: 'The ID of the email'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete "emails/:email_id" do
         email = current_user.emails.find_by(id: params[:email_id])
         not_found!('Email') unless email
@@ -719,12 +786,14 @@ module API
           Emails::DestroyService.new(current_user, user: current_user).execute(email)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Get a list of user activities'
       params do
         optional :from, type: DateTime, default: 6.months.ago, desc: 'Date string in the format YEAR-MONTH-DAY'
         use :pagination
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get "activities" do
         authenticated_as_admin!
 
@@ -734,6 +803,31 @@ module API
 
         present paginate(activities), with: Entities::UserActivity
       end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      desc 'Set the status of the current user' do
+        success Entities::UserStatus
+      end
+      params do
+        optional :emoji, type: String, desc: "The emoji to set on the status"
+        optional :message, type: String, desc: "The status message to set"
+      end
+      put "status" do
+        forbidden! unless can?(current_user, :update_user_status, current_user)
+
+        if ::Users::SetStatusService.new(current_user, declared_params).execute
+          present current_user.status, with: Entities::UserStatus
+        else
+          render_validation_error!(current_user.status)
+        end
+      end
+
+      desc 'get the status of the current user' do
+        success Entities::UserStatus
+      end
+      get 'status' do
+        present current_user.status || {}, with: Entities::UserStatus
+      end
     end
   end
 end
diff --git a/lib/api/validations/types/safe_file.rb b/lib/api/validations/types/safe_file.rb
new file mode 100644
index 00000000000..53b5790bfa2
--- /dev/null
+++ b/lib/api/validations/types/safe_file.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# This module overrides the Grape type validator defined in
+# https://github.com/ruby-grape/grape/blob/master/lib/grape/validations/types/file.rb
+module API
+  module Validations
+    module Types
+      class SafeFile < ::Grape::Validations::Types::File
+        def value_coerced?(value)
+          super && value[:tempfile].is_a?(Tempfile)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/variables.rb b/lib/api/variables.rb
index a34de9410e8..148deb86c4c 100644
--- a/lib/api/variables.rb
+++ b/lib/api/variables.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Variables < Grape::API
     include PaginationParams
@@ -9,7 +11,7 @@ module API
       requires :id, type: String, desc: 'The ID of a project'
     end
 
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS  do
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       desc 'Get project variables' do
         success Entities::Variable
       end
@@ -27,6 +29,7 @@ module API
       params do
         requires :key, type: String, desc: 'The key of the variable'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       get ':id/variables/:key' do
         key = params[:key]
         variable = user_project.variables.find_by(key: key)
@@ -35,6 +38,7 @@ module API
 
         present variable, with: Entities::Variable
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Create a new variable in a project' do
         success Entities::Variable
@@ -64,6 +68,7 @@ module API
         optional :value, type: String, desc: 'The value of the variable'
         optional :protected, type: String, desc: 'Whether the variable is protected'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       put ':id/variables/:key' do
         variable = user_project.variables.find_by(key: params[:key])
 
@@ -77,6 +82,7 @@ module API
           render_validation_error!(variable)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Delete an existing variable from a project' do
         success Entities::Variable
@@ -84,6 +90,7 @@ module API
       params do
         requires :key, type: String, desc: 'The key of the variable'
       end
+      # rubocop: disable CodeReuse/ActiveRecord
       delete ':id/variables/:key' do
         variable = user_project.variables.find_by(key: params[:key])
         not_found!('Variable') unless variable
@@ -92,6 +99,7 @@ module API
         status 204
         variable.destroy
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/api/version.rb b/lib/api/version.rb
index 3b10bfa6a7d..74cd857f447 100644
--- a/lib/api/version.rb
+++ b/lib/api/version.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module API
   class Version < Grape::API
     before { authenticate! }
diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb
index b3fc4e876ad..ef0e3decc2c 100644
--- a/lib/api/wikis.rb
+++ b/lib/api/wikis.rb
@@ -1,6 +1,16 @@
+# frozen_string_literal: true
+
 module API
   class Wikis < Grape::API
     helpers do
+      def commit_params(attrs)
+        {
+          file_name: attrs[:file][:filename],
+          file_content: attrs[:file][:tempfile].read,
+          branch_name: attrs[:branch]
+        }
+      end
+
       params :wiki_page_params do
         requires :content, type: String, desc: 'Content of a wiki page'
         requires :title, type: String, desc: 'Title of a wiki page'
@@ -12,7 +22,9 @@ module API
       end
     end
 
-    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+    WIKI_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(slug: API::NO_SLASH_URL_PART_REGEX)
+
+    resource :projects, requirements: WIKI_ENDPOINT_REQUIREMENTS do
       desc 'Get a list of wiki pages' do
         success Entities::WikiPageBasic
       end
@@ -84,6 +96,29 @@ module API
         status 204
         WikiPages::DestroyService.new(user_project, current_user).execute(wiki_page)
       end
+
+      desc 'Upload an attachment to the wiki repository' do
+        detail 'This feature was introduced in GitLab 11.3.'
+        success Entities::WikiAttachment
+      end
+      params do
+        requires :file, type: ::API::Validations::Types::SafeFile, desc: 'The attachment file to be uploaded'
+        optional :branch, type: String, desc: 'The name of the branch'
+      end
+      post ":id/wikis/attachments" do
+        authorize! :create_wiki, user_project
+
+        result = ::Wikis::CreateAttachmentService.new(user_project,
+                                                      current_user,
+                                                      commit_params(declared_params(include_missing: false))).execute
+
+        if result[:status] == :success
+          status(201)
+          present OpenStruct.new(result[:result]), with: Entities::WikiAttachment
+        else
+          render_api_error!(result[:message], 400)
+        end
+      end
     end
   end
 end
diff --git a/lib/backup.rb b/lib/backup.rb
new file mode 100644
index 00000000000..2712b33b4b4
--- /dev/null
+++ b/lib/backup.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Backup
+  Error = Class.new(StandardError)
+end
diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb
index 45a935ab352..33658ae225f 100644
--- a/lib/backup/artifacts.rb
+++ b/lib/backup/artifacts.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'backup/files'
 
 module Backup
diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb
index adf85ca4719..5e795a449de 100644
--- a/lib/backup/builds.rb
+++ b/lib/backup/builds.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'backup/files'
 
 module Backup
diff --git a/lib/backup/database.rb b/lib/backup/database.rb
index 1608f7ad02d..e6bf3d1856f 100644
--- a/lib/backup/database.rb
+++ b/lib/backup/database.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'yaml'
 
 module Backup
@@ -44,7 +46,7 @@ module Backup
       end
 
       report_success(success)
-      abort 'Backup failed' unless success
+      raise Backup::Error, 'Backup failed' unless success
     end
 
     def restore
@@ -72,7 +74,7 @@ module Backup
       end
 
       report_success(success)
-      abort 'Restore failed' unless success
+      abort Backup::Error, 'Restore failed' unless success
     end
 
     protected
diff --git a/lib/backup/files.rb b/lib/backup/files.rb
index 9895db9e451..2bac84846c5 100644
--- a/lib/backup/files.rb
+++ b/lib/backup/files.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'open3'
 require_relative 'helper'
 
@@ -26,20 +28,29 @@ module Backup
 
         unless status.zero?
           puts output
-          abort 'Backup failed'
+          raise Backup::Error, 'Backup failed'
         end
 
-        run_pipeline!([%W(tar --exclude=lost+found -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600])
+        run_pipeline!([%W(#{tar} --exclude=lost+found -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600])
         FileUtils.rm_rf(@backup_files_dir)
       else
-        run_pipeline!([%W(tar --exclude=lost+found -C #{app_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600])
+        run_pipeline!([%W(#{tar} --exclude=lost+found -C #{app_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600])
       end
     end
 
     def restore
       backup_existing_files_dir
 
-      run_pipeline!([%w(gzip -cd), %W(tar --unlink-first --recursive-unlink -C #{app_files_dir} -xf -)], in: backup_tarball)
+      run_pipeline!([%w(gzip -cd), %W(#{tar} --unlink-first --recursive-unlink -C #{app_files_dir} -xf -)], in: backup_tarball)
+    end
+
+    def tar
+      if system(*%w[gtar --version], out: '/dev/null')
+        # It looks like we can get GNU tar by running 'gtar'
+        'gtar'
+      else
+        'tar'
+      end
     end
 
     def backup_existing_files_dir
@@ -60,8 +71,14 @@ module Backup
     end
 
     def run_pipeline!(cmd_list, options = {})
-      status_list = Open3.pipeline(*cmd_list, options)
-      abort 'Backup failed' unless status_list.compact.all?(&:success?)
+      err_r, err_w = IO.pipe
+      options[:err] = err_w
+      status = Open3.pipeline(*cmd_list, options)
+      err_w.close
+      return if status.compact.all?(&:success?)
+
+      regex = /^g?tar: \.: Cannot mkdir: No such file or directory$/
+      raise Backup::Error, 'Backup failed' unless err_r.read =~ regex
     end
   end
 end
diff --git a/lib/backup/helper.rb b/lib/backup/helper.rb
index 54b9ce10b4d..22f00aef569 100644
--- a/lib/backup/helper.rb
+++ b/lib/backup/helper.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Backup
   module Helper
     def access_denied_error(path)
diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb
index 185ff8ae6bd..0dfe56e214f 100644
--- a/lib/backup/lfs.rb
+++ b/lib/backup/lfs.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'backup/files'
 
 module Backup
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index a8da0c7edef..06b0338b1ed 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Backup
   class Manager
     ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs registry].freeze
@@ -27,7 +29,7 @@ module Backup
           progress.puts "done".color(:green)
         else
           puts "creating archive #{tar_file} failed".color(:red)
-          abort 'Backup failed'
+          raise Backup::Error, 'Backup failed'
         end
 
         upload
@@ -48,11 +50,12 @@ module Backup
       if directory.files.create(key: remote_target, body: File.open(tar_file), public: false,
                                 multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size,
                                 encryption: Gitlab.config.backup.upload.encryption,
+                                encryption_key: Gitlab.config.backup.upload.encryption_key,
                                 storage_class: Gitlab.config.backup.upload.storage_class)
         progress.puts "done".color(:green)
       else
         puts "uploading backup to #{remote_directory} failed".color(:red)
-        abort 'Backup failed'
+        raise Backup::Error, 'Backup failed'
       end
     end
 
@@ -66,7 +69,7 @@ module Backup
           progress.puts "done".color(:green)
         else
           puts "deleting tmp directory '#{dir}' failed".color(:red)
-          abort 'Backup failed'
+          raise Backup::Error, 'Backup failed'
         end
       end
     end
@@ -162,7 +165,7 @@ module Backup
 
     def tar_version
       tar_version, _ = Gitlab::Popen.popen(%w(tar --version))
-      tar_version.force_encoding('locale').split("\n").first
+      tar_version.dup.force_encoding('locale').split("\n").first
     end
 
     def skipped?(item)
@@ -193,7 +196,7 @@ module Backup
       if connection.service == ::Fog::Storage::Local
         connection.directories.create(key: remote_directory)
       else
-        connection.directories.get(remote_directory)
+        connection.directories.new(key: remote_directory)
       end
     end
 
@@ -241,6 +244,7 @@ module Backup
         backup_created_at: Time.now,
         gitlab_version: Gitlab::VERSION,
         tar_version: tar_version,
+        installation_type: Gitlab::INSTALLATION_TYPE,
         skipped: ENV["SKIP"]
       }
     end
diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb
index 542e35a7c7c..a4be728df08 100644
--- a/lib/backup/pages.rb
+++ b/lib/backup/pages.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'backup/files'
 
 module Backup
diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb
index 35821805797..d16ed2facf1 100644
--- a/lib/backup/registry.rb
+++ b/lib/backup/registry.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'backup/files'
 
 module Backup
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index 84670d6582e..184c7418e75 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -1,11 +1,10 @@
+# frozen_string_literal: true
+
 require 'yaml'
-require_relative 'helper'
 
 module Backup
   class Repository
-    include Backup::Helper
-    # rubocop:disable Metrics/AbcSize
-
+    include Gitlab::ShellAdapter
     attr_reader :progress
 
     def initialize(progress)
@@ -17,113 +16,66 @@ module Backup
 
       Project.find_each(batch_size: 1000) do |project|
         progress.print " * #{display_repo_path(project)} ... "
-        path_to_project_repo = path_to_repo(project)
-        path_to_project_bundle = path_to_bundle(project)
 
-        # Create namespace dir or hashed path if missing
         if project.hashed_storage?(:repository)
           FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path)))
         else
           FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace
         end
 
-        if empty_repo?(project)
-          progress.puts "[SKIPPED]".color(:cyan)
+        if !empty_repo?(project)
+          backup_project(project)
+          progress.puts "[DONE]".color(:green)
         else
-          in_path(path_to_project_repo) do |dir|
-            FileUtils.mkdir_p(path_to_tars(project))
-            cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
-            output, status = Gitlab::Popen.popen(cmd)
-
-            unless status.zero?
-              progress_warn(project, cmd.join(' '), output)
-            end
-          end
-
-          cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
-          output, status = Gitlab::Popen.popen(cmd)
-
-          if status.zero?
-            progress.puts "[DONE]".color(:green)
-          else
-            progress_warn(project, cmd.join(' '), output)
-          end
+          progress.puts "[SKIPPED]".color(:cyan)
         end
 
         wiki = ProjectWiki.new(project)
-        path_to_wiki_repo = path_to_repo(wiki)
-        path_to_wiki_bundle = path_to_bundle(wiki)
 
-        if File.exist?(path_to_wiki_repo)
-          progress.print " * #{display_repo_path(wiki)} ... "
-
-          if empty_repo?(wiki)
-            progress.puts " [SKIPPED]".color(:cyan)
-          else
-            cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all)
-            output, status = Gitlab::Popen.popen(cmd)
-            if status.zero?
-              progress.puts " [DONE]".color(:green)
-            else
-              progress_warn(wiki, cmd.join(' '), output)
-            end
-          end
+        if !empty_repo?(wiki)
+          backup_project(wiki)
+          progress.puts "[DONE] Wiki".color(:green)
+        else
+          progress.puts "[SKIPPED] Wiki".color(:cyan)
         end
       end
     end
 
     def prepare_directories
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        delete_all_repositories(name, repository_storage)
+      Gitlab.config.repositories.storages.each do |name, _repository_storage|
+        Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories
       end
     end
 
-    def delete_all_repositories(name, repository_storage)
-      gitaly_migrate(:delete_all_repositories) do |is_enabled|
-        if is_enabled
-          Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories
-        else
-          local_delete_all_repositories(name, repository_storage)
-        end
-      end
-    end
+    def backup_project(project)
+      path_to_project_bundle = path_to_bundle(project)
+      Gitlab::GitalyClient::RepositoryService.new(project.repository)
+        .create_bundle(path_to_project_bundle)
 
-    def local_delete_all_repositories(name, repository_storage)
-      path = repository_storage.legacy_disk_path
-      return unless File.exist?(path)
+      backup_custom_hooks(project)
+    rescue => e
+      progress_warn(project, e, 'Failed to backup repo')
+    end
 
-      # Move all files in the existing repos directory except . and .. to
-      # repositories.old.<timestamp> directory
-      bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s)
-      FileUtils.mkdir_p(bk_repos_path, mode: 0700)
-      files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")]
+    def backup_custom_hooks(project)
+      FileUtils.mkdir_p(project_backup_path(project))
 
-      begin
-        FileUtils.mv(files, bk_repos_path)
-      rescue Errno::EACCES
-        access_denied_error(path)
-      rescue Errno::EBUSY
-        resource_busy_error(path)
-      end
+      custom_hooks_path = custom_hooks_tar(project)
+      Gitlab::GitalyClient::RepositoryService.new(project.repository)
+        .backup_custom_hooks(custom_hooks_path)
     end
 
     def restore_custom_hooks(project)
-      # TODO: Need to find a way to do this for gitaly
-      # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195
-      in_path(path_to_tars(project)) do |dir|
-        path_to_project_repo = path_to_repo(project)
-        cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
-
-        output, status = Gitlab::Popen.popen(cmd)
-        unless status.zero?
-          progress_warn(project, cmd.join(' '), output)
-        end
-      end
+      return unless Dir.exist?(project_backup_path(project))
+      return if Dir.glob("#{project_backup_path(project)}/custom_hooks*").none?
+
+      custom_hooks_path = custom_hooks_tar(project)
+      Gitlab::GitalyClient::RepositoryService.new(project.repository)
+        .restore_custom_hooks(custom_hooks_path)
     end
 
     def restore
       prepare_directories
-      gitlab_shell = Gitlab::Shell.new
 
       Project.find_each(batch_size: 1000) do |project|
         progress.print " * #{project.full_path} ... "
@@ -133,7 +85,8 @@ module Backup
         restore_repo_success = nil
         if File.exist?(path_to_project_bundle)
           begin
-            project.repository.create_from_bundle path_to_project_bundle
+            project.repository.create_from_bundle(path_to_project_bundle)
+            restore_custom_hooks(project)
             restore_repo_success = true
           rescue => e
             restore_repo_success = false
@@ -149,8 +102,6 @@ module Backup
           progress.puts "[Failed] restoring #{project.full_path} repository".color(:red)
         end
 
-        restore_custom_hooks(project)
-
         wiki = ProjectWiki.new(project)
         path_to_wiki_bundle = path_to_bundle(wiki)
 
@@ -158,6 +109,8 @@ module Backup
           progress.print " * #{wiki.full_path} ... "
           begin
             wiki.repository.create_from_bundle(path_to_wiki_bundle)
+            restore_custom_hooks(wiki)
+
             progress.puts "[DONE]".color(:green)
           rescue => e
             progress.puts "[Failed] restoring #{wiki.full_path} wiki".color(:red)
@@ -165,55 +118,34 @@ module Backup
           end
         end
       end
+
+      restore_object_pools
     end
-    # rubocop:enable Metrics/AbcSize
 
     protected
 
-    def path_to_repo(project)
-      project.repository.path_to_repo
-    end
-
     def path_to_bundle(project)
       File.join(backup_repos_path, project.disk_path + '.bundle')
     end
 
-    def path_to_tars(project, dir = nil)
-      path = File.join(backup_repos_path, project.disk_path)
+    def project_backup_path(project)
+      File.join(backup_repos_path, project.disk_path)
+    end
 
-      if dir
-        File.join(path, "#{dir}.tar")
-      else
-        path
-      end
+    def custom_hooks_tar(project)
+      File.join(project_backup_path(project), "custom_hooks.tar")
     end
 
     def backup_repos_path
       File.join(Gitlab.config.backup.path, 'repositories')
     end
 
-    def in_path(path)
-      return unless Dir.exist?(path)
-
-      dir_entries = Dir.entries(path)
-
-      if dir_entries.include?('custom_hooks') || dir_entries.include?('custom_hooks.tar')
-        yield('custom_hooks')
-      end
-    end
-
     def prepare
       FileUtils.rm_rf(backup_repos_path)
-      # Ensure the parent dir of backup_repos_path exists
       FileUtils.mkdir_p(Gitlab.config.backup.path)
-      # Fail if somebody raced to create backup_repos_path before us
       FileUtils.mkdir(backup_repos_path, mode: 0700)
     end
 
-    def silent
-      { err: '/dev/null', out: '/dev/null' }
-    end
-
     private
 
     def progress_warn(project, cmd, output)
@@ -222,23 +154,24 @@ module Backup
     end
 
     def empty_repo?(project_or_wiki)
-      # Protect against stale caches
       project_or_wiki.repository.expire_emptiness_caches
       project_or_wiki.repository.empty?
     end
 
-    def repository_storage_paths_args
-      Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path }
-    end
-
     def display_repo_path(project)
       project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path
     end
 
-    def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
-      Gitlab::GitalyClient.migrate(method, status: status, &block)
-    rescue GRPC::NotFound, GRPC::BadStatus => e
-      raise Error, e
+    def restore_object_pools
+      PoolRepository.includes(:source_project).find_each do |pool|
+        progress.puts " - Object pool #{pool.disk_path}..."
+
+        pool.source_project ||= pool.member_projects.first.root_of_fork_network
+        pool.state = 'none'
+        pool.save
+
+        pool.schedule
+      end
     end
   end
 end
diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb
index 49b117a7ee3..9577df2634a 100644
--- a/lib/backup/uploads.rb
+++ b/lib/backup/uploads.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'backup/files'
 
 module Backup
diff --git a/lib/banzai.rb b/lib/banzai.rb
index 5df98f66f3b..1eb41ff7133 100644
--- a/lib/banzai.rb
+++ b/lib/banzai.rb
@@ -1,4 +1,13 @@
+# frozen_string_literal: true
+
 module Banzai
+  # if you need to render markdown, then you probably need to post_process as well,
+  # such as removing references that the current user doesn't have
+  # permission to make
+  def self.render_and_post_process(text, context = {})
+    post_process(render(text, context), context)
+  end
+
   def self.render(text, context = {})
     Renderer.render(text, context)
   end
diff --git a/lib/banzai/color_parser.rb b/lib/banzai/color_parser.rb
index 355c364b07b..6d01d51955c 100644
--- a/lib/banzai/color_parser.rb
+++ b/lib/banzai/color_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module ColorParser
     ALPHA = /0(?:\.\d+)?|\.\d+|1(?:\.0+)?/ # 0.0..1.0
diff --git a/lib/banzai/commit_renderer.rb b/lib/banzai/commit_renderer.rb
index c351a155ae5..f346151a3c1 100644
--- a/lib/banzai/commit_renderer.rb
+++ b/lib/banzai/commit_renderer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module CommitRenderer
     ATTRIBUTES = [:description, :title].freeze
diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb
index 3f1e95d4cc0..b7344808989 100644
--- a/lib/banzai/cross_project_reference.rb
+++ b/lib/banzai/cross_project_reference.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   # Common methods for ReferenceFilters that support an optional cross-project
   # reference.
@@ -13,6 +15,7 @@ module Banzai
     # Returns a Project, or nil if the reference can't be found
     def parent_from_ref(ref)
       return context[:project] || context[:group] unless ref
+      return context[:project] if context[:project]&.full_path == ref
 
       Project.find_by_full_path(ref)
     end
diff --git a/lib/banzai/filter.rb b/lib/banzai/filter.rb
index 3eb544dfef9..7d9766c906c 100644
--- a/lib/banzai/filter.rb
+++ b/lib/banzai/filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     def self.[](name)
diff --git a/lib/banzai/filter/absolute_link_filter.rb b/lib/banzai/filter/absolute_link_filter.rb
index 1ec6201523f..a9bdb004c4b 100644
--- a/lib/banzai/filter/absolute_link_filter.rb
+++ b/lib/banzai/filter/absolute_link_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'uri'
 
 module Banzai
@@ -27,6 +29,7 @@ module Banzai
       end
 
       def absolute_link_attr(uri)
+        # Here we really want to expand relative path to absolute path
         URI.join(Gitlab.config.gitlab.url, uri).to_s
       end
     end
diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb
index 60a12dca9d3..4764f8e1e19 100644
--- a/lib/banzai/filter/abstract_reference_filter.rb
+++ b/lib/banzai/filter/abstract_reference_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # Issues, Merge Requests, Snippets, Commits and Commit Ranges share
@@ -100,6 +102,11 @@ module Banzai
         ref_pattern = object_class.reference_pattern
         link_pattern = object_class.link_reference_pattern
 
+        # Compile often used regexps only once outside of the loop
+        ref_pattern_anchor = /\A#{ref_pattern}\z/
+        link_pattern_start = /\A#{link_pattern}/
+        link_pattern_anchor = /\A#{link_pattern}\z/
+
         nodes.each do |node|
           if text_node?(node) && ref_pattern
             replace_text_when_pattern_matches(node, ref_pattern) do |content|
@@ -108,7 +115,7 @@ module Banzai
 
           elsif element_node?(node)
             yield_valid_link(node) do |link, inner_html|
-              if ref_pattern && link =~ /\A#{ref_pattern}\z/
+              if ref_pattern && link =~ ref_pattern_anchor
                 replace_link_node_with_href(node, link) do
                   object_link_filter(link, ref_pattern, link_content: inner_html)
                 end
@@ -118,7 +125,7 @@ module Banzai
 
               next unless link_pattern
 
-              if link == inner_html && inner_html =~ /\A#{link_pattern}/
+              if link == inner_html && inner_html =~ link_pattern_start
                 replace_link_node_with_text(node, link) do
                   object_link_filter(inner_html, link_pattern, link_reference: true)
                 end
@@ -126,7 +133,7 @@ module Banzai
                 next
               end
 
-              if link =~ /\A#{link_pattern}\z/
+              if link =~ link_pattern_anchor
                 replace_link_node_with_href(node, link) do
                   object_link_filter(link, link_pattern, link_content: inner_html, link_reference: true)
                 end
@@ -289,7 +296,7 @@ module Banzai
 
       # Returns projects for the given paths.
       def find_for_paths(paths)
-        if RequestStore.active?
+        if Gitlab::SafeRequestStore.active?
           cache = refs_cache
           to_query = paths - cache.keys
 
@@ -333,7 +340,7 @@ module Banzai
       end
 
       def refs_cache
-        RequestStore["banzai_#{parent_type}_refs".to_sym] ||= {}
+        Gitlab::SafeRequestStore["banzai_#{parent_type}_refs".to_sym] ||= {}
       end
 
       def parent_type
diff --git a/lib/banzai/filter/ascii_doc_post_processing_filter.rb b/lib/banzai/filter/ascii_doc_post_processing_filter.rb
index c9fcf057c5f..88439f06b5f 100644
--- a/lib/banzai/filter/ascii_doc_post_processing_filter.rb
+++ b/lib/banzai/filter/ascii_doc_post_processing_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     class AsciiDocPostProcessingFilter < HTML::Pipeline::Filter
diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb
index 4a143baeef6..f3061bad4ff 100644
--- a/lib/banzai/filter/autolink_filter.rb
+++ b/lib/banzai/filter/autolink_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'uri'
 
 module Banzai
@@ -6,6 +8,10 @@ module Banzai
     #
     # Based on HTML::Pipeline::AutolinkFilter
     #
+    # Note that our CommonMark parser, `commonmarker` (using the autolink extension)
+    # handles standard autolinking, like http/https. We detect additional
+    # schemes (smb, rdar, etc).
+    #
     # Context options:
     #   :autolink  - Boolean, skips all processing done by this filter when false
     #   :link_attr - Hash of attributes for the generated links
@@ -105,10 +111,13 @@ module Banzai
           end
         end
 
-        # match has come from node.to_html above, so we know it's encoded
-        # correctly.
+        # Since this came from a Text node, make sure the new href is encoded.
+        # `commonmarker` percent encodes the domains of links it handles, so
+        # do the same (instead of using `normalized_encode`).
+        href_safe = Addressable::URI.encode(match).html_safe
+
         html_safe_match = match.html_safe
-        options = link_options.merge(href: html_safe_match)
+        options         = link_options.merge(href: href_safe)
 
         content_tag(:a, html_safe_match, options) + dropped
       end
diff --git a/lib/banzai/filter/blockquote_fence_filter.rb b/lib/banzai/filter/blockquote_fence_filter.rb
index d2c4b1e4d76..ad367cc5efe 100644
--- a/lib/banzai/filter/blockquote_fence_filter.rb
+++ b/lib/banzai/filter/blockquote_fence_filter.rb
@@ -1,28 +1,10 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     class BlockquoteFenceFilter < HTML::Pipeline::TextFilter
       REGEX = %r{
-          (?<code>
-            # Code blocks:
-            # ```
-            # Anything, including `>>>` blocks which are ignored by this filter
-            # ```
-
-            ^```
-            .+?
-            \n```$
-          )
-        |
-          (?<html>
-            # HTML block:
-            # <tag>
-            # Anything, including `>>>` blocks which are ignored by this filter
-            # </tag>
-
-            ^<[^>]+?>\n
-            .+?
-            \n<\/[^>]+?>$
-          )
+          #{::Gitlab::Regex.markdown_code_or_html_blocks}
         |
           (?:
             # Blockquote:
@@ -30,14 +12,14 @@ module Banzai
             # Anything, including code and HTML blocks
             # >>>
 
-            ^>>>\n
+            ^>>>\ *\n
             (?<quote>
               (?:
                   # Any character that doesn't introduce a code or HTML block
                   (?!
                       ^```
                     |
-                      ^<[^>]+?>\n
+                      ^<[^>]+?>\ *\n
                   )
                   .
                 |
@@ -48,7 +30,7 @@ module Banzai
                   \g<html>
               )+?
             )
-            \n>>>$
+            \n>>>\ *$
           )
       }mx.freeze
 
diff --git a/lib/banzai/filter/color_filter.rb b/lib/banzai/filter/color_filter.rb
index 6ab29ac281f..6d9bdb9cbd3 100644
--- a/lib/banzai/filter/color_filter.rb
+++ b/lib/banzai/filter/color_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # HTML filter that renders `color` followed by a color "chip".
diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb
index 01b3b0dafb9..d6b46236a49 100644
--- a/lib/banzai/filter/commit_range_reference_filter.rb
+++ b/lib/banzai/filter/commit_range_reference_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # HTML filter that replaces commit range references with links.
diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb
index 8cd92a1adba..c3e5ac41cb8 100644
--- a/lib/banzai/filter/commit_reference_filter.rb
+++ b/lib/banzai/filter/commit_reference_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # HTML filter that replaces commit references with links.
diff --git a/lib/banzai/filter/commit_trailers_filter.rb b/lib/banzai/filter/commit_trailers_filter.rb
index 7b55e8b36f6..f49c4b403db 100644
--- a/lib/banzai/filter/commit_trailers_filter.rb
+++ b/lib/banzai/filter/commit_trailers_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # HTML filter that replaces users' names and emails in commit trailers
diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb
index e1261e7bbbe..fa1690f73ad 100644
--- a/lib/banzai/filter/emoji_filter.rb
+++ b/lib/banzai/filter/emoji_filter.rb
@@ -1,12 +1,11 @@
+# frozen_string_literal: true
+
+# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/emoji.js
 module Banzai
   module Filter
     # HTML filter that replaces :emoji: and unicode with images.
     #
     # Based on HTML::Pipeline::EmojiFilter
-    #
-    # Context options:
-    #   :asset_root
-    #   :asset_host
     class EmojiFilter < HTML::Pipeline::Filter
       IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set
 
diff --git a/lib/banzai/filter/epic_reference_filter.rb b/lib/banzai/filter/epic_reference_filter.rb
index 265924abe24..26bcf5c04b4 100644
--- a/lib/banzai/filter/epic_reference_filter.rb
+++ b/lib/banzai/filter/epic_reference_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # The actual filter is implemented in the EE mixin
@@ -7,6 +9,12 @@ module Banzai
       def self.object_class
         Epic
       end
+
+      private
+
+      def group
+        context[:group] || context[:project]&.group
+      end
     end
   end
 end
diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb
index ed01a72ff9f..8159dcfed72 100644
--- a/lib/banzai/filter/external_issue_reference_filter.rb
+++ b/lib/banzai/filter/external_issue_reference_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # HTML filter that replaces external issue tracker references with links.
@@ -95,9 +97,7 @@ module Banzai
       private
 
       def external_issues_cached(attribute)
-        return project.public_send(attribute) unless RequestStore.active? # rubocop:disable GitlabSecurity/PublicSend
-
-        cached_attributes = RequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} }
+        cached_attributes = Gitlab::SafeRequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} }
         cached_attributes[project.id][attribute] = project.public_send(attribute) if cached_attributes[project.id][attribute].nil? # rubocop:disable GitlabSecurity/PublicSend
         cached_attributes[project.id][attribute]
       end
diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb
index d6327ef31cb..61ee3eac216 100644
--- a/lib/banzai/filter/external_link_filter.rb
+++ b/lib/banzai/filter/external_link_filter.rb
@@ -1,19 +1,32 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # HTML Filter to modify the attributes of external links
     class ExternalLinkFilter < HTML::Pipeline::Filter
-      SCHEMES = ['http', 'https', nil].freeze
+      SCHEMES      = ['http', 'https', nil].freeze
+      RTLO         = "\u202E".freeze
+      ENCODED_RTLO = '%E2%80%AE'.freeze
 
       def call
         links.each do |node|
-          uri = uri(node['href'].to_s)
-          next unless uri
-
-          node.set_attribute('href', uri.to_s)
+          # URI.parse does stricter checking on the url than Addressable,
+          # such as on `mailto:` links. Since we've been using it, do an
+          # initial parse for validity and then use Addressable
+          # for IDN support, etc
+          uri = uri_strict(node['href'].to_s)
+          if uri
+            node.set_attribute('href', uri.to_s)
+            addressable_uri = addressable_uri(node['href'])
+          else
+            addressable_uri = nil
+          end
 
-          if SCHEMES.include?(uri.scheme) && external_url?(uri)
-            node.set_attribute('rel', 'nofollow noreferrer noopener')
-            node.set_attribute('target', '_blank')
+          unless internal_url?(addressable_uri)
+            punycode_autolink_node!(addressable_uri, node)
+            sanitize_link_text!(node)
+            add_malicious_tooltip!(addressable_uri, node)
+            add_nofollow!(addressable_uri, node)
           end
         end
 
@@ -22,27 +35,85 @@ module Banzai
 
       private
 
-      def uri(href)
+      def uri_strict(href)
         URI.parse(href)
       rescue URI::Error
         nil
       end
 
+      def addressable_uri(href)
+        Addressable::URI.parse(href)
+      rescue Addressable::URI::InvalidURIError
+        nil
+      end
+
       def links
         query = 'descendant-or-self::a[@href and not(@href = "")]'
         doc.xpath(query)
       end
 
-      def external_url?(uri)
+      def internal_url?(uri)
+        return false if uri.nil?
         # Relative URLs miss a hostname
-        return false unless uri.hostname
+        return true unless uri.hostname
 
-        uri.hostname != internal_url.hostname
+        uri.hostname == internal_url.hostname
       end
 
       def internal_url
         @internal_url ||= URI.parse(Gitlab.config.gitlab.url)
       end
+
+      # Only replace an autolink with an IDN with it's punycode
+      # version if we need emailable links.  Otherwise let it
+      # be shown normally and the tooltips will show the
+      # punycode version.
+      def punycode_autolink_node!(uri, node)
+        return unless uri
+        return unless context[:emailable_links]
+
+        unencoded_uri_str = Addressable::URI.unencode(node['href'])
+
+        if unencoded_uri_str == node.content && idn?(uri)
+          node.content = uri.normalize
+        end
+      end
+
+      # escape any right-to-left (RTLO) characters in link text
+      def sanitize_link_text!(node)
+        node.inner_html = node.inner_html.gsub(RTLO, ENCODED_RTLO)
+      end
+
+      # If the domain is an international domain name (IDN),
+      # let's expose with a tooltip in case it's intended
+      # to be malicious. This is particularly useful for links
+      # where the link text is not the same as the actual link.
+      # We will continue to show the unicode version of the domain
+      # in autolinked link text, which could contain emojis, etc.
+      #
+      # Also show the tooltip if the url contains the RTLO character,
+      # as this is an indicator of a malicious link
+      def add_malicious_tooltip!(uri, node)
+        if idn?(uri) || has_encoded_rtlo?(uri)
+          node.add_class('has-tooltip')
+          node.set_attribute('title', uri.normalize)
+        end
+      end
+
+      def add_nofollow!(uri, node)
+        if SCHEMES.include?(uri&.scheme)
+          node.set_attribute('rel', 'nofollow noreferrer noopener')
+          node.set_attribute('target', '_blank')
+        end
+      end
+
+      def idn?(uri)
+        uri&.normalized_host&.start_with?('xn--')
+      end
+
+      def has_encoded_rtlo?(uri)
+        uri&.to_s&.include?(ENCODED_RTLO)
+      end
     end
   end
 end
diff --git a/lib/banzai/filter/footnote_filter.rb b/lib/banzai/filter/footnote_filter.rb
new file mode 100644
index 00000000000..97527976437
--- /dev/null
+++ b/lib/banzai/filter/footnote_filter.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Banzai
+  module Filter
+    # HTML Filter for footnotes
+    #
+    # Footnotes are supported in CommonMark.  However we were stripping
+    # the ids during sanitization.  Those are now allowed.
+    #
+    # Footnotes are numbered the same - the first one has `id=fn1`, the
+    # second is `id=fn2`, etc.  In order to allow footnotes when rendering
+    # multiple markdown blocks on a page, we need to make each footnote
+    # reference unique.
+    #
+    # This filter adds a random number to each footnote (the same number
+    # can be used for a single render). So you get `id=fn1-4335` and `id=fn2-4335`.
+    #
+    class FootnoteFilter < HTML::Pipeline::Filter
+      INTEGER_PATTERN                 = /\A\d+\z/.freeze
+      FOOTNOTE_ID_PREFIX              = 'fn'.freeze
+      FOOTNOTE_LINK_ID_PREFIX         = 'fnref'.freeze
+      FOOTNOTE_LI_REFERENCE_PATTERN   = /\A#{FOOTNOTE_ID_PREFIX}\d+\z/.freeze
+      FOOTNOTE_LINK_REFERENCE_PATTERN = /\A#{FOOTNOTE_LINK_ID_PREFIX}\d+\z/.freeze
+      FOOTNOTE_START_NUMBER           = 1
+
+      def call
+        return doc unless first_footnote = doc.at_css("ol > li[id=#{fn_id(FOOTNOTE_START_NUMBER)}]")
+
+        # Sanitization stripped off the section wrapper - add it back in
+        first_footnote.parent.wrap('<section class="footnotes">')
+        rand_suffix = "-#{random_number}"
+
+        doc.css('sup > a[id]').each do |link_node|
+          ref_num       = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX)
+          footnote_node = doc.at_css("li[id=#{fn_id(ref_num)}]")
+          backref_node  = footnote_node.at_css("a[href=\"##{fnref_id(ref_num)}\"]")
+
+          if ref_num =~ INTEGER_PATTERN && footnote_node && backref_node
+            link_node[:href]    += rand_suffix
+            link_node[:id]      += rand_suffix
+            footnote_node[:id]  += rand_suffix
+            backref_node[:href] += rand_suffix
+
+            # Sanitization stripped off class - add it back in
+            link_node.parent.append_class('footnote-ref')
+            backref_node.append_class('footnote-backref')
+          end
+        end
+
+        doc
+      end
+
+      private
+
+      def random_number
+        @random_number ||= rand(10000)
+      end
+
+      def fn_id(num)
+        "#{FOOTNOTE_ID_PREFIX}#{num}"
+      end
+
+      def fnref_id(num)
+        "#{FOOTNOTE_LINK_ID_PREFIX}#{num}"
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/front_matter_filter.rb b/lib/banzai/filter/front_matter_filter.rb
new file mode 100644
index 00000000000..a27d18facd1
--- /dev/null
+++ b/lib/banzai/filter/front_matter_filter.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Banzai
+  module Filter
+    class FrontMatterFilter < HTML::Pipeline::Filter
+      DELIM_LANG = {
+        '---' => 'yaml',
+        '+++' => 'toml',
+        ';;;' => 'json'
+      }.freeze
+
+      DELIM = Regexp.union(DELIM_LANG.keys)
+
+      PATTERN = %r{
+        \A(?:[^\r\n]*coding:[^\r\n]*)?         # optional encoding line
+        \s*
+        ^(?<delim>#{DELIM})[ \t]*(?<lang>\S*)  # opening front matter marker (optional language specifier)
+        \s*
+        ^(?<front_matter>.*?)                  # front matter (not greedy)
+        \s*
+        ^\k<delim>                             # closing front matter marker
+        \s*
+      }mx
+
+      def call
+        html.sub(PATTERN) do |_match|
+          lang = $~[:lang].presence || DELIM_LANG[$~[:delim]]
+
+          ["```#{lang}", $~[:front_matter], "```", "\n"].join("\n")
+        end
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb
index 4bc82ecb4d6..0c1bbd2d250 100644
--- a/lib/banzai/filter/gollum_tags_filter.rb
+++ b/lib/banzai/filter/gollum_tags_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # HTML Filter for parsing Gollum's tags in HTML. It's only parses the
@@ -56,10 +58,12 @@ module Banzai
       # Pattern to match allowed image extensions
       ALLOWED_IMAGE_EXTENSIONS = /.+(jpg|png|gif|svg|bmp)\z/i.freeze
 
+      # Do not perform linking inside these tags.
+      IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set
+
       def call
         doc.search(".//text()").each do |node|
-          # Do not perform linking inside <code> blocks
-          next unless node.ancestors('code').empty?
+          next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS)
 
           # A Gollum ToC tag is `[[_TOC_]]`, but due to MarkdownFilter running
           # before this one, it will be converted into `[[<em>TOC</em>]]`, so it
diff --git a/lib/banzai/filter/html_entity_filter.rb b/lib/banzai/filter/html_entity_filter.rb
index e008fd428b0..406c2d3c96b 100644
--- a/lib/banzai/filter/html_entity_filter.rb
+++ b/lib/banzai/filter/html_entity_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'erb'
 
 module Banzai
diff --git a/lib/banzai/filter/image_lazy_load_filter.rb b/lib/banzai/filter/image_lazy_load_filter.rb
index 4cd9b02b76c..d8b9eb29cf5 100644
--- a/lib/banzai/filter/image_lazy_load_filter.rb
+++ b/lib/banzai/filter/image_lazy_load_filter.rb
@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+
+# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/image.js
 module Banzai
   module Filter
     # HTML filter that moves the value of image `src` attributes to `data-src`
@@ -5,7 +8,7 @@ module Banzai
     class ImageLazyLoadFilter < HTML::Pipeline::Filter
       def call
         doc.xpath('descendant-or-self::img').each do |img|
-          img['class'] ||= '' << 'lazy'
+          img.add_class('lazy')
           img['data-src'] = img['src']
           img['src'] = LazyImageTagHelper.placeholder_image
         end
diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb
index f318c425962..01237303c27 100644
--- a/lib/banzai/filter/image_link_filter.rb
+++ b/lib/banzai/filter/image_link_filter.rb
@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+
+# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/image.js
 module Banzai
   module Filter
     # HTML filter that wraps links around inline images.
diff --git a/lib/banzai/filter/inline_diff_filter.rb b/lib/banzai/filter/inline_diff_filter.rb
index 73e82a4d7e3..5a1c0bee32d 100644
--- a/lib/banzai/filter/inline_diff_filter.rb
+++ b/lib/banzai/filter/inline_diff_filter.rb
@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+
+# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/marks/inline_diff.js
 module Banzai
   module Filter
     class InlineDiffFilter < HTML::Pipeline::Filter
diff --git a/lib/banzai/filter/issuable_reference_filter.rb b/lib/banzai/filter/issuable_reference_filter.rb
index 7addf09be73..2963cba91e8 100644
--- a/lib/banzai/filter/issuable_reference_filter.rb
+++ b/lib/banzai/filter/issuable_reference_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     class IssuableReferenceFilter < AbstractReferenceFilter
diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb
index 1a415232545..8e2358694d4 100644
--- a/lib/banzai/filter/issuable_state_filter.rb
+++ b/lib/banzai/filter/issuable_state_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # HTML filter that appends state information to issuable links.
@@ -16,7 +18,7 @@ module Banzai
         issuables = extractor.extract([doc])
 
         issuables.each do |node, issuable|
-          next if !can_read_cross_project? && issuable.project != project
+          next if !can_read_cross_project? && cross_reference?(issuable)
 
           if VISIBLE_STATES.include?(issuable.state) && issuable_reference?(node.inner_html, issuable)
             node.content += " (#{issuable.state})"
@@ -29,7 +31,14 @@ module Banzai
       private
 
       def issuable_reference?(text, issuable)
-        text == issuable.reference_link_text(project || group)
+        CGI.unescapeHTML(text) == issuable.reference_link_text(project || group)
+      end
+
+      def cross_reference?(issuable)
+        return true if issuable.project != project
+        return true if issuable.respond_to?(:group) && issuable.group != group
+
+        false
       end
 
       def can_read_cross_project?
diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb
index 6877cae8c55..f85be042999 100644
--- a/lib/banzai/filter/issue_reference_filter.rb
+++ b/lib/banzai/filter/issue_reference_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # HTML filter that replaces issue references with links. References to
diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb
index a5f38046a43..f90a35952e5 100644
--- a/lib/banzai/filter/label_reference_filter.rb
+++ b/lib/banzai/filter/label_reference_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # HTML filter that replaces label references with links.
@@ -27,7 +29,7 @@ module Banzai
           if label
             yield match, label.id, project, namespace, $~
           else
-            match
+            escape_html_entities(match)
           end
         end
       end
@@ -46,7 +48,7 @@ module Banzai
                      include_ancestor_groups: true,
                      only_group_labels: true }
                  else
-                   { project_id: parent.id,
+                   { project: parent,
                      include_ancestor_groups: true }
                  end
 
@@ -100,6 +102,10 @@ module Banzai
         CGI.unescapeHTML(text.to_s)
       end
 
+      def escape_html_entities(text)
+        CGI.escapeHTML(text.to_s)
+      end
+
       def object_link_title(object, matches)
         # use title of wrapped element instead
         nil
diff --git a/lib/banzai/filter/markdown_engines/common_mark.rb b/lib/banzai/filter/markdown_engines/common_mark.rb
index bc9597df894..d3af776db05 100644
--- a/lib/banzai/filter/markdown_engines/common_mark.rb
+++ b/lib/banzai/filter/markdown_engines/common_mark.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # `CommonMark` markdown engine for GitLab's Banzai markdown filter.
 # This module is used in Banzai::Filter::MarkdownFilter.
 # Used gem is `commonmarker` which is a ruby wrapper for libcmark (CommonMark parser)
@@ -18,7 +20,7 @@ module Banzai
         PARSE_OPTIONS = [
           :FOOTNOTES,                  # parse footnotes.
           :STRIKETHROUGH_DOUBLE_TILDE, # parse strikethroughs by double tildes (as redcarpet does).
-          :VALIDATE_UTF8	             # replace illegal sequences with the replacement character U+FFFD.
+          :VALIDATE_UTF8               # replace illegal sequences with the replacement character U+FFFD.
         ].freeze
 
         # The `:GITHUB_PRE_LANG` option is not used intentionally because
@@ -30,8 +32,13 @@ module Banzai
           :DEFAULT # default rendering system. Nothing special.
         ].freeze
 
-        def initialize
-          @renderer = Banzai::Renderer::CommonMark::HTML.new(options: RENDER_OPTIONS)
+        RENDER_OPTIONS_SOURCEPOS = RENDER_OPTIONS + [
+          :SOURCEPOS # enable embedding of source position information
+        ].freeze
+
+        def initialize(context)
+          @context  = context
+          @renderer = Banzai::Renderer::CommonMark::HTML.new(options: render_options)
         end
 
         def render(text)
@@ -39,6 +46,12 @@ module Banzai
 
           @renderer.render(doc)
         end
+
+        private
+
+        def render_options
+          @context[:no_sourcepos] ? RENDER_OPTIONS : RENDER_OPTIONS_SOURCEPOS
+        end
       end
     end
   end
diff --git a/lib/banzai/filter/markdown_engines/redcarpet.rb b/lib/banzai/filter/markdown_engines/redcarpet.rb
index ac99941fefa..5b3f75096b1 100644
--- a/lib/banzai/filter/markdown_engines/redcarpet.rb
+++ b/lib/banzai/filter/markdown_engines/redcarpet.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # `Redcarpet` markdown engine for GitLab's Banzai markdown filter.
 # This module is used in Banzai::Filter::MarkdownFilter.
 # Used gem is `redcarpet` which is a ruby library for markdown processing.
@@ -18,7 +20,7 @@ module Banzai
           tables:              true
         }.freeze
 
-        def initialize
+        def initialize(context = nil)
           html_renderer = Banzai::Renderer::Redcarpet::HTML.new
           @renderer = ::Redcarpet::Markdown.new(html_renderer, OPTIONS)
         end
diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb
index c1e2b680240..242e39f5495 100644
--- a/lib/banzai/filter/markdown_filter.rb
+++ b/lib/banzai/filter/markdown_filter.rb
@@ -1,10 +1,12 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     class MarkdownFilter < HTML::Pipeline::TextFilter
       def initialize(text, context = nil, result = nil)
         super(text, context, result)
 
-        @renderer = renderer(context[:markdown_engine]).new
+        @renderer = renderer(context[:markdown_engine]).new(context)
         @text = @text.delete("\r")
       end
 
@@ -14,7 +16,7 @@ module Banzai
 
       private
 
-      DEFAULT_ENGINE = :redcarpet
+      DEFAULT_ENGINE = :common_mark
 
       def engine(engine_from_context)
         engine_from_context ||= DEFAULT_ENGINE
diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb
index b6e784c886b..8dd5a8979c8 100644
--- a/lib/banzai/filter/math_filter.rb
+++ b/lib/banzai/filter/math_filter.rb
@@ -1,5 +1,10 @@
+# frozen_string_literal: true
+
 require 'uri'
 
+# Generated HTML is transformed back to GFM by:
+# - app/assets/javascripts/behaviors/markdown/marks/math.js
+# - app/assets/javascripts/behaviors/markdown/nodes/code_block.js
 module Banzai
   module Filter
     # HTML filter that adds class="code math" and removes the dollar sign in $`2+2`$.
diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb
index 5cbdb01c130..7098767b583 100644
--- a/lib/banzai/filter/merge_request_reference_filter.rb
+++ b/lib/banzai/filter/merge_request_reference_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # HTML filter that replaces merge request references with links. References
@@ -25,7 +27,10 @@ module Banzai
         extras = super
 
         if commit_ref = object_link_commit_ref(object, matches)
-          return extras.unshift(commit_ref)
+          klass = reference_class(:commit, tooltip: false)
+          commit_ref_tag = %(<span class="#{klass}">#{commit_ref}</span>)
+
+          return extras.unshift(commit_ref_tag)
         end
 
         path = matches[:path] if matches.names.include?("path")
diff --git a/lib/banzai/filter/mermaid_filter.rb b/lib/banzai/filter/mermaid_filter.rb
index 65c131e08d9..f0adb83af8a 100644
--- a/lib/banzai/filter/mermaid_filter.rb
+++ b/lib/banzai/filter/mermaid_filter.rb
@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+
+# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/code_block.js
 module Banzai
   module Filter
     class MermaidFilter < HTML::Pipeline::Filter
diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb
index b144bd8cf54..fce042e8946 100644
--- a/lib/banzai/filter/milestone_reference_filter.rb
+++ b/lib/banzai/filter/milestone_reference_filter.rb
@@ -1,7 +1,11 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # HTML filter that replaces milestone references with links.
     class MilestoneReferenceFilter < AbstractReferenceFilter
+      include Gitlab::Utils::StrongMemoize
+
       self.reference_type = :milestone
 
       def self.object_class
@@ -11,16 +15,34 @@ module Banzai
       # Links to project milestones contain the IID, but when we're handling
       # 'regular' references, we need to use the global ID to disambiguate
       # between group and project milestones.
-      def find_object(project, id)
-        return unless project.is_a?(Project)
+      def find_object(parent, id)
+        return unless valid_context?(parent)
+
+        find_milestone_with_finder(parent, id: id)
+      end
+
+      def find_object_from_link(parent, iid)
+        return unless valid_context?(parent)
 
-        find_milestone_with_finder(project, id: id)
+        find_milestone_with_finder(parent, iid: iid)
       end
 
-      def find_object_from_link(project, iid)
-        return unless project.is_a?(Project)
+      def valid_context?(parent)
+        strong_memoize(:valid_context) do
+          group_context?(parent) || project_context?(parent)
+        end
+      end
 
-        find_milestone_with_finder(project, iid: iid)
+      def group_context?(parent)
+        strong_memoize(:group_context) do
+          parent.is_a?(Group)
+        end
+      end
+
+      def project_context?(parent)
+        strong_memoize(:project_context) do
+          parent.is_a?(Project)
+        end
       end
 
       def references_in(text, pattern = Milestone.reference_pattern)
@@ -42,13 +64,15 @@ module Banzai
 
       def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name)
         project_path = full_project_path(namespace_ref, project_ref)
-        project = parent_from_ref(project_path)
 
-        return unless project && project.is_a?(Project)
+        # Returns group if project is not found by path
+        parent = parent_from_ref(project_path)
+
+        return unless parent
 
         milestone_params = milestone_params(milestone_id, milestone_name)
 
-        find_milestone_with_finder(project, milestone_params)
+        find_milestone_with_finder(parent, milestone_params)
       end
 
       def milestone_params(iid, name)
@@ -59,16 +83,28 @@ module Banzai
         end
       end
 
-      def find_milestone_with_finder(project, params)
-        finder_params = { project_ids: [project.id], order: nil, state: 'all' }
+      def find_milestone_with_finder(parent, params)
+        finder_params = milestone_finder_params(parent, params[:iid].present?)
+
+        MilestonesFinder.new(finder_params).find_by(params)
+      end
 
-        # We don't support IID lookups for group milestones, because IIDs can
-        # clash between group and project milestones.
-        if project.group && !params[:iid]
-          finder_params[:group_ids] = [project.group.id]
+      def milestone_finder_params(parent, find_by_iid)
+        { order: nil, state: 'all' }.tap do |params|
+          params[:project_ids] = parent.id if project_context?(parent)
+
+          # We don't support IID lookups because IIDs can clash between
+          # group/project milestones and group/subgroup milestones.
+          params[:group_ids] = self_and_ancestors_ids(parent) unless find_by_iid
         end
+      end
 
-        MilestonesFinder.new(finder_params).find_by(params)
+      def self_and_ancestors_ids(parent)
+        if group_context?(parent)
+          parent.self_and_ancestors.select(:id)
+        elsif project_context?(parent)
+          parent.group&.self_and_ancestors&.select(:id)
+        end
       end
 
       def url_for_object(milestone, project)
diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb
index 28933c78966..caba8955bac 100644
--- a/lib/banzai/filter/plantuml_filter.rb
+++ b/lib/banzai/filter/plantuml_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "nokogiri"
 require "asciidoctor-plantuml/plantuml"
 
diff --git a/lib/banzai/filter/project_reference_filter.rb b/lib/banzai/filter/project_reference_filter.rb
new file mode 100644
index 00000000000..83cf45097ed
--- /dev/null
+++ b/lib/banzai/filter/project_reference_filter.rb
@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Banzai
+  module Filter
+    # HTML filter that replaces project references with links.
+    class ProjectReferenceFilter < ReferenceFilter
+      self.reference_type = :project
+
+      # Public: Find `namespace/project>` project references in text
+      #
+      #   ProjectReferenceFilter.references_in(text) do |match, project|
+      #     "<a href=...>#{project}></a>"
+      #   end
+      #
+      # text - String text to search.
+      #
+      # Yields the String match, and the String project name.
+      #
+      # Returns a String replaced with the return of the block.
+      def self.references_in(text)
+        text.gsub(Project.markdown_reference_pattern) do |match|
+          yield match, "#{$~[:namespace]}/#{$~[:project]}"
+        end
+      end
+
+      def call
+        ref_pattern = Project.markdown_reference_pattern
+        ref_pattern_start = /\A#{ref_pattern}\z/
+
+        nodes.each do |node|
+          if text_node?(node)
+            replace_text_when_pattern_matches(node, ref_pattern) do |content|
+              project_link_filter(content)
+            end
+          elsif element_node?(node)
+            yield_valid_link(node) do |link, inner_html|
+              if link =~ ref_pattern_start
+                replace_link_node_with_href(node, link) do
+                  project_link_filter(link, link_content: inner_html)
+                end
+              end
+            end
+          end
+        end
+
+        doc
+      end
+
+      # Replace `namespace/project>` project references in text with links to the referenced
+      # project page.
+      #
+      # text - String text to replace references in.
+      # link_content - Original content of the link being replaced.
+      #
+      # Returns a String with `namespace/project>` references replaced with links. All links
+      # have `gfm` and `gfm-project` class names attached for styling.
+      def project_link_filter(text, link_content: nil)
+        self.class.references_in(text) do |match, project_path|
+          cached_call(:banzai_url_for_object, match, path: [Project, project_path.downcase]) do
+            if project = projects_hash[project_path.downcase]
+              link_to_project(project, link_content: link_content) || match
+            else
+              match
+            end
+          end
+        end
+      end
+
+      # Returns a Hash containing all Project objects for the project
+      # references in the current document.
+      #
+      # The keys of this Hash are the project paths, the values the
+      # corresponding Project objects.
+      def projects_hash
+        @projects ||= Project.eager_load(:route, namespace: [:route])
+                             .where_full_path_in(projects)
+                             .index_by(&:full_path)
+                             .transform_keys(&:downcase)
+      end
+
+      # Returns all projects referenced in the current document.
+      def projects
+        refs = Set.new
+
+        nodes.each do |node|
+          node.to_html.scan(Project.markdown_reference_pattern) do
+            refs << "#{$~[:namespace]}/#{$~[:project]}"
+          end
+        end
+
+        refs.to_a
+      end
+
+      private
+
+      def urls
+        Gitlab::Routing.url_helpers
+      end
+
+      def link_class
+        reference_class(:project)
+      end
+
+      def link_to_project(project, link_content: nil)
+        url = urls.project_url(project, only_path: context[:only_path])
+        data = data_attribute(project: project.id)
+        content = link_content || project.to_reference_with_postfix
+
+        link_tag(url, data, content, project.name)
+      end
+
+      def link_tag(url, data, link_content, title)
+        %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{link_content}</a>)
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb
index caf11fe94c4..1f091f594f8 100644
--- a/lib/banzai/filter/redactor_filter.rb
+++ b/lib/banzai/filter/redactor_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # HTML filter that removes references to records that the current user does
diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb
index 2f023f4f242..42f9b3a689c 100644
--- a/lib/banzai/filter/reference_filter.rb
+++ b/lib/banzai/filter/reference_filter.rb
@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+
+# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/reference.js
 module Banzai
   module Filter
     # Base class for GitLab Flavored Markdown reference filters.
@@ -65,8 +68,12 @@ module Banzai
         context[:skip_project_check]
       end
 
-      def reference_class(type)
-        "gfm gfm-#{type} has-tooltip"
+      def reference_class(type, tooltip: true)
+        gfm_klass = "gfm gfm-#{type}"
+
+        return gfm_klass unless tooltip
+
+        "#{gfm_klass} has-tooltip"
       end
 
       # Ensure that a :project key exists in context
diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb
index 262458a872a..93e6d6470f1 100644
--- a/lib/banzai/filter/relative_link_filter.rb
+++ b/lib/banzai/filter/relative_link_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'uri'
 
 module Banzai
@@ -56,9 +58,15 @@ module Banzai
           path_parts.unshift(relative_url_root, 'groups', group.full_path, '-')
         elsif project
           path_parts.unshift(relative_url_root, project.full_path)
+        else
+          path_parts.unshift(relative_url_root)
         end
 
-        path = Addressable::URI.escape(File.join(*path_parts))
+        begin
+          path = Addressable::URI.escape(File.join(*path_parts))
+        rescue Addressable::URI::InvalidURIError
+          return
+        end
 
         html_attr.value =
           if context[:only_path]
diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb
index 6786b9d07b6..a4a06eae7b7 100644
--- a/lib/banzai/filter/sanitization_filter.rb
+++ b/lib/banzai/filter/sanitization_filter.rb
@@ -1,34 +1,30 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # Sanitize HTML
     #
     # Extends HTML::Pipeline::SanitizationFilter with a custom whitelist.
     class SanitizationFilter < HTML::Pipeline::SanitizationFilter
-      UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze
-      TABLE_ALIGNMENT_PATTERN = /text-align: (?<alignment>center|left|right)/
-
-      def whitelist
-        whitelist = super
+      include Gitlab::Utils::StrongMemoize
 
-        customize_whitelist(whitelist)
+      UNSAFE_PROTOCOLS        = %w(data javascript vbscript).freeze
+      TABLE_ALIGNMENT_PATTERN = /text-align: (?<alignment>center|left|right)/.freeze
 
-        whitelist
+      def whitelist
+        strong_memoize(:whitelist) do
+          customize_whitelist(super.deep_dup)
+        end
       end
 
       private
 
-      def customized?(transformers)
-        transformers.last.source_location[0] == __FILE__
-      end
-
       def customize_whitelist(whitelist)
-        # Only push these customizations once
-        return if customized?(whitelist[:transformers])
-
-        # Allow table alignment; we whitelist specific style properties in a
+        # Allow table alignment; we whitelist specific text-align values in a
         # transformer below
         whitelist[:attributes]['th'] = %w(style)
         whitelist[:attributes]['td'] = %w(style)
+        whitelist[:css] = { properties: ['text-align'] }
 
         # Allow span elements
         whitelist[:elements].push('span')
@@ -45,14 +41,16 @@ module Banzai
         whitelist[:elements].push('abbr')
         whitelist[:attributes]['abbr'] = %w(title)
 
+        # Allow the 'data-sourcepos' from CommonMark on all elements
+        whitelist[:attributes][:all].push('data-sourcepos')
+
         # Disallow `name` attribute globally, allow on `a`
         whitelist[:attributes][:all].delete('name')
         whitelist[:attributes]['a'].push('name')
 
-        # Allow any protocol in `a` elements...
+        # Allow any protocol in `a` elements
+        # and then remove links with unsafe protocols
         whitelist[:protocols].delete('a')
-
-        # ...but then remove links with unsafe protocols
         whitelist[:transformers].push(self.class.remove_unsafe_links)
 
         # Remove `rel` attribute from `a` elements
@@ -61,6 +59,12 @@ module Banzai
         # Remove any `style` properties not required for table alignment
         whitelist[:transformers].push(self.class.remove_unsafe_table_style)
 
+        # Allow `id` in a and li elements for footnotes
+        # and remove any `id` properties not matching for footnotes
+        whitelist[:attributes]['a'].push('id')
+        whitelist[:attributes]['li'] = %w(id)
+        whitelist[:transformers].push(self.class.remove_non_footnote_ids)
+
         whitelist
       end
 
@@ -116,6 +120,20 @@ module Banzai
             end
           end
         end
+
+        def remove_non_footnote_ids
+          lambda do |env|
+            node = env[:node]
+
+            return unless node.name == 'a' || node.name == 'li'
+            return unless node.has_attribute?('id')
+
+            return if node.name == 'a' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LINK_REFERENCE_PATTERN
+            return if node.name == 'li' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LI_REFERENCE_PATTERN
+
+            node.remove_attribute('id')
+          end
+        end
       end
     end
   end
diff --git a/lib/banzai/filter/set_direction_filter.rb b/lib/banzai/filter/set_direction_filter.rb
index c2976aeb7c6..45b259a2faf 100644
--- a/lib/banzai/filter/set_direction_filter.rb
+++ b/lib/banzai/filter/set_direction_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # HTML filter that sets dir="auto" for RTL languages support
diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb
index 881e10afb9f..f4b6edb6174 100644
--- a/lib/banzai/filter/snippet_reference_filter.rb
+++ b/lib/banzai/filter/snippet_reference_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # HTML filter that replaces snippet references with links. References to
diff --git a/lib/banzai/filter/spaced_link_filter.rb b/lib/banzai/filter/spaced_link_filter.rb
new file mode 100644
index 00000000000..00dbf2d3130
--- /dev/null
+++ b/lib/banzai/filter/spaced_link_filter.rb
@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+module Banzai
+  module Filter
+    # HTML Filter for markdown links with spaces in the URLs
+    #
+    # Based on Banzai::Filter::AutolinkFilter
+    #
+    # CommonMark does not allow spaces in the url portion of a link/url.
+    # For example, `[example](page slug)` is not valid.
+    # Neither is `![example](test image.jpg)`. However, particularly
+    # in our wikis, we support (via RedCarpet) this type of link, allowing
+    # wiki pages to be easily linked by their title.  This filter adds that functionality.
+    #
+    # This is a small extension to the CommonMark spec.  If they start allowing
+    # spaces in urls, we could then remove this filter.
+    #
+    # Note: Filter::SanitizationFilter should always be run sometime after this filter
+    # to prevent XSS attacks
+    #
+    class SpacedLinkFilter < HTML::Pipeline::Filter
+      include ActionView::Helpers::TagHelper
+
+      # Pattern to match a standard markdown link
+      #
+      # Rubular: http://rubular.com/r/2EXEQ49rg5
+      LINK_OR_IMAGE_PATTERN = %r{
+        (?<preview_operator>!)?
+        \[(?<text>.+?)\]
+        \(
+          (?<new_link>.+?)
+          (?<title>\ ".+?")?
+        \)
+      }x
+
+      # Text matching LINK_OR_IMAGE_PATTERN inside these elements will not be linked
+      IGNORE_PARENTS = %w(a code kbd pre script style).to_set
+
+      # The XPath query to use for finding text nodes to parse.
+      TEXT_QUERY = %Q(descendant-or-self::text()[
+        not(#{IGNORE_PARENTS.map { |p| "ancestor::#{p}" }.join(' or ')})
+        and contains(., ']\(')
+      ]).freeze
+
+      def call
+        return doc if context[:markdown_engine] == :redcarpet
+
+        doc.xpath(TEXT_QUERY).each do |node|
+          content = node.to_html
+
+          next unless content.match(LINK_OR_IMAGE_PATTERN)
+
+          html = spaced_link_filter(content)
+
+          next if html == content
+
+          node.replace(html)
+        end
+
+        doc
+      end
+
+      private
+
+      def spaced_link_match(link)
+        match = LINK_OR_IMAGE_PATTERN.match(link)
+        return link unless match
+
+        # escape the spaces in the url so that it's a valid markdown link,
+        # then run it through the markdown processor again, let it do its magic
+        html = Banzai::Filter::MarkdownFilter.call(transform_markdown(match), context)
+
+        # link is wrapped in a <p>, so strip that off
+        p_node = Nokogiri::HTML.fragment(html).at_css('p')
+        p_node ? p_node.children.to_html : html
+      end
+
+      def spaced_link_filter(text)
+        Gitlab::StringRegexMarker.new(CGI.unescapeHTML(text), text.html_safe).mark(LINK_OR_IMAGE_PATTERN) do |link, left:, right:|
+          spaced_link_match(link)
+        end
+      end
+
+      def transform_markdown(match)
+        preview_operator, text, new_link, title = process_match(match)
+
+        "#{preview_operator}[#{text}](#{new_link}#{title})"
+      end
+
+      def process_match(match)
+        [
+          match[:preview_operator],
+          match[:text],
+          match[:new_link].gsub(' ', '%20'),
+          match[:title]
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/suggestion_filter.rb b/lib/banzai/filter/suggestion_filter.rb
new file mode 100644
index 00000000000..9950db373d8
--- /dev/null
+++ b/lib/banzai/filter/suggestion_filter.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/code_block.js
+module Banzai
+  module Filter
+    class SuggestionFilter < HTML::Pipeline::Filter
+      # Class used for tagging elements that should be rendered
+      TAG_CLASS = 'js-render-suggestion'.freeze
+
+      def call
+        return doc unless suggestions_filter_enabled?
+
+        doc.search('pre.suggestion > code').each do |node|
+          node.add_class(TAG_CLASS)
+        end
+
+        doc
+      end
+
+      def suggestions_filter_enabled?
+        context[:suggestions_filter_enabled]
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb
index 6dbf0d68fe8..bcf77861f10 100644
--- a/lib/banzai/filter/syntax_highlight_filter.rb
+++ b/lib/banzai/filter/syntax_highlight_filter.rb
@@ -1,6 +1,9 @@
+# frozen_string_literal: true
+
 require 'rouge/plugins/common_mark'
 require 'rouge/plugins/redcarpet'
 
+# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/code_block.js
 module Banzai
   module Filter
     # HTML Filter to highlight fenced code blocks
@@ -15,7 +18,7 @@ module Banzai
       end
 
       def highlight_node(node)
-        css_classes = 'code highlight js-syntax-highlight'
+        css_classes = +'code highlight js-syntax-highlight'
         lang = node.attr('lang')
         retried = false
 
@@ -67,7 +70,7 @@ module Banzai
       end
 
       def use_rouge?(language)
-        %w(math mermaid plantuml).exclude?(language)
+        %w(math mermaid plantuml suggestion).exclude?(language)
       end
     end
   end
diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb
index 97244159985..f2ae17b44fa 100644
--- a/lib/banzai/filter/table_of_contents_filter.rb
+++ b/lib/banzai/filter/table_of_contents_filter.rb
@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+
+# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/table_of_contents.js
 module Banzai
   module Filter
     # HTML filter that adds an anchor child element to all Headers in a
@@ -19,7 +22,7 @@ module Banzai
       def call
         return doc if context[:no_header_anchors]
 
-        result[:toc] = ""
+        result[:toc] = +""
 
         headers = Hash.new(0)
         header_root = current_header = HeaderNode.new
@@ -92,7 +95,7 @@ module Banzai
         def text
           return '' unless node
 
-          @text ||= node.text
+          @text ||= EscapeUtils.escape_html(node.text)
         end
 
         private
diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb
index 9fa5f589f3e..c6b402575cb 100644
--- a/lib/banzai/filter/task_list_filter.rb
+++ b/lib/banzai/filter/task_list_filter.rb
@@ -1,5 +1,11 @@
+# frozen_string_literal: true
+
 require 'task_list/filter'
 
+# Generated HTML is transformed back to GFM by:
+# - app/assets/javascripts/behaviors/markdown/nodes/ordered_task_list.js
+# - app/assets/javascripts/behaviors/markdown/nodes/task_list.js
+# - app/assets/javascripts/behaviors/markdown/nodes/task_list_item.js
 module Banzai
   module Filter
     class TaskListFilter < TaskList::Filter
diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb
index c7fa8a8119f..8cda67867a8 100644
--- a/lib/banzai/filter/user_reference_filter.rb
+++ b/lib/banzai/filter/user_reference_filter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     # HTML filter that replaces user or group references with links.
@@ -104,7 +106,7 @@ module Banzai
       end
 
       def link_class
-        reference_class(:project_member)
+        reference_class(:project_member, tooltip: false)
       end
 
       def link_to_all(link_content: nil)
diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb
index 35cb10eae5d..0fff104cf91 100644
--- a/lib/banzai/filter/video_link_filter.rb
+++ b/lib/banzai/filter/video_link_filter.rb
@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+
+# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/video.js
 module Banzai
   module Filter
     # Find every image that isn't already wrapped in an `a` tag, and that has
diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb
index 269d5bf74fa..1728a442533 100644
--- a/lib/banzai/filter/wiki_link_filter.rb
+++ b/lib/banzai/filter/wiki_link_filter.rb
@@ -1,4 +1,4 @@
-require 'uri'
+# frozen_string_literal: true
 
 module Banzai
   module Filter
@@ -11,8 +11,12 @@ module Banzai
       def call
         return doc unless project_wiki?
 
-        doc.search('a:not(.gfm)').each do |el|
-          process_link_attr el.attribute('href')
+        doc.search('a:not(.gfm)').each { |el| process_link_attr(el.attribute('href')) }
+        doc.search('video').each { |el| process_link_attr(el.attribute('src')) }
+        doc.search('img').each do |el|
+          attr = el.attribute('data-src') || el.attribute('src')
+
+          process_link_attr(attr)
         end
 
         doc
diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb
index 072d24e5a11..f4cc8beeb52 100644
--- a/lib/banzai/filter/wiki_link_filter/rewriter.rb
+++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Filter
     class WikiLinkFilter < HTML::Pipeline::Filter
@@ -10,11 +12,16 @@ module Banzai
 
         def apply_rules
           # Special case: relative URLs beginning with `/uploads/` refer to
-          # user-uploaded files and will be handled elsewhere.
-          return @uri.to_s if @uri.relative? && @uri.path.starts_with?('/uploads/')
+          # user-uploaded files will be handled elsewhere.
+          return @uri.to_s if public_upload?
+
+          # Special case: relative URLs beginning with Wikis::CreateAttachmentService::ATTACHMENT_PATH
+          # refer to user-uploaded files to the wiki repository.
+          unless repository_upload?
+            apply_file_link_rules!
+            apply_hierarchical_link_rules!
+          end
 
-          apply_file_link_rules!
-          apply_hierarchical_link_rules!
           apply_relative_link_rules!
           @uri.to_s
         end
@@ -39,6 +46,14 @@ module Banzai
             @uri = Addressable::URI.parse(link)
           end
         end
+
+        def public_upload?
+          @uri.relative? && @uri.path.starts_with?('/uploads/')
+        end
+
+        def repository_upload?
+          @uri.relative? && @uri.path.starts_with?(Wikis::CreateAttachmentService::ATTACHMENT_PATH)
+        end
       end
     end
   end
diff --git a/lib/banzai/filter/yaml_front_matter_filter.rb b/lib/banzai/filter/yaml_front_matter_filter.rb
deleted file mode 100644
index 58e3e81209e..00000000000
--- a/lib/banzai/filter/yaml_front_matter_filter.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-module Banzai
-  module Filter
-    class YamlFrontMatterFilter < HTML::Pipeline::Filter
-      DELIM = '---'.freeze
-
-      # Hat-tip to Middleman: https://git.io/v2e0z
-      PATTERN = %r{
-        \A(?:[^\r\n]*coding:[^\r\n]*\r?\n)?
-        (?<start>#{DELIM})[ ]*\r?\n
-        (?<frontmatter>.*?)[ ]*\r?\n?
-        ^(?<stop>#{DELIM})[ ]*\r?\n?
-        \r?\n?
-        (?<content>.*)
-      }mx.freeze
-
-      def call
-        match = PATTERN.match(html)
-
-        return html unless match
-
-        "```yaml\n#{match['frontmatter']}\n```\n\n#{match['content']}"
-      end
-    end
-  end
-end
diff --git a/lib/banzai/filter_array.rb b/lib/banzai/filter_array.rb
index 77835a14027..818af4643a7 100644
--- a/lib/banzai/filter_array.rb
+++ b/lib/banzai/filter_array.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   class FilterArray < Array
     # Insert a value immediately after another value
diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb
index ae7dc71e7eb..341dbb74fe0 100644
--- a/lib/banzai/issuable_extractor.rb
+++ b/lib/banzai/issuable_extractor.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   # Extract references to issuables from multiple documents
 
@@ -7,13 +9,11 @@ module Banzai
   # so we can avoid N+1 queries problem
 
   class IssuableExtractor
-    QUERY = %q(
-      descendant-or-self::a[contains(concat(" ", @class, " "), " gfm ")]
-      [@data-reference-type="issue" or @data-reference-type="merge_request"]
-    ).freeze
-
     attr_reader :context
 
+    ISSUE_REFERENCE_TYPE = '@data-reference-type="issue"'.freeze
+    MERGE_REQUEST_REFERENCE_TYPE = '@data-reference-type="merge_request"'.freeze
+
     # context - An instance of Banzai::RenderContext.
     def initialize(context)
       @context = context
@@ -22,21 +22,38 @@ module Banzai
     # Returns Hash in the form { node => issuable_instance }
     def extract(documents)
       nodes = documents.flat_map do |document|
-        document.xpath(QUERY)
+        document.xpath(query)
       end
 
-      issue_parser = Banzai::ReferenceParser::IssueParser.new(context)
+      # The project or group for the issuable might be pending for deletion!
+      # Filter them out because we don't care about them.
+      issuables_for_nodes(nodes).select { |node, issuable| issuable.project || issuable.group }
+    end
+
+    private
 
-      merge_request_parser =
+    def issuables_for_nodes(nodes)
+      parsers.each_with_object({}) do |parser, result|
+        result.merge!(parser.records_for_nodes(nodes))
+      end
+    end
+
+    def parsers
+      [
+        Banzai::ReferenceParser::IssueParser.new(context),
         Banzai::ReferenceParser::MergeRequestParser.new(context)
+      ]
+    end
 
-      issuables_for_nodes = issue_parser.records_for_nodes(nodes).merge(
-        merge_request_parser.records_for_nodes(nodes)
+    def query
+      %Q(
+        descendant-or-self::a[contains(concat(" ", @class, " "), " gfm ")]
+        [#{reference_types.join(' or ')}]
       )
+    end
 
-      # The project for the issue/MR might be pending for deletion!
-      # Filter them out because we don't care about them.
-      issuables_for_nodes.select { |node, issuable| issuable.project }
+    def reference_types
+      [ISSUE_REFERENCE_TYPE, MERGE_REQUEST_REFERENCE_TYPE]
     end
   end
 end
diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb
index a176f1e261b..75661ffa233 100644
--- a/lib/banzai/object_renderer.rb
+++ b/lib/banzai/object_renderer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   # Class for rendering multiple objects (e.g. Note instances) in a single pass,
   # using +render_field+ to benefit from caching in the database. Rendering and
@@ -38,6 +40,7 @@ module Banzai
         redacted_data = redacted[index]
         object.__send__("redacted_#{attribute}_html=", redacted_data[:document].to_html(save_options).html_safe) # rubocop:disable GitlabSecurity/PublicSend
         object.user_visible_reference_count = redacted_data[:visible_reference_count] if object.respond_to?(:user_visible_reference_count)
+        object.total_reference_count = redacted_data[:total_reference_count] if object.respond_to?(:total_reference_count)
       end
     end
 
diff --git a/lib/banzai/pipeline.rb b/lib/banzai/pipeline.rb
index 142a9962eb1..e8a81bebaa9 100644
--- a/lib/banzai/pipeline.rb
+++ b/lib/banzai/pipeline.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     def self.[](name)
diff --git a/lib/banzai/pipeline/ascii_doc_pipeline.rb b/lib/banzai/pipeline/ascii_doc_pipeline.rb
index 1048b927cd3..cc4af280872 100644
--- a/lib/banzai/pipeline/ascii_doc_pipeline.rb
+++ b/lib/banzai/pipeline/ascii_doc_pipeline.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     class AsciiDocPipeline < BasePipeline
diff --git a/lib/banzai/pipeline/atom_pipeline.rb b/lib/banzai/pipeline/atom_pipeline.rb
index 9694e4bc23f..c632910585d 100644
--- a/lib/banzai/pipeline/atom_pipeline.rb
+++ b/lib/banzai/pipeline/atom_pipeline.rb
@@ -1,10 +1,13 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     class AtomPipeline < FullPipeline
       def self.transform_context(context)
         super(context).merge(
           only_path: false,
-          xhtml: true
+          xhtml: true,
+          no_sourcepos: true
         )
       end
     end
diff --git a/lib/banzai/pipeline/base_pipeline.rb b/lib/banzai/pipeline/base_pipeline.rb
index 3ae3bed570d..87d1cf9912f 100644
--- a/lib/banzai/pipeline/base_pipeline.rb
+++ b/lib/banzai/pipeline/base_pipeline.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     class BasePipeline
diff --git a/lib/banzai/pipeline/broadcast_message_pipeline.rb b/lib/banzai/pipeline/broadcast_message_pipeline.rb
index 5dd572de3a1..580b5b72474 100644
--- a/lib/banzai/pipeline/broadcast_message_pipeline.rb
+++ b/lib/banzai/pipeline/broadcast_message_pipeline.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     class BroadcastMessagePipeline < DescriptionPipeline
@@ -12,6 +14,12 @@ module Banzai
           Filter::ExternalLinkFilter
         ]
       end
+
+      def self.transform_context(context)
+        super(context).merge(
+          no_sourcepos: true
+        )
+      end
     end
   end
 end
diff --git a/lib/banzai/pipeline/combined_pipeline.rb b/lib/banzai/pipeline/combined_pipeline.rb
index 60190f8d9dd..56b424dc8e0 100644
--- a/lib/banzai/pipeline/combined_pipeline.rb
+++ b/lib/banzai/pipeline/combined_pipeline.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     module CombinedPipeline
diff --git a/lib/banzai/pipeline/commit_description_pipeline.rb b/lib/banzai/pipeline/commit_description_pipeline.rb
index 607c2731ed3..e8ec7453f0f 100644
--- a/lib/banzai/pipeline/commit_description_pipeline.rb
+++ b/lib/banzai/pipeline/commit_description_pipeline.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     class CommitDescriptionPipeline < SingleLinePipeline
diff --git a/lib/banzai/pipeline/description_pipeline.rb b/lib/banzai/pipeline/description_pipeline.rb
index 042fb2e6e14..d5ff9b025cc 100644
--- a/lib/banzai/pipeline/description_pipeline.rb
+++ b/lib/banzai/pipeline/description_pipeline.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     class DescriptionPipeline < FullPipeline
diff --git a/lib/banzai/pipeline/email_pipeline.rb b/lib/banzai/pipeline/email_pipeline.rb
index 8f5f144d582..13e6a990407 100644
--- a/lib/banzai/pipeline/email_pipeline.rb
+++ b/lib/banzai/pipeline/email_pipeline.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     class EmailPipeline < FullPipeline
@@ -9,7 +11,9 @@ module Banzai
 
       def self.transform_context(context)
         super(context).merge(
-          only_path: false
+          only_path: false,
+          emailable_links: true,
+          no_sourcepos: true
         )
       end
     end
diff --git a/lib/banzai/pipeline/emoji_pipeline.rb b/lib/banzai/pipeline/emoji_pipeline.rb
new file mode 100644
index 00000000000..a1b522f4303
--- /dev/null
+++ b/lib/banzai/pipeline/emoji_pipeline.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Banzai
+  module Pipeline
+    class EmojiPipeline < BasePipeline
+      # These filters will only perform sanitization of the content, preventing
+      # XSS, and replace emoji.
+      def self.filters
+        @filters ||= FilterArray[
+          Filter::HtmlEntityFilter,
+          Filter::SanitizationFilter,
+          Filter::EmojiFilter
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/banzai/pipeline/full_pipeline.rb b/lib/banzai/pipeline/full_pipeline.rb
index 3c974f73176..a5b1cbdd030 100644
--- a/lib/banzai/pipeline/full_pipeline.rb
+++ b/lib/banzai/pipeline/full_pipeline.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     class FullPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GfmPipeline)
diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb
index a1f24e8b093..30cafd11834 100644
--- a/lib/banzai/pipeline/gfm_pipeline.rb
+++ b/lib/banzai/pipeline/gfm_pipeline.rb
@@ -1,15 +1,21 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     class GfmPipeline < BasePipeline
-      # These filters convert GitLab Flavored Markdown (GFM) to HTML.
-      # The handlers defined in app/assets/javascripts/behaviors/markdown/copy_as_gfm.js
-      # consequently convert that same HTML to GFM to be copied to the clipboard.
-      # Every filter that generates HTML from GFM should have a handler in
-      # app/assets/javascripts/behaviors/markdown/copy_as_gfm.js, in reverse order.
+      # These filters transform GitLab Flavored Markdown (GFM) to HTML.
+      # The nodes and marks referenced in app/assets/javascripts/behaviors/markdown/editor_extensions.js
+      # consequently transform that same HTML to GFM to be copied to the clipboard.
+      # Every filter that generates HTML from GFM should have a node or mark in
+      # app/assets/javascripts/behaviors/markdown/editor_extensions.js.
       # The GFM-to-HTML-to-GFM cycle is tested in spec/features/copy_as_gfm_spec.rb.
       def self.filters
         @filters ||= FilterArray[
           Filter::PlantumlFilter,
+
+          # Must always be before the SanitizationFilter to prevent XSS attacks
+          Filter::SpacedLinkFilter,
+
           Filter::SanitizationFilter,
           Filter::SyntaxHighlightFilter,
 
@@ -23,8 +29,22 @@ module Banzai
           Filter::TableOfContentsFilter,
           Filter::AutolinkFilter,
           Filter::ExternalLinkFilter,
+          Filter::SuggestionFilter,
+          Filter::FootnoteFilter,
+
+          *reference_filters,
+
+          Filter::TaskListFilter,
+          Filter::InlineDiffFilter,
+
+          Filter::SetDirectionFilter
+        ]
+      end
 
+      def self.reference_filters
+        [
           Filter::UserReferenceFilter,
+          Filter::ProjectReferenceFilter,
           Filter::IssueReferenceFilter,
           Filter::ExternalIssueReferenceFilter,
           Filter::MergeRequestReferenceFilter,
@@ -32,23 +52,14 @@ module Banzai
           Filter::CommitRangeReferenceFilter,
           Filter::CommitReferenceFilter,
           Filter::LabelReferenceFilter,
-          Filter::MilestoneReferenceFilter,
-
-          Filter::TaskListFilter,
-          Filter::InlineDiffFilter,
-
-          Filter::SetDirectionFilter
+          Filter::MilestoneReferenceFilter
         ]
       end
 
       def self.transform_context(context)
         context[:only_path] = true unless context.key?(:only_path)
 
-        context.merge(
-          # EmojiFilter
-          asset_host: Gitlab::Application.config.asset_host,
-          asset_root: Gitlab.config.gitlab.base_url
-        )
+        context
       end
     end
   end
diff --git a/lib/banzai/pipeline/label_pipeline.rb b/lib/banzai/pipeline/label_pipeline.rb
new file mode 100644
index 00000000000..725cccc4b2b
--- /dev/null
+++ b/lib/banzai/pipeline/label_pipeline.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Banzai
+  module Pipeline
+    class LabelPipeline < BasePipeline
+      def self.filters
+        @filters ||= FilterArray[
+          Filter::SanitizationFilter,
+          Filter::LabelReferenceFilter
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/banzai/pipeline/markup_pipeline.rb b/lib/banzai/pipeline/markup_pipeline.rb
index c56d908009f..db79a22549c 100644
--- a/lib/banzai/pipeline/markup_pipeline.rb
+++ b/lib/banzai/pipeline/markup_pipeline.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     class MarkupPipeline < BasePipeline
diff --git a/lib/banzai/pipeline/note_pipeline.rb b/lib/banzai/pipeline/note_pipeline.rb
index 7890f20f716..4480d7ede05 100644
--- a/lib/banzai/pipeline/note_pipeline.rb
+++ b/lib/banzai/pipeline/note_pipeline.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     class NotePipeline < FullPipeline
diff --git a/lib/banzai/pipeline/plain_markdown_pipeline.rb b/lib/banzai/pipeline/plain_markdown_pipeline.rb
index 3f45db21869..b64f13cde47 100644
--- a/lib/banzai/pipeline/plain_markdown_pipeline.rb
+++ b/lib/banzai/pipeline/plain_markdown_pipeline.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     class PlainMarkdownPipeline < BasePipeline
diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb
index dcd52bc03c7..7eaad6d7560 100644
--- a/lib/banzai/pipeline/post_process_pipeline.rb
+++ b/lib/banzai/pipeline/post_process_pipeline.rb
@@ -1,12 +1,21 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     class PostProcessPipeline < BasePipeline
       def self.filters
-        FilterArray[
+        @filters ||= FilterArray[
+          *internal_link_filters,
+          Filter::AbsoluteLinkFilter
+        ]
+      end
+
+      def self.internal_link_filters
+        [
           Filter::RedactorFilter,
           Filter::RelativeLinkFilter,
           Filter::IssuableStateFilter,
-          Filter::AbsoluteLinkFilter
+          Filter::SuggestionFilter
         ]
       end
 
diff --git a/lib/banzai/pipeline/pre_process_pipeline.rb b/lib/banzai/pipeline/pre_process_pipeline.rb
index 6cf219661d3..4c2b4ca1665 100644
--- a/lib/banzai/pipeline/pre_process_pipeline.rb
+++ b/lib/banzai/pipeline/pre_process_pipeline.rb
@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     class PreProcessPipeline < BasePipeline
       def self.filters
         FilterArray[
-          Filter::YamlFrontMatterFilter,
+          Filter::FrontMatterFilter,
           Filter::BlockquoteFenceFilter,
         ]
       end
diff --git a/lib/banzai/pipeline/relative_link_pipeline.rb b/lib/banzai/pipeline/relative_link_pipeline.rb
index 270990e7ab4..88651892acc 100644
--- a/lib/banzai/pipeline/relative_link_pipeline.rb
+++ b/lib/banzai/pipeline/relative_link_pipeline.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     class RelativeLinkPipeline < BasePipeline
diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb
index 1929099931b..72374207a8f 100644
--- a/lib/banzai/pipeline/single_line_pipeline.rb
+++ b/lib/banzai/pipeline/single_line_pipeline.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     class SingleLinePipeline < GfmPipeline
@@ -10,15 +12,27 @@ module Banzai
           Filter::AutolinkFilter,
           Filter::ExternalLinkFilter,
 
+          *reference_filters
+        ]
+      end
+
+      def self.reference_filters
+        [
           Filter::UserReferenceFilter,
           Filter::IssueReferenceFilter,
           Filter::ExternalIssueReferenceFilter,
           Filter::MergeRequestReferenceFilter,
           Filter::SnippetReferenceFilter,
           Filter::CommitRangeReferenceFilter,
-          Filter::CommitReferenceFilter,
+          Filter::CommitReferenceFilter
         ]
       end
+
+      def self.transform_context(context)
+        super(context).merge(
+          no_sourcepos: true
+        )
+      end
     end
   end
 end
diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb
index c37b8e71cb0..97a03895ff3 100644
--- a/lib/banzai/pipeline/wiki_pipeline.rb
+++ b/lib/banzai/pipeline/wiki_pipeline.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Pipeline
     class WikiPipeline < FullPipeline
diff --git a/lib/banzai/querying.rb b/lib/banzai/querying.rb
index a19a05e8c0d..55aa5fa66c3 100644
--- a/lib/banzai/querying.rb
+++ b/lib/banzai/querying.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Querying
     module_function
diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb
index 28928d6f376..7db5f5e1f7d 100644
--- a/lib/banzai/redactor.rb
+++ b/lib/banzai/redactor.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   # Class for removing Markdown references a certain user is not allowed to
   # view.
@@ -37,7 +39,13 @@ module Banzai
 
       all_document_nodes.each do |entry|
         nodes_for_document = entry[:nodes]
-        doc_data = { document: entry[:document], visible_reference_count: nodes_for_document.count }
+
+        doc_data = {
+          document:                entry[:document],
+          total_reference_count:   nodes_for_document.count,
+          visible_reference_count: nodes_for_document.count
+        }
+
         metadata << doc_data
 
         nodes_for_document.each do |node|
diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb
index 78588299c18..3fc3ae02088 100644
--- a/lib/banzai/reference_extractor.rb
+++ b/lib/banzai/reference_extractor.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   # Extract possible GFM references from an arbitrary String for further processing.
   class ReferenceExtractor
diff --git a/lib/banzai/reference_parser.rb b/lib/banzai/reference_parser.rb
index 557bec4316e..efe15096f08 100644
--- a/lib/banzai/reference_parser.rb
+++ b/lib/banzai/reference_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module ReferenceParser
     # Returns the reference parser class for the given type
diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb
index 68752f5bb5a..8419769085a 100644
--- a/lib/banzai/reference_parser/base_parser.rb
+++ b/lib/banzai/reference_parser/base_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module ReferenceParser
     # Base class for reference parsing classes.
@@ -166,7 +168,7 @@ module Banzai
       # objects that have not yet been queried. For objects that have already
       # been queried the object is returned from the cache.
       def collection_objects_for_ids(collection, ids)
-        if RequestStore.active?
+        if Gitlab::SafeRequestStore.active?
           ids = ids.map(&:to_i)
           cache = collection_cache[collection_cache_key(collection)]
           to_query = ids - cache.keys
@@ -215,7 +217,7 @@ module Banzai
       #
       def projects_for_nodes(nodes)
         @projects_for_nodes ||=
-          grouped_objects_for_nodes(nodes, Project, 'data-project')
+          grouped_objects_for_nodes(nodes, Project.includes(:project_feature), 'data-project')
       end
 
       def can?(user, permission, subject = :global)
@@ -248,7 +250,7 @@ module Banzai
       end
 
       def collection_cache
-        RequestStore[:banzai_collection_cache] ||= Hash.new do |hash, key|
+        Gitlab::SafeRequestStore[:banzai_collection_cache] ||= Hash.new do |hash, key|
           hash[key] = {}
         end
       end
diff --git a/lib/banzai/reference_parser/commit_parser.rb b/lib/banzai/reference_parser/commit_parser.rb
index 30dc87248b4..0bfb6a92020 100644
--- a/lib/banzai/reference_parser/commit_parser.rb
+++ b/lib/banzai/reference_parser/commit_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module ReferenceParser
     class CommitParser < BaseParser
diff --git a/lib/banzai/reference_parser/commit_range_parser.rb b/lib/banzai/reference_parser/commit_range_parser.rb
index 2920e886938..480eefd5c4d 100644
--- a/lib/banzai/reference_parser/commit_range_parser.rb
+++ b/lib/banzai/reference_parser/commit_range_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module ReferenceParser
     class CommitRangeParser < BaseParser
diff --git a/lib/banzai/reference_parser/directly_addressed_user_parser.rb b/lib/banzai/reference_parser/directly_addressed_user_parser.rb
index 77df9bbd024..1f18f82b916 100644
--- a/lib/banzai/reference_parser/directly_addressed_user_parser.rb
+++ b/lib/banzai/reference_parser/directly_addressed_user_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module ReferenceParser
     class DirectlyAddressedUserParser < UserParser
diff --git a/lib/banzai/reference_parser/epic_parser.rb b/lib/banzai/reference_parser/epic_parser.rb
index 08b8a4c9a0f..7f366f0f8ab 100644
--- a/lib/banzai/reference_parser/epic_parser.rb
+++ b/lib/banzai/reference_parser/epic_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module ReferenceParser
     # The actual parser is implemented in the EE mixin
diff --git a/lib/banzai/reference_parser/external_issue_parser.rb b/lib/banzai/reference_parser/external_issue_parser.rb
index 1802cd04854..029b09dcd25 100644
--- a/lib/banzai/reference_parser/external_issue_parser.rb
+++ b/lib/banzai/reference_parser/external_issue_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module ReferenceParser
     class ExternalIssueParser < BaseParser
diff --git a/lib/banzai/reference_parser/issuable_parser.rb b/lib/banzai/reference_parser/issuable_parser.rb
index fad127d7e5b..f8c26288017 100644
--- a/lib/banzai/reference_parser/issuable_parser.rb
+++ b/lib/banzai/reference_parser/issuable_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module ReferenceParser
     class IssuableParser < BaseParser
diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb
index 7b5915899cf..97c7173ac0f 100644
--- a/lib/banzai/reference_parser/issue_parser.rb
+++ b/lib/banzai/reference_parser/issue_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module ReferenceParser
     class IssueParser < IssuableParser
diff --git a/lib/banzai/reference_parser/label_parser.rb b/lib/banzai/reference_parser/label_parser.rb
index 30e2a012f09..398cc45fea0 100644
--- a/lib/banzai/reference_parser/label_parser.rb
+++ b/lib/banzai/reference_parser/label_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module ReferenceParser
     class LabelParser < BaseParser
diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb
index a370ff5b5b3..e8147ac591a 100644
--- a/lib/banzai/reference_parser/merge_request_parser.rb
+++ b/lib/banzai/reference_parser/merge_request_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module ReferenceParser
     class MergeRequestParser < IssuableParser
@@ -14,11 +16,12 @@ module Banzai
               # Eager loading these ensures we don't end up running dozens of
               # queries in this process.
               target_project: [
-                { namespace: :owner },
+                { namespace: [:owner, :route] },
                 { group: [:owners, :group_members] },
                 :invited_groups,
                 :project_members,
-                :project_feature
+                :project_feature,
+                :route
               ]
             }),
           self.class.data_attribute
diff --git a/lib/banzai/reference_parser/milestone_parser.rb b/lib/banzai/reference_parser/milestone_parser.rb
index 68675abe22a..925d736fb9a 100644
--- a/lib/banzai/reference_parser/milestone_parser.rb
+++ b/lib/banzai/reference_parser/milestone_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module ReferenceParser
     class MilestoneParser < BaseParser
diff --git a/lib/banzai/reference_parser/project_parser.rb b/lib/banzai/reference_parser/project_parser.rb
new file mode 100644
index 00000000000..b4e3a55b4f1
--- /dev/null
+++ b/lib/banzai/reference_parser/project_parser.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Banzai
+  module ReferenceParser
+    class ProjectParser < BaseParser
+      include Gitlab::Utils::StrongMemoize
+
+      self.reference_type = :project
+
+      def references_relation
+        Project
+      end
+
+      private
+
+      # Returns an Array of Project ids that can be read by the given user.
+      #
+      # user - The User for which to check the projects
+      def readable_project_ids_for(user)
+        @project_ids_by_user ||= {}
+        @project_ids_by_user[user] ||=
+          Project.public_or_visible_to_user(user).where("projects.id IN (?)", @projects_for_nodes.values.map(&:id)).pluck(:id)
+      end
+
+      def can_read_reference?(user, ref_project, node)
+        readable_project_ids_for(user).include?(ref_project.try(:id))
+      end
+    end
+  end
+end
diff --git a/lib/banzai/reference_parser/snippet_parser.rb b/lib/banzai/reference_parser/snippet_parser.rb
index 3ade168b566..6f6ac08de04 100644
--- a/lib/banzai/reference_parser/snippet_parser.rb
+++ b/lib/banzai/reference_parser/snippet_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module ReferenceParser
     class SnippetParser < BaseParser
diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb
index ceb7f1d165c..067b06b7590 100644
--- a/lib/banzai/reference_parser/user_parser.rb
+++ b/lib/banzai/reference_parser/user_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module ReferenceParser
     class UserParser < BaseParser
diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb
index 0050295eeda..c7239a5eaa6 100644
--- a/lib/banzai/renderer.rb
+++ b/lib/banzai/renderer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Renderer
     # Convert a Markdown String into an HTML-safe String of HTML
diff --git a/lib/banzai/renderer/common_mark/html.rb b/lib/banzai/renderer/common_mark/html.rb
index 46b609c36b0..837665451a1 100644
--- a/lib/banzai/renderer/common_mark/html.rb
+++ b/lib/banzai/renderer/common_mark/html.rb
@@ -1,18 +1,16 @@
+# frozen_string_literal: true
+
 module Banzai
   module Renderer
     module CommonMark
       class HTML < CommonMarker::HtmlRenderer
         def code_block(node)
           block do
-            code      = node.string_content
-            lang      = node.fence_info
-            lang_attr = lang.present? ? %Q{ lang="#{lang}"} : ''
-            result    =
-              "<pre>" \
-                "<code#{lang_attr}>#{ERB::Util.html_escape(code)}</code>" \
-              "</pre>"
-
-            out(result)
+            out("<pre#{sourcepos(node)}><code")
+            out(' lang="', node.fence_info, '"') if node.fence_info.present?
+            out('>')
+            out(escape_html(node.string_content))
+            out('</code></pre>')
           end
         end
       end
diff --git a/lib/banzai/renderer/redcarpet/html.rb b/lib/banzai/renderer/redcarpet/html.rb
index 30e815f1224..84931fdc784 100644
--- a/lib/banzai/renderer/redcarpet/html.rb
+++ b/lib/banzai/renderer/redcarpet/html.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Banzai
   module Renderer
     module Redcarpet
diff --git a/lib/banzai/request_store_reference_cache.rb b/lib/banzai/request_store_reference_cache.rb
index 426131442a2..91fb489b72d 100644
--- a/lib/banzai/request_store_reference_cache.rb
+++ b/lib/banzai/request_store_reference_cache.rb
@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 module Banzai
   module RequestStoreReferenceCache
     def cached_call(request_store_key, cache_key, path: [])
-      if RequestStore.active?
-        cache = RequestStore[request_store_key] ||= Hash.new do |hash, key|
+      if Gitlab::SafeRequestStore.active?
+        cache = Gitlab::SafeRequestStore[request_store_key] ||= Hash.new do |hash, key|
           hash[key] = Hash.new { |h, k| h[k] = {} }
         end
 
diff --git a/lib/banzai/suggestions_parser.rb b/lib/banzai/suggestions_parser.rb
new file mode 100644
index 00000000000..09f36635020
--- /dev/null
+++ b/lib/banzai/suggestions_parser.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Banzai
+  module SuggestionsParser
+    # Returns the content of each suggestion code block.
+    #
+    def self.parse(text)
+      html = Banzai.render(text, project: nil, no_original_data: true)
+      doc = Nokogiri::HTML(html)
+
+      doc.search('pre.suggestion').map { |node| node.text }
+    end
+  end
+end
diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb
index f8ee7e0f9ae..1343f424c51 100644
--- a/lib/bitbucket/client.rb
+++ b/lib/bitbucket/client.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bitbucket
   class Client
     attr_reader :connection
diff --git a/lib/bitbucket/collection.rb b/lib/bitbucket/collection.rb
index a78495dbf5e..9c496daccaa 100644
--- a/lib/bitbucket/collection.rb
+++ b/lib/bitbucket/collection.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bitbucket
   class Collection < Enumerator
     def initialize(paginator)
diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb
index ba5a9e2f04c..0041634f9e3 100644
--- a/lib/bitbucket/connection.rb
+++ b/lib/bitbucket/connection.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bitbucket
   class Connection
     DEFAULT_API_VERSION = '2.0'.freeze
diff --git a/lib/bitbucket/error/unauthorized.rb b/lib/bitbucket/error/unauthorized.rb
index efe10542f19..3cde11babee 100644
--- a/lib/bitbucket/error/unauthorized.rb
+++ b/lib/bitbucket/error/unauthorized.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bitbucket
   module Error
     Unauthorized = Class.new(StandardError)
diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb
index 2b0a3fe7b1a..7cc1342ad65 100644
--- a/lib/bitbucket/page.rb
+++ b/lib/bitbucket/page.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bitbucket
   class Page
     attr_reader :attrs, :items
diff --git a/lib/bitbucket/paginator.rb b/lib/bitbucket/paginator.rb
index 135d0d55674..0d004592e67 100644
--- a/lib/bitbucket/paginator.rb
+++ b/lib/bitbucket/paginator.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bitbucket
   class Paginator
     PAGE_LENGTH = 50 # The minimum length is 10 and the maximum is 100.
diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb
index 800d5a075c6..bb8dcd91ad5 100644
--- a/lib/bitbucket/representation/base.rb
+++ b/lib/bitbucket/representation/base.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bitbucket
   module Representation
     class Base
diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb
index 4937aa9728f..1b8dc27793a 100644
--- a/lib/bitbucket/representation/comment.rb
+++ b/lib/bitbucket/representation/comment.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bitbucket
   module Representation
     class Comment < Representation::Base
diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb
index 44bcbc250b3..a88797cdab9 100644
--- a/lib/bitbucket/representation/issue.rb
+++ b/lib/bitbucket/representation/issue.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bitbucket
   module Representation
     class Issue < Representation::Base
diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb
index eebf8093380..6a0e8b354bf 100644
--- a/lib/bitbucket/representation/pull_request.rb
+++ b/lib/bitbucket/representation/pull_request.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bitbucket
   module Representation
     class PullRequest < Representation::Base
diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb
index c52acbc3ddc..34dbf9ad22d 100644
--- a/lib/bitbucket/representation/pull_request_comment.rb
+++ b/lib/bitbucket/representation/pull_request_comment.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bitbucket
   module Representation
     class PullRequestComment < Comment
diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb
index 59b0fda8e14..c5bfc91e43d 100644
--- a/lib/bitbucket/representation/repo.rb
+++ b/lib/bitbucket/representation/repo.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bitbucket
   module Representation
     class Repo < Representation::Base
diff --git a/lib/bitbucket/representation/user.rb b/lib/bitbucket/representation/user.rb
index ba6b7667b49..2b45d751e70 100644
--- a/lib/bitbucket/representation/user.rb
+++ b/lib/bitbucket/representation/user.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bitbucket
   module Representation
     class User < Representation::Base
diff --git a/lib/bitbucket_server/client.rb b/lib/bitbucket_server/client.rb
new file mode 100644
index 00000000000..6a608058813
--- /dev/null
+++ b/lib/bitbucket_server/client.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module BitbucketServer
+  class Client
+    attr_reader :connection
+
+    def initialize(options = {})
+      @connection = Connection.new(options)
+    end
+
+    def pull_requests(project_key, repo)
+      path = "/projects/#{project_key}/repos/#{repo}/pull-requests?state=ALL"
+      get_collection(path, :pull_request)
+    end
+
+    def activities(project_key, repo, pull_request_id)
+      path = "/projects/#{project_key}/repos/#{repo}/pull-requests/#{pull_request_id}/activities"
+      get_collection(path, :activity)
+    end
+
+    def repo(project, repo_name)
+      parsed_response = connection.get("/projects/#{project}/repos/#{repo_name}")
+      BitbucketServer::Representation::Repo.new(parsed_response)
+    end
+
+    def repos(page_offset: 0, limit: nil)
+      path = "/repos"
+      get_collection(path, :repo, page_offset: page_offset, limit: limit)
+    end
+
+    def create_branch(project_key, repo, branch_name, sha)
+      payload = {
+        name: branch_name,
+        startPoint: sha,
+        message: 'GitLab temporary branch for import'
+      }
+
+      connection.post("/projects/#{project_key}/repos/#{repo}/branches", payload.to_json)
+    end
+
+    def delete_branch(project_key, repo, branch_name, sha)
+      payload = {
+        name: Gitlab::Git::BRANCH_REF_PREFIX + branch_name,
+        dryRun: false
+      }
+
+      connection.delete(:branches, "/projects/#{project_key}/repos/#{repo}/branches", payload.to_json)
+    end
+
+    private
+
+    def get_collection(path, type, page_offset: 0, limit: nil)
+      paginator = BitbucketServer::Paginator.new(connection, Addressable::URI.escape(path), type, page_offset: page_offset, limit: limit)
+      BitbucketServer::Collection.new(paginator)
+    end
+  end
+end
diff --git a/lib/bitbucket_server/collection.rb b/lib/bitbucket_server/collection.rb
new file mode 100644
index 00000000000..7e4b2277bbe
--- /dev/null
+++ b/lib/bitbucket_server/collection.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module BitbucketServer
+  class Collection < Enumerator
+    attr_reader :paginator
+
+    delegate :page_offset, :has_next_page?, to: :paginator
+
+    def initialize(paginator)
+      @paginator = paginator
+
+      super() do |yielder|
+        loop do
+          paginator.items.each { |item| yielder << item }
+        end
+      end
+
+      lazy
+    end
+
+    def current_page
+      return 1 if page_offset <= 1
+
+      [1, page_offset].max
+    end
+
+    def prev_page
+      return nil unless current_page > 1
+
+      current_page - 1
+    end
+
+    def next_page
+      return nil unless has_next_page?
+
+      current_page + 1
+    end
+
+    def method_missing(method, *args)
+      return super unless self.respond_to?(method)
+
+      self.__send__(method, *args) do |item| # rubocop:disable GitlabSecurity/PublicSend
+        block_given? ? yield(item) : item
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket_server/connection.rb b/lib/bitbucket_server/connection.rb
new file mode 100644
index 00000000000..9c14b26c65a
--- /dev/null
+++ b/lib/bitbucket_server/connection.rb
@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module BitbucketServer
+  class Connection
+    include ActionView::Helpers::SanitizeHelper
+
+    DEFAULT_API_VERSION = '1.0'
+    SEPARATOR = '/'
+
+    NETWORK_ERRORS = [
+      SocketError,
+      OpenSSL::SSL::SSLError,
+      Errno::ECONNRESET,
+      Errno::ECONNREFUSED,
+      Errno::EHOSTUNREACH,
+      Net::OpenTimeout,
+      Net::ReadTimeout,
+      Gitlab::HTTP::BlockedUrlError
+    ].freeze
+
+    attr_reader :api_version, :base_uri, :username, :token
+
+    ConnectionError = Class.new(StandardError)
+
+    def initialize(options = {})
+      @api_version = options.fetch(:api_version, DEFAULT_API_VERSION)
+      @base_uri = options[:base_uri]
+      @username = options[:user]
+      @token = options[:password]
+    end
+
+    def get(path, extra_query = {})
+      response = Gitlab::HTTP.get(build_url(path),
+                                  basic_auth: auth,
+                                  headers: accept_headers,
+                                  query: extra_query)
+
+      check_errors!(response)
+
+      response.parsed_response
+    rescue *NETWORK_ERRORS => e
+      raise ConnectionError, e
+    end
+
+    def post(path, body)
+      response = Gitlab::HTTP.post(build_url(path),
+                                   basic_auth: auth,
+                                   headers: post_headers,
+                                   body: body)
+
+      check_errors!(response)
+
+      response.parsed_response
+    rescue *NETWORK_ERRORS => e
+      raise ConnectionError, e
+    end
+
+    # We need to support two different APIs for deletion:
+    #
+    # /rest/api/1.0/projects/{projectKey}/repos/{repositorySlug}/branches/default
+    # /rest/branch-utils/1.0/projects/{projectKey}/repos/{repositorySlug}/branches
+    def delete(resource, path, body)
+      url = delete_url(resource, path)
+
+      response = Gitlab::HTTP.delete(url,
+                                     basic_auth: auth,
+                                     headers: post_headers,
+                                     body: body)
+
+      check_errors!(response)
+
+      response.parsed_response
+    rescue *NETWORK_ERRORS => e
+      raise ConnectionError, e
+    end
+
+    private
+
+    def check_errors!(response)
+      raise ConnectionError, "Response is not valid JSON" unless response.parsed_response.is_a?(Hash)
+
+      return if response.code >= 200 && response.code < 300
+
+      details = sanitize(response.parsed_response.dig('errors', 0, 'message'))
+      message = "Error #{response.code}"
+      message += ": #{details}" if details
+
+      raise ConnectionError, message
+    rescue JSON::ParserError
+      raise ConnectionError, "Unable to parse the server response as JSON"
+    end
+
+    def auth
+      @auth ||= { username: username, password: token }
+    end
+
+    def accept_headers
+      @accept_headers ||= { 'Accept' => 'application/json' }
+    end
+
+    def post_headers
+      @post_headers ||= accept_headers.merge({ 'Content-Type' => 'application/json' })
+    end
+
+    def build_url(path)
+      return path if path.starts_with?(root_url)
+
+      Gitlab::Utils.append_path(root_url, path)
+    end
+
+    def root_url
+      Gitlab::Utils.append_path(base_uri, "rest/api/#{api_version}")
+    end
+
+    def delete_url(resource, path)
+      if resource == :branches
+        Gitlab::Utils.append_path(base_uri, "rest/branch-utils/#{api_version}#{path}")
+      else
+        build_url(path)
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket_server/page.rb b/lib/bitbucket_server/page.rb
new file mode 100644
index 00000000000..5d9a3168876
--- /dev/null
+++ b/lib/bitbucket_server/page.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module BitbucketServer
+  class Page
+    attr_reader :attrs, :items
+
+    def initialize(raw, type)
+      @attrs = parse_attrs(raw)
+      @items = parse_values(raw, representation_class(type))
+    end
+
+    def next?
+      !attrs.fetch(:isLastPage, true)
+    end
+
+    def next
+      attrs.fetch(:nextPageStart)
+    end
+
+    private
+
+    def parse_attrs(raw)
+      raw.slice('size', 'nextPageStart', 'isLastPage').symbolize_keys
+    end
+
+    def parse_values(raw, bitbucket_rep_class)
+      return [] unless raw['values'] && raw['values'].is_a?(Array)
+
+      bitbucket_rep_class.decorate(raw['values'])
+    end
+
+    def representation_class(type)
+      BitbucketServer::Representation.const_get(type.to_s.camelize)
+    end
+  end
+end
diff --git a/lib/bitbucket_server/paginator.rb b/lib/bitbucket_server/paginator.rb
new file mode 100644
index 00000000000..9eda1c921b2
--- /dev/null
+++ b/lib/bitbucket_server/paginator.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module BitbucketServer
+  class Paginator
+    PAGE_LENGTH = 25
+
+    attr_reader :page_offset
+
+    def initialize(connection, url, type, page_offset: 0, limit: nil)
+      @connection = connection
+      @type = type
+      @url = url
+      @page = nil
+      @page_offset = page_offset
+      @limit = limit
+      @total = 0
+    end
+
+    def items
+      raise StopIteration unless has_next_page?
+      raise StopIteration if over_limit?
+
+      @page = fetch_next_page
+      @total += @page.items.count
+      @page.items
+    end
+
+    def has_next_page?
+      page.nil? || page.next?
+    end
+
+    private
+
+    attr_reader :connection, :page, :url, :type, :limit
+
+    def over_limit?
+      return false unless @limit
+
+      @limit.positive? && @total >= @limit
+    end
+
+    def next_offset
+      page.nil? ? starting_offset : page.next
+    end
+
+    def starting_offset
+      [0, page_offset - 1].max * max_per_page
+    end
+
+    def max_per_page
+      limit || PAGE_LENGTH
+    end
+
+    def fetch_next_page
+      parsed_response = connection.get(@url, start: next_offset, limit: max_per_page)
+      Page.new(parsed_response, type)
+    end
+  end
+end
diff --git a/lib/bitbucket_server/representation/activity.rb b/lib/bitbucket_server/representation/activity.rb
new file mode 100644
index 00000000000..08bf30a5d1e
--- /dev/null
+++ b/lib/bitbucket_server/representation/activity.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module BitbucketServer
+  module Representation
+    class Activity < Representation::Base
+      def comment?
+        action == 'COMMENTED'
+      end
+
+      def inline_comment?
+        !!(comment? && comment_anchor)
+      end
+
+      def comment
+        return unless comment?
+
+        @comment ||=
+          if inline_comment?
+            PullRequestComment.new(raw)
+          else
+            Comment.new(raw)
+          end
+      end
+
+      # TODO Move this into MergeEvent
+      def merge_event?
+        action == 'MERGED'
+      end
+
+      def committer_user
+        commit.dig('committer', 'displayName')
+      end
+
+      def committer_email
+        commit.dig('committer', 'emailAddress')
+      end
+
+      def merge_timestamp
+        timestamp = commit['committerTimestamp']
+
+        self.class.convert_timestamp(timestamp)
+      end
+
+      def merge_commit
+        commit['id']
+      end
+
+      def created_at
+        self.class.convert_timestamp(created_date)
+      end
+
+      private
+
+      def commit
+        raw.fetch('commit', {})
+      end
+
+      def action
+        raw['action']
+      end
+
+      def comment_anchor
+        raw['commentAnchor']
+      end
+
+      def created_date
+        raw['createdDate']
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket_server/representation/base.rb b/lib/bitbucket_server/representation/base.rb
new file mode 100644
index 00000000000..a1961bae6ef
--- /dev/null
+++ b/lib/bitbucket_server/representation/base.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module BitbucketServer
+  module Representation
+    class Base
+      attr_reader :raw
+
+      def initialize(raw)
+        @raw = raw
+      end
+
+      def self.decorate(entries)
+        entries.map { |entry| new(entry)}
+      end
+
+      def self.convert_timestamp(time_usec)
+        Time.at(time_usec / 1000) if time_usec.is_a?(Integer)
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket_server/representation/comment.rb b/lib/bitbucket_server/representation/comment.rb
new file mode 100644
index 00000000000..99b97a3b181
--- /dev/null
+++ b/lib/bitbucket_server/representation/comment.rb
@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module BitbucketServer
+  module Representation
+    # A general comment with the structure:
+    # "comment": {
+    #   "author": {
+    #               "active": true,
+    #               "displayName": "root",
+    #               "emailAddress": "stanhu+bitbucket@gitlab.com",
+    #               "id": 1,
+    #               "links": {
+    #                 "self": [
+    #                   {
+    #                     "href": "http://localhost:7990/users/root"
+    #                   }
+    #                 ]
+    #                },
+    #                "name": "root",
+    #                "slug": "root",
+    #                "type": "NORMAL"
+    #               }
+    #   }
+    # }
+    class Comment < Representation::Base
+      attr_reader :parent_comment
+
+      CommentNode = Struct.new(:raw_comments, :parent)
+
+      def initialize(raw, parent_comment: nil)
+        super(raw)
+
+        @parent_comment = parent_comment
+      end
+
+      def id
+        raw_comment['id']
+      end
+
+      def author_username
+        author['displayName']
+      end
+
+      def author_email
+        author['emailAddress']
+      end
+
+      def note
+        raw_comment['text']
+      end
+
+      def created_at
+        self.class.convert_timestamp(created_date)
+      end
+
+      def updated_at
+        self.class.convert_timestamp(created_date)
+      end
+
+      # Bitbucket Server supports the ability to reply to any comment
+      # and created multiple threads. It represents these as a linked list
+      # of comments within comments. For example:
+      #
+      # "comments": [
+      #    {
+      #       "author" : ...
+      #       "comments": [
+      #          {
+      #             "author": ...
+      #
+      # Since GitLab only supports a single thread, we flatten all these
+      # comments into a single discussion.
+      def comments
+        @comments ||= flatten_comments
+      end
+
+      private
+
+      # In order to provide context for each reply, we need to track
+      # the parent of each comment. This method works as follows:
+      #
+      # 1. Insert the root comment into the workset. The root element is the current note.
+      # 2. For each node in the workset:
+      #    a. Examine if it has replies to that comment. If it does,
+      #       insert that node into the workset.
+      #    b. Parse that note into a Comment structure and add it to a flat list.
+      def flatten_comments
+        comments = raw_comment['comments']
+        workset =
+          if comments
+            [CommentNode.new(comments, self)]
+          else
+            []
+          end
+
+        all_comments = []
+
+        until workset.empty?
+          node = workset.pop
+          parent = node.parent
+
+          node.raw_comments.each do |comment|
+            new_comments = comment.delete('comments')
+            current_comment = Comment.new({ 'comment' => comment }, parent_comment: parent)
+            all_comments << current_comment
+            workset << CommentNode.new(new_comments, current_comment) if new_comments
+          end
+        end
+
+        all_comments
+      end
+
+      def raw_comment
+        raw.fetch('comment', {})
+      end
+
+      def author
+        raw_comment['author']
+      end
+
+      def created_date
+        raw_comment['createdDate']
+      end
+
+      def updated_date
+        raw_comment['updatedDate']
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket_server/representation/pull_request.rb b/lib/bitbucket_server/representation/pull_request.rb
new file mode 100644
index 00000000000..c3e927d8de7
--- /dev/null
+++ b/lib/bitbucket_server/representation/pull_request.rb
@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module BitbucketServer
+  module Representation
+    class PullRequest < Representation::Base
+      def author
+        raw.dig('author', 'user', 'name')
+      end
+
+      def author_email
+        raw.dig('author', 'user', 'emailAddress')
+      end
+
+      def description
+        raw['description']
+      end
+
+      def iid
+        raw['id']
+      end
+
+      def state
+        case raw['state']
+        when 'MERGED'
+          'merged'
+        when 'DECLINED'
+          'closed'
+        else
+          'opened'
+        end
+      end
+
+      def merged?
+        state == 'merged'
+      end
+
+      def created_at
+        self.class.convert_timestamp(created_date)
+      end
+
+      def updated_at
+        self.class.convert_timestamp(updated_date)
+      end
+
+      def title
+        raw['title']
+      end
+
+      def source_branch_name
+        raw.dig('fromRef', 'id')
+      end
+
+      def source_branch_sha
+        raw.dig('fromRef', 'latestCommit')
+      end
+
+      def target_branch_name
+        raw.dig('toRef', 'id')
+      end
+
+      def target_branch_sha
+        raw.dig('toRef', 'latestCommit')
+      end
+
+      private
+
+      def created_date
+        raw['createdDate']
+      end
+
+      def updated_date
+        raw['updatedDate']
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket_server/representation/pull_request_comment.rb b/lib/bitbucket_server/representation/pull_request_comment.rb
new file mode 100644
index 00000000000..a2b3873a397
--- /dev/null
+++ b/lib/bitbucket_server/representation/pull_request_comment.rb
@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module BitbucketServer
+  module Representation
+    # An inline comment with the following structure that identifies
+    # the part of the diff:
+    #
+    # "commentAnchor": {
+    #   "diffType": "EFFECTIVE",
+    #   "fileType": "TO",
+    #   "fromHash": "c5f4288162e2e6218180779c7f6ac1735bb56eab",
+    #   "line": 1,
+    #   "lineType": "ADDED",
+    #   "orphaned": false,
+    #   "path": "CHANGELOG.md",
+    #   "toHash": "a4c2164330f2549f67c13f36a93884cf66e976be"
+    #  }
+    #
+    # More details in https://docs.atlassian.com/bitbucket-server/rest/5.12.0/bitbucket-rest.html.
+    class PullRequestComment < Comment
+      def from_sha
+        comment_anchor['fromHash']
+      end
+
+      def to_sha
+        comment_anchor['toHash']
+      end
+
+      def to?
+        file_type == 'TO'
+      end
+
+      def from?
+        file_type == 'FROM'
+      end
+
+      def added?
+        line_type == 'ADDED'
+      end
+
+      def removed?
+        line_type == 'REMOVED'
+      end
+
+      # There are three line comment types: added, removed, or context.
+      #
+      # 1. An added type means a new line was inserted, so there is no old position.
+      # 2. A removed type means a line was removed, so there is no new position.
+      # 3. A context type means the line was unmodified, so there is both a
+      #    old and new position.
+      def new_pos
+        return if removed?
+        return unless line_position
+
+        line_position[1]
+      end
+
+      def old_pos
+        return if added?
+        return unless line_position
+
+        line_position[0]
+      end
+
+      def file_path
+        comment_anchor.fetch('path')
+      end
+
+      private
+
+      def file_type
+        comment_anchor['fileType']
+      end
+
+      def line_type
+        comment_anchor['lineType']
+      end
+
+      # Each comment contains the following information about the diff:
+      #
+      # hunks: [
+      #     {
+      #         segments: [
+      #             {
+      #                 "lines": [
+      #                     {
+      #                         "commentIds": [ N ],
+      #                         "source": X,
+      #                         "destination": Y
+      #                     }, ...
+      #                   ] ....
+      #
+      # To determine the line position of a comment, we search all the lines
+      # entries until we find this comment ID.
+      def line_position
+        @line_position ||= diff_hunks.each do |hunk|
+          segments = hunk.fetch('segments', [])
+          segments.each do |segment|
+            lines = segment.fetch('lines', [])
+            lines.each do |line|
+              if line['commentIds']&.include?(id)
+                return [line['source'], line['destination']]
+              end
+            end
+          end
+        end
+      end
+
+      def comment_anchor
+        raw.fetch('commentAnchor', {})
+      end
+
+      def diff
+        raw.fetch('diff', {})
+      end
+
+      def diff_hunks
+        diff.fetch('hunks', [])
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket_server/representation/repo.rb b/lib/bitbucket_server/representation/repo.rb
new file mode 100644
index 00000000000..6c494b79166
--- /dev/null
+++ b/lib/bitbucket_server/representation/repo.rb
@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module BitbucketServer
+  module Representation
+    class Repo < Representation::Base
+      def initialize(raw)
+        super(raw)
+      end
+
+      def project_key
+        raw.dig('project', 'key')
+      end
+
+      def project_name
+        raw.dig('project', 'name')
+      end
+
+      def slug
+        raw['slug']
+      end
+
+      def browse_url
+        # The JSON reponse contains an array of 1 element. Not sure if there
+        # are cases where multiple links would be provided.
+        raw.dig('links', 'self').first.fetch('href')
+      end
+
+      def clone_url
+        raw['links']['clone'].find { |link| link['name'].starts_with?('http') }.fetch('href')
+      end
+
+      def description
+        project['description']
+      end
+
+      def full_name
+        "#{project_name}/#{name}"
+      end
+
+      def issues_enabled?
+        true
+      end
+
+      def name
+        raw['name']
+      end
+
+      def valid?
+        raw['scmId'] == 'git'
+      end
+
+      def visibility_level
+        if project['public']
+          Gitlab::VisibilityLevel::PUBLIC
+        else
+          Gitlab::VisibilityLevel::PRIVATE
+        end
+      end
+
+      def project
+        raw['project']
+      end
+
+      def to_s
+        full_name
+      end
+    end
+  end
+end
diff --git a/lib/carrier_wave_string_file.rb b/lib/carrier_wave_string_file.rb
index 6c848902e4a..c9a64d9e631 100644
--- a/lib/carrier_wave_string_file.rb
+++ b/lib/carrier_wave_string_file.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class CarrierWaveStringFile < StringIO
   def original_filename
     ""
diff --git a/lib/constraints/feature_constrainer.rb b/lib/constraints/feature_constrainer.rb
new file mode 100644
index 00000000000..cd246cf37a4
--- /dev/null
+++ b/lib/constraints/feature_constrainer.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Constraints
+  class FeatureConstrainer
+    attr_reader :args
+
+    def initialize(*args)
+      @args = args
+    end
+
+    def matches?(_request)
+      Feature.enabled?(*args)
+    end
+  end
+end
diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb
index 87649c50424..8a3f8d2faaf 100644
--- a/lib/constraints/group_url_constrainer.rb
+++ b/lib/constraints/group_url_constrainer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Constraints
   class GroupUrlConstrainer
     def matches?(request)
diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb
index 32aea98f0f7..eadfbf7bc01 100644
--- a/lib/constraints/project_url_constrainer.rb
+++ b/lib/constraints/project_url_constrainer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Constraints
   class ProjectUrlConstrainer
     def matches?(request)
diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb
index 8afa04d29a4..e763569cb2e 100644
--- a/lib/constraints/user_url_constrainer.rb
+++ b/lib/constraints/user_url_constrainer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Constraints
   class UserUrlConstrainer
     def matches?(request)
diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb
index d5f85f9fcad..837b22c3082 100644
--- a/lib/container_registry/blob.rb
+++ b/lib/container_registry/blob.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ContainerRegistry
   class Blob
     attr_reader :repository, :config
diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb
index 010ca1ec27b..c80f49f5ae0 100644
--- a/lib/container_registry/client.rb
+++ b/lib/container_registry/client.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'faraday'
 require 'faraday_middleware'
 
diff --git a/lib/container_registry/config.rb b/lib/container_registry/config.rb
index 589f9f4380a..740c0e13da0 100644
--- a/lib/container_registry/config.rb
+++ b/lib/container_registry/config.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ContainerRegistry
   class Config
     attr_reader :tag, :blob, :data
diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb
index 61849a40383..9b2a61cdedc 100644
--- a/lib/container_registry/path.rb
+++ b/lib/container_registry/path.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ContainerRegistry
   ##
   # Class responsible for extracting project and repository name from
@@ -28,6 +30,7 @@ module ContainerRegistry
       @components ||= @path.split('/')
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def nodes
       raise InvalidRegistryPathError unless valid?
 
@@ -35,17 +38,20 @@ module ContainerRegistry
         components.take(length).join('/')
       end
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     def has_project?
       repository_project.present?
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def has_repository?
       return false unless has_project?
 
       repository_project.container_repositories
         .where(name: repository_name).any?
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     def root_repository?
       @path == project_path
diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb
index f90d711474a..523364ac7c7 100644
--- a/lib/container_registry/registry.rb
+++ b/lib/container_registry/registry.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ContainerRegistry
   class Registry
     attr_reader :uri, :client, :path
diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb
index 728deea224f..ef41dc560c9 100644
--- a/lib/container_registry/tag.rb
+++ b/lib/container_registry/tag.rb
@@ -1,5 +1,9 @@
+# frozen_string_literal: true
+
 module ContainerRegistry
   class Tag
+    include Gitlab::Utils::StrongMemoize
+
     attr_reader :repository, :name
 
     delegate :registry, :client, to: :repository
@@ -13,6 +17,10 @@ module ContainerRegistry
       manifest.present?
     end
 
+    def latest?
+      name == "latest"
+    end
+
     def v1?
       manifest && manifest['schemaVersion'] == 1
     end
@@ -22,7 +30,9 @@ module ContainerRegistry
     end
 
     def manifest
-      @manifest ||= client.repository_manifest(repository.path, name)
+      strong_memoize(:manifest) do
+        client.repository_manifest(repository.path, name)
+      end
     end
 
     def path
@@ -40,44 +50,54 @@ module ContainerRegistry
     end
 
     def digest
-      @digest ||= client.repository_tag_digest(repository.path, name)
+      strong_memoize(:digest) do
+        client.repository_tag_digest(repository.path, name)
+      end
     end
 
     def config_blob
-      return @config_blob if defined?(@config_blob)
       return unless manifest && manifest['config']
 
-      @config_blob = repository.blob(manifest['config'])
+      strong_memoize(:config_blob) do
+        repository.blob(manifest['config'])
+      end
     end
 
     def config
-      return unless config_blob
+      return unless config_blob&.data
 
-      @config ||= ContainerRegistry::Config.new(self, config_blob) if config_blob.data
+      strong_memoize(:config) do
+        ContainerRegistry::Config.new(self, config_blob)
+      end
     end
 
     def created_at
       return unless config
 
-      @created_at ||= DateTime.rfc3339(config['created'])
+      strong_memoize(:created_at) do
+        DateTime.rfc3339(config['created'])
+      end
     end
 
     def layers
-      return @layers if defined?(@layers)
       return unless manifest
 
-      layers = manifest['layers'] || manifest['fsLayers']
+      strong_memoize(:layers) do
+        layers = manifest['layers'] || manifest['fsLayers']
 
-      @layers = layers.map do |layer|
-        repository.blob(layer)
+        layers.map do |layer|
+          repository.blob(layer)
+        end
       end
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def total_size
       return unless layers
 
       layers.map(&:size).sum if v2?
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     def delete
       return unless digest
diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb
index 1dd2855063d..7ba48ae9c79 100644
--- a/lib/declarative_policy.rb
+++ b/lib/declarative_policy.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_dependency 'declarative_policy/cache'
 require_dependency 'declarative_policy/condition'
 require_dependency 'declarative_policy/delegate_dsl'
@@ -10,8 +12,6 @@ require_dependency 'declarative_policy/step'
 
 require_dependency 'declarative_policy/base'
 
-require 'thread'
-
 module DeclarativePolicy
   CLASS_CACHE_MUTEX = Mutex.new
   CLASS_CACHE_IVAR = :@__DeclarativePolicy_CLASS_CACHE
@@ -21,7 +21,13 @@ module DeclarativePolicy
       cache = opts[:cache] || {}
       key = Cache.policy_key(user, subject)
 
-      cache[key] ||= class_for(subject).new(user, subject, opts)
+      cache[key] ||=
+        # to avoid deadlocks in multi-threaded environment when
+        # autoloading is enabled, we allow concurrent loads,
+        # https://gitlab.com/gitlab-org/gitlab-ce/issues/48263
+        ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+          class_for(subject).new(user, subject, opts)
+        end
     end
 
     def class_for(subject)
diff --git a/lib/declarative_policy/base.rb b/lib/declarative_policy/base.rb
index 47542194497..cd6e1606f22 100644
--- a/lib/declarative_policy/base.rb
+++ b/lib/declarative_policy/base.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module DeclarativePolicy
   class Base
     # A map of ability => list of rules together with :enable
@@ -119,8 +121,8 @@ module DeclarativePolicy
       # a PolicyDsl which is used for registering the rule with
       # this class. PolicyDsl will call back into Base.enable_when,
       # Base.prevent_when, and Base.prevent_all_when.
-      def rule(&b)
-        rule = RuleDsl.new(self).instance_eval(&b)
+      def rule(&block)
+        rule = RuleDsl.new(self).instance_eval(&block)
         PolicyDsl.new(self, rule)
       end
 
@@ -222,8 +224,8 @@ module DeclarativePolicy
 
     # computes the given ability and prints a helpful debugging output
     # showing which
-    def debug(ability, *a)
-      runner(ability).debug(*a)
+    def debug(ability, *args)
+      runner(ability).debug(*args)
     end
 
     desc "Unknown user"
@@ -274,7 +276,7 @@ module DeclarativePolicy
     #
     # NOTE we can't use ||= here because the value might be the
     # boolean `false`
-    def cache(key, &b)
+    def cache(key)
       return @cache[key] if cached?(key)
 
       @cache[key] = yield
diff --git a/lib/declarative_policy/cache.rb b/lib/declarative_policy/cache.rb
index 780d8f707bd..13006e56454 100644
--- a/lib/declarative_policy/cache.rb
+++ b/lib/declarative_policy/cache.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module DeclarativePolicy
   module Cache
     class << self
diff --git a/lib/declarative_policy/condition.rb b/lib/declarative_policy/condition.rb
index 51c4a8b2bbe..b77f40b1093 100644
--- a/lib/declarative_policy/condition.rb
+++ b/lib/declarative_policy/condition.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module DeclarativePolicy
   # A Condition is the data structure that is created by the
   # `condition` declaration on DeclarativePolicy::Base. It is
diff --git a/lib/declarative_policy/delegate_dsl.rb b/lib/declarative_policy/delegate_dsl.rb
index f544dffe888..67e3429b696 100644
--- a/lib/declarative_policy/delegate_dsl.rb
+++ b/lib/declarative_policy/delegate_dsl.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module DeclarativePolicy
   # Used when the name of a delegate is mentioned in
   # the rule DSL.
@@ -7,10 +9,10 @@ module DeclarativePolicy
       @delegate_name = delegate_name
     end
 
-    def method_missing(m, *a, &b)
-      return super unless a.empty? && !block_given?
+    def method_missing(msg, *args)
+      return super unless args.empty? && !block_given?
 
-      @rule_dsl.delegate(@delegate_name, m)
+      @rule_dsl.delegate(@delegate_name, msg)
     end
   end
 end
diff --git a/lib/declarative_policy/policy_dsl.rb b/lib/declarative_policy/policy_dsl.rb
index f11b6e9f730..96741c0478e 100644
--- a/lib/declarative_policy/policy_dsl.rb
+++ b/lib/declarative_policy/policy_dsl.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module DeclarativePolicy
   # The return value of a rule { ... } declaration.
   # Can call back to register rules with the containing
@@ -15,8 +17,8 @@ module DeclarativePolicy
       @rule = rule
     end
 
-    def policy(&b)
-      instance_eval(&b)
+    def policy(&block)
+      instance_eval(&block)
     end
 
     def enable(*abilities)
@@ -31,14 +33,14 @@ module DeclarativePolicy
       @context_class.prevent_all_when(@rule)
     end
 
-    def method_missing(m, *a, &b)
-      return super unless @context_class.respond_to?(m)
+    def method_missing(msg, *args, &block)
+      return super unless @context_class.respond_to?(msg)
 
-      @context_class.__send__(m, *a, &b) # rubocop:disable GitlabSecurity/PublicSend
+      @context_class.__send__(msg, *args, &block) # rubocop:disable GitlabSecurity/PublicSend
     end
 
-    def respond_to_missing?(m)
-      @context_class.respond_to?(m) || super
+    def respond_to_missing?(msg)
+      @context_class.respond_to?(msg) || super
     end
   end
 end
diff --git a/lib/declarative_policy/preferred_scope.rb b/lib/declarative_policy/preferred_scope.rb
index 5c214408dd0..239780d8626 100644
--- a/lib/declarative_policy/preferred_scope.rb
+++ b/lib/declarative_policy/preferred_scope.rb
@@ -1,8 +1,11 @@
-module DeclarativePolicy # rubocop:disable Naming/FileName
+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+module DeclarativePolicy
   PREFERRED_SCOPE_KEY = :"DeclarativePolicy.preferred_scope"
 
   class << self
-    def with_preferred_scope(scope, &b)
+    def with_preferred_scope(scope)
       Thread.current[PREFERRED_SCOPE_KEY], old_scope = scope, Thread.current[PREFERRED_SCOPE_KEY]
       yield
     ensure
@@ -13,12 +16,12 @@ module DeclarativePolicy # rubocop:disable Naming/FileName
       Thread.current[PREFERRED_SCOPE_KEY]
     end
 
-    def user_scope(&b)
-      with_preferred_scope(:user, &b)
+    def user_scope(&block)
+      with_preferred_scope(:user, &block)
     end
 
-    def subject_scope(&b)
-      with_preferred_scope(:subject, &b)
+    def subject_scope(&block)
+      with_preferred_scope(:subject, &block)
     end
 
     def preferred_scope=(scope)
diff --git a/lib/declarative_policy/rule.rb b/lib/declarative_policy/rule.rb
index e309244a3b3..f38f4f0a50f 100644
--- a/lib/declarative_policy/rule.rb
+++ b/lib/declarative_policy/rule.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module DeclarativePolicy
   module Rule
     # A Rule is the object that results from the `rule` declaration,
@@ -8,8 +10,8 @@ module DeclarativePolicy
     # how that affects the actual ability decision - for that, a
     # `Step` is used.
     class Base
-      def self.make(*a)
-        new(*a).simplify
+      def self.make(*args)
+        new(*args).simplify
       end
 
       # true or false whether this rule passes.
diff --git a/lib/declarative_policy/rule_dsl.rb b/lib/declarative_policy/rule_dsl.rb
index e948b7f2de1..85da7f261fa 100644
--- a/lib/declarative_policy/rule_dsl.rb
+++ b/lib/declarative_policy/rule_dsl.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module DeclarativePolicy
   # The DSL evaluation context inside rule { ... } blocks.
   # Responsible for creating and combining Rule objects.
@@ -32,13 +34,13 @@ module DeclarativePolicy
       Rule::DelegatedCondition.new(delegate_name, condition)
     end
 
-    def method_missing(m, *a, &b)
-      return super unless a.empty? && !block_given?
+    def method_missing(msg, *args)
+      return super unless args.empty? && !block_given?
 
-      if @context_class.delegations.key?(m)
-        DelegateDsl.new(self, m)
+      if @context_class.delegations.key?(msg)
+        DelegateDsl.new(self, msg)
       else
-        cond(m.to_sym)
+        cond(msg.to_sym)
       end
     end
   end
diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb
index 87f14b3b0d2..f739fe5e16e 100644
--- a/lib/declarative_policy/runner.rb
+++ b/lib/declarative_policy/runner.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module DeclarativePolicy
   class Runner
     class State
@@ -127,7 +129,7 @@ module DeclarativePolicy
     #
     # For each step, we yield the step object along with the computed score
     # for debugging purposes.
-    def steps_by_score(&b)
+    def steps_by_score
       flatten_steps!
 
       if @steps.size > 50
diff --git a/lib/declarative_policy/step.rb b/lib/declarative_policy/step.rb
index 3469fe9f991..c289c17cc19 100644
--- a/lib/declarative_policy/step.rb
+++ b/lib/declarative_policy/step.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module DeclarativePolicy
   # This object represents one step in the runtime decision of whether
   # an ability is allowed. It contains a Rule and a context (instance
diff --git a/lib/disable_email_interceptor.rb b/lib/disable_email_interceptor.rb
deleted file mode 100644
index cee664b8951..00000000000
--- a/lib/disable_email_interceptor.rb
+++ /dev/null
@@ -1,7 +0,0 @@
-# Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails
-class DisableEmailInterceptor
-  def self.delivering_email(message)
-    message.perform_deliveries = false
-    Rails.logger.info "Emails disabled! Interceptor prevented sending mail #{message.subject}"
-  end
-end
diff --git a/lib/email_template_interceptor.rb b/lib/email_template_interceptor.rb
deleted file mode 100644
index 3978a6d9fe4..00000000000
--- a/lib/email_template_interceptor.rb
+++ /dev/null
@@ -1,11 +0,0 @@
-# Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails
-class EmailTemplateInterceptor
-  def self.delivering_email(message)
-    # Remove HTML part if HTML emails are disabled.
-    unless Gitlab::CurrentSettings.html_emails_enabled
-      message.parts.delete_if do |part|
-        part.content_type.start_with?('text/html')
-      end
-    end
-  end
-end
diff --git a/lib/event_filter.rb b/lib/event_filter.rb
index 515095af1c2..24fdcd6fbb1 100644
--- a/lib/event_filter.rb
+++ b/lib/event_filter.rb
@@ -1,74 +1,42 @@
-class EventFilter
-  attr_accessor :params
-
-  class << self
-    def all
-      'all'
-    end
-
-    def push
-      'push'
-    end
-
-    def merged
-      'merged'
-    end
+# frozen_string_literal: true
 
-    def issue
-      'issue'
-    end
-
-    def comments
-      'comments'
-    end
-
-    def team
-      'team'
-    end
+class EventFilter
+  attr_accessor :filter
+
+  ALL = 'all'
+  PUSH = 'push'
+  MERGED = 'merged'
+  ISSUE = 'issue'
+  COMMENTS = 'comments'
+  TEAM = 'team'
+  FILTERS = [ALL, PUSH, MERGED, ISSUE, COMMENTS, TEAM].freeze
+
+  def initialize(filter)
+    # Split using comma to maintain backward compatibility Ex/ "filter1,filter2"
+    filter = filter.to_s.split(',')[0].to_s
+    @filter = FILTERS.include?(filter) ? filter : ALL
   end
 
-  def initialize(params)
-    @params = if params
-                params.dup
-              else
-                [] # EventFilter.default_filter
-              end
+  def active?(key)
+    filter == key.to_s
   end
 
+  # rubocop: disable CodeReuse/ActiveRecord
   def apply_filter(events)
-    return events if params.blank? || params == EventFilter.all
-
-    case params
-    when EventFilter.push
+    case filter
+    when PUSH
       events.where(action: Event::PUSHED)
-    when EventFilter.merged
+    when MERGED
       events.where(action: Event::MERGED)
-    when EventFilter.comments
+    when COMMENTS
       events.where(action: Event::COMMENTED)
-    when EventFilter.team
+    when TEAM
       events.where(action: [Event::JOINED, Event::LEFT, Event::EXPIRED])
-    when EventFilter.issue
+    when ISSUE
       events.where(action: [Event::CREATED, Event::UPDATED, Event::CLOSED, Event::REOPENED])
-    end
-  end
-
-  def options(key)
-    filter = params.dup
-
-    if filter.include? key
-      filter.delete key
-    else
-      filter << key
-    end
-
-    filter
-  end
-
-  def active?(key)
-    if params.present?
-      params.include? key
     else
-      key == EventFilter.all
+      events
     end
   end
+  # rubocop: enable CodeReuse/ActiveRecord
 end
diff --git a/lib/expand_variables.rb b/lib/expand_variables.rb
index 7b1533d0d32..c83cec9dc4a 100644
--- a/lib/expand_variables.rb
+++ b/lib/expand_variables.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ExpandVariables
   class << self
     def expand(value, variables)
diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb
index a9b04c183ad..b2c8d46ede1 100644
--- a/lib/extracts_path.rb
+++ b/lib/extracts_path.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Module providing methods for dealing with separating a tree-ish string and a
 # file path string when combined in a request parameter
 module ExtractsPath
@@ -50,11 +52,13 @@ module ExtractsPath
       # branches and tags
 
       # Append a trailing slash if we only get a ref and no file path
-      id += '/' unless id.ends_with?('/')
+      unless id.ends_with?('/')
+        id = [id, '/'].join
+      end
 
       valid_refs = ref_names.select { |v| id.start_with?("#{v}/") }
 
-      if valid_refs.length == 0
+      if valid_refs.empty?
         # No exact ref match, so just try our best
         pair = id.match(%r{([^/]+)(.*)}).captures
       else
@@ -106,11 +110,6 @@ module ExtractsPath
   # resolved (e.g., when a user inserts an invalid path or ref).
   # rubocop:disable Gitlab/ModuleWithInstanceVariables
   def assign_ref_vars
-    # assign allowed options
-    allowed_options = ["filter_ref"]
-    @options = params.select {|key, value| allowed_options.include?(key) && !value.blank? }
-    @options = HashWithIndifferentAccess.new(@options)
-
     @id = get_id
     @ref, @path = extract_ref(@id)
     @repo = @project.repository
@@ -139,16 +138,21 @@ module ExtractsPath
 
   def lfs_blob_ids
     blob_ids = tree.blobs.map(&:id)
+
+    # When current endpoint is a Blob then `tree.blobs` will be empty, it means we need to analyze
+    # the current Blob in order to determine if it's a LFS object
+    blob_ids = Array.wrap(@repo.blob_at(@commit.id, @path)&.id) if blob_ids.empty? # rubocop:disable Gitlab/ModuleWithInstanceVariables
+
     @lfs_blob_ids = Gitlab::Git::Blob.batch_lfs_pointers(@project.repository, blob_ids).map(&:id) # rubocop:disable Gitlab/ModuleWithInstanceVariables
   end
 
   private
 
-  # overriden in subclasses, do not remove
+  # overridden in subclasses, do not remove
   def get_id
-    id = params[:id] || params[:ref]
-    id += "/" + params[:path] unless params[:path].blank?
-    id
+    id = [params[:id] || params[:ref]]
+    id << "/" + params[:path] unless params[:path].blank?
+    id.join
   end
 
   def ref_names
diff --git a/lib/feature.rb b/lib/feature.rb
index 314ae224d90..e59cd70f822 100644
--- a/lib/feature.rb
+++ b/lib/feature.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'flipper/adapters/active_record'
 require 'flipper/adapters/active_support_cache_store'
 
@@ -28,22 +30,31 @@ class Feature
     end
 
     def persisted_names
-      if RequestStore.active?
-        RequestStore[:flipper_persisted_names] ||= FlipperFeature.feature_names
-      else
-        FlipperFeature.feature_names
-      end
+      Gitlab::SafeRequestStore[:flipper_persisted_names] ||= FlipperFeature.feature_names
     end
 
     def persisted?(feature)
       # Flipper creates on-memory features when asked for a not-yet-created one.
       # If we want to check if a feature has been actually set, we look for it
       # on the persisted features list.
-      persisted_names.include?(feature.name)
+      persisted_names.include?(feature.name.to_s)
+    end
+
+    # use `default_enabled: true` to default the flag to being `enabled`
+    # unless set explicitly.  The default is `disabled`
+    def enabled?(key, thing = nil, default_enabled: false)
+      feature = Feature.get(key)
+
+      # If we're not default enabling the flag or the feature has been set, always evaluate.
+      # `persisted?` can potentially generate DB queries and also checks for inclusion
+      # in an array of feature names (177 at last count), possibly reducing performance by half.
+      # So we only perform the `persisted` check if `default_enabled: true`
+      !default_enabled || Feature.persisted?(feature) ? feature.enabled?(thing) : true
     end
 
-    def enabled?(key, thing = nil)
-      get(key).enabled?(thing)
+    def disabled?(key, thing = nil, default_enabled: false)
+      # we need to make different method calls to make it easy to mock / define expectations in test mode
+      thing.nil? ? !enabled?(key, default_enabled: default_enabled) : !enabled?(key, thing, default_enabled: default_enabled)
     end
 
     def enable(key, thing = true)
@@ -63,8 +74,8 @@ class Feature
     end
 
     def flipper
-      if RequestStore.active?
-        RequestStore[:flipper] ||= build_flipper_instance
+      if Gitlab::SafeRequestStore.active?
+        Gitlab::SafeRequestStore[:flipper] ||= build_flipper_instance
       else
         @flipper ||= build_flipper_instance
       end
@@ -91,4 +102,42 @@ class Feature
         expires_in: 1.hour)
     end
   end
+
+  class Target
+    attr_reader :params
+
+    def initialize(params)
+      @params = params
+    end
+
+    def gate_specified?
+      %i(user project feature_group).any? { |key| params.key?(key) }
+    end
+
+    def targets
+      [feature_group, user, project].compact
+    end
+
+    private
+
+    # rubocop: disable CodeReuse/ActiveRecord
+    def feature_group
+      return unless params.key?(:feature_group)
+
+      Feature.group(params[:feature_group])
+    end
+    # rubocop: enable CodeReuse/ActiveRecord
+
+    def user
+      return unless params.key?(:user)
+
+      UserFinder.new(params[:user]).find_by_username!
+    end
+
+    def project
+      return unless params.key?(:project)
+
+      Project.find_by_full_path(params[:project])
+    end
+  end
 end
diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb
index 69d981e8be9..70a145cd5bd 100644
--- a/lib/file_size_validator.rb
+++ b/lib/file_size_validator.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class FileSizeValidator < ActiveModel::EachValidator
   MESSAGES = { is: :wrong_size, minimum: :size_too_small, maximum: :size_too_big }.freeze
   CHECKS   = { is: :==, minimum: :>=, maximum: :<= }.freeze
@@ -32,6 +34,7 @@ class FileSizeValidator < ActiveModel::EachValidator
     end
   end
 
+  # rubocop: disable CodeReuse/ActiveRecord
   def validate_each(record, attribute, value)
     raise(ArgumentError, "A CarrierWave::Uploader::Base object was expected") unless value.is_a? CarrierWave::Uploader::Base
 
@@ -62,6 +65,7 @@ class FileSizeValidator < ActiveModel::EachValidator
       record.errors.add(attribute, MESSAGES[key], errors_options)
     end
   end
+  # rubocop: enable CodeReuse/ActiveRecord
 
   def help
     Helper.instance
diff --git a/lib/flowdock/git.rb b/lib/flowdock/git.rb
new file mode 100644
index 00000000000..f165ecfc1fa
--- /dev/null
+++ b/lib/flowdock/git.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+require 'flowdock'
+require 'flowdock/git/builder'
+
+module Flowdock
+  class Git
+    TokenError = Class.new(StandardError)
+
+    DEFAULT_PERMANENT_REFS = [
+      Regexp.new('refs/heads/master')
+    ].freeze
+
+    class << self
+      def post(ref, from, to, options = {})
+        Git.new(ref, from, to, options).post
+      end
+    end
+
+    def initialize(ref, from, to, options = {})
+      raise TokenError.new("Flowdock API token not found") unless options[:token]
+
+      @ref = ref
+      @from = from
+      @to = to
+      @options = options
+      @token = options[:token]
+      @commit_url = options[:commit_url]
+      @diff_url = options[:diff_url]
+      @repo_url = options[:repo_url]
+      @repo_name = options[:repo_name]
+      @permanent_refs = options.fetch(:permanent_refs, DEFAULT_PERMANENT_REFS)
+    end
+
+    # Send git push notification to Flowdock
+    def post
+      messages.each do |message|
+        Flowdock::Client.new(flow_token: @token).post_to_thread(message)
+      end
+    end
+
+    def repo
+      @options[:repo]
+    end
+
+    private
+
+    def messages
+      Git::Builder.new(repo: repo,
+                       ref: @ref,
+                       before: @from,
+                       after: @to,
+                       commit_url: @commit_url,
+                       branch_url: @branch_url,
+                       diff_url: @diff_url,
+                       repo_url: @repo_url,
+                       repo_name: @repo_name,
+                       permanent_refs: @permanent_refs,
+                       tags: tags
+                      ).to_hashes
+    end
+
+    # Flowdock tags attached to the push notification
+    def tags
+      Array(@options[:tags]).map { |tag| CGI.escape(tag) }
+    end
+  end
+end
diff --git a/lib/flowdock/git/builder.rb b/lib/flowdock/git/builder.rb
new file mode 100644
index 00000000000..6f4428d1f42
--- /dev/null
+++ b/lib/flowdock/git/builder.rb
@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+module Flowdock
+  class Git
+    class Commit
+      def initialize(external_thread_id, thread, tags, commit)
+        @commit = commit
+        @external_thread_id = external_thread_id
+        @thread = thread
+        @tags = tags
+      end
+
+      def to_hash
+        hash = {
+          external_thread_id: @external_thread_id,
+          event: "activity",
+          author: {
+            name: @commit[:author][:name],
+            email: @commit[:author][:email]
+          },
+          title: title,
+          thread: @thread,
+          body: body
+        }
+        hash[:tags] = @tags if @tags
+        encode(hash)
+      end
+
+      private
+
+      def encode(hash)
+        return hash unless "".respond_to?(:encode)
+
+        encode_as_utf8(hash)
+      end
+
+      # This only works on Ruby 1.9
+      def encode_as_utf8(obj)
+        if obj.is_a? Hash
+          obj.each_pair do |key, val|
+            encode_as_utf8(val)
+          end
+        elsif obj.is_a?(Array)
+          obj.each do |val|
+            encode_as_utf8(val)
+          end
+        elsif obj.is_a?(String) && obj.encoding != Encoding::UTF_8
+          unless obj.force_encoding("UTF-8").valid_encoding?
+            obj.force_encoding("ISO-8859-1").encode!(Encoding::UTF_8, invalid: :replace, undef: :replace)
+          end
+        end
+      end
+
+      def body
+        content = @commit[:message][first_line.size..-1]
+        content.strip! if content
+        "<pre>#{content}</pre>" unless content.empty?
+      end
+
+      def first_line
+        @first_line ||= (@commit[:message].split("\n")[0] || @commit[:message])
+      end
+
+      def title
+        commit_id = @commit[:id][0, 7]
+        if @commit[:url]
+          "<a href=\"#{@commit[:url]}\">#{commit_id}</a> #{message_title}"
+        else
+          "#{commit_id} #{message_title}"
+        end
+      end
+
+      def message_title
+        CGI.escape_html(first_line.strip)
+      end
+    end
+
+    # Class used to build Git payload
+    class Builder
+      include ::Gitlab::Utils::StrongMemoize
+
+      def initialize(opts)
+        @repo = opts[:repo]
+        @ref = opts[:ref]
+        @before = opts[:before]
+        @after = opts[:after]
+        @opts = opts
+      end
+
+      def commits
+        @repo.commits_between(@before, @after).map do |commit|
+          {
+            url: @opts[:commit_url] ? @opts[:commit_url] % [commit.sha] : nil,
+            id: commit.sha,
+            message: commit.message,
+            author: {
+              name: commit.author_name,
+              email: commit.author_email
+            }
+          }
+        end
+      end
+
+      def ref_name
+        @ref.to_s.sub(%r{\Arefs/(heads|tags)/}, '')
+      end
+
+      def to_hashes
+        commits.map do |commit|
+          Commit.new(external_thread_id, thread, @opts[:tags], commit).to_hash
+        end
+      end
+
+      private
+
+      def thread
+        @thread ||= {
+          title: thread_title,
+          external_url: @opts[:repo_url]
+        }
+      end
+
+      def permanent?
+        strong_memoize(:permanent) do
+          @opts[:permanent_refs].any? { |regex| regex.match(@ref) }
+        end
+      end
+
+      def thread_title
+        action = "updated" if permanent?
+        type = @ref =~ %r(^refs/heads/) ? "branch" : "tag"
+
+        [@opts[:repo_name], type, ref_name, action].compact.join(" ")
+      end
+
+      def external_thread_id
+        @external_thread_id ||=
+          if permanent?
+            SecureRandom.hex
+          else
+            @ref
+          end
+      end
+    end
+  end
+end
diff --git a/lib/forever.rb b/lib/forever.rb
index 7df17912544..0a37118fe68 100644
--- a/lib/forever.rb
+++ b/lib/forever.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class Forever
   POSTGRESQL_DATE = DateTime.new(3000, 1, 1)
   MYSQL_DATE = DateTime.new(2038, 01, 19)
diff --git a/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb b/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb
index 91175b49c79..15cdd25e711 100644
--- a/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb
+++ b/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rails/generators'
 
 module Rails
diff --git a/lib/gitaly/server.rb b/lib/gitaly/server.rb
index 605e93022e7..7b238623418 100644
--- a/lib/gitaly/server.rb
+++ b/lib/gitaly/server.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitaly
   class Server
     def self.all
@@ -22,6 +24,18 @@ module Gitaly
       server_version == Gitlab::GitalyClient.expected_server_version
     end
 
+    def read_writeable?
+      readable? && writeable?
+    end
+
+    def readable?
+      storage_status&.readable
+    end
+
+    def writeable?
+      storage_status&.writeable
+    end
+
     def address
       Gitlab::GitalyClient.address(@storage)
     rescue RuntimeError => e
@@ -30,13 +44,17 @@ module Gitaly
 
     private
 
+    def storage_status
+      @storage_status ||= info.storage_statuses.find { |s| s.storage_name == storage }
+    end
+
     def info
       @info ||=
         begin
           Gitlab::GitalyClient::ServerService.new(@storage).info
-        rescue GRPC::Unavailable, GRPC::GRPC::DeadlineExceeded
+        rescue GRPC::Unavailable, GRPC::DeadlineExceeded
           # This will show the server as being out of date
-          Gitaly::ServerInfoResponse.new(git_version: '', server_version: '')
+          Gitaly::ServerInfoResponse.new(git_version: '', server_version: '', storage_statuses: [])
         end
     end
   end
diff --git a/lib/gitlab.rb b/lib/gitlab.rb
index a129746e2c6..e073450283b 100644
--- a/lib/gitlab.rb
+++ b/lib/gitlab.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_dependency 'gitlab/popen'
 
 module Gitlab
@@ -5,12 +7,16 @@ module Gitlab
     Pathname.new(File.expand_path('..', __dir__))
   end
 
-  def self.config
-    Settings
+  def self.version_info
+    Gitlab::VersionInfo.parse(Gitlab::VERSION)
+  end
+
+  def self.pre_release?
+    VERSION.include?('pre')
   end
 
-  def self.migrations_hash
-    @_migrations_hash ||= Digest::MD5.hexdigest(ActiveRecord::Migrator.get_all_versions.to_s)
+  def self.config
+    Settings
   end
 
   def self.revision
@@ -33,6 +39,7 @@ module Gitlab
   APP_DIRS_PATTERN = %r{^/?(app|config|ee|lib|spec|\(\w*\))}
   SUBDOMAIN_REGEX = %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z}
   VERSION = File.read(root.join("VERSION")).strip.freeze
+  INSTALLATION_TYPE = File.read(root.join("INSTALLATION_TYPE")).strip.freeze
 
   def self.com?
     # Check `gl_subdomain?` as well to keep parity with gitlab.com
@@ -50,4 +57,12 @@ module Gitlab
   def self.dev_env_or_com?
     Rails.env.development? || org? || com?
   end
+
+  def self.process_name
+    return 'sidekiq' if Sidekiq.server?
+    return 'console' if defined?(Rails::Console)
+    return 'test' if Rails.env.test?
+
+    'web'
+  end
 end
diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb
index 7127948cf00..ec090aea784 100644
--- a/lib/gitlab/access.rb
+++ b/lib/gitlab/access.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Gitlab::Access module
 #
 # Define allowed roles that can be used
@@ -7,12 +9,14 @@ module Gitlab
   module Access
     AccessDeniedError = Class.new(StandardError)
 
-    NO_ACCESS = 0
-    GUEST     = 10
-    REPORTER  = 20
-    DEVELOPER = 30
-    MASTER    = 40
-    OWNER     = 50
+    NO_ACCESS  = 0
+    GUEST      = 10
+    REPORTER   = 20
+    DEVELOPER  = 30
+    MAINTAINER = 40
+    # @deprecated
+    MASTER     = MAINTAINER
+    OWNER      = 50
 
     # Branch protection settings
     PROTECTION_NONE          = 0
@@ -29,10 +33,10 @@ module Gitlab
 
       def options
         {
-          "Guest"     => GUEST,
-          "Reporter"  => REPORTER,
-          "Developer" => DEVELOPER,
-          "Master"    => MASTER
+          "Guest"      => GUEST,
+          "Reporter"   => REPORTER,
+          "Developer"  => DEVELOPER,
+          "Maintainer" => MAINTAINER
         }
       end
 
@@ -44,10 +48,10 @@ module Gitlab
 
       def sym_options
         {
-          guest:     GUEST,
-          reporter:  REPORTER,
-          developer: DEVELOPER,
-          master:    MASTER
+          guest:      GUEST,
+          reporter:   REPORTER,
+          developer:  DEVELOPER,
+          maintainer: MAINTAINER
         }
       end
 
@@ -57,10 +61,10 @@ module Gitlab
 
       def protection_options
         {
-          "Not protected: Both developers and masters can push new commits, force push, or delete the branch." => PROTECTION_NONE,
-          "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch. Masters can push to the branch." => PROTECTION_DEV_CAN_MERGE,
-          "Partially protected: Both developers and masters can push new commits, but cannot force push or delete the branch." => PROTECTION_DEV_CAN_PUSH,
-          "Fully protected: Developers cannot push new commits, but masters can. No-one can force push or delete the branch." => PROTECTION_FULL
+          "Not protected: Both developers and maintainers can push new commits, force push, or delete the branch." => PROTECTION_NONE,
+          "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch. Maintainers can push to the branch." => PROTECTION_DEV_CAN_MERGE,
+          "Partially protected: Both developers and maintainers can push new commits, but cannot force push or delete the branch." => PROTECTION_DEV_CAN_PUSH,
+          "Fully protected: Developers cannot push new commits, but maintainers can. No-one can force push or delete the branch." => PROTECTION_FULL
         }
       end
 
diff --git a/lib/gitlab/access/branch_protection.rb b/lib/gitlab/access/branch_protection.rb
new file mode 100644
index 00000000000..f039e5c011f
--- /dev/null
+++ b/lib/gitlab/access/branch_protection.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Access
+    # A wrapper around Integer based branch protection levels.
+    #
+    # This wrapper can be used to work with branch protection levels without
+    # having to directly refer to the constants. For example, instead of this:
+    #
+    #     if access_level == Gitlab::Access::PROTECTION_DEV_CAN_PUSH
+    #       ...
+    #     end
+    #
+    # You can write this instead:
+    #
+    #     protection = BranchProtection.new(access_level)
+    #
+    #     if protection.developer_can_push?
+    #       ...
+    #     end
+    class BranchProtection
+      attr_reader :level
+
+      # @param [Integer] level The branch protection level as an Integer.
+      def initialize(level)
+        @level = level
+      end
+
+      def any?
+        level != PROTECTION_NONE
+      end
+
+      def developer_can_push?
+        level == PROTECTION_DEV_CAN_PUSH
+      end
+
+      def developer_can_merge?
+        level == PROTECTION_DEV_CAN_MERGE
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/action_rate_limiter.rb b/lib/gitlab/action_rate_limiter.rb
index 4cd3bdefda3..c442211e073 100644
--- a/lib/gitlab/action_rate_limiter.rb
+++ b/lib/gitlab/action_rate_limiter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   # This class implements a simple rate limiter that can be used to throttle
   # certain actions. Unlike Rack Attack and Rack::Throttle, which operate at
diff --git a/lib/gitlab/allowable.rb b/lib/gitlab/allowable.rb
index 45c2b01dd8f..4518c8a862c 100644
--- a/lib/gitlab/allowable.rb
+++ b/lib/gitlab/allowable.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Allowable
     def can?(*args)
diff --git a/lib/gitlab/app_logger.rb b/lib/gitlab/app_logger.rb
index dddcb2538f9..5edec8b3efe 100644
--- a/lib/gitlab/app_logger.rb
+++ b/lib/gitlab/app_logger.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class AppLogger < Gitlab::Logger
     def self.file_name_noext
diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb
index 62c41801d75..df8f0470063 100644
--- a/lib/gitlab/asciidoc.rb
+++ b/lib/gitlab/asciidoc.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'asciidoctor'
 require 'asciidoctor/converter/html5'
 require "asciidoctor-plantuml"
diff --git a/lib/gitlab/audit_json_logger.rb b/lib/gitlab/audit_json_logger.rb
new file mode 100644
index 00000000000..12e0645f3e4
--- /dev/null
+++ b/lib/gitlab/audit_json_logger.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class AuditJsonLogger < Gitlab::JsonLogger
+    def self.file_name_noext
+      'audit_json'
+    end
+  end
+end
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 0f7a7b0ce8d..7aa02009aa0 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Auth
     MissingPersonalAccessTokenError = Class.new(StandardError)
@@ -14,23 +16,8 @@ module Gitlab
     DEFAULT_SCOPES = [:api].freeze
 
     class << self
-      def omniauth_customized_providers
-        @omniauth_customized_providers ||= %w[bitbucket jwt]
-      end
-
-      def omniauth_setup_providers(provider_names)
-        provider_names.each do |provider|
-          omniauth_setup_a_provider(provider)
-        end
-      end
-
-      def omniauth_setup_a_provider(provider)
-        case provider
-        when 'kerberos'
-          require 'omniauth-kerberos'
-        when *omniauth_customized_providers
-          require_dependency "omni_auth/strategies/#{provider}"
-        end
+      def omniauth_enabled?
+        Gitlab.config.omniauth.enabled
       end
 
       def find_for_git_client(login, password, project:, ip:)
@@ -151,6 +138,7 @@ module Gitlab
         Gitlab::Auth::Result.new(user, nil, :gitlab_or_ldap, full_authentication_abilities)
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def oauth_access_token_check(login, password)
         if login == "oauth2" && password.present?
           token = Doorkeeper::AccessToken.by_token(password)
@@ -161,11 +149,12 @@ module Gitlab
           end
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def personal_access_token_check(password)
         return unless password.present?
 
-        token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password)
+        token = PersonalAccessTokensFinder.new(state: 'active').find_by_token(password)
 
         if token && valid_scoped_token?(token, available_scopes)
           Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scopes(token.scopes))
@@ -192,6 +181,7 @@ module Gitlab
         end.uniq
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def deploy_token_check(login, password)
         return unless password.present?
 
@@ -207,8 +197,9 @@ module Gitlab
           Gitlab::Auth::Result.new(token, token.project, :deploy_token, scopes)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
-      def lfs_token_check(login, password, project)
+      def lfs_token_check(login, encoded_token, project)
         deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/)
 
         actor =
@@ -231,7 +222,7 @@ module Gitlab
             read_authentication_abilities
           end
 
-        if Devise.secure_compare(token_handler.token, password)
+        if token_handler.token_valid?(encoded_token)
           Gitlab::Auth::Result.new(actor, nil, token_handler.type, authentication_abilities)
         end
       end
@@ -240,7 +231,7 @@ module Gitlab
         return unless login == 'gitlab-ci-token'
         return unless password
 
-        build = ::Ci::Build.running.find_by_token(password)
+        build = find_build_by_token(password)
         return unless build
         return unless build.project.builds_enabled?
 
@@ -301,6 +292,12 @@ module Gitlab
 
         REGISTRY_SCOPES
       end
+
+      private
+
+      def find_build_by_token(token)
+        ::Ci::Build.running.find_by_token(token)
+      end
     end
   end
 end
diff --git a/lib/gitlab/auth/activity.rb b/lib/gitlab/auth/activity.rb
new file mode 100644
index 00000000000..558628b5422
--- /dev/null
+++ b/lib/gitlab/auth/activity.rb
@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Auth
+    ##
+    # Metrics and logging for user authentication activity.
+    #
+    class Activity
+      extend Gitlab::Utils::StrongMemoize
+
+      COUNTERS = {
+        user_authenticated: 'Counter of successful authentication events',
+        user_unauthenticated: 'Counter of authentication failures',
+        user_not_found: 'Counter of failed log-ins when user is unknown',
+        user_password_invalid: 'Counter of failed log-ins with invalid password',
+        user_session_override: 'Counter of manual log-ins and sessions overrides',
+        user_session_destroyed: 'Counter of user sessions being destroyed',
+        user_two_factor_authenticated: 'Counter of two factor authentications',
+        user_sessionless_authentication: 'Counter of sessionless authentications',
+        user_blocked: 'Counter of sign in attempts when user is blocked'
+      }.freeze
+
+      def initialize(opts)
+        @opts = opts
+      end
+
+      def user_authentication_failed!
+        self.class.user_unauthenticated_counter_increment!
+
+        case @opts[:message]
+        when :not_found_in_database
+          self.class.user_not_found_counter_increment!
+        when :invalid
+          self.class.user_password_invalid_counter_increment!
+        end
+      end
+
+      def user_authenticated!
+        self.class.user_authenticated_counter_increment!
+      end
+
+      def user_session_override!
+        self.class.user_session_override_counter_increment!
+
+        case @opts[:message]
+        when :two_factor_authenticated
+          self.class.user_two_factor_authenticated_counter_increment!
+        when :sessionless_sign_in
+          self.class.user_sessionless_authentication_counter_increment!
+        end
+      end
+
+      def user_blocked!
+        self.class.user_blocked_counter_increment!
+      end
+
+      def user_session_destroyed!
+        self.class.user_session_destroyed_counter_increment!
+      end
+
+      def self.each_counter
+        COUNTERS.each_pair do |metric, description|
+          yield "#{metric}_counter", metric, description
+        end
+      end
+
+      each_counter do |counter, metric, description|
+        define_singleton_method(counter) do
+          strong_memoize(counter) do
+            Gitlab::Metrics.counter("gitlab_auth_#{metric}_total".to_sym, description)
+          end
+        end
+
+        define_singleton_method("#{counter}_increment!") do
+          public_send(counter).increment # rubocop:disable GitlabSecurity/PublicSend
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/auth/blocked_user_tracker.rb b/lib/gitlab/auth/blocked_user_tracker.rb
index 7609a7b04f6..50712d7eac2 100644
--- a/lib/gitlab/auth/blocked_user_tracker.rb
+++ b/lib/gitlab/auth/blocked_user_tracker.rb
@@ -2,35 +2,19 @@
 module Gitlab
   module Auth
     class BlockedUserTracker
-      ACTIVE_RECORD_REQUEST_PARAMS = 'action_dispatch.request.request_parameters'
-
-      def self.log_if_user_blocked(env)
-        message = env.dig('warden.options', :message)
-
-        # Devise calls User#active_for_authentication? on the User model and then
-        # throws an exception to Warden with User#inactive_message:
-        # https://github.com/plataformatec/devise/blob/v4.2.1/lib/devise/hooks/activatable.rb#L8
-        #
-        # Since Warden doesn't pass the user record to the failure handler, we
-        # need to do a database lookup with the username. We can limit the
-        # lookups to happen when the user was blocked by checking the inactive
-        # message passed along by Warden.
-        return unless message == User::BLOCKED_MESSAGE
-
-        # Check for either LDAP or regular GitLab account logins
-        login = env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'username') ||
-          env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'user', 'login')
-
-        return unless login.present?
-
-        user = User.by_login(login)
+      def initialize(user, auth)
+        @user = user
+        @auth = auth
+      end
 
-        return unless user&.blocked?
+      def log_activity!
+        return unless @user.blocked?
 
-        Gitlab::AppLogger.info("Failed login for blocked user: user=#{user.username} ip=#{env['REMOTE_ADDR']}")
-        SystemHooksService.new.execute_hooks_for(user, :failed_login)
+        Gitlab::AppLogger.info <<~INFO
+          "Failed login for blocked user: user=#{@user.username} ip=#{@auth.request.ip}")
+        INFO
 
-        true
+        SystemHooksService.new.execute_hooks_for(@user, :failed_login)
       rescue TypeError
       end
     end
diff --git a/lib/gitlab/auth/database/authentication.rb b/lib/gitlab/auth/database/authentication.rb
index 1234ace0334..c0dc2b0875f 100644
--- a/lib/gitlab/auth/database/authentication.rb
+++ b/lib/gitlab/auth/database/authentication.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # These calls help to authenticate to OAuth provider by providing username and password
 #
 
diff --git a/lib/gitlab/auth/ip_rate_limiter.rb b/lib/gitlab/auth/ip_rate_limiter.rb
index e6173d45af3..81e616fa20a 100644
--- a/lib/gitlab/auth/ip_rate_limiter.rb
+++ b/lib/gitlab/auth/ip_rate_limiter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Auth
     class IpRateLimiter
diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb
index 865185eb5db..c875bba4bcb 100644
--- a/lib/gitlab/auth/ldap/access.rb
+++ b/lib/gitlab/auth/ldap/access.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # LDAP authorization model
 #
 # * Check if we are allowed access (not blocked)
@@ -19,8 +21,10 @@ module Gitlab
             # Whether user is allowed, or not, we should update
             # permissions to keep things clean
             if access.allowed?
-              access.update_user
-              Users::UpdateService.new(user, user: user, last_credential_check_at: Time.now).execute
+              unless Gitlab::Database.read_only?
+                access.update_user
+                Users::UpdateService.new(user, user: user, last_credential_check_at: Time.now).execute
+              end
 
               true
             else
@@ -60,6 +64,12 @@ module Gitlab
           false
         end
 
+        def update_user
+          # no-op in CE
+        end
+
+        private
+
         def adapter
           @adapter ||= Gitlab::Auth::LDAP::Adapter.new(provider)
         end
@@ -68,28 +78,28 @@ module Gitlab
           Gitlab::Auth::LDAP::Config.new(provider)
         end
 
-        def find_ldap_user
-          Gitlab::Auth::LDAP::Person.find_by_dn(ldap_identity.extern_uid, adapter)
-        end
-
         def ldap_user
           return unless provider
 
           @ldap_user ||= find_ldap_user
         end
 
+        def find_ldap_user
+          Gitlab::Auth::LDAP::Person.find_by_dn(ldap_identity.extern_uid, adapter)
+        end
+
         def block_user(user, reason)
           user.ldap_block
 
           if provider
             Gitlab::AppLogger.info(
               "LDAP account \"#{ldap_identity.extern_uid}\" #{reason}, " \
-              "blocking Gitlab user \"#{user.name}\" (#{user.email})"
+              "blocking GitLab user \"#{user.name}\" (#{user.email})"
             )
           else
             Gitlab::AppLogger.info(
               "Account is not provided by LDAP, " \
-              "blocking Gitlab user \"#{user.name}\" (#{user.email})"
+              "blocking GitLab user \"#{user.name}\" (#{user.email})"
             )
           end
         end
@@ -99,13 +109,9 @@ module Gitlab
 
           Gitlab::AppLogger.info(
             "LDAP account \"#{ldap_identity.extern_uid}\" #{reason}, " \
-            "unblocking Gitlab user \"#{user.name}\" (#{user.email})"
+            "unblocking GitLab user \"#{user.name}\" (#{user.email})"
           )
         end
-
-        def update_user
-          # no-op in CE
-        end
       end
     end
   end
diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb
index 82ff1e77e5c..15b9d5ad6e9 100644
--- a/lib/gitlab/auth/ldap/adapter.rb
+++ b/lib/gitlab/auth/ldap/adapter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Auth
     module LDAP
@@ -28,14 +30,7 @@ module Gitlab
 
         def users(fields, value, limit = nil)
           options = user_options(Array(fields), value, limit)
-
-          entries = ldap_search(options).select do |entry|
-            entry.respond_to? config.uid
-          end
-
-          entries.map do |entry|
-            Gitlab::Auth::LDAP::Person.new(entry, provider)
-          end
+          users_search(options)
         end
 
         def user(*args)
@@ -88,6 +83,16 @@ module Gitlab
           SEARCH_RETRY_FACTOR[retry_number] * config.timeout
         end
 
+        def users_search(options)
+          entries = ldap_search(options).select do |entry|
+            entry.respond_to? config.uid
+          end
+
+          entries.map do |entry|
+            Gitlab::Auth::LDAP::Person.new(entry, provider)
+          end
+        end
+
         def user_options(fields, value, limit)
           options = {
             attributes: Gitlab::Auth::LDAP::Person.ldap_attributes(config),
diff --git a/lib/gitlab/auth/ldap/auth_hash.rb b/lib/gitlab/auth/ldap/auth_hash.rb
index ac5c14d374d..83fdc8a8c76 100644
--- a/lib/gitlab/auth/ldap/auth_hash.rb
+++ b/lib/gitlab/auth/ldap/auth_hash.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Class to parse and transform the info provided by omniauth
 #
 module Gitlab
diff --git a/lib/gitlab/auth/ldap/authentication.rb b/lib/gitlab/auth/ldap/authentication.rb
index 7c134fb6438..174e81dd603 100644
--- a/lib/gitlab/auth/ldap/authentication.rb
+++ b/lib/gitlab/auth/ldap/authentication.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # These calls help to authenticate to LDAP by providing username and password
 #
 # Since multiple LDAP servers are supported, it will loop through all of them
diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb
index d4415eaa6dc..7ceb96f502b 100644
--- a/lib/gitlab/auth/ldap/config.rb
+++ b/lib/gitlab/auth/ldap/config.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Load a specific server configuration
 module Gitlab
   module Auth
diff --git a/lib/gitlab/auth/ldap/dn.rb b/lib/gitlab/auth/ldap/dn.rb
index 1fa5338f5a6..5df914aa367 100644
--- a/lib/gitlab/auth/ldap/dn.rb
+++ b/lib/gitlab/auth/ldap/dn.rb
@@ -1,4 +1,5 @@
 # -*- ruby encoding: utf-8 -*-
+# frozen_string_literal: true
 
 # Based on the `ruby-net-ldap` gem's `Net::LDAP::DN`
 #
diff --git a/lib/gitlab/auth/ldap/ldap_connection_error.rb b/lib/gitlab/auth/ldap/ldap_connection_error.rb
index ef0a695742b..d0e5f24d203 100644
--- a/lib/gitlab/auth/ldap/ldap_connection_error.rb
+++ b/lib/gitlab/auth/ldap/ldap_connection_error.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Auth
     module LDAP
diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb
index 8dfae3ee541..48d134f91b0 100644
--- a/lib/gitlab/auth/ldap/person.rb
+++ b/lib/gitlab/auth/ldap/person.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Auth
     module LDAP
@@ -96,9 +98,7 @@ module Gitlab
 
         private
 
-        def entry
-          @entry
-        end
+        attr_reader :entry
 
         def config
           @config ||= Gitlab::Auth::LDAP::Config.new(provider)
diff --git a/lib/gitlab/auth/ldap/user.rb b/lib/gitlab/auth/ldap/user.rb
index 922d0567d99..9c71671f409 100644
--- a/lib/gitlab/auth/ldap/user.rb
+++ b/lib/gitlab/auth/ldap/user.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # LDAP extension for User model
 #
 # * Find or create user from omniauth.auth data
@@ -11,11 +13,13 @@ module Gitlab
         extend ::Gitlab::Utils::Override
 
         class << self
+          # rubocop: disable CodeReuse/ActiveRecord
           def find_by_uid_and_provider(uid, provider)
             identity = ::Identity.with_extern_uid(provider, uid).take
 
             identity && identity.user
           end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
 
         def save
diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb
index ed8fba94305..36fc8061d92 100644
--- a/lib/gitlab/auth/o_auth/auth_hash.rb
+++ b/lib/gitlab/auth/o_auth/auth_hash.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Class to parse and transform the info provided by omniauth
 #
 module Gitlab
@@ -78,7 +80,7 @@ module Gitlab
         end
 
         # Get the first part of the email address (before @)
-        # In addtion in removes illegal characters
+        # In addition in removes illegal characters
         def generate_username(email)
           email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/, '').to_s
         end
diff --git a/lib/gitlab/auth/o_auth/authentication.rb b/lib/gitlab/auth/o_auth/authentication.rb
index d4e7f35c857..5f008678bd1 100644
--- a/lib/gitlab/auth/o_auth/authentication.rb
+++ b/lib/gitlab/auth/o_auth/authentication.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # These calls help to authenticate to OAuth provider by providing username and password
 #
 
diff --git a/lib/gitlab/auth/o_auth/identity_linker.rb b/lib/gitlab/auth/o_auth/identity_linker.rb
index de92d7a214d..e69c2bb54dc 100644
--- a/lib/gitlab/auth/o_auth/identity_linker.rb
+++ b/lib/gitlab/auth/o_auth/identity_linker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Auth
     module OAuth
diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb
index 5fb61ffe00d..9fdf3324db3 100644
--- a/lib/gitlab/auth/o_auth/provider.rb
+++ b/lib/gitlab/auth/o_auth/provider.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Auth
     module OAuth
@@ -29,8 +31,9 @@ module Gitlab
 
         def self.enabled?(name)
           return true if name == 'database'
+          return true if self.ldap_provider?(name) && providers.include?(name.to_sym)
 
-          providers.include?(name.to_sym)
+          Gitlab::Auth.omniauth_enabled? && providers.include?(name.to_sym)
         end
 
         def self.ldap_provider?(name)
diff --git a/lib/gitlab/auth/o_auth/session.rb b/lib/gitlab/auth/o_auth/session.rb
index 8f2b4d58552..4925b107042 100644
--- a/lib/gitlab/auth/o_auth/session.rb
+++ b/lib/gitlab/auth/o_auth/session.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # :nocov:
 module Gitlab
   module Auth
diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb
index 6c5d0788a0a..f38c5d57c44 100644
--- a/lib/gitlab/auth/o_auth/user.rb
+++ b/lib/gitlab/auth/o_auth/user.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # OAuth extension for User model
 #
 # * Find GitLab user based on omniauth uid and provider
@@ -44,11 +46,11 @@ module Gitlab
 
           gl_user.block if block_after_save
 
-          log.info "(#{provider}) saving user #{auth_hash.email} from login with extern_uid => #{auth_hash.uid}"
+          log.info "(#{provider}) saving user #{auth_hash.email} from login with admin => #{gl_user.admin}, extern_uid => #{auth_hash.uid}"
           gl_user
         rescue ActiveRecord::RecordInvalid => e
           log.info "(#{provider}) Error saving user #{auth_hash.uid} (#{auth_hash.email}): #{gl_user.errors.full_messages}"
-          return self, e.record.errors
+          [self, e.record.errors]
         end
 
         def gl_user
@@ -74,6 +76,10 @@ module Gitlab
           gl_user
         end
 
+        def bypass_two_factor?
+          false
+        end
+
         protected
 
         def should_save?
@@ -108,11 +114,13 @@ module Gitlab
           build_new_user
         end
 
+        # rubocop: disable CodeReuse/ActiveRecord
         def find_by_email
           return unless auth_hash.has_attribute?(:email)
 
           ::User.find_by(email: auth_hash.email.downcase)
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         def auto_link_ldap_user?
           Gitlab.config.omniauth.auto_link_ldap_user
@@ -176,10 +184,12 @@ module Gitlab
           @auth_hash = AuthHash.new(auth_hash)
         end
 
+        # rubocop: disable CodeReuse/ActiveRecord
         def find_by_uid_and_provider
           identity = Identity.with_extern_uid(auth_hash.provider, auth_hash.uid).take
           identity&.user
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         def build_new_user
           user_params = user_attributes.merge(skip_confirmation: true)
diff --git a/lib/gitlab/auth/omniauth_identity_linker_base.rb b/lib/gitlab/auth/omniauth_identity_linker_base.rb
index f79ce6bb809..253445570f2 100644
--- a/lib/gitlab/auth/omniauth_identity_linker_base.rb
+++ b/lib/gitlab/auth/omniauth_identity_linker_base.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Auth
     class OmniauthIdentityLinkerBase
@@ -33,11 +35,13 @@ module Gitlab
         @changed = identity.save
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def identity
         @identity ||= current_user.identities
                                   .with_extern_uid(provider, uid)
                                   .first_or_initialize(extern_uid: uid)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def provider
         oauth['provider']
diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb
index 66de52506ce..176766d1a8b 100644
--- a/lib/gitlab/auth/request_authenticator.rb
+++ b/lib/gitlab/auth/request_authenticator.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Use for authentication only, in particular for Rack::Attack.
 # Does not perform authorization of scopes, etc.
 module Gitlab
@@ -11,12 +13,18 @@ module Gitlab
         @request = request
       end
 
-      def user
-        find_sessionless_user || find_user_from_warden
+      def user(request_formats)
+        request_formats.each do |format|
+          user = find_sessionless_user(format)
+
+          return user if user
+        end
+
+        find_user_from_warden
       end
 
-      def find_sessionless_user
-        find_user_from_access_token || find_user_from_feed_token
+      def find_sessionless_user(request_format)
+        find_user_from_web_access_token(request_format) || find_user_from_feed_token(request_format)
       rescue Gitlab::Auth::AuthenticationError
         nil
       end
diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb
index 00cdc94a9ef..78fa25c5516 100644
--- a/lib/gitlab/auth/result.rb
+++ b/lib/gitlab/auth/result.rb
@@ -1,4 +1,7 @@
-module Gitlab # rubocop:disable Naming/FileName
+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+module Gitlab
   module Auth
     Result = Struct.new(:actor, :project, :type, :authentication_abilities) do
       def ci?(for_project)
diff --git a/lib/gitlab/auth/saml/auth_hash.rb b/lib/gitlab/auth/saml/auth_hash.rb
index c345a7e3f6c..1af9fa40c3a 100644
--- a/lib/gitlab/auth/saml/auth_hash.rb
+++ b/lib/gitlab/auth/saml/auth_hash.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Auth
     module Saml
@@ -6,6 +8,17 @@ module Gitlab
           Array.wrap(get_raw(Gitlab::Auth::Saml::Config.groups))
         end
 
+        def authn_context
+          response_object = auth_hash.extra[:response_object]
+          return nil if response_object.blank?
+
+          document = response_object.decrypted_document
+          document ||= response_object.document
+          return nil if document.blank?
+
+          extract_authn_context(document)
+        end
+
         private
 
         def get_raw(key)
@@ -13,6 +26,10 @@ module Gitlab
           # otherwise just the first value is returned
           auth_hash.extra[:raw_info].all[key]
         end
+
+        def extract_authn_context(document)
+          REXML::XPath.first(document, "//*[name()='saml:AuthnStatement' or name()='saml2:AuthnStatement']/*[name()='saml:AuthnContext' or name()='saml2:AuthnContext']/*[name()='saml:AuthnContextClassRef' or name()='saml2:AuthnContextClassRef']/text()").to_s
+        end
       end
     end
   end
diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb
index 5fa9581f837..8cb999f50d4 100644
--- a/lib/gitlab/auth/saml/config.rb
+++ b/lib/gitlab/auth/saml/config.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Auth
     module Saml
@@ -7,6 +9,10 @@ module Gitlab
             Gitlab::Auth::OAuth::Provider.config_for('saml')
           end
 
+          def upstream_two_factor_authn_contexts
+            options.args[:upstream_two_factor_authn_contexts]
+          end
+
           def groups
             options[:groups_attribute]
           end
diff --git a/lib/gitlab/auth/saml/identity_linker.rb b/lib/gitlab/auth/saml/identity_linker.rb
index 7e4b191d512..ae0d6dded4e 100644
--- a/lib/gitlab/auth/saml/identity_linker.rb
+++ b/lib/gitlab/auth/saml/identity_linker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Auth
     module Saml
diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb
index b8c84c37cd5..ec95bc46791 100644
--- a/lib/gitlab/auth/saml/user.rb
+++ b/lib/gitlab/auth/saml/user.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # SAML extension for User model
 #
 # * Find GitLab user based on SAML uid and provider
@@ -34,6 +36,10 @@ module Gitlab
           gl_user.changed? || gl_user.identities.any?(&:changed?)
         end
 
+        def bypass_two_factor?
+          saml_config.upstream_two_factor_authn_contexts&.include?(auth_hash.authn_context)
+        end
+
         protected
 
         def saml_config
diff --git a/lib/gitlab/auth/too_many_ips.rb b/lib/gitlab/auth/too_many_ips.rb
index ed862791551..ee4d80e6b89 100644
--- a/lib/gitlab/auth/too_many_ips.rb
+++ b/lib/gitlab/auth/too_many_ips.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Auth
     class TooManyIps < StandardError
diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb
index baa1f802d8a..31dd61ae6cf 100644
--- a/lib/gitlab/auth/unique_ips_limiter.rb
+++ b/lib/gitlab/auth/unique_ips_limiter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Auth
     class UniqueIpsLimiter
diff --git a/lib/gitlab/auth/user_access_denied_reason.rb b/lib/gitlab/auth/user_access_denied_reason.rb
index 1893cb001b2..fd09fe76c02 100644
--- a/lib/gitlab/auth/user_access_denied_reason.rb
+++ b/lib/gitlab/auth/user_access_denied_reason.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Auth
     class UserAccessDeniedReason
diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb
index c7993665421..a5efe33bdc6 100644
--- a/lib/gitlab/auth/user_auth_finders.rb
+++ b/lib/gitlab/auth/user_auth_finders.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Auth
     AuthenticationError = Class.new(StandardError)
@@ -5,6 +7,7 @@ module Gitlab
     TokenNotFoundError = Class.new(AuthenticationError)
     ExpiredError = Class.new(AuthenticationError)
     RevokedError = Class.new(AuthenticationError)
+    ImpersonationDisabled = Class.new(AuthenticationError)
     UnauthorizedError = Class.new(AuthenticationError)
 
     class InsufficientScopeError < AuthenticationError
@@ -25,8 +28,8 @@ module Gitlab
         current_request.env['warden']&.authenticate if verified_request?
       end
 
-      def find_user_from_feed_token
-        return unless rss_request? || ics_request?
+      def find_user_from_feed_token(request_format)
+        return unless valid_rss_format?(request_format)
 
         # NOTE: feed_token was renamed from rss_token but both needs to be supported because
         #       users might have already added the feed to their RSS reader before the rename
@@ -36,6 +39,17 @@ module Gitlab
         User.find_by_feed_token(token) || raise(UnauthorizedError)
       end
 
+      # We only allow Private Access Tokens with `api` scope to be used by web
+      # requests on RSS feeds or ICS files for backwards compatibility.
+      # It is also used by GraphQL/API requests.
+      def find_user_from_web_access_token(request_format)
+        return unless access_token && valid_web_access_format?(request_format)
+
+        validate_access_token!(scopes: [:api])
+
+        access_token.user || raise(UnauthorizedError)
+      end
+
       def find_user_from_access_token
         return unless access_token
 
@@ -54,6 +68,8 @@ module Gitlab
           raise ExpiredError
         when AccessTokenValidationService::REVOKED
           raise RevokedError
+        when AccessTokenValidationService::IMPERSONATION_DISABLED
+          raise ImpersonationDisabled
         end
       end
 
@@ -79,7 +95,7 @@ module Gitlab
         return unless token
 
         # Expiration, revocation and scopes are verified in `validate_access_token!`
-        PersonalAccessToken.find_by(token: token) || raise(UnauthorizedError)
+        PersonalAccessToken.find_by_token(token) || raise(UnauthorizedError)
       end
 
       def find_oauth_access_token
@@ -107,6 +123,26 @@ module Gitlab
         @current_request ||= ensure_action_dispatch_request(request)
       end
 
+      def valid_web_access_format?(request_format)
+        case request_format
+        when :rss
+          rss_request?
+        when :ics
+          ics_request?
+        when :api
+          api_request?
+        end
+      end
+
+      def valid_rss_format?(request_format)
+        case request_format
+        when :rss
+          rss_request?
+        when :ics
+          ics_request?
+        end
+      end
+
       def rss_request?
         current_request.path.ends_with?('.atom') || current_request.format.atom?
       end
@@ -114,6 +150,10 @@ module Gitlab
       def ics_request?
         current_request.path.ends_with?('.ics') || current_request.format.ics?
       end
+
+      def api_request?
+        current_request.path.starts_with?("/api/")
+      end
     end
   end
 end
diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb
index d3f66877672..5251e0fadf9 100644
--- a/lib/gitlab/background_migration.rb
+++ b/lib/gitlab/background_migration.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module BackgroundMigration
     def self.queue
@@ -14,11 +16,18 @@ module Gitlab
     # re-raises the exception.
     #
     # steal_class - The name of the class for which to steal jobs.
-    def self.steal(steal_class)
-      enqueued = Sidekiq::Queue.new(self.queue)
-      scheduled = Sidekiq::ScheduledSet.new
+    def self.steal(steal_class, retry_dead_jobs: false)
+      queues = [
+        Sidekiq::ScheduledSet.new,
+        Sidekiq::Queue.new(self.queue)
+      ]
+
+      if retry_dead_jobs
+        queues << Sidekiq::RetrySet.new
+        queues << Sidekiq::DeadSet.new
+      end
 
-      [scheduled, enqueued].each do |queue|
+      queues.each do |queue|
         queue.each do |job|
           migration_class, migration_args = job.args
 
@@ -46,7 +55,24 @@ module Gitlab
     # arguments - The arguments to pass to the background migration's "perform"
     #             method.
     def self.perform(class_name, arguments)
-      const_get(class_name).new.perform(*arguments)
+      migration_class_for(class_name).new.perform(*arguments)
+    end
+
+    def self.exists?(migration_class)
+      enqueued = Sidekiq::Queue.new(self.queue)
+      scheduled = Sidekiq::ScheduledSet.new
+
+      [enqueued, scheduled].each do |queue|
+        queue.each do |job|
+          return true if job.queue == self.queue && job.args.first == migration_class
+        end
+      end
+
+      false
+    end
+
+    def self.migration_class_for(class_name)
+      const_get(class_name)
     end
   end
 end
diff --git a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb
index d5cf9e0d53a..cb2bdea755c 100644
--- a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb
+++ b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb
@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 # rubocop:disable Style/Documentation
-# rubocop:disable Metrics/LineLength
 
 module Gitlab
   module BackgroundMigration
diff --git a/lib/gitlab/background_migration/archive_legacy_traces.rb b/lib/gitlab/background_migration/archive_legacy_traces.rb
new file mode 100644
index 00000000000..92096e29ef1
--- /dev/null
+++ b/lib/gitlab/background_migration/archive_legacy_traces.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class ArchiveLegacyTraces
+      def perform(start_id, stop_id)
+        # This background migration directly refers to ::Ci::Build model which is defined in application code.
+        # In general, migration code should be isolated as much as possible in order to be idempotent.
+        # However, `archive!` method is too complicated to be replicated by coping its subsequent code.
+        # So we chose a way to use ::Ci::Build directly and we don't change the `archive!` method until 11.1
+        ::Ci::Build.finished.without_archived_trace
+          .where(id: start_id..stop_id).find_each do |build|
+            begin
+              build.trace.archive!
+            rescue => e
+              Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}"
+            end
+          end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb b/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb
new file mode 100644
index 00000000000..a6194616663
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Class that will fill the project_repositories table for projects that
+    # are on hashed storage and an entry is is missing in this table.
+    class BackfillHashedProjectRepositories < BackfillProjectRepositories
+      private
+
+      def projects
+        Project.on_hashed_storage
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/backfill_legacy_project_repositories.rb b/lib/gitlab/background_migration/backfill_legacy_project_repositories.rb
new file mode 100644
index 00000000000..6dc92672929
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_legacy_project_repositories.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Class that will fill the project_repositories table for projects that
+    # are on legacy storage and an entry is is missing in this table.
+    class BackfillLegacyProjectRepositories < BackfillProjectRepositories
+      private
+
+      def projects
+        Project.with_parent.on_legacy_storage
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb b/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb
new file mode 100644
index 00000000000..29fa0f18448
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb
@@ -0,0 +1,209 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This module is used to write the full path of all projects to
+    # the git repository config file.
+    # Storing the full project path in the git config allows admins to
+    # easily identify a project when it is using hashed storage.
+    module BackfillProjectFullpathInRepoConfig
+      OrphanedNamespaceError = Class.new(StandardError)
+
+      module Storage
+        # Class that returns the disk path for a project using hashed storage
+        class HashedProject
+          attr_accessor :project
+
+          ROOT_PATH_PREFIX = '@hashed'
+
+          def initialize(project)
+            @project = project
+          end
+
+          def disk_path
+            "#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}/#{disk_hash}"
+          end
+
+          def disk_hash
+            @disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) if project.id
+          end
+        end
+
+        # Class that returns the disk path for a project using legacy storage
+        class LegacyProject
+          attr_accessor :project
+
+          def initialize(project)
+            @project = project
+          end
+
+          def disk_path
+            project.full_path
+          end
+        end
+      end
+
+      # Concern used by Project and Namespace to determine the full
+      # route to the project
+      module Routable
+        extend ActiveSupport::Concern
+
+        def full_path
+          @full_path ||= build_full_path
+        end
+
+        def build_full_path
+          return path unless has_parent?
+
+          raise OrphanedNamespaceError if parent.nil?
+
+          parent.full_path + '/' + path
+        end
+
+        def has_parent?
+          read_attribute(association(:parent).reflection.foreign_key)
+        end
+      end
+
+      # Class used to interact with repository using Gitaly
+      class Repository
+        attr_reader :storage
+
+        def initialize(storage, relative_path)
+          @storage = storage
+          @relative_path = relative_path
+        end
+
+        def gitaly_repository
+          Gitaly::Repository.new(storage_name: @storage, relative_path: @relative_path)
+        end
+      end
+
+      # Namespace can be a user or group. It can be the root or a
+      # child of another namespace.
+      class Namespace < ActiveRecord::Base
+        self.table_name = 'namespaces'
+        self.inheritance_column = nil
+
+        include Routable
+
+        belongs_to :parent, class_name: 'Namespace', inverse_of: 'namespaces'
+        has_many :projects, inverse_of: :parent
+        has_many :namespaces, inverse_of: :parent
+      end
+
+      # Project is where the repository (etc.) is stored
+      class Project < ActiveRecord::Base
+        self.table_name = 'projects'
+
+        include Routable
+        include EachBatch
+
+        FULLPATH_CONFIG_KEY = 'gitlab.fullpath'
+
+        belongs_to :parent, class_name: 'Namespace', foreign_key: :namespace_id, inverse_of: 'projects'
+        delegate :disk_path, to: :storage
+
+        def add_fullpath_config
+          entries = { FULLPATH_CONFIG_KEY => full_path }
+
+          repository_service.set_config(entries)
+        end
+
+        def remove_fullpath_config
+          repository_service.delete_config([FULLPATH_CONFIG_KEY])
+        end
+
+        def cleanup_repository
+          repository_service.cleanup
+        end
+
+        def storage
+          @storage ||=
+            if hashed_storage?
+              Storage::HashedProject.new(self)
+            else
+              Storage::LegacyProject.new(self)
+            end
+        end
+
+        def hashed_storage?
+          self.storage_version && self.storage_version >= 1
+        end
+
+        def repository
+          @repository ||= Repository.new(repository_storage, disk_path + '.git')
+        end
+
+        def repository_service
+          @repository_service ||= Gitlab::GitalyClient::RepositoryService.new(repository)
+        end
+      end
+
+      # Base class for Up and Down migration classes
+      class BackfillFullpathMigration
+        RETRY_DELAY = 15.minutes
+        MAX_RETRIES = 2
+
+        # Base class for retrying one project
+        class BaseRetryOne
+          def perform(project_id, retry_count)
+            project = Project.find(project_id)
+
+            return unless project
+
+            migration_class.new.safe_perform_one(project, retry_count)
+          end
+        end
+
+        def perform(start_id, end_id)
+          Project.includes(:parent).where(id: start_id..end_id).each do |project|
+            safe_perform_one(project)
+          end
+        end
+
+        def safe_perform_one(project, retry_count = 0)
+          perform_one(project)
+        rescue GRPC::NotFound, GRPC::InvalidArgument, OrphanedNamespaceError
+          nil
+        rescue GRPC::BadStatus
+          schedule_retry(project, retry_count + 1) if retry_count < MAX_RETRIES
+        end
+
+        def schedule_retry(project, retry_count)
+          BackgroundMigrationWorker.perform_in(RETRY_DELAY, self.class::RetryOne.name, [project.id, retry_count])
+        end
+      end
+
+      # Class to add the fullpath to the git repo config
+      class Up < BackfillFullpathMigration
+        # Class used to retry
+        class RetryOne < BaseRetryOne
+          def migration_class
+            Up
+          end
+        end
+
+        def perform_one(project)
+          project.cleanup_repository
+          project.add_fullpath_config
+        end
+      end
+
+      # Class to rollback adding the fullpath to the git repo config
+      class Down < BackfillFullpathMigration
+        # Class used to retry
+        class RetryOne < BaseRetryOne
+          def migration_class
+            Down
+          end
+        end
+
+        def perform_one(project)
+          project.cleanup_repository
+          project.remove_fullpath_config
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb
new file mode 100644
index 00000000000..c8d83cc1803
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_project_repositories.rb
@@ -0,0 +1,229 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Class that will create fill the project_repositories table
+    # for projects an entry is is missing in this table.
+    class BackfillProjectRepositories
+      OrphanedNamespaceError = Class.new(StandardError)
+
+      # Shard model
+      class Shard < ActiveRecord::Base
+        self.table_name = 'shards'
+      end
+
+      # Class that will find or create the shard by name.
+      # There is only a small set of shards, which would
+      # not change quickly, so look them up from memory
+      # instead of hitting the DB each time.
+      class ShardFinder
+        def find_shard_id(name)
+          shard_id = shards.fetch(name, nil)
+          return shard_id if shard_id.present?
+
+          Shard.transaction(requires_new: true) do
+            create!(name)
+          end
+        rescue ActiveRecord::RecordNotUnique
+          reload!
+          retry
+        end
+
+        private
+
+        def create!(name)
+          Shard.create!(name: name).tap { |shard| @shards[name] = shard.id }
+        end
+
+        def shards
+          @shards ||= reload!
+        end
+
+        def reload!
+          @shards = Hash[*Shard.all.map { |shard| [shard.name, shard.id] }.flatten]
+        end
+      end
+
+      module Storage
+        # Class that returns the disk path for a project using hashed storage
+        class HashedProject
+          attr_accessor :project
+
+          ROOT_PATH_PREFIX = '@hashed'
+
+          def initialize(project)
+            @project = project
+          end
+
+          def disk_path
+            "#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}/#{disk_hash}"
+          end
+
+          def disk_hash
+            @disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s)
+          end
+        end
+
+        # Class that returns the disk path for a project using legacy storage
+        class LegacyProject
+          attr_accessor :project
+
+          def initialize(project)
+            @project = project
+          end
+
+          def disk_path
+            project.full_path
+          end
+        end
+      end
+
+      # Concern used by Project and Namespace to determine the full route to the project
+      module Routable
+        extend ActiveSupport::Concern
+
+        def full_path
+          route&.path || build_full_path
+        end
+
+        def build_full_path
+          return path unless has_parent?
+
+          raise OrphanedNamespaceError if parent.nil?
+
+          parent.full_path + '/' + path
+        end
+
+        def has_parent?
+          read_attribute(association(:parent).reflection.foreign_key)
+        end
+      end
+
+      # Route model
+      class Route < ActiveRecord::Base
+        belongs_to :source, inverse_of: :route, polymorphic: true
+      end
+
+      # Namespace model
+      class Namespace < ActiveRecord::Base
+        self.table_name = 'namespaces'
+        self.inheritance_column = nil
+
+        include Routable
+
+        belongs_to :parent, class_name: 'Namespace', inverse_of: 'namespaces'
+
+        has_one :route, -> { where(source_type: 'Namespace') }, inverse_of: :source, foreign_key: :source_id
+
+        has_many :projects, inverse_of: :parent
+        has_many :namespaces, inverse_of: :parent
+      end
+
+      # ProjectRegistry model
+      class ProjectRepository < ActiveRecord::Base
+        self.table_name = 'project_repositories'
+
+        belongs_to :project, inverse_of: :project_repository
+      end
+
+      # Project model
+      class Project < ActiveRecord::Base
+        self.table_name = 'projects'
+
+        include Routable
+
+        HASHED_STORAGE_FEATURES = {
+          repository: 1,
+          attachments: 2
+        }.freeze
+
+        scope :with_parent, -> { includes(:parent) }
+
+        belongs_to :parent, class_name: 'Namespace', foreign_key: :namespace_id, inverse_of: 'projects'
+
+        has_one :route, -> { where(source_type: 'Project') }, inverse_of: :source, foreign_key: :source_id
+        has_one :project_repository, inverse_of: :project
+
+        delegate :disk_path, to: :storage
+
+        class << self
+          def on_hashed_storage
+            where(Project.arel_table[:storage_version]
+              .gteq(HASHED_STORAGE_FEATURES[:repository]))
+          end
+
+          def on_legacy_storage
+            where(Project.arel_table[:storage_version].eq(nil)
+              .or(Project.arel_table[:storage_version].eq(0)))
+          end
+
+          def without_project_repository
+            joins(left_outer_join_project_repository)
+              .where(ProjectRepository.arel_table[:project_id].eq(nil))
+          end
+
+          def left_outer_join_project_repository
+            projects_table = Project.arel_table
+            repository_table = ProjectRepository.arel_table
+
+            projects_table
+              .join(repository_table, Arel::Nodes::OuterJoin)
+              .on(projects_table[:id].eq(repository_table[:project_id]))
+              .join_sources
+          end
+        end
+
+        def storage
+          @storage ||=
+            if hashed_storage?
+              Storage::HashedProject.new(self)
+            else
+              Storage::LegacyProject.new(self)
+            end
+        end
+
+        def hashed_storage?
+          self.storage_version &&
+            self.storage_version >= HASHED_STORAGE_FEATURES[:repository]
+        end
+      end
+
+      def perform(start_id, stop_id)
+        Gitlab::Database.bulk_insert(:project_repositories, project_repositories(start_id, stop_id))
+      end
+
+      private
+
+      def projects
+        raise NotImplementedError,
+          "#{self.class} does not implement #{__method__}"
+      end
+
+      def project_repositories(start_id, stop_id)
+        projects
+          .without_project_repository
+          .includes(:route, parent: [:route]).references(:routes)
+          .includes(:parent).references(:namespaces)
+          .where(id: start_id..stop_id)
+          .map { |project| build_attributes_for_project(project) }
+          .compact
+      end
+
+      def build_attributes_for_project(project)
+        {
+          project_id: project.id,
+          shard_id:   find_shard_id(project.repository_storage),
+          disk_path:  project.disk_path
+        }
+      end
+
+      def find_shard_id(repository_storage)
+        shard_finder.find_shard_id(repository_storage)
+      end
+
+      def shard_finder
+        @shard_finder ||= ShardFinder.new
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/cleanup_concurrent_rename.rb b/lib/gitlab/background_migration/cleanup_concurrent_rename.rb
new file mode 100644
index 00000000000..d3f366f3480
--- /dev/null
+++ b/lib/gitlab/background_migration/cleanup_concurrent_rename.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Background migration for cleaning up a concurrent column rename.
+    class CleanupConcurrentRename < CleanupConcurrentSchemaChange
+      RESCHEDULE_DELAY = 10.minutes
+
+      def cleanup_concurrent_schema_change(table, old_column, new_column)
+        cleanup_concurrent_column_rename(table, old_column, new_column)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb
new file mode 100644
index 00000000000..54f77f184d5
--- /dev/null
+++ b/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Base class for cleaning up concurrent schema changes.
+    class CleanupConcurrentSchemaChange
+      include Database::MigrationHelpers
+
+      # table - The name of the table the migration is performed for.
+      # old_column - The name of the old (to drop) column.
+      # new_column - The name of the new column.
+      def perform(table, old_column, new_column)
+        return unless column_exists?(table, new_column)
+
+        rows_to_migrate = define_model_for(table)
+          .where(new_column => nil)
+          .where
+          .not(old_column => nil)
+
+        if rows_to_migrate.any?
+          BackgroundMigrationWorker.perform_in(
+            RESCHEDULE_DELAY,
+            self.class.name,
+            [table, old_column, new_column]
+          )
+        else
+          cleanup_concurrent_schema_change(table, old_column, new_column)
+        end
+      end
+
+      # These methods are necessary so we can re-use the migration helpers in
+      # this class.
+      def connection
+        ActiveRecord::Base.connection
+      end
+
+      def method_missing(name, *args, &block)
+        connection.__send__(name, *args, &block) # rubocop: disable GitlabSecurity/PublicSend
+      end
+
+      def respond_to_missing?(*args)
+        connection.respond_to?(*args) || super
+      end
+
+      def define_model_for(table)
+        Class.new(ActiveRecord::Base) do
+          self.table_name = table
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb
index de622f657b2..48411095dbb 100644
--- a/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb
+++ b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb
@@ -2,52 +2,12 @@
 
 module Gitlab
   module BackgroundMigration
-    # Background migration for cleaning up a concurrent column rename.
-    class CleanupConcurrentTypeChange
-      include Database::MigrationHelpers
-
+    # Background migration for cleaning up a concurrent column type changeb.
+    class CleanupConcurrentTypeChange < CleanupConcurrentSchemaChange
       RESCHEDULE_DELAY = 10.minutes
 
-      # table - The name of the table the migration is performed for.
-      # old_column - The name of the old (to drop) column.
-      # new_column - The name of the new column.
-      def perform(table, old_column, new_column)
-        return unless column_exists?(:issues, new_column)
-
-        rows_to_migrate = define_model_for(table)
-          .where(new_column => nil)
-          .where
-          .not(old_column => nil)
-
-        if rows_to_migrate.any?
-          BackgroundMigrationWorker.perform_in(
-            RESCHEDULE_DELAY,
-            'CleanupConcurrentTypeChange',
-            [table, old_column, new_column]
-          )
-        else
-          cleanup_concurrent_column_type_change(table, old_column)
-        end
-      end
-
-      # These methods are necessary so we can re-use the migration helpers in
-      # this class.
-      def connection
-        ActiveRecord::Base.connection
-      end
-
-      def method_missing(name, *args, &block)
-        connection.__send__(name, *args, &block) # rubocop: disable GitlabSecurity/PublicSend
-      end
-
-      def respond_to_missing?(*args)
-        connection.respond_to?(*args) || super
-      end
-
-      def define_model_for(table)
-        Class.new(ActiveRecord::Base) do
-          self.table_name = table
-        end
+      def cleanup_concurrent_schema_change(table, old_column, new_column)
+        cleanup_concurrent_column_type_change(table, old_column)
       end
     end
   end
diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb
index 1b4a9e8a194..ccd1f9b4dba 100644
--- a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb
+++ b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-# rubocop:disable Metrics/LineLength
 # rubocop:disable Style/Documentation
 
 module Gitlab
diff --git a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb
index c2bf42f846d..da8265a3a5f 100644
--- a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb
+++ b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-# rubocop:disable Metrics/LineLength
 # rubocop:disable Style/Documentation
 
 class Gitlab::BackgroundMigration::CreateGpgKeySubkeysFromGpgKeys
diff --git a/lib/gitlab/background_migration/delete_diff_files.rb b/lib/gitlab/background_migration/delete_diff_files.rb
new file mode 100644
index 00000000000..664ead1af44
--- /dev/null
+++ b/lib/gitlab/background_migration/delete_diff_files.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class DeleteDiffFiles
+      class MergeRequestDiff < ActiveRecord::Base
+        self.table_name = 'merge_request_diffs'
+
+        belongs_to :merge_request
+        has_many :merge_request_diff_files
+      end
+
+      class MergeRequestDiffFile < ActiveRecord::Base
+        self.table_name = 'merge_request_diff_files'
+      end
+
+      DEAD_TUPLES_THRESHOLD = 50_000
+      VACUUM_WAIT_TIME = 5.minutes
+
+      def perform(ids)
+        @ids = ids
+
+        # We should reschedule until deadtuples get in a desirable
+        # state (e.g. < 50_000). That may take more than one reschedule.
+        #
+        if should_wait_deadtuple_vacuum?
+          reschedule
+          return
+        end
+
+        prune_diff_files
+      end
+
+      def should_wait_deadtuple_vacuum?
+        return false unless Gitlab::Database.postgresql?
+
+        diff_files_dead_tuples_count >= DEAD_TUPLES_THRESHOLD
+      end
+
+      private
+
+      def reschedule
+        BackgroundMigrationWorker.perform_in(VACUUM_WAIT_TIME, self.class.name.demodulize, [@ids])
+      end
+
+      def diffs_collection
+        MergeRequestDiff.where(id: @ids)
+      end
+
+      def diff_files_dead_tuples_count
+        dead_tuple =
+          execute_statement("SELECT n_dead_tup FROM pg_stat_all_tables "\
+                            "WHERE relname = 'merge_request_diff_files'")[0]
+
+        dead_tuple&.fetch('n_dead_tup', 0).to_i
+      end
+
+      def prune_diff_files
+        removed = 0
+        updated = 0
+
+        MergeRequestDiff.transaction do
+          updated = diffs_collection.update_all(state: 'without_files')
+          removed = MergeRequestDiffFile.where(merge_request_diff_id: @ids).delete_all
+        end
+
+        log_info("Removed #{removed} merge_request_diff_files rows, "\
+                 "updated #{updated} merge_request_diffs rows")
+      end
+
+      def execute_statement(sql)
+        ActiveRecord::Base.connection.execute(sql)
+      end
+
+      def log_info(message)
+        Rails.logger.info("BackgroundMigration::DeleteDiffFiles - #{message}")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb
index a357538a885..58df74cfa9b 100644
--- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb
+++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb
@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 # rubocop:disable Metrics/MethodLength
-# rubocop:disable Metrics/LineLength
 # rubocop:disable Metrics/AbcSize
 # rubocop:disable Style/Documentation
 
diff --git a/lib/gitlab/background_migration/digest_column.rb b/lib/gitlab/background_migration/digest_column.rb
new file mode 100644
index 00000000000..22a3bb8f8f3
--- /dev/null
+++ b/lib/gitlab/background_migration/digest_column.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# rubocop:disable Style/Documentation
+module Gitlab
+  module BackgroundMigration
+    class DigestColumn
+      class PersonalAccessToken < ActiveRecord::Base
+        self.table_name = 'personal_access_tokens'
+      end
+
+      def perform(model, attribute_from, attribute_to, start_id, stop_id)
+        model = model.constantize if model.is_a?(String)
+
+        model.transaction do
+          relation = model.where(id: start_id..stop_id).where.not(attribute_from => nil).lock
+
+          relation.each do |instance|
+            instance.update_columns(attribute_to => Gitlab::CryptoHelper.sha256(instance.read_attribute(attribute_from)),
+                                    attribute_from => nil)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/encrypt_columns.rb b/lib/gitlab/background_migration/encrypt_columns.rb
new file mode 100644
index 00000000000..b9ad8267e37
--- /dev/null
+++ b/lib/gitlab/background_migration/encrypt_columns.rb
@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # EncryptColumn migrates data from an unencrypted column - `foo`, say - to
+    # an encrypted column - `encrypted_foo`, say.
+    #
+    # To avoid depending on a particular version of the model in app/, add a
+    # model to `lib/gitlab/background_migration/models/encrypt_columns` and use
+    # it in the migration that enqueues the jobs, so code can be shared.
+    #
+    # For this background migration to work, the table that is migrated _has_ to
+    # have an `id` column as the primary key. Additionally, the encrypted column
+    # should be managed by attr_encrypted, and map to an attribute with the same
+    # name as the unencrypted column (i.e., the unencrypted column should be
+    # shadowed), unless you want to define specific methods / accessors in the
+    # temporary model in `/models/encrypt_columns/your_model.rb`.
+    #
+    class EncryptColumns
+      def perform(model, attributes, from, to)
+        model = model.constantize if model.is_a?(String)
+
+        # If sidekiq hasn't undergone a restart, its idea of what columns are
+        # present may be inaccurate, so ensure this is as fresh as possible
+        model.reset_column_information
+        model.define_attribute_methods
+
+        attributes = expand_attributes(model, Array(attributes).map(&:to_sym))
+
+        model.transaction do
+          # Use SELECT ... FOR UPDATE to prevent the value being changed while
+          # we are encrypting it
+          relation = model.where(id: from..to).lock
+
+          relation.each do |instance|
+            encrypt!(instance, attributes)
+          end
+        end
+      end
+
+      def clear_migrated_values?
+        true
+      end
+
+      private
+
+      # Build a hash of { attribute => encrypted column name }
+      def expand_attributes(klass, attributes)
+        expanded = attributes.flat_map do |attribute|
+          attr_config = klass.encrypted_attributes[attribute]
+          crypt_column_name = attr_config&.fetch(:attribute)
+
+          raise "Couldn't determine encrypted column for #{klass}##{attribute}" if
+            crypt_column_name.nil?
+
+          raise "#{klass} source column: #{attribute} is missing" unless
+            klass.column_names.include?(attribute.to_s)
+
+          # Running the migration without the destination column being present
+          # leads to data loss
+          raise "#{klass} destination column: #{crypt_column_name} is missing" unless
+            klass.column_names.include?(crypt_column_name.to_s)
+
+          [attribute, crypt_column_name]
+        end
+
+        Hash[*expanded]
+      end
+
+      # Generate ciphertext for each column and update the database
+      def encrypt!(instance, attributes)
+        to_clear = attributes
+          .map { |plain, crypt| apply_attribute!(instance, plain, crypt) }
+          .compact
+          .flat_map { |plain| [plain, nil] }
+
+        to_clear = Hash[*to_clear]
+
+        if instance.changed?
+          instance.save!
+
+          if clear_migrated_values?
+            instance.update_columns(to_clear)
+          end
+        end
+      end
+
+      def apply_attribute!(instance, plain_column, crypt_column)
+        plaintext = instance[plain_column]
+        ciphertext = instance[crypt_column]
+
+        # No need to do anything if the plaintext is nil, or an encrypted
+        # value already exists
+        return nil unless plaintext.present? && !ciphertext.present?
+
+        # attr_encrypted will calculate and set the expected value for us
+        instance.public_send("#{plain_column}=", plaintext) # rubocop:disable GitlabSecurity/PublicSend
+
+        plain_column
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/encrypt_runners_tokens.rb b/lib/gitlab/background_migration/encrypt_runners_tokens.rb
new file mode 100644
index 00000000000..91e559a8765
--- /dev/null
+++ b/lib/gitlab/background_migration/encrypt_runners_tokens.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # EncryptColumn migrates data from an unencrypted column - `foo`, say - to
+    # an encrypted column - `encrypted_foo`, say.
+    #
+    # We only create a subclass here because we want to isolate this migration
+    # (migrating unencrypted runner registration tokens to encrypted columns)
+    # from other `EncryptColumns` migration. This class name is going to be
+    # serialized and stored in Redis and later picked by Sidekiq, so we need to
+    # create a separate class name in order to isolate these migration tasks.
+    #
+    # We can solve this differently, see tech debt issue:
+    #
+    # https://gitlab.com/gitlab-org/gitlab-ce/issues/54328
+    #
+    class EncryptRunnersTokens < EncryptColumns
+      def perform(model, from, to)
+        resource = "::Gitlab::BackgroundMigration::Models::EncryptColumns::#{model.to_s.capitalize}"
+        model = resource.constantize
+        attributes = model.encrypted_attributes.keys
+
+        super(model, attributes, from, to)
+      end
+
+      def clear_migrated_values?
+        false
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb
index 22b0ac71920..103bd98af14 100644
--- a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb
+++ b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-# rubocop:disable Metrics/AbcSize
 # rubocop:disable Style/Documentation
 
 module Gitlab
diff --git a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb
index d0816ae3ed5..77c1f1ffaf0 100644
--- a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb
+++ b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-# rubocop:disable Metrics/AbcSize
 # rubocop:disable Style/Documentation
 
 module Gitlab
diff --git a/lib/gitlab/background_migration/fill_store_upload.rb b/lib/gitlab/background_migration/fill_store_upload.rb
index 94c65459a67..cba3e21cea6 100644
--- a/lib/gitlab/background_migration/fill_store_upload.rb
+++ b/lib/gitlab/background_migration/fill_store_upload.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-# rubocop:disable Metrics/AbcSize
 # rubocop:disable Style/Documentation
 
 module Gitlab
diff --git a/lib/gitlab/background_migration/fix_cross_project_label_links.rb b/lib/gitlab/background_migration/fix_cross_project_label_links.rb
new file mode 100644
index 00000000000..0a12401c35f
--- /dev/null
+++ b/lib/gitlab/background_migration/fix_cross_project_label_links.rb
@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class FixCrossProjectLabelLinks
+      GROUP_NESTED_LEVEL = 10.freeze
+
+      class Project < ActiveRecord::Base
+        self.table_name = 'projects'
+      end
+
+      class Label < ActiveRecord::Base
+        self.inheritance_column = :_type_disabled
+        self.table_name = 'labels'
+      end
+
+      class LabelLink < ActiveRecord::Base
+        self.table_name = 'label_links'
+      end
+
+      class Issue < ActiveRecord::Base
+        self.table_name = 'issues'
+      end
+
+      class MergeRequest < ActiveRecord::Base
+        self.table_name = 'merge_requests'
+      end
+
+      class Namespace < ActiveRecord::Base
+        self.inheritance_column = :_type_disabled
+        self.table_name = 'namespaces'
+
+        def self.groups_with_descendants_ids(start_id, stop_id)
+          # To isolate migration code, we avoid usage of
+          # Gitlab::GroupHierarchy#base_and_descendants which already
+          # does this job better
+          ids = Namespace.where(type: 'Group', id: Label.where(type: 'GroupLabel').select('distinct group_id')).where(id: start_id..stop_id).pluck(:id)
+          group_ids = ids
+
+          GROUP_NESTED_LEVEL.times do
+            ids = Namespace.where(type: 'Group', parent_id: ids).pluck(:id)
+            break if ids.empty?
+
+            group_ids += ids
+          end
+
+          group_ids.uniq
+        end
+      end
+
+      def perform(start_id, stop_id)
+        group_ids = Namespace.groups_with_descendants_ids(start_id, stop_id)
+        project_ids = Project.where(namespace_id: group_ids).select(:id)
+
+        fix_issues(project_ids)
+        fix_merge_requests(project_ids)
+      end
+
+      private
+
+      # select IDs of issues which reference a label which is:
+      # a) a project label of a different project, or
+      # b) a group label of a different group than issue's project group
+      def fix_issues(project_ids)
+        issue_ids = Label
+          .joins('INNER JOIN label_links ON label_links.label_id = labels.id AND label_links.target_type = \'Issue\'
+                  INNER JOIN issues ON issues.id = label_links.target_id
+                  INNER JOIN projects ON projects.id = issues.project_id')
+          .where('issues.project_id in (?)', project_ids)
+          .where('(labels.project_id is not null and labels.project_id != issues.project_id) '\
+                 'or (labels.group_id is not null and labels.group_id != projects.namespace_id)')
+          .select('distinct issues.id')
+
+        Issue.where(id: issue_ids).find_each { |issue| check_resource_labels(issue, issue.project_id) }
+      end
+
+      # select IDs of MRs which reference a label which is:
+      # a) a project label of a different project, or
+      # b) a group label of a different group than MR's project group
+      def fix_merge_requests(project_ids)
+        mr_ids = Label
+          .joins('INNER JOIN label_links ON label_links.label_id = labels.id AND label_links.target_type = \'MergeRequest\'
+                  INNER JOIN merge_requests ON merge_requests.id = label_links.target_id
+                  INNER JOIN projects ON projects.id = merge_requests.target_project_id')
+          .where('merge_requests.target_project_id in (?)', project_ids)
+          .where('(labels.project_id is not null and labels.project_id != merge_requests.target_project_id) '\
+                 'or (labels.group_id is not null and labels.group_id != projects.namespace_id)')
+          .select('distinct merge_requests.id')
+
+        MergeRequest.where(id: mr_ids).find_each { |merge_request| check_resource_labels(merge_request, merge_request.target_project_id) }
+      end
+
+      def check_resource_labels(resource, project_id)
+        local_labels = available_labels(project_id)
+
+        # get all label links for the given resource (issue/MR)
+        # which reference a label not included in avaiable_labels
+        # (other than its project labels and labels of ancestor groups)
+        cross_labels = LabelLink
+          .select('label_id, labels.title as title, labels.color as color, label_links.id as label_link_id')
+          .joins('INNER JOIN labels ON labels.id = label_links.label_id')
+          .where(target_type: resource.class.name.demodulize, target_id: resource.id)
+          .where('labels.id not in (?)', local_labels.select(:id))
+
+        cross_labels.each do |label|
+          matching_label = local_labels.find {|l| l.title == label.title && l.color == label.color}
+
+          next unless matching_label
+
+          Rails.logger.info "#{resource.class.name.demodulize} #{resource.id}: replacing #{label.label_id} with #{matching_label.id}"
+          LabelLink.update(label.label_link_id, label_id: matching_label.id)
+        end
+      end
+
+      # get all labels available for the project (including
+      # group labels of ancestor groups)
+      def available_labels(project_id)
+        @labels ||= {}
+        @labels[project_id] ||= Label
+          .where("(type = 'GroupLabel' and group_id in (?)) or (type = 'ProjectLabel' and id = ?)",
+                 project_group_ids(project_id),
+                 project_id)
+      end
+
+      def project_group_ids(project_id)
+        ids = [Project.find(project_id).namespace_id]
+
+        GROUP_NESTED_LEVEL.times do
+          group = Namespace.find(ids.last)
+          break unless group.parent_id
+
+          ids << group.parent_id
+        end
+
+        ids
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/migrate_build_stage.rb b/lib/gitlab/background_migration/migrate_build_stage.rb
index 242e3143e71..268c6083d3c 100644
--- a/lib/gitlab/background_migration/migrate_build_stage.rb
+++ b/lib/gitlab/background_migration/migrate_build_stage.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-# rubocop:disable Metrics/AbcSize
 # rubocop:disable Style/Documentation
 
 module Gitlab
diff --git a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb
index 7088aa0860a..38fecac1bfe 100644
--- a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb
+++ b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-# rubocop:disable Metrics/LineLength
 # rubocop:disable Style/Documentation
 
 module Gitlab
diff --git a/lib/gitlab/background_migration/migrate_legacy_artifacts.rb b/lib/gitlab/background_migration/migrate_legacy_artifacts.rb
new file mode 100644
index 00000000000..5cd638083b0
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_legacy_artifacts.rb
@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+# rubocop:disable Metrics/ClassLength
+
+module Gitlab
+  module BackgroundMigration
+    ##
+    # The class to migrate job artifacts from `ci_builds` to `ci_job_artifacts`
+    class MigrateLegacyArtifacts
+      FILE_LOCAL_STORE = 1 # equal to ObjectStorage::Store::LOCAL
+      ARCHIVE_FILE_TYPE = 1 # equal to Ci::JobArtifact.file_types['archive']
+      METADATA_FILE_TYPE = 2 # equal to Ci::JobArtifact.file_types['metadata']
+      LEGACY_PATH_FILE_LOCATION = 1 # equal to Ci::JobArtifact.file_location['legacy_path']
+
+      def perform(start_id, stop_id)
+        ActiveRecord::Base.transaction do
+          insert_archives(start_id, stop_id)
+          insert_metadatas(start_id, stop_id)
+          delete_legacy_artifacts(start_id, stop_id)
+        end
+      end
+
+      private
+
+      def insert_archives(start_id, stop_id)
+        ActiveRecord::Base.connection.execute <<~SQL
+          INSERT INTO
+              ci_job_artifacts (
+                  project_id,
+                  job_id,
+                  expire_at,
+                  file_location,
+                  created_at,
+                  updated_at,
+                  file,
+                  size,
+                  file_store,
+                  file_type
+              )
+          SELECT
+              project_id,
+              id,
+              artifacts_expire_at,
+              #{LEGACY_PATH_FILE_LOCATION},
+              created_at,
+              created_at,
+              artifacts_file,
+              artifacts_size,
+              COALESCE(artifacts_file_store, #{FILE_LOCAL_STORE}),
+              #{ARCHIVE_FILE_TYPE}
+          FROM
+              ci_builds
+          WHERE
+              id BETWEEN #{start_id.to_i} AND #{stop_id.to_i}
+              AND artifacts_file <> ''
+              AND NOT EXISTS (
+                  SELECT
+                      1
+                  FROM
+                      ci_job_artifacts
+                  WHERE
+                      ci_builds.id = ci_job_artifacts.job_id
+                      AND ci_job_artifacts.file_type = #{ARCHIVE_FILE_TYPE})
+        SQL
+      end
+
+      def insert_metadatas(start_id, stop_id)
+        ActiveRecord::Base.connection.execute <<~SQL
+          INSERT INTO
+              ci_job_artifacts (
+                  project_id,
+                  job_id,
+                  expire_at,
+                  file_location,
+                  created_at,
+                  updated_at,
+                  file,
+                  size,
+                  file_store,
+                  file_type
+              )
+          SELECT
+              project_id,
+              id,
+              artifacts_expire_at,
+              #{LEGACY_PATH_FILE_LOCATION},
+              created_at,
+              created_at,
+              artifacts_metadata,
+              NULL,
+              COALESCE(artifacts_metadata_store, #{FILE_LOCAL_STORE}),
+              #{METADATA_FILE_TYPE}
+          FROM
+              ci_builds
+          WHERE
+              id BETWEEN #{start_id.to_i} AND #{stop_id.to_i}
+              AND artifacts_file <> ''
+              AND artifacts_metadata <> ''
+              AND NOT EXISTS (
+                  SELECT
+                      1
+                  FROM
+                      ci_job_artifacts
+                  WHERE
+                      ci_builds.id = ci_job_artifacts.job_id
+                      AND ci_job_artifacts.file_type = #{METADATA_FILE_TYPE})
+        SQL
+      end
+
+      def delete_legacy_artifacts(start_id, stop_id)
+        ActiveRecord::Base.connection.execute <<~SQL
+          UPDATE
+              ci_builds
+          SET
+              artifacts_file = NULL,
+              artifacts_file_store = NULL,
+              artifacts_size = NULL,
+              artifacts_metadata = NULL,
+              artifacts_metadata_store = NULL
+          WHERE
+              id BETWEEN #{start_id.to_i} AND #{stop_id.to_i}
+              AND artifacts_file <> ''
+        SQL
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/migrate_stage_status.rb b/lib/gitlab/background_migration/migrate_stage_status.rb
index 0e5c7f092f2..6a29a632577 100644
--- a/lib/gitlab/background_migration/migrate_stage_status.rb
+++ b/lib/gitlab/background_migration/migrate_stage_status.rb
@@ -16,10 +16,10 @@ module Gitlab
         scope :running, -> { where(status: 'running') }
         scope :pending, -> { where(status: 'pending') }
         scope :success, -> { where(status: 'success') }
-        scope :failed, -> { where(status: 'failed')  }
-        scope :canceled, -> { where(status: 'canceled')  }
-        scope :skipped, -> { where(status: 'skipped')  }
-        scope :manual, -> { where(status: 'manual')  }
+        scope :failed, -> { where(status: 'failed') }
+        scope :canceled, -> { where(status: 'canceled') }
+        scope :skipped, -> { where(status: 'skipped') }
+        scope :manual, -> { where(status: 'manual') }
 
         scope :failed_but_allowed, -> do
           where(allow_failure: true, status: [:failed, :canceled])
diff --git a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb
index 7f243073fd0..ef50fe4adb1 100644
--- a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb
+++ b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-# rubocop:disable Metrics/LineLength
 # rubocop:disable Style/Documentation
 
 module Gitlab
diff --git a/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb b/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb
new file mode 100644
index 00000000000..41f18979d76
--- /dev/null
+++ b/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module Models
+      module EncryptColumns
+        # This model is shared between synchronous and background migrations to
+        # encrypt the `runners_token` column in `namespaces` table.
+        #
+        class Namespace < ActiveRecord::Base
+          include ::EachBatch
+
+          self.table_name = 'namespaces'
+          self.inheritance_column = :_type_disabled
+
+          def runners_token=(value)
+            self.runners_token_encrypted =
+              ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value)
+          end
+
+          def self.encrypted_attributes
+            { runners_token: { attribute: :runners_token_encrypted } }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/models/encrypt_columns/project.rb b/lib/gitlab/background_migration/models/encrypt_columns/project.rb
new file mode 100644
index 00000000000..bfeae14584d
--- /dev/null
+++ b/lib/gitlab/background_migration/models/encrypt_columns/project.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module Models
+      module EncryptColumns
+        # This model is shared between synchronous and background migrations to
+        # encrypt the `runners_token` column in `projects` table.
+        #
+        class Project < ActiveRecord::Base
+          include ::EachBatch
+
+          self.table_name = 'projects'
+          self.inheritance_column = :_type_disabled
+
+          def runners_token=(value)
+            self.runners_token_encrypted =
+              ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value)
+          end
+
+          def self.encrypted_attributes
+            { runners_token: { attribute: :runners_token_encrypted } }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/models/encrypt_columns/runner.rb b/lib/gitlab/background_migration/models/encrypt_columns/runner.rb
new file mode 100644
index 00000000000..14ddce4b147
--- /dev/null
+++ b/lib/gitlab/background_migration/models/encrypt_columns/runner.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module Models
+      module EncryptColumns
+        # This model is shared between synchronous and background migrations to
+        # encrypt the `token` column in `ci_runners` table.
+        #
+        class Runner < ActiveRecord::Base
+          include ::EachBatch
+
+          self.table_name = 'ci_runners'
+          self.inheritance_column = :_type_disabled
+
+          def token=(value)
+            self.token_encrypted =
+              ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value)
+          end
+
+          def self.encrypted_attributes
+            { token: { attribute: :token_encrypted } }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/models/encrypt_columns/settings.rb b/lib/gitlab/background_migration/models/encrypt_columns/settings.rb
new file mode 100644
index 00000000000..08ae35c0671
--- /dev/null
+++ b/lib/gitlab/background_migration/models/encrypt_columns/settings.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module Models
+      module EncryptColumns
+        # This model is shared between synchronous and background migrations to
+        # encrypt the `runners_token` column in `application_settings` table.
+        #
+        class Settings < ActiveRecord::Base
+          include ::EachBatch
+          include ::CacheableAttributes
+
+          self.table_name = 'application_settings'
+          self.inheritance_column = :_type_disabled
+
+          after_commit do
+            ::ApplicationSetting.expire
+          end
+
+          def runners_registration_token=(value)
+            self.runners_registration_token_encrypted =
+              ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value)
+          end
+
+          def self.encrypted_attributes
+            {
+              runners_registration_token: {
+                attribute: :runners_registration_token_encrypted
+              }
+            }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb
new file mode 100644
index 00000000000..34e72fd9f34
--- /dev/null
+++ b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module Models
+      module EncryptColumns
+        # This model is shared between synchronous and background migrations to
+        # encrypt the `token` and `url` columns
+        class WebHook < ActiveRecord::Base
+          include ::EachBatch
+
+          self.table_name = 'web_hooks'
+          self.inheritance_column = :_type_disabled
+
+          attr_encrypted :token,
+                         mode:      :per_attribute_iv,
+                         algorithm: 'aes-256-gcm',
+                         key:       ::Settings.attr_encrypted_db_key_base_32
+
+          attr_encrypted :url,
+                         mode:      :per_attribute_iv,
+                         algorithm: 'aes-256-gcm',
+                         key:       ::Settings.attr_encrypted_db_key_base_32
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/move_personal_snippet_files.rb b/lib/gitlab/background_migration/move_personal_snippet_files.rb
index a4ef51fd0e8..5b2b2af718a 100644
--- a/lib/gitlab/background_migration/move_personal_snippet_files.rb
+++ b/lib/gitlab/background_migration/move_personal_snippet_files.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-# rubocop:disable Metrics/LineLength
 # rubocop:disable Style/Documentation
 
 module Gitlab
diff --git a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb
index d9d3d2e667b..698f5e46c0c 100644
--- a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb
+++ b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb
@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 # rubocop:disable Metrics/MethodLength
-# rubocop:disable Metrics/LineLength
 # rubocop:disable Metrics/ClassLength
 # rubocop:disable Metrics/BlockLength
 # rubocop:disable Style/Documentation
diff --git a/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb b/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb
new file mode 100644
index 00000000000..35bfc381180
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb
@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+#
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class PopulateClusterKubernetesNamespaceTable
+      include Gitlab::Database::MigrationHelpers
+
+      BATCH_SIZE = 1_000
+
+      module Migratable
+        class KubernetesNamespace < ActiveRecord::Base
+          self.table_name = 'clusters_kubernetes_namespaces'
+        end
+
+        class ClusterProject < ActiveRecord::Base
+          include EachBatch
+
+          self.table_name = 'cluster_projects'
+
+          belongs_to :project
+
+          def self.with_no_kubernetes_namespace
+            where.not(id: Migratable::KubernetesNamespace.select(:cluster_project_id))
+          end
+
+          def namespace
+            slug = "#{project.path}-#{project.id}".downcase
+            slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '')
+          end
+
+          def service_account
+            "#{namespace}-service-account"
+          end
+        end
+
+        class Project < ActiveRecord::Base
+          self.table_name = 'projects'
+        end
+      end
+
+      def perform
+        cluster_projects_with_no_kubernetes_namespace.each_batch(of: BATCH_SIZE) do |cluster_projects_batch, index|
+          sql_values = sql_values_for(cluster_projects_batch)
+
+          insert_into_cluster_kubernetes_namespace(sql_values)
+        end
+      end
+
+      private
+
+      def cluster_projects_with_no_kubernetes_namespace
+        Migratable::ClusterProject.with_no_kubernetes_namespace
+      end
+
+      def sql_values_for(cluster_projects)
+        cluster_projects.map do |cluster_project|
+          values_for_cluster_project(cluster_project)
+        end
+      end
+
+      def values_for_cluster_project(cluster_project)
+        {
+          cluster_project_id: cluster_project.id,
+          cluster_id: cluster_project.cluster_id,
+          project_id: cluster_project.project_id,
+          namespace: cluster_project.namespace,
+          service_account_name: cluster_project.service_account,
+          created_at: 'NOW()',
+          updated_at: 'NOW()'
+        }
+      end
+
+      def insert_into_cluster_kubernetes_namespace(rows)
+        Gitlab::Database.bulk_insert(Migratable::KubernetesNamespace.table_name,
+                                     rows,
+                                     disable_quote: [:created_at, :updated_at])
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/populate_external_pipeline_source.rb b/lib/gitlab/background_migration/populate_external_pipeline_source.rb
new file mode 100644
index 00000000000..036fe641757
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_external_pipeline_source.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class PopulateExternalPipelineSource
+      module Migratable
+        class Pipeline < ActiveRecord::Base
+          self.table_name = 'ci_pipelines'
+
+          def self.sources
+            {
+              unknown: nil,
+              push: 1,
+              web: 2,
+              trigger: 3,
+              schedule: 4,
+              api: 5,
+              external: 6
+            }
+          end
+        end
+
+        class CommitStatus < ActiveRecord::Base
+          self.table_name = 'ci_builds'
+          self.inheritance_column = :_type_disabled
+
+          scope :has_pipeline, -> { where('ci_builds.commit_id=ci_pipelines.id') }
+          scope :of_type, -> (type) { where('type=?', type) }
+        end
+      end
+
+      def perform(start_id, stop_id)
+        external_pipelines(start_id, stop_id)
+          .update_all(source: Migratable::Pipeline.sources[:external])
+      end
+
+      private
+
+      def external_pipelines(start_id, stop_id)
+        Migratable::Pipeline.where(id: (start_id..stop_id))
+          .where(
+            'EXISTS (?) AND NOT EXISTS (?)',
+            Migratable::CommitStatus.of_type('GenericCommitStatus').has_pipeline.select(1),
+            Migratable::CommitStatus.of_type('Ci::Build').has_pipeline.select(1)
+          )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/populate_fork_networks_range.rb b/lib/gitlab/background_migration/populate_fork_networks_range.rb
index a976cb4c243..aa4f130538c 100644
--- a/lib/gitlab/background_migration/populate_fork_networks_range.rb
+++ b/lib/gitlab/background_migration/populate_fork_networks_range.rb
@@ -19,7 +19,7 @@ module Gitlab
         create_fork_networks_for_missing_projects(start_id, end_id)
         create_fork_networks_memberships_for_root_projects(start_id, end_id)
 
-        delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY # rubocop:disable Metrics/LineLength
+        delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY
         BackgroundMigrationWorker.perform_in(
           delay, "CreateForkNetworkMembershipsRange", [start_id, end_id]
         )
diff --git a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb
index 8a901a9bf39..d89ce358bb9 100644
--- a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb
+++ b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb
@@ -1,7 +1,4 @@
 # frozen_string_literal: true
-# rubocop:disable Metrics/LineLength
-# rubocop:disable Metrics/MethodLength
-# rubocop:disable Metrics/ClassLength
 # rubocop:disable Style/Documentation
 
 module Gitlab
diff --git a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data_improved.rb b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data_improved.rb
new file mode 100644
index 00000000000..37592d67dd9
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data_improved.rb
@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class PopulateMergeRequestMetricsWithEventsDataImproved
+      CLOSED_EVENT_ACTION = 3
+      MERGED_EVENT_ACTION = 7
+
+      def perform(min_merge_request_id, max_merge_request_id)
+        insert_metrics_for_range(min_merge_request_id, max_merge_request_id)
+        update_metrics_with_events_data(min_merge_request_id, max_merge_request_id)
+      end
+
+      # Inserts merge_request_metrics records for merge_requests without it for
+      # a given merge request batch.
+      def insert_metrics_for_range(min, max)
+        metrics_not_exists_clause =
+          <<-SQL.strip_heredoc
+            NOT EXISTS (SELECT 1 FROM merge_request_metrics
+                        WHERE merge_request_metrics.merge_request_id = merge_requests.id)
+          SQL
+
+        MergeRequest.where(metrics_not_exists_clause).where(id: min..max).each_batch do |batch|
+          select_sql = batch.select(:id, :created_at, :updated_at).to_sql
+
+          execute("INSERT INTO merge_request_metrics (merge_request_id, created_at, updated_at) #{select_sql}")
+        end
+      end
+
+      def update_metrics_with_events_data(min, max)
+        if Gitlab::Database.postgresql?
+          psql_update_metrics_with_events_data(min, max)
+        else
+          mysql_update_metrics_with_events_data(min, max)
+        end
+      end
+
+      def psql_update_metrics_with_events_data(min, max)
+        update_sql = <<-SQL.strip_heredoc
+          UPDATE merge_request_metrics
+          SET (latest_closed_at,
+             latest_closed_by_id) =
+          ( SELECT updated_at,
+                   author_id
+           FROM events
+           WHERE target_id = merge_request_id
+             AND target_type = 'MergeRequest'
+             AND action = #{CLOSED_EVENT_ACTION}
+           ORDER BY id DESC
+           LIMIT 1 ),
+             merged_by_id =
+          ( SELECT author_id
+           FROM events
+           WHERE target_id = merge_request_id
+             AND target_type = 'MergeRequest'
+             AND action = #{MERGED_EVENT_ACTION}
+           ORDER BY id DESC
+           LIMIT 1 )
+          WHERE merge_request_id BETWEEN #{min} AND #{max}
+        SQL
+
+        execute(update_sql)
+      end
+
+      def mysql_update_metrics_with_events_data(min, max)
+        closed_updated_at_subquery = mysql_events_select(:updated_at, CLOSED_EVENT_ACTION)
+        closed_author_id_subquery = mysql_events_select(:author_id, CLOSED_EVENT_ACTION)
+        merged_author_id_subquery = mysql_events_select(:author_id, MERGED_EVENT_ACTION)
+
+        update_sql = <<-SQL.strip_heredoc
+          UPDATE merge_request_metrics
+          SET latest_closed_at = (#{closed_updated_at_subquery}),
+              latest_closed_by_id = (#{closed_author_id_subquery}),
+              merged_by_id = (#{merged_author_id_subquery})
+          WHERE merge_request_id BETWEEN #{min} AND #{max}
+        SQL
+
+        execute(update_sql)
+      end
+
+      def mysql_events_select(column, action)
+        <<-SQL.strip_heredoc
+          SELECT #{column} FROM events
+           WHERE target_id = merge_request_id
+             AND target_type = 'MergeRequest'
+             AND action = #{action}
+           ORDER BY id DESC
+           LIMIT 1
+        SQL
+      end
+
+      def execute(sql)
+        @connection ||= ActiveRecord::Base.connection
+        @connection.execute(sql)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb
index 9232f20a063..a19dc9747fb 100644
--- a/lib/gitlab/background_migration/populate_untracked_uploads.rb
+++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb
@@ -4,7 +4,7 @@ module Gitlab
   module BackgroundMigration
     # This class processes a batch of rows in `untracked_files_for_uploads` by
     # adding each file to the `uploads` table if it does not exist.
-    class PopulateUntrackedUploads # rubocop:disable Metrics/ClassLength
+    class PopulateUntrackedUploads
       def perform(start_id, end_id)
         return unless migrate?
 
diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb
index a2c5acbde71..4a9a62aaeb5 100644
--- a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb
+++ b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb
@@ -4,7 +4,7 @@ module Gitlab
     module PopulateUntrackedUploadsDependencies
       # This class is responsible for producing the attributes necessary to
       # track an uploaded file in the `uploads` table.
-      class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength, Metrics/LineLength
+      class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength
         self.table_name = 'untracked_files_for_uploads'
 
         # Ends with /:random_hex/:filename
@@ -134,7 +134,7 @@ module Gitlab
 
         # Not including a leading slash
         def path_relative_to_upload_dir
-          upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR # rubocop:disable Metrics/LineLength
+          upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR
           base = %r{\A#{Regexp.escape(upload_dir)}/}
           @path_relative_to_upload_dir ||= path.sub(base, '')
         end
diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb
index 914a9e48a2f..81ca2b0a9b7 100644
--- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb
+++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb
@@ -54,7 +54,8 @@ module Gitlab
 
       def ensure_temporary_tracking_table_exists
         table_name = :untracked_files_for_uploads
-        unless UntrackedFile.connection.table_exists?(table_name)
+
+        unless ActiveRecord::Base.connection.data_source_exists?(table_name)
           UntrackedFile.connection.create_table table_name do |t|
             t.string :path, limit: 600, null: false
             t.index :path, unique: true
@@ -143,7 +144,7 @@ module Gitlab
 
       def table_columns_and_values_for_insert(file_paths)
         values = file_paths.map do |file_path|
-          ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend, Metrics/LineLength
+          ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend
         end.join(', ')
 
         "#{UntrackedFile.table_name} (path) VALUES #{values}"
diff --git a/lib/gitlab/background_migration/redact_links.rb b/lib/gitlab/background_migration/redact_links.rb
new file mode 100644
index 00000000000..92256e59a6c
--- /dev/null
+++ b/lib/gitlab/background_migration/redact_links.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+require_relative 'redact_links/redactable'
+
+module Gitlab
+  module BackgroundMigration
+    class RedactLinks
+      class Note < ActiveRecord::Base
+        include EachBatch
+        include ::Gitlab::BackgroundMigration::RedactLinks::Redactable
+
+        self.table_name = 'notes'
+        self.inheritance_column = :_type_disabled
+      end
+
+      class Issue < ActiveRecord::Base
+        include EachBatch
+        include ::Gitlab::BackgroundMigration::RedactLinks::Redactable
+
+        self.table_name = 'issues'
+        self.inheritance_column = :_type_disabled
+      end
+
+      class MergeRequest < ActiveRecord::Base
+        include EachBatch
+        include ::Gitlab::BackgroundMigration::RedactLinks::Redactable
+
+        self.table_name = 'merge_requests'
+        self.inheritance_column = :_type_disabled
+      end
+
+      class Snippet < ActiveRecord::Base
+        include EachBatch
+        include ::Gitlab::BackgroundMigration::RedactLinks::Redactable
+
+        self.table_name = 'snippets'
+        self.inheritance_column = :_type_disabled
+      end
+
+      def perform(model_name, field, start_id, stop_id)
+        link_pattern = "%/sent_notifications/" + ("_" * 32) + "/unsubscribe%"
+        model = "Gitlab::BackgroundMigration::RedactLinks::#{model_name}".constantize
+
+        model.where("#{field} like ?", link_pattern).where(id: start_id..stop_id).each do |resource|
+          resource.redact_field!(field)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/redact_links/redactable.rb b/lib/gitlab/background_migration/redact_links/redactable.rb
new file mode 100644
index 00000000000..baab34221f1
--- /dev/null
+++ b/lib/gitlab/background_migration/redact_links/redactable.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class RedactLinks
+      module Redactable
+        extend ActiveSupport::Concern
+
+        def redact_field!(field)
+          self[field].gsub!(%r{/sent_notifications/\h{32}/unsubscribe}, '/sent_notifications/REDACTED/unsubscribe')
+
+          if self.changed?
+            self.update_columns(field => self[field],
+                                "#{field}_html" => nil)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/remove_restricted_todos.rb b/lib/gitlab/background_migration/remove_restricted_todos.rb
new file mode 100644
index 00000000000..47579d46c1b
--- /dev/null
+++ b/lib/gitlab/background_migration/remove_restricted_todos.rb
@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+# rubocop:disable Metrics/ClassLength
+
+module Gitlab
+  module BackgroundMigration
+    class RemoveRestrictedTodos
+      PRIVATE_FEATURE = 10
+      PRIVATE_PROJECT = 0
+
+      class Project < ActiveRecord::Base
+        self.table_name = 'projects'
+      end
+
+      class ProjectAuthorization < ActiveRecord::Base
+        self.table_name = 'project_authorizations'
+      end
+
+      class ProjectFeature < ActiveRecord::Base
+        self.table_name = 'project_features'
+      end
+
+      class Todo < ActiveRecord::Base
+        include EachBatch
+
+        self.table_name = 'todos'
+      end
+
+      class Issue < ActiveRecord::Base
+        include EachBatch
+
+        self.table_name = 'issues'
+      end
+
+      def perform(start_id, stop_id)
+        projects = Project.where('EXISTS (SELECT 1 FROM todos WHERE todos.project_id = projects.id)')
+          .where(id: start_id..stop_id)
+
+        projects.each do |project|
+          remove_confidential_issue_todos(project.id)
+
+          if project.visibility_level == PRIVATE_PROJECT
+            remove_non_members_todos(project.id)
+          else
+            remove_restricted_features_todos(project.id)
+          end
+        end
+      end
+
+      private
+
+      def remove_non_members_todos(project_id)
+        if Gitlab::Database.postgresql?
+          batch_remove_todos_cte(project_id)
+        else
+          unauthorized_project_todos(project_id)
+            .each_batch(of: 5000) do |batch|
+              batch.delete_all
+            end
+        end
+      end
+
+      def remove_confidential_issue_todos(project_id)
+        # min access level to access a confidential issue is reporter
+        min_reporters = authorized_users(project_id)
+          .select(:user_id)
+          .where('access_level >= ?', 20)
+
+        confidential_issues = Issue.select(:id, :author_id).where(confidential: true, project_id: project_id)
+        confidential_issues.each_batch(of: 100, order_hint: :confidential) do |batch|
+          batch.each do |issue|
+            assigned_users = IssueAssignee.select(:user_id).where(issue_id: issue.id)
+
+            todos = Todo.where(target_type: 'Issue', target_id: issue.id)
+              .where('user_id NOT IN (?)', min_reporters)
+              .where('user_id NOT IN (?)', assigned_users)
+            todos = todos.where('user_id != ?', issue.author_id) if issue.author_id
+
+            todos.delete_all
+          end
+        end
+      end
+
+      def remove_restricted_features_todos(project_id)
+        ProjectFeature.where(project_id: project_id).each do |project_features|
+          target_types = []
+          target_types << 'Issue' if private?(project_features.issues_access_level)
+          target_types << 'MergeRequest' if private?(project_features.merge_requests_access_level)
+          target_types << 'Commit' if private?(project_features.repository_access_level)
+
+          next if target_types.empty?
+
+          if Gitlab::Database.postgresql?
+            batch_remove_todos_cte(project_id, target_types)
+          else
+            unauthorized_project_todos(project_id)
+              .where(target_type: target_types)
+              .delete_all
+          end
+        end
+      end
+
+      def private?(feature_level)
+        feature_level == PRIVATE_FEATURE
+      end
+
+      def authorized_users(project_id)
+        ProjectAuthorization.select(:user_id).where(project_id: project_id)
+      end
+
+      def unauthorized_project_todos(project_id)
+        Todo.where(project_id: project_id)
+          .where('user_id NOT IN (?)', authorized_users(project_id))
+      end
+
+      def batch_remove_todos_cte(project_id, target_types = nil)
+        loop do
+          count = remove_todos_cte(project_id, target_types)
+
+          break if count == 0
+        end
+      end
+
+      def remove_todos_cte(project_id, target_types = nil)
+        sql = []
+        sql << with_all_todos_sql(project_id, target_types)
+        sql << as_deleted_sql
+        sql << "SELECT count(*) FROM deleted"
+
+        result = Todo.connection.exec_query(sql.join(' '))
+        result.rows[0][0].to_i
+      end
+
+      def with_all_todos_sql(project_id, target_types = nil)
+        if target_types
+          table = Arel::Table.new(:todos)
+          in_target = table[:target_type].in(target_types)
+          target_types_sql = " AND #{in_target.to_sql}"
+        end
+
+        <<-SQL
+          WITH all_todos AS (
+          	SELECT id
+          	FROM "todos"
+          	WHERE "todos"."project_id" = #{project_id}
+            AND (user_id NOT IN (
+              SELECT "project_authorizations"."user_id"
+              FROM "project_authorizations"
+              WHERE "project_authorizations"."project_id" = #{project_id})
+              #{target_types_sql}
+            )
+        	),
+        SQL
+      end
+
+      def as_deleted_sql
+        <<-SQL
+          deleted AS (
+            DELETE FROM todos
+            WHERE id IN (
+              SELECT id
+              FROM all_todos
+              LIMIT 5000
+            )
+            RETURNING id
+          )
+        SQL
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/schedule_diff_files_deletion.rb b/lib/gitlab/background_migration/schedule_diff_files_deletion.rb
new file mode 100644
index 00000000000..609cf19187c
--- /dev/null
+++ b/lib/gitlab/background_migration/schedule_diff_files_deletion.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class ScheduleDiffFilesDeletion
+      class MergeRequestDiff < ActiveRecord::Base
+        self.table_name = 'merge_request_diffs'
+
+        belongs_to :merge_request
+
+        include EachBatch
+      end
+
+      DIFF_BATCH_SIZE = 5_000
+      INTERVAL = 5.minutes
+      MIGRATION = 'DeleteDiffFiles'
+
+      def perform
+        diffs = MergeRequestDiff
+          .from("(#{diffs_collection.to_sql}) merge_request_diffs")
+          .where('merge_request_diffs.id != merge_request_diffs.latest_merge_request_diff_id')
+          .select(:id)
+
+        diffs.each_batch(of: DIFF_BATCH_SIZE) do |relation, index|
+          ids = relation.pluck(:id)
+
+          BackgroundMigrationWorker.perform_in(index * INTERVAL, MIGRATION, [ids])
+        end
+      end
+
+      private
+
+      def diffs_collection
+        MergeRequestDiff
+          .joins(:merge_request)
+          .where("merge_requests.state = 'merged'")
+          .where('merge_requests.latest_merge_request_diff_id IS NOT NULL')
+          .where("merge_request_diffs.state NOT IN ('without_files', 'empty')")
+          .select('merge_requests.latest_merge_request_diff_id, merge_request_diffs.id')
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb
index e5e8837221e..bc434b0cb64 100644
--- a/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb
+++ b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb
@@ -3,8 +3,8 @@
 
 module Gitlab
   module BackgroundMigration
-    # Ensures services which previously recieved all notes events continue
-    # to recieve confidential ones.
+    # Ensures services which previously received all notes events continue
+    # to receive confidential ones.
     class SetConfidentialNoteEventsOnServices
       class Service < ActiveRecord::Base
         self.table_name = 'services'
diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb
index 171c8ef21b7..28d8d2c640b 100644
--- a/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb
+++ b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb
@@ -3,8 +3,8 @@
 
 module Gitlab
   module BackgroundMigration
-    # Ensures hooks which previously recieved all notes events continue
-    # to recieve confidential ones.
+    # Ensures hooks which previously received all notes events continue
+    # to receive confidential ones.
     class SetConfidentialNoteEventsOnWebhooks
       class WebHook < ActiveRecord::Base
         self.table_name = 'web_hooks'
diff --git a/lib/gitlab/badge/base.rb b/lib/gitlab/badge/base.rb
index 909fa24fa90..fb55b9e2f1f 100644
--- a/lib/gitlab/badge/base.rb
+++ b/lib/gitlab/badge/base.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Badge
     class Base
diff --git a/lib/gitlab/badge/coverage/metadata.rb b/lib/gitlab/badge/coverage/metadata.rb
index e898f5d790e..9181ba2d4b0 100644
--- a/lib/gitlab/badge/coverage/metadata.rb
+++ b/lib/gitlab/badge/coverage/metadata.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Badge
     module Coverage
diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb
index 778d78185ff..7f7cc62c8ef 100644
--- a/lib/gitlab/badge/coverage/report.rb
+++ b/lib/gitlab/badge/coverage/report.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Badge
     module Coverage
@@ -12,7 +14,7 @@ module Gitlab
           @ref = ref
           @job = job
 
-          @pipeline = @project.pipelines.latest_successful_for(@ref)
+          @pipeline = @project.ci_pipelines.latest_successful_for(@ref)
         end
 
         def entity
@@ -36,6 +38,7 @@ module Gitlab
 
         private
 
+        # rubocop: disable CodeReuse/ActiveRecord
         def raw_coverage
           return unless @pipeline
 
@@ -47,6 +50,7 @@ module Gitlab
               .try(:coverage)
           end
         end
+        # rubocop: enable CodeReuse/ActiveRecord
       end
     end
   end
diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/badge/coverage/template.rb
index afbf9dd17e3..817dc28f84a 100644
--- a/lib/gitlab/badge/coverage/template.rb
+++ b/lib/gitlab/badge/coverage/template.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Badge
     module Coverage
diff --git a/lib/gitlab/badge/metadata.rb b/lib/gitlab/badge/metadata.rb
index 8ad6f3cb986..b9ae68134b0 100644
--- a/lib/gitlab/badge/metadata.rb
+++ b/lib/gitlab/badge/metadata.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Badge
     ##
diff --git a/lib/gitlab/badge/pipeline/metadata.rb b/lib/gitlab/badge/pipeline/metadata.rb
index db1e9f8cfb8..d4d789558c9 100644
--- a/lib/gitlab/badge/pipeline/metadata.rb
+++ b/lib/gitlab/badge/pipeline/metadata.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Badge
     module Pipeline
diff --git a/lib/gitlab/badge/pipeline/status.rb b/lib/gitlab/badge/pipeline/status.rb
index 5fee7a93475..a403d839517 100644
--- a/lib/gitlab/badge/pipeline/status.rb
+++ b/lib/gitlab/badge/pipeline/status.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Badge
     module Pipeline
@@ -18,11 +20,13 @@ module Gitlab
           'pipeline'
         end
 
+        # rubocop: disable CodeReuse/ActiveRecord
         def status
-          @project.pipelines
+          @project.ci_pipelines
             .where(sha: @sha)
             .latest_status(@ref) || 'unknown'
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         def metadata
           @metadata ||= Pipeline::Metadata.new(self)
diff --git a/lib/gitlab/badge/pipeline/template.rb b/lib/gitlab/badge/pipeline/template.rb
index e09db32262d..64c3dfcd10b 100644
--- a/lib/gitlab/badge/pipeline/template.rb
+++ b/lib/gitlab/badge/pipeline/template.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Badge
     module Pipeline
diff --git a/lib/gitlab/badge/template.rb b/lib/gitlab/badge/template.rb
index bfeb0052642..ed2ec50b197 100644
--- a/lib/gitlab/badge/template.rb
+++ b/lib/gitlab/badge/template.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Badge
     ##
diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb
index 4ca5a78e068..3cd327f5109 100644
--- a/lib/gitlab/bare_repository_import/importer.rb
+++ b/lib/gitlab/bare_repository_import/importer.rb
@@ -1,10 +1,15 @@
+# frozen_string_literal: true
+
 module Gitlab
   module BareRepositoryImport
     class Importer
       NoAdminError = Class.new(StandardError)
 
       def self.execute(import_path)
-        import_path << '/' unless import_path.ends_with?('/')
+        unless import_path.ends_with?('/')
+          import_path = "#{import_path}/"
+        end
+
         repos_to_import = Dir.glob(import_path + '**/*.git')
 
         unless user = User.admins.order_id_asc.first
@@ -26,6 +31,12 @@ module Gitlab
         end
       end
 
+      # This is called from within a rake task only used by Admins, so allow writing
+      # to STDOUT
+      def self.log(message)
+        puts message # rubocop:disable Rails/Output
+      end
+
       attr_reader :user, :project_name, :bare_repo
 
       delegate :log, to: :class
@@ -59,11 +70,10 @@ module Gitlab
                                               import_type: 'bare_repository',
                                               namespace_id: group&.id).execute
 
-        if project.persisted? && mv_repo(project)
+        if project.persisted? && mv_repositories(project)
           log " * Created #{project.name} (#{project_full_path})".color(:green)
 
           project.write_repository_config
-          Gitlab::Git::Repository.create_hooks(project.repository.path_to_repo, Gitlab.config.gitlab_shell.hooks_path)
 
           ProjectCacheWorker.perform_async(project.id)
         else
@@ -74,12 +84,11 @@ module Gitlab
         project
       end
 
-      def mv_repo(project)
-        storage_path = storage_path_for_shard(project.repository_storage)
-        FileUtils.mv(repo_path, project.repository.path_to_repo)
+      def mv_repositories(project)
+        mv_repo(bare_repo.repo_path, project.repository)
 
         if bare_repo.wiki_exists?
-          FileUtils.mv(wiki_path, File.join(storage_path, project.disk_path + '.wiki.git'))
+          mv_repo(bare_repo.wiki_path, project.wiki.repository)
         end
 
         true
@@ -89,6 +98,11 @@ module Gitlab
         false
       end
 
+      def mv_repo(path, repository)
+        repository.create_from_bundle(bundle(path))
+        FileUtils.rm_rf(path)
+      end
+
       def storage_path_for_shard(shard)
         Gitlab.config.repositories.storages[shard].legacy_disk_path
       end
@@ -101,10 +115,17 @@ module Gitlab
         Groups::NestedCreateService.new(user, group_path: group_path).execute
       end
 
-      # This is called from within a rake task only used by Admins, so allow writing
-      # to STDOUT
-      def self.log(message)
-        puts message # rubocop:disable Rails/Output
+      def bundle(repo_path)
+        # TODO: we could save some time and disk space by using
+        # `git bundle create - --all` and streaming the bundle directly to
+        # Gitaly, rather than writing it on disk first
+        bundle_path = "#{repo_path}.bundle"
+        cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)
+        output, status = Gitlab::Popen.popen(cmd)
+
+        raise output unless status.zero?
+
+        bundle_path
       end
     end
   end
diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb
index fe267248275..b903c581aac 100644
--- a/lib/gitlab/bare_repository_import/repository.rb
+++ b/lib/gitlab/bare_repository_import/repository.rb
@@ -1,5 +1,5 @@
-# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/953
-#
+# frozen_string_literal: true
+
 module Gitlab
   module BareRepositoryImport
     class Repository
@@ -8,9 +8,12 @@ module Gitlab
       attr_reader :group_path, :project_name, :repo_path
 
       def initialize(root_path, repo_path)
+        unless root_path.ends_with?('/')
+          root_path = "#{root_path}/"
+        end
+
         @root_path = root_path
         @repo_path = repo_path
-        @root_path << '/' unless root_path.ends_with?('/')
 
         full_path =
           if hashed? && !wiki?
diff --git a/lib/gitlab/base_doorkeeper_controller.rb b/lib/gitlab/base_doorkeeper_controller.rb
index e4227af25d2..b78993aba30 100644
--- a/lib/gitlab/base_doorkeeper_controller.rb
+++ b/lib/gitlab/base_doorkeeper_controller.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # This is a base controller for doorkeeper.
 # It adds the `can?` helper used in the views.
 module Gitlab
diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb
index f3999e690fa..eaead41a720 100644
--- a/lib/gitlab/bitbucket_import/importer.rb
+++ b/lib/gitlab/bitbucket_import/importer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module BitbucketImport
     class Importer
@@ -33,7 +35,7 @@ module Gitlab
       def handle_errors
         return unless errors.any?
 
-        project.update_column(:import_error, {
+        project.import_state.update_column(:last_error, {
           message: 'The remote data could not be fully imported.',
           errors: errors
         }.to_json)
@@ -43,6 +45,7 @@ module Gitlab
         find_user_id(username) || project.creator_id
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def find_user_id(username)
         return nil unless username
 
@@ -53,6 +56,7 @@ module Gitlab
                               .find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username)
                               .try(:id)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def repo
         @repo ||= client.repo(project.import_source)
@@ -68,6 +72,7 @@ module Gitlab
         errors << { type: :wiki, errors: e.message }
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def import_issues
         return unless repo.issues_enabled?
 
@@ -101,6 +106,7 @@ module Gitlab
           end
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def import_issue_comments(issue, gitlab_issue)
         client.issue_comments(repo, issue.iid).each do |comment|
@@ -188,7 +194,8 @@ module Gitlab
       end
 
       def import_inline_comments(inline_comments, pull_request, merge_request)
-        line_code_map = {}
+        position_map = {}
+        discussion_map = {}
 
         children, parents = inline_comments.partition(&:has_parent?)
 
@@ -196,22 +203,28 @@ module Gitlab
         # relationships. We assume that the child can appear in any order in
         # the JSON.
         parents.each do |comment|
-          line_code_map[comment.iid] = generate_line_code(comment)
+          position_map[comment.iid] = build_position(merge_request, comment)
         end
 
         children.each do |comment|
-          line_code_map[comment.iid] = line_code_map.fetch(comment.parent_id, nil)
+          position_map[comment.iid] = position_map.fetch(comment.parent_id, nil)
         end
 
         inline_comments.each do |comment|
           begin
             attributes = pull_request_comment_attributes(comment)
+            attributes[:discussion_id] = discussion_map[comment.parent_id] if comment.has_parent?
+
             attributes.merge!(
-              position: build_position(merge_request, comment),
-              line_code: line_code_map.fetch(comment.iid),
+              position: position_map[comment.iid],
               type: 'DiffNote')
 
-            merge_request.notes.create!(attributes)
+            note = merge_request.notes.create!(attributes)
+
+            # We can't store a discussion ID until a note is created, so if
+            # replies are created before the parent the discussion ID won't be
+            # linked properly.
+            discussion_map[comment.iid] = note.discussion_id
           rescue StandardError => e
             errors << { type: :pull_request, iid: comment.iid, errors: e.message }
           end
@@ -240,10 +253,6 @@ module Gitlab
         end
       end
 
-      def generate_line_code(pr_comment)
-        Gitlab::Git.diff_line_code(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos)
-      end
-
       def pull_request_comment_attributes(comment)
         {
           project: project,
diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb
index d94f70fd1fb..11070a68e02 100644
--- a/lib/gitlab/bitbucket_import/project_creator.rb
+++ b/lib/gitlab/bitbucket_import/project_creator.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module BitbucketImport
     class ProjectCreator
diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb
new file mode 100644
index 00000000000..dbbedd5dcbe
--- /dev/null
+++ b/lib/gitlab/bitbucket_server_import/importer.rb
@@ -0,0 +1,386 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BitbucketServerImport
+    class Importer
+      attr_reader :recover_missing_commits
+      attr_reader :project, :project_key, :repository_slug, :client, :errors, :users
+      attr_accessor :logger
+
+      REMOTE_NAME = 'bitbucket_server'.freeze
+      BATCH_SIZE = 100
+
+      TempBranch = Struct.new(:name, :sha)
+
+      def self.imports_repository?
+        true
+      end
+
+      def self.refmap
+        [:heads, :tags, '+refs/pull-requests/*/to:refs/merge-requests/*/head']
+      end
+
+      # Unlike GitHub, you can't grab the commit SHAs for pull requests that
+      # have been closed but not merged even though Bitbucket has these
+      # commits internally. We can recover these pull requests by creating a
+      # branch with the Bitbucket REST API, but by default we turn this
+      # behavior off.
+      def initialize(project, recover_missing_commits: false)
+        @project = project
+        @recover_missing_commits = recover_missing_commits
+        @project_key = project.import_data.data['project_key']
+        @repository_slug = project.import_data.data['repo_slug']
+        @client = BitbucketServer::Client.new(project.import_data.credentials)
+        @formatter = Gitlab::ImportFormatter.new
+        @errors = []
+        @users = {}
+        @temp_branches = []
+        @logger = Gitlab::Import::Logger.build
+      end
+
+      def execute
+        import_repository
+        import_pull_requests
+        delete_temp_branches
+        handle_errors
+
+        log_info(stage: "complete")
+
+        true
+      end
+
+      private
+
+      def handle_errors
+        return unless errors.any?
+
+        project.import_state.update_column(:last_error, {
+          message: 'The remote data could not be fully imported.',
+          errors: errors
+        }.to_json)
+      end
+
+      def gitlab_user_id(email)
+        find_user_id(email) || project.creator_id
+      end
+
+      def find_user_id(email)
+        return nil unless email
+
+        return users[email] if users.key?(email)
+
+        user = User.find_by_any_email(email, confirmed: true)
+        users[email] = user&.id
+
+        user&.id
+      end
+
+      def repo
+        @repo ||= client.repo(project_key, repository_slug)
+      end
+
+      def sha_exists?(sha)
+        project.repository.commit(sha)
+      end
+
+      def temp_branch_name(pull_request, suffix)
+        "gitlab/import/pull-request/#{pull_request.iid}/#{suffix}"
+      end
+
+      # This method restores required SHAs that GitLab needs to create diffs
+      # into branch names as the following:
+      #
+      # gitlab/import/pull-request/N/{to,from}
+      def restore_branches(pull_requests)
+        shas_to_restore = []
+
+        pull_requests.each do |pull_request|
+          shas_to_restore << TempBranch.new(temp_branch_name(pull_request, :from),
+                                            pull_request.source_branch_sha)
+          shas_to_restore << TempBranch.new(temp_branch_name(pull_request, :to),
+                                            pull_request.target_branch_sha)
+        end
+
+        # Create the branches on the Bitbucket Server first
+        created_branches = restore_branch_shas(shas_to_restore)
+
+        @temp_branches += created_branches
+        # Now sync the repository so we get the new branches
+        import_repository unless created_branches.empty?
+      end
+
+      def restore_branch_shas(shas_to_restore)
+        shas_to_restore.each_with_object([]) do |temp_branch, branches_created|
+          branch_name = temp_branch.name
+          sha = temp_branch.sha
+
+          next if sha_exists?(sha)
+
+          begin
+            client.create_branch(project_key, repository_slug, branch_name, sha)
+            branches_created << temp_branch
+          rescue BitbucketServer::Connection::ConnectionError => e
+            log_warn(message: "Unable to recreate branch", sha: sha, error: e.message)
+          end
+        end
+      end
+
+      def import_repository
+        log_info(stage: 'import_repository', message: 'starting import')
+
+        project.ensure_repository
+        project.repository.fetch_as_mirror(project.import_url, refmap: self.class.refmap, remote_name: REMOTE_NAME)
+
+        log_info(stage: 'import_repository', message: 'finished import')
+      rescue Gitlab::Shell::Error => e
+        log_error(stage: 'import_repository', message: 'failed import', error: e.message)
+
+        # Expire cache to prevent scenarios such as:
+        # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true
+        # 2. Retried import, repo is broken or not imported but +exists?+ still returns true
+        project.repository.expire_content_cache if project.repository_exists?
+
+        raise
+      end
+
+      # Bitbucket Server keeps tracks of references for open pull requests in
+      # refs/heads/pull-requests, but closed and merged requests get moved
+      # into hidden internal refs under stash-refs/pull-requests. Unless the
+      # SHAs involved are at the tip of a branch or tag, there is no way to
+      # retrieve the server for those commits.
+      #
+      # To avoid losing history, we use the Bitbucket API to re-create the branch
+      # on the remote server. Then we have to issue a `git fetch` to download these
+      # branches.
+      def import_pull_requests
+        pull_requests = client.pull_requests(project_key, repository_slug).to_a
+
+        # Creating branches on the server and fetching the newly-created branches
+        # may take a number of network round-trips. Do this in batches so that we can
+        # avoid doing a git fetch for every new branch.
+        pull_requests.each_slice(BATCH_SIZE) do |batch|
+          restore_branches(batch) if recover_missing_commits
+
+          batch.each do |pull_request|
+            begin
+              import_bitbucket_pull_request(pull_request)
+            rescue StandardError => e
+              backtrace = Gitlab::Profiler.clean_backtrace(e.backtrace)
+              log_error(stage: 'import_pull_requests', iid: pull_request.iid, error: e.message, backtrace: backtrace)
+
+              errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, backtrace: backtrace.join("\n"), raw_response: pull_request.raw }
+            end
+          end
+        end
+      end
+
+      def delete_temp_branches
+        @temp_branches.each do |branch|
+          begin
+            client.delete_branch(project_key, repository_slug, branch.name, branch.sha)
+            project.repository.delete_branch(branch.name)
+          rescue BitbucketServer::Connection::ConnectionError => e
+            log_error(stage: 'delete_temp_branches', branch: branch.name, error: e.message)
+            @errors << { type: :delete_temp_branches, branch_name: branch.name, errors: e.message }
+          end
+        end
+      end
+
+      def import_bitbucket_pull_request(pull_request)
+        log_info(stage: 'import_bitbucket_pull_requests', message: 'starting', iid: pull_request.iid)
+
+        description = ''
+        description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author_email)
+        description += pull_request.description if pull_request.description
+        author_id = gitlab_user_id(pull_request.author_email)
+
+        attributes = {
+          iid: pull_request.iid,
+          title: pull_request.title,
+          description: description,
+          source_project_id: project.id,
+          source_branch: Gitlab::Git.ref_name(pull_request.source_branch_name),
+          source_branch_sha: pull_request.source_branch_sha,
+          target_project_id: project.id,
+          target_branch: Gitlab::Git.ref_name(pull_request.target_branch_name),
+          target_branch_sha: pull_request.target_branch_sha,
+          state: pull_request.state,
+          author_id: author_id,
+          assignee_id: nil,
+          created_at: pull_request.created_at,
+          updated_at: pull_request.updated_at
+        }
+
+        creator = Gitlab::Import::MergeRequestCreator.new(project)
+        merge_request = creator.execute(attributes)
+
+        import_pull_request_comments(pull_request, merge_request) if merge_request.persisted?
+
+        log_info(stage: 'import_bitbucket_pull_requests', message: 'finished', iid: pull_request.iid)
+      end
+
+      def import_pull_request_comments(pull_request, merge_request)
+        log_info(stage: 'import_pull_request_comments', message: 'starting', iid: merge_request.iid)
+
+        comments, other_activities = client.activities(project_key, repository_slug, pull_request.iid).partition(&:comment?)
+
+        merge_event = other_activities.find(&:merge_event?)
+        import_merge_event(merge_request, merge_event) if merge_event
+
+        inline_comments, pr_comments = comments.partition(&:inline_comment?)
+
+        import_inline_comments(inline_comments.map(&:comment), merge_request)
+        import_standalone_pr_comments(pr_comments.map(&:comment), merge_request)
+
+        log_info(stage: 'import_pull_request_comments', message: 'finished', iid: merge_request.iid,
+                 merge_event_found: merge_event.present?,
+                 inline_comments_count: inline_comments.count,
+                 standalone_pr_comments: pr_comments.count)
+      end
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def import_merge_event(merge_request, merge_event)
+        log_info(stage: 'import_merge_event', message: 'starting', iid: merge_request.iid)
+
+        committer = merge_event.committer_email
+
+        user_id = gitlab_user_id(committer)
+        timestamp = merge_event.merge_timestamp
+        merge_request.update({ merge_commit_sha: merge_event.merge_commit })
+        metric = MergeRequest::Metrics.find_or_initialize_by(merge_request: merge_request)
+        metric.update(merged_by_id: user_id, merged_at: timestamp)
+
+        log_info(stage: 'import_merge_event', message: 'finished', iid: merge_request.iid)
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      def import_inline_comments(inline_comments, merge_request)
+        log_info(stage: 'import_inline_comments', message: 'starting', iid: merge_request.iid)
+
+        inline_comments.each do |comment|
+          position = build_position(merge_request, comment)
+          parent = create_diff_note(merge_request, comment, position)
+
+          next unless parent&.persisted?
+
+          discussion_id = parent.discussion_id
+
+          comment.comments.each do |reply|
+            create_diff_note(merge_request, reply, position, discussion_id)
+          end
+        end
+
+        log_info(stage: 'import_inline_comments', message: 'finished', iid: merge_request.iid)
+      end
+
+      def create_diff_note(merge_request, comment, position, discussion_id = nil)
+        attributes = pull_request_comment_attributes(comment)
+        attributes.merge!(position: position, type: 'DiffNote')
+        attributes[:discussion_id] = discussion_id if discussion_id
+
+        note = merge_request.notes.build(attributes)
+
+        if note.valid?
+          note.save
+          return note
+        end
+
+        log_info(stage: 'create_diff_note', message: 'creating fallback DiffNote', iid: merge_request.iid)
+
+        # Bitbucket Server supports the ability to comment on any line, not just the
+        # line in the diff. If we can't add the note as a DiffNote, fallback to creating
+        # a regular note.
+        create_fallback_diff_note(merge_request, comment, position)
+      rescue StandardError => e
+        log_error(stage: 'create_diff_note', comment_id: comment.id, error: e.message)
+        errors << { type: :pull_request, id: comment.id, errors: e.message }
+        nil
+      end
+
+      def create_fallback_diff_note(merge_request, comment, position)
+        attributes = pull_request_comment_attributes(comment)
+        note = "*Comment on"
+
+        note += " #{position.old_path}:#{position.old_line} -->" if position.old_line
+        note += " #{position.new_path}:#{position.new_line}" if position.new_line
+        note += "*\n\n#{comment.note}"
+
+        attributes[:note] = note
+        merge_request.notes.create!(attributes)
+      end
+
+      def build_position(merge_request, pr_comment)
+        params = {
+          diff_refs: merge_request.diff_refs,
+          old_path: pr_comment.file_path,
+          new_path: pr_comment.file_path,
+          old_line: pr_comment.old_pos,
+          new_line: pr_comment.new_pos
+        }
+
+        Gitlab::Diff::Position.new(params)
+      end
+
+      def import_standalone_pr_comments(pr_comments, merge_request)
+        pr_comments.each do |comment|
+          begin
+            merge_request.notes.create!(pull_request_comment_attributes(comment))
+
+            comment.comments.each do |replies|
+              merge_request.notes.create!(pull_request_comment_attributes(replies))
+            end
+          rescue StandardError => e
+            log_error(stage: 'import_standalone_pr_comments', merge_request_id: merge_request.id, comment_id: comment.id, error: e.message)
+            errors << { type: :pull_request, comment_id: comment.id, errors: e.message }
+          end
+        end
+      end
+
+      def pull_request_comment_attributes(comment)
+        author = find_user_id(comment.author_email)
+        note = ''
+
+        unless author
+          author = project.creator_id
+          note = "*By #{comment.author_username} (#{comment.author_email})*\n\n"
+        end
+
+        note +=
+          # Provide some context for replying
+          if comment.parent_comment
+            "> #{comment.parent_comment.note.truncate(80)}\n\n#{comment.note}"
+          else
+            comment.note
+          end
+
+        {
+          project: project,
+          note: note,
+          author_id: author,
+          created_at: comment.created_at,
+          updated_at: comment.updated_at
+        }
+      end
+
+      def log_info(details)
+        logger.info(log_base_data.merge(details))
+      end
+
+      def log_error(details)
+        logger.error(log_base_data.merge(details))
+      end
+
+      def log_warn(details)
+        logger.warn(log_base_data.merge(details))
+      end
+
+      def log_base_data
+        {
+          class: self.class.name,
+          project_id: project.id,
+          project_path: project.full_path
+        }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/bitbucket_server_import/project_creator.rb b/lib/gitlab/bitbucket_server_import/project_creator.rb
new file mode 100644
index 00000000000..48ca4951957
--- /dev/null
+++ b/lib/gitlab/bitbucket_server_import/project_creator.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BitbucketServerImport
+    class ProjectCreator
+      attr_reader :project_key, :repo_slug, :repo, :name, :namespace, :current_user, :session_data
+
+      def initialize(project_key, repo_slug, repo, name, namespace, current_user, session_data)
+        @project_key = project_key
+        @repo_slug = repo_slug
+        @repo = repo
+        @name = name
+        @namespace = namespace
+        @current_user = current_user
+        @session_data = session_data
+      end
+
+      def execute
+        ::Projects::CreateService.new(
+          current_user,
+          name: name,
+          path: name,
+          description: repo.description,
+          namespace_id: namespace.id,
+          visibility_level: repo.visibility_level,
+          import_type: 'bitbucket_server',
+          import_source: repo.browse_url,
+          import_url: repo.clone_url,
+          import_data: {
+            credentials: session_data,
+            data: { project_key: project_key, repo_slug: repo_slug }
+          },
+          skip_wiki: true
+        ).execute
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb
index 169aac79854..f1a653a9d95 100644
--- a/lib/gitlab/blame.rb
+++ b/lib/gitlab/blame.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class Blame
     attr_accessor :blob, :commit
@@ -41,8 +43,7 @@ module Gitlab
 
     def highlighted_lines
       @blob.load_all_data!
-      @highlighted_lines ||=
-        Gitlab::Highlight.highlight(@blob.path, @blob.data, repository: repository).lines
+      @highlighted_lines ||= @blob.present.highlight.lines
     end
 
     def project
diff --git a/lib/gitlab/blob_helper.rb b/lib/gitlab/blob_helper.rb
new file mode 100644
index 00000000000..d3e15a79a8b
--- /dev/null
+++ b/lib/gitlab/blob_helper.rb
@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+# This has been extracted from https://github.com/github/linguist/blob/master/lib/linguist/blob_helper.rb
+module Gitlab
+  module BlobHelper
+    def extname
+      File.extname(name.to_s)
+    end
+
+    def known_extension?
+      LanguageData.extensions.include?(extname)
+    end
+
+    def viewable?
+      !large? && text_in_repo?
+    end
+
+    MEGABYTE = 1024 * 1024
+
+    def large?
+      size.to_i > MEGABYTE
+    end
+
+    def binary_in_repo?
+      # Large blobs aren't even loaded into memory
+      if data.nil?
+        true
+
+      # Treat blank files as text
+      elsif data == ""
+        false
+
+      # Charlock doesn't know what to think
+      elsif encoding.nil?
+        true
+
+      # If Charlock says its binary
+      else
+        detect_encoding[:type] == :binary
+      end
+    end
+
+    def text_in_repo?
+      !binary_in_repo?
+    end
+
+    def image?
+      ['.png', '.jpg', '.jpeg', '.gif'].include?(extname.downcase)
+    end
+
+    # Internal: Lookup mime type for extension.
+    #
+    # Returns a MIME::Type
+    # rubocop:disable Gitlab/ModuleWithInstanceVariables
+    def _mime_type
+      if defined? @_mime_type
+        @_mime_type
+      else
+        guesses = ::MIME::Types.type_for(extname.to_s)
+
+        # Prefer text mime types over binary
+        @_mime_type = guesses.detect { |type| type.ascii? } || guesses.first
+      end
+    end
+    # rubocop:enable Gitlab/ModuleWithInstanceVariables
+
+    # Public: Get the actual blob mime type
+    #
+    # Examples
+    #
+    #   # => 'text/plain'
+    #   # => 'text/html'
+    #
+    # Returns a mime type String.
+    def mime_type
+      _mime_type ? _mime_type.to_s : 'text/plain'
+    end
+
+    def binary_mime_type?
+      _mime_type ? _mime_type.binary? : false
+    end
+
+    def lines
+      @lines ||=
+        if viewable? && data
+          # `data` is usually encoded as ASCII-8BIT even when the content has
+          # been detected as a different encoding. However, we are not allowed
+          # to change the encoding of `data` because we've made the implicit
+          # guarantee that each entry in `lines` is encoded the same way as
+          # `data`.
+          #
+          # Instead, we re-encode each possible newline sequence as the
+          # detected encoding, then force them back to the encoding of `data`
+          # (usually a binary encoding like ASCII-8BIT). This means that the
+          # byte sequence will match how newlines are likely encoded in the
+          # file, but we don't have to change the encoding of `data` as far as
+          # Ruby is concerned. This allows us to correctly parse out each line
+          # without changing the encoding of `data`, and
+          # also--importantly--without having to duplicate many (potentially
+          # large) strings.
+          begin
+            data.split(encoded_newlines_re, -1)
+          rescue Encoding::ConverterNotFoundError
+            # The data is not splittable in the detected encoding.  Assume it's
+            # one big line.
+            [data]
+          end
+        else
+          []
+        end
+    end
+
+    def content_type
+      # rubocop:disable Style/MultilineTernaryOperator
+      # rubocop:disable Style/NestedTernaryOperator
+      @content_type ||= binary_mime_type? || binary_in_repo? ? mime_type :
+                          (encoding ? "text/plain; charset=#{encoding.downcase}" : "text/plain")
+      # rubocop:enable Style/NestedTernaryOperator
+      # rubocop:enable Style/MultilineTernaryOperator
+    end
+
+    def encoded_newlines_re
+      @encoded_newlines_re ||=
+        Regexp.union(["\r\n", "\r", "\n"].map { |nl| nl.encode(ruby_encoding, "ASCII-8BIT").force_encoding(data.encoding) })
+    end
+
+    def ruby_encoding
+      if hash = detect_encoding
+        hash[:ruby_encoding]
+      end
+    end
+
+    def encoding
+      if hash = detect_encoding
+        hash[:encoding]
+      end
+    end
+
+    def detect_encoding
+      @detect_encoding ||= CharlockHolmes::EncodingDetector.new.detect(data) if data # rubocop:disable Gitlab/ModuleWithInstanceVariables
+    end
+
+    def empty?
+      data.nil? || data == ""
+    end
+  end
+end
diff --git a/lib/gitlab/branch_push_merge_commit_analyzer.rb b/lib/gitlab/branch_push_merge_commit_analyzer.rb
new file mode 100644
index 00000000000..a8f601f2451
--- /dev/null
+++ b/lib/gitlab/branch_push_merge_commit_analyzer.rb
@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module Gitlab
+  # Analyse a graph of commits from a push to a branch,
+  # for each commit, analyze that if it is the head of a merge request,
+  # then what should its merge_commit be, relative to the branch.
+  #
+  # A----->B----->C----->D   target branch
+  # |             ^
+  # |             |
+  # +-->E----->F--+          merged branch
+  #     |     ^
+  #     |     |
+  #     +->G--+
+  #
+  # (See merge-commit-analyze-after branch in gitlab-test)
+  #
+  # Assuming
+  # - A is already in remote
+  # - B~D are all in its own branch with its own merge request, targeting the target branch
+  #
+  # When D is finally pushed to the target branch,
+  # what are the merge commits for all the other merge requests?
+  #
+  # We can walk backwards from the HEAD commit D,
+  # and find status of its parents.
+  # First we determine if commit belongs to the target branch (i.e. A, B, C, D),
+  # and then determine its merge commit.
+  #
+  # +--------+-----------------+--------------+
+  # | Commit | Direct ancestor | Merge commit |
+  # +--------+-----------------+--------------+
+  # | D      | Y               | D            |
+  # +--------+-----------------+--------------+
+  # | C      | Y               | C            |
+  # +--------+-----------------+--------------+
+  # | F      |                 | C            |
+  # +--------+-----------------+--------------+
+  # | B      | Y               | B            |
+  # +--------+-----------------+--------------+
+  # | E      |                 | C            |
+  # +--------+-----------------+--------------+
+  # | G      |                 | C            |
+  # +--------+-----------------+--------------+
+  #
+  # By examining the result, it can be said that
+  #
+  # - If commit is direct ancestor of HEAD, its merge commit is itself.
+  # - Otherwise, the merge commit is the same as its child's merge commit.
+  #
+  class BranchPushMergeCommitAnalyzer
+    class CommitDecorator < SimpleDelegator
+      attr_accessor :merge_commit
+      attr_writer :direct_ancestor # boolean
+
+      def direct_ancestor?
+        @direct_ancestor
+      end
+
+      # @param child_commit [CommitDecorator]
+      # @param first_parent [Boolean] whether `self` is the first parent of `child_commit`
+      def set_merge_commit(child_commit:)
+        @merge_commit ||= direct_ancestor? ? self : child_commit.merge_commit
+      end
+    end
+
+    # @param commits [Array] list of commits, must be ordered from the child (tip) of the graph back to the ancestors
+    def initialize(commits, relevant_commit_ids: nil)
+      @commits = commits
+      @id_to_commit = {}
+      @commits.each do |commit|
+        @id_to_commit[commit.id] = CommitDecorator.new(commit)
+
+        if relevant_commit_ids
+          relevant_commit_ids.delete(commit.id)
+          break if relevant_commit_ids.empty? # Only limit the analyze up to relevant_commit_ids
+        end
+      end
+
+      analyze
+    end
+
+    def get_merge_commit(id)
+      get_commit(id).merge_commit.id
+    end
+
+    private
+
+    def analyze
+      head_commit = get_commit(@commits.first.id)
+      head_commit.direct_ancestor = true
+      head_commit.merge_commit = head_commit
+
+      mark_all_direct_ancestors(head_commit)
+
+      # Analyzing a commit requires its child commit be analyzed first,
+      # which is the case here since commits are ordered from child to parent.
+      @id_to_commit.each_value do |commit|
+        analyze_parents(commit)
+      end
+    end
+
+    def analyze_parents(commit)
+      commit.parent_ids.each do |parent_commit_id|
+        parent_commit = get_commit(parent_commit_id)
+
+        next unless parent_commit # parent commit may not be part of new commits
+
+        parent_commit.set_merge_commit(child_commit: commit)
+      end
+    end
+
+    # Mark all direct ancestors.
+    # If child commit is a direct ancestor, its first parent is also a direct ancestor.
+    # We assume direct ancestors matches the trail of the target branch over time,
+    # This assumption is correct most of the time, especially for gitlab managed merges,
+    # but there are exception cases which can't be solved (https://stackoverflow.com/a/49754723/474597)
+    def mark_all_direct_ancestors(commit)
+      loop do
+        commit = get_commit(commit.parent_ids.first)
+
+        break unless commit
+
+        commit.direct_ancestor = true
+      end
+    end
+
+    def get_commit(id)
+      @id_to_commit[id]
+    end
+  end
+end
diff --git a/lib/gitlab/build_access.rb b/lib/gitlab/build_access.rb
index 08a8f846ca5..37e79413541 100644
--- a/lib/gitlab/build_access.rb
+++ b/lib/gitlab/build_access.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class BuildAccess < UserAccess
     attr_accessor :user, :project
diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb
index add048d671e..ea7013db2ce 100644
--- a/lib/gitlab/cache/ci/project_pipeline_status.rb
+++ b/lib/gitlab/cache/ci/project_pipeline_status.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # This class is not backed by a table in the main database.
 # It loads the latest Pipeline for the HEAD of a repository, and caches that
 # in Redis.
@@ -5,9 +7,9 @@ module Gitlab
   module Cache
     module Ci
       class ProjectPipelineStatus
-        attr_accessor :sha, :status, :ref, :project, :loaded
+        include Gitlab::Utils::StrongMemoize
 
-        delegate :commit, to: :project
+        attr_accessor :sha, :status, :ref, :project, :loaded
 
         def self.load_for_project(project)
           new(project).tap do |status|
@@ -16,33 +18,12 @@ module Gitlab
         end
 
         def self.load_in_batch_for_projects(projects)
-          cached_results_for_projects(projects).zip(projects).each do |result, project|
-            project.pipeline_status = new(project, result)
+          projects.each do |project|
+            project.pipeline_status = new(project)
             project.pipeline_status.load_status
           end
         end
 
-        def self.cached_results_for_projects(projects)
-          result = Gitlab::Redis::Cache.with do |redis|
-            redis.multi do
-              projects.each do |project|
-                cache_key = cache_key_for_project(project)
-                redis.exists(cache_key)
-                redis.hmget(cache_key, :sha, :status, :ref)
-              end
-            end
-          end
-
-          result.each_slice(2).map do |(cache_key_exists, (sha, status, ref))|
-            pipeline_info = { sha: sha, status: status, ref: ref }
-            { loaded_from_cache: cache_key_exists, pipeline_info: pipeline_info }
-          end
-        end
-
-        def self.cache_key_for_project(project)
-          "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:projects/#{project.id}/pipeline_status"
-        end
-
         def self.update_for_pipeline(pipeline)
           pipeline_info = {
             sha: pipeline.sha,
@@ -68,6 +49,7 @@ module Gitlab
 
         def load_status
           return if loaded?
+          return unless commit
 
           if has_cache?
             load_from_cache
@@ -80,11 +62,7 @@ module Gitlab
         end
 
         def load_from_project
-          return unless commit
-
-          self.sha = commit.sha
-          self.status = commit.status
-          self.ref = project.default_branch
+          self.sha, self.status, self.ref = commit.sha, commit.status, project.default_branch
         end
 
         # We only cache the status for the HEAD commit of a project
@@ -102,6 +80,8 @@ module Gitlab
         def load_from_cache
           Gitlab::Redis::Cache.with do |redis|
             self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref)
+
+            self.status = nil if self.status.empty?
           end
         end
 
@@ -130,7 +110,13 @@ module Gitlab
         end
 
         def cache_key
-          self.class.cache_key_for_project(project)
+          "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:project:#{project.id}:pipeline_status:#{commit&.sha}"
+        end
+
+        def commit
+          strong_memoize(:commit) do
+            project.commit
+          end
         end
       end
     end
diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb
index ecc85f847d4..4c658dc0b8d 100644
--- a/lib/gitlab/cache/request_cache.rb
+++ b/lib/gitlab/cache/request_cache.rb
@@ -1,41 +1,8 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Cache
-    # This module provides a simple way to cache values in RequestStore,
-    # and the cache key would be based on the class name, method name,
-    # optionally customized instance level values, optionally customized
-    # method level values, and optional method arguments.
-    #
-    # A simple example:
-    #
-    # class UserAccess
-    #   extend Gitlab::Cache::RequestCache
-    #
-    #   request_cache_key do
-    #     [user&.id, project&.id]
-    #   end
-    #
-    #   request_cache def can_push_to_branch?(ref)
-    #     # ...
-    #   end
-    # end
-    #
-    # This way, the result of `can_push_to_branch?` would be cached in
-    # `RequestStore.store` based on the cache key. If RequestStore is not
-    # currently active, then it would be stored in a hash saved in an
-    # instance variable, so the cache logic would be the same.
-    # Here's another example using customized method level values:
-    #
-    # class Commit
-    #   extend Gitlab::Cache::RequestCache
-    #
-    #   def author
-    #     User.find_by_any_email(author_email.downcase)
-    #   end
-    #   request_cache(:author) { author_email.downcase }
-    # end
-    #
-    # So that we could have different strategies for different methods
-    #
+    # See https://docs.gitlab.com/ee/development/utilities.html#requestcache
     module RequestCache
       def self.extended(klass)
         return if klass < self
@@ -61,8 +28,8 @@ module Gitlab
 
           define_method(method_name) do |*args|
             store =
-              if RequestStore.active?
-                RequestStore.store
+              if Gitlab::SafeRequestStore.active?
+                Gitlab::SafeRequestStore.store
               else
                 ivar_name = # ! and ? cannot be used as ivar name
                   "@cache_#{method_name.to_s.tr('!?', "\u2605\u2606")}"
diff --git a/lib/gitlab/changes_list.rb b/lib/gitlab/changes_list.rb
index 9c9e6668e6f..fb75a78a978 100644
--- a/lib/gitlab/changes_list.rb
+++ b/lib/gitlab/changes_list.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class ChangesList
     include Enumerable
diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb
index e63e5437331..8b3c5dc9e8b 100644
--- a/lib/gitlab/chat_name_token.rb
+++ b/lib/gitlab/chat_name_token.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'json'
 
 module Gitlab
diff --git a/lib/gitlab/checks/base_checker.rb b/lib/gitlab/checks/base_checker.rb
new file mode 100644
index 00000000000..09b17b5b76b
--- /dev/null
+++ b/lib/gitlab/checks/base_checker.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Checks
+    class BaseChecker
+      include Gitlab::Utils::StrongMemoize
+
+      attr_reader :change_access
+      delegate(*ChangeAccess::ATTRIBUTES, to: :change_access)
+
+      def initialize(change_access)
+        @change_access = change_access
+      end
+
+      def validate!
+        raise NotImplementedError
+      end
+
+      private
+
+      def creation?
+        Gitlab::Git.blank_ref?(oldrev)
+      end
+
+      def deletion?
+        Gitlab::Git.blank_ref?(newrev)
+      end
+
+      def update?
+        !creation? && !deletion?
+      end
+
+      def updated_from_web?
+        protocol == 'web'
+      end
+
+      def tag_exists?
+        project.repository.tag_exists?(tag_name)
+      end
+
+      def validate_once(resource)
+        Gitlab::SafeRequestStore.fetch(cache_key_for_resource(resource)) do
+          yield(resource)
+
+          true
+        end
+      end
+
+      def cache_key_for_resource(resource)
+        "git_access:#{checker_cache_key}:#{resource.cache_key}"
+      end
+
+      def checker_cache_key
+        self.class.name.demodulize.underscore
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb
new file mode 100644
index 00000000000..d06b2df36f2
--- /dev/null
+++ b/lib/gitlab/checks/branch_check.rb
@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Checks
+    class BranchCheck < BaseChecker
+      ERROR_MESSAGES = {
+        delete_default_branch: 'The default branch of a project cannot be deleted.',
+        force_push_protected_branch: 'You are not allowed to force push code to a protected branch on this project.',
+        non_master_delete_protected_branch: 'You are not allowed to delete protected branches from this project. Only a project maintainer or owner can delete a protected branch.',
+        non_web_delete_protected_branch: 'You can only delete protected branches using the web interface.',
+        merge_protected_branch: 'You are not allowed to merge code into protected branches on this project.',
+        push_protected_branch: 'You are not allowed to push code to protected branches on this project.'
+      }.freeze
+
+      LOG_MESSAGES = {
+        delete_default_branch_check: "Checking if default branch is being deleted...",
+        protected_branch_checks: "Checking if you are force pushing to a protected branch...",
+        protected_branch_push_checks: "Checking if you are allowed to push to the protected branch...",
+        protected_branch_deletion_checks: "Checking if you are allowed to delete the protected branch..."
+      }.freeze
+
+      def validate!
+        return unless branch_name
+
+        logger.log_timed(LOG_MESSAGES[:delete_default_branch_check]) do
+          if deletion? && branch_name == project.default_branch
+            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch]
+          end
+        end
+
+        protected_branch_checks
+      end
+
+      private
+
+      def protected_branch_checks
+        logger.log_timed(LOG_MESSAGES[:protected_branch_checks]) do
+          return unless ProtectedBranch.protected?(project, branch_name) # rubocop:disable Cop/AvoidReturnFromBlocks
+
+          if forced_push?
+            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch]
+          end
+        end
+
+        if deletion?
+          protected_branch_deletion_checks
+        else
+          protected_branch_push_checks
+        end
+      end
+
+      def protected_branch_deletion_checks
+        logger.log_timed(LOG_MESSAGES[:protected_branch_deletion_checks]) do
+          unless user_access.can_delete_branch?(branch_name)
+            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch]
+          end
+
+          unless updated_from_web?
+            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch]
+          end
+        end
+      end
+
+      def protected_branch_push_checks
+        logger.log_timed(LOG_MESSAGES[:protected_branch_push_checks]) do
+          if matching_merge_request?
+            unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name)
+              raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch]
+            end
+          else
+            unless user_access.can_push_to_branch?(branch_name)
+              raise GitAccess::UnauthorizedError, push_to_protected_branch_rejected_message
+            end
+          end
+        end
+      end
+
+      def push_to_protected_branch_rejected_message
+        if project.empty_repo?
+          empty_project_push_message
+        else
+          ERROR_MESSAGES[:push_protected_branch]
+        end
+      end
+
+      def empty_project_push_message
+        <<~MESSAGE
+
+        A default branch (e.g. master) does not yet exist for #{project.full_path}
+        Ask a project Owner or Maintainer to create a default branch:
+
+          #{project_members_url}
+
+        MESSAGE
+      end
+
+      def project_members_url
+        Gitlab::Routing.url_helpers.project_project_members_url(project)
+      end
+
+      def matching_merge_request?
+        Checks::MatchingMergeRequest.new(newrev, branch_name, project).match?
+      end
+
+      def forced_push?
+        Gitlab::Checks::ForcePush.force_push?(project, oldrev, newrev)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb
index 51ba09aa129..8a57a3a6d9a 100644
--- a/lib/gitlab/checks/change_access.rb
+++ b/lib/gitlab/checks/change_access.rb
@@ -1,192 +1,54 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Checks
     class ChangeAccess
-      ERROR_MESSAGES = {
-        push_code: 'You are not allowed to push code to this project.',
-        delete_default_branch: 'The default branch of a project cannot be deleted.',
-        force_push_protected_branch: 'You are not allowed to force push code to a protected branch on this project.',
-        non_master_delete_protected_branch: 'You are not allowed to delete protected branches from this project. Only a project master or owner can delete a protected branch.',
-        non_web_delete_protected_branch: 'You can only delete protected branches using the web interface.',
-        merge_protected_branch: 'You are not allowed to merge code into protected branches on this project.',
-        push_protected_branch: 'You are not allowed to push code to protected branches on this project.',
-        change_existing_tags: 'You are not allowed to change existing tags on this project.',
-        update_protected_tag: 'Protected tags cannot be updated.',
-        delete_protected_tag: 'Protected tags cannot be deleted.',
-        create_protected_tag: 'You are not allowed to create this tag as it is protected.',
-        lfs_objects_missing: 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".'
-      }.freeze
+      ATTRIBUTES = %i[user_access project skip_authorization
+                      skip_lfs_integrity_check protocol oldrev newrev ref
+                      branch_name tag_name logger commits].freeze
 
-      attr_reader :user_access, :project, :skip_authorization, :skip_lfs_integrity_check, :protocol, :oldrev, :newrev, :ref, :branch_name, :tag_name
+      attr_reader(*ATTRIBUTES)
 
       def initialize(
-        change, user_access:, project:, skip_authorization: false,
-        skip_lfs_integrity_check: false, protocol:
+        change, user_access:, project:,
+        skip_lfs_integrity_check: false, protocol:, logger:
       )
         @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref)
         @branch_name = Gitlab::Git.branch_name(@ref)
         @tag_name = Gitlab::Git.tag_name(@ref)
         @user_access = user_access
         @project = project
-        @skip_authorization = skip_authorization
         @skip_lfs_integrity_check = skip_lfs_integrity_check
         @protocol = protocol
-      end
-
-      def exec(skip_commits_check: false)
-        return true if skip_authorization
-
-        push_checks
-        branch_checks
-        tag_checks
-        lfs_objects_exist_check unless skip_lfs_integrity_check
-        commits_check unless skip_commits_check
-
-        true
-      end
-
-      protected
-
-      def push_checks
-        unless can_push?
-          raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code]
-        end
-      end
-
-      def branch_checks
-        return unless branch_name
-
-        if deletion? && branch_name == project.default_branch
-          raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch]
-        end
-
-        protected_branch_checks
-      end
-
-      def protected_branch_checks
-        return unless ProtectedBranch.protected?(project, branch_name)
 
-        if forced_push?
-          raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch]
-        end
-
-        if deletion?
-          protected_branch_deletion_checks
-        else
-          protected_branch_push_checks
-        end
+        @logger = logger
+        @logger.append_message("Running checks for ref: #{@branch_name || @tag_name}")
       end
 
-      def protected_branch_deletion_checks
-        unless user_access.can_delete_branch?(branch_name)
-          raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch]
-        end
+      def exec
+        ref_level_checks
+        # Check of commits should happen as the last step
+        # given they're expensive in terms of performance
+        commits_check
 
-        unless updated_from_web?
-          raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch]
-        end
-      end
-
-      def protected_branch_push_checks
-        if matching_merge_request?
-          unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name)
-            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch]
-          end
-        else
-          unless user_access.can_push_to_branch?(branch_name)
-            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_protected_branch]
-          end
-        end
+        true
       end
 
-      def tag_checks
-        return unless tag_name
-
-        if tag_exists? && user_access.cannot_do_action?(:admin_project)
-          raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags]
-        end
-
-        protected_tag_checks
+      def commits
+        @commits ||= project.repository.new_commits(newrev)
       end
 
-      def protected_tag_checks
-        return unless ProtectedTag.protected?(project, tag_name)
-
-        raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update?
-        raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion?
+      protected
 
-        unless user_access.can_create_tag?(tag_name)
-          raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag]
-        end
+      def ref_level_checks
+        Gitlab::Checks::PushCheck.new(self).validate!
+        Gitlab::Checks::BranchCheck.new(self).validate!
+        Gitlab::Checks::TagCheck.new(self).validate!
+        Gitlab::Checks::LfsCheck.new(self).validate!
       end
 
       def commits_check
-        return if deletion? || newrev.nil?
-        return unless should_run_commit_validations?
-
-        # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593
-        ::Gitlab::GitalyClient.allow_n_plus_1_calls do
-          commits.each do |commit|
-            commit_check.validate(commit, validations_for_commit(commit))
-          end
-        end
-
-        commit_check.validate_file_paths
-      end
-
-      # Method overwritten in EE to inject custom validations
-      def validations_for_commit(_)
-        []
-      end
-
-      private
-
-      def should_run_commit_validations?
-        commit_check.validate_lfs_file_locks?
-      end
-
-      def updated_from_web?
-        protocol == 'web'
-      end
-
-      def tag_exists?
-        project.repository.tag_exists?(tag_name)
-      end
-
-      def forced_push?
-        Gitlab::Checks::ForcePush.force_push?(project, oldrev, newrev)
-      end
-
-      def update?
-        !Gitlab::Git.blank_ref?(oldrev) && !deletion?
-      end
-
-      def deletion?
-        Gitlab::Git.blank_ref?(newrev)
-      end
-
-      def matching_merge_request?
-        Checks::MatchingMergeRequest.new(newrev, branch_name, project).match?
-      end
-
-      def lfs_objects_exist_check
-        lfs_check = Checks::LfsIntegrity.new(project, newrev)
-
-        if lfs_check.objects_missing?
-          raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:lfs_objects_missing]
-        end
-      end
-
-      def commit_check
-        @commit_check ||= Gitlab::Checks::CommitCheck.new(project, user_access.user, newrev, oldrev)
-      end
-
-      def commits
-        @commits ||= project.repository.new_commits(newrev)
-      end
-
-      def can_push?
-        user_access.can_do_action?(:push_code) ||
-          user_access.can_push_to_branch?(branch_name)
+        Gitlab::Checks::DiffCheck.new(self).validate!
       end
     end
   end
diff --git a/lib/gitlab/checks/commit_check.rb b/lib/gitlab/checks/commit_check.rb
deleted file mode 100644
index 43a52b493bb..00000000000
--- a/lib/gitlab/checks/commit_check.rb
+++ /dev/null
@@ -1,61 +0,0 @@
-module Gitlab
-  module Checks
-    class CommitCheck
-      include Gitlab::Utils::StrongMemoize
-
-      attr_reader :project, :user, :newrev, :oldrev
-
-      def initialize(project, user, newrev, oldrev)
-        @project = project
-        @user = user
-        @newrev = user
-        @oldrev = user
-        @file_paths = []
-      end
-
-      def validate(commit, validations)
-        return if validations.empty? && path_validations.empty?
-
-        commit.raw_deltas.each do |diff|
-          @file_paths << (diff.new_path || diff.old_path)
-
-          validations.each do |validation|
-            if error = validation.call(diff)
-              raise ::Gitlab::GitAccess::UnauthorizedError, error
-            end
-          end
-        end
-      end
-
-      def validate_file_paths
-        path_validations.each do |validation|
-          if error = validation.call(@file_paths)
-            raise ::Gitlab::GitAccess::UnauthorizedError, error
-          end
-        end
-      end
-
-      def validate_lfs_file_locks?
-        strong_memoize(:validate_lfs_file_locks) do
-          project.lfs_enabled? && project.lfs_file_locks.any? && newrev && oldrev
-        end
-      end
-
-      private
-
-      def lfs_file_locks_validation
-        lambda do |paths|
-          lfs_lock = project.lfs_file_locks.where(path: paths).where.not(user_id: user.id).first
-
-          if lfs_lock
-            return "The path '#{lfs_lock.path}' is locked in Git LFS by #{lfs_lock.user.name}"
-          end
-        end
-      end
-
-      def path_validations
-        validate_lfs_file_locks? ? [lfs_file_locks_validation] : []
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb
new file mode 100644
index 00000000000..ea0d8c85a66
--- /dev/null
+++ b/lib/gitlab/checks/diff_check.rb
@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Checks
+    class DiffCheck < BaseChecker
+      include Gitlab::Utils::StrongMemoize
+
+      LOG_MESSAGES = {
+        validate_file_paths: "Validating diffs' file paths...",
+        diff_content_check: "Validating diff contents..."
+      }.freeze
+
+      def validate!
+        return if deletion?
+        return unless should_run_diff_validations?
+        return if commits.empty?
+
+        file_paths = []
+
+        process_commits do |commit|
+          validate_once(commit) do
+            commit.raw_deltas.each do |diff|
+              file_paths << (diff.new_path || diff.old_path)
+
+              validate_diff(diff)
+            end
+          end
+        end
+
+        validate_file_paths(file_paths)
+      end
+
+      private
+
+      def validate_lfs_file_locks?
+        strong_memoize(:validate_lfs_file_locks) do
+          project.lfs_enabled? && project.any_lfs_file_locks?
+        end
+      end
+
+      def should_run_diff_validations?
+        validations_for_diff.present? || path_validations.present?
+      end
+
+      def validate_diff(diff)
+        validations_for_diff.each do |validation|
+          if error = validation.call(diff)
+            raise ::Gitlab::GitAccess::UnauthorizedError, error
+          end
+        end
+      end
+
+      # Method overwritten in EE to inject custom validations
+      def validations_for_diff
+        []
+      end
+
+      def path_validations
+        validate_lfs_file_locks? ? [lfs_file_locks_validation] : []
+      end
+
+      def process_commits
+        logger.log_timed(LOG_MESSAGES[:diff_content_check]) do
+          # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593
+          ::Gitlab::GitalyClient.allow_n_plus_1_calls do
+            commits.each do |commit|
+              logger.check_timeout_reached
+
+              yield(commit)
+            end
+          end
+        end
+      end
+
+      def validate_file_paths(file_paths)
+        logger.log_timed(LOG_MESSAGES[__method__]) do
+          path_validations.each do |validation|
+            if error = validation.call(file_paths)
+              raise ::Gitlab::GitAccess::UnauthorizedError, error
+            end
+          end
+        end
+      end
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def lfs_file_locks_validation
+        lambda do |paths|
+          lfs_lock = project.lfs_file_locks.where(path: paths).where.not(user_id: user_access.user.id).take
+
+          if lfs_lock
+            return "The path '#{lfs_lock.path}' is locked in Git LFS by #{lfs_lock.user.name}"
+          end
+        end
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+    end
+  end
+end
diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb
index c9c3050cfc2..263972923ed 100644
--- a/lib/gitlab/checks/force_push.rb
+++ b/lib/gitlab/checks/force_push.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Checks
     class ForcePush
@@ -7,18 +9,10 @@ module Gitlab
         # Created or deleted branch
         return false if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev)
 
-        GitalyClient.migrate(:force_push) do |is_enabled|
-          if is_enabled
-            !project
-              .repository
-              .gitaly_commit_client
-              .ancestor?(oldrev, newrev)
-          else
-            Gitlab::Git::RevList.new(
-              project.repository.raw, oldrev: oldrev, newrev: newrev
-            ).missed_ref.present?
-          end
-        end
+        !project
+          .repository
+          .gitaly_commit_client
+          .ancestor?(oldrev, newrev)
       end
     end
   end
diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb
new file mode 100644
index 00000000000..cc6a14d2d9a
--- /dev/null
+++ b/lib/gitlab/checks/lfs_check.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Checks
+    class LfsCheck < BaseChecker
+      LOG_MESSAGE = "Scanning repository for blobs stored in LFS and verifying their files have been uploaded to GitLab...".freeze
+      ERROR_MESSAGE = 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".'.freeze
+
+      def validate!
+        return unless project.lfs_enabled?
+        return if skip_lfs_integrity_check
+
+        logger.log_timed(LOG_MESSAGE) do
+          lfs_check = Checks::LfsIntegrity.new(project, newrev, logger.time_left)
+
+          if lfs_check.objects_missing?
+            raise GitAccess::UnauthorizedError, ERROR_MESSAGE
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/checks/lfs_integrity.rb b/lib/gitlab/checks/lfs_integrity.rb
index f0e5773ec3c..1652d5a30a4 100644
--- a/lib/gitlab/checks/lfs_integrity.rb
+++ b/lib/gitlab/checks/lfs_integrity.rb
@@ -1,17 +1,20 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Checks
     class LfsIntegrity
-      REV_LIST_OBJECT_LIMIT = 2_000
-
-      def initialize(project, newrev)
+      def initialize(project, newrev, time_left)
         @project = project
         @newrev = newrev
+        @time_left = time_left
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def objects_missing?
         return false unless @newrev && @project.lfs_enabled?
 
-        new_lfs_pointers = Gitlab::Git::LfsChanges.new(@project.repository, @newrev).new_pointers(object_limit: REV_LIST_OBJECT_LIMIT)
+        new_lfs_pointers = Gitlab::Git::LfsChanges.new(@project.repository, @newrev)
+                                                  .new_pointers(object_limit: ::Gitlab::Git::Repository::REV_LIST_COMMIT_LIMIT, dynamic_timeout: @time_left)
 
         return false unless new_lfs_pointers.present?
 
@@ -21,6 +24,7 @@ module Gitlab
 
         existing_count != new_lfs_pointers.count
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/gitlab/checks/matching_merge_request.rb b/lib/gitlab/checks/matching_merge_request.rb
index 849848515da..71361b12d07 100644
--- a/lib/gitlab/checks/matching_merge_request.rb
+++ b/lib/gitlab/checks/matching_merge_request.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Checks
     class MatchingMergeRequest
@@ -7,12 +9,14 @@ module Gitlab
         @project = project
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def match?
         @project.merge_requests
           .with_state(:locked)
           .where(in_progress_merge_commit_sha: @newrev, target_branch: @branch_name)
           .exists?
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/gitlab/checks/post_push_message.rb b/lib/gitlab/checks/post_push_message.rb
index 473c0385b34..492dbb5a596 100644
--- a/lib/gitlab/checks/post_push_message.rb
+++ b/lib/gitlab/checks/post_push_message.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Checks
     class PostPushMessage
diff --git a/lib/gitlab/checks/project_created.rb b/lib/gitlab/checks/project_created.rb
index cec270d6a58..0058a402a62 100644
--- a/lib/gitlab/checks/project_created.rb
+++ b/lib/gitlab/checks/project_created.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Checks
     class ProjectCreated < PostPushMessage
diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb
index 3a197078d08..cb3b7acaaad 100644
--- a/lib/gitlab/checks/project_moved.rb
+++ b/lib/gitlab/checks/project_moved.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Checks
     class ProjectMoved < PostPushMessage
diff --git a/lib/gitlab/checks/push_check.rb b/lib/gitlab/checks/push_check.rb
new file mode 100644
index 00000000000..91f8d0bdbc8
--- /dev/null
+++ b/lib/gitlab/checks/push_check.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Checks
+    class PushCheck < BaseChecker
+      def validate!
+        logger.log_timed("Checking if you are allowed to push...") do
+          unless can_push?
+            raise GitAccess::UnauthorizedError, GitAccess::ERROR_MESSAGES[:push_code]
+          end
+        end
+      end
+
+      private
+
+      def can_push?
+        user_access.can_do_action?(:push_code) ||
+          project.branch_allows_collaboration?(user_access.user, branch_name)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/checks/tag_check.rb b/lib/gitlab/checks/tag_check.rb
new file mode 100644
index 00000000000..2a75c8059bd
--- /dev/null
+++ b/lib/gitlab/checks/tag_check.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Checks
+    class TagCheck < BaseChecker
+      ERROR_MESSAGES = {
+        change_existing_tags: 'You are not allowed to change existing tags on this project.',
+        update_protected_tag: 'Protected tags cannot be updated.',
+        delete_protected_tag: 'Protected tags cannot be deleted.',
+        create_protected_tag: 'You are not allowed to create this tag as it is protected.'
+      }.freeze
+
+      LOG_MESSAGES = {
+        tag_checks: "Checking if you are allowed to change existing tags...",
+        protected_tag_checks: "Checking if you are creating, updating or deleting a protected tag..."
+      }.freeze
+
+      def validate!
+        return unless tag_name
+
+        logger.log_timed(LOG_MESSAGES[:tag_checks]) do
+          if tag_exists? && user_access.cannot_do_action?(:admin_project)
+            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags]
+          end
+        end
+
+        protected_tag_checks
+      end
+
+      private
+
+      def protected_tag_checks
+        logger.log_timed(LOG_MESSAGES[__method__]) do
+          return unless ProtectedTag.protected?(project, tag_name) # rubocop:disable Cop/AvoidReturnFromBlocks
+
+          raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update?
+          raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion?
+
+          unless user_access.can_create_tag?(tag_name)
+            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag]
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/checks/timed_logger.rb b/lib/gitlab/checks/timed_logger.rb
new file mode 100644
index 00000000000..f365e0a43f6
--- /dev/null
+++ b/lib/gitlab/checks/timed_logger.rb
@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Checks
+    class TimedLogger
+      TimeoutError = Class.new(StandardError)
+
+      attr_reader :start_time, :header, :log, :timeout
+
+      def initialize(start_time: Time.now, log: [], header: "", timeout:)
+        @start_time = start_time
+        @timeout = timeout
+        @header = header
+        @log = log
+      end
+
+      # Adds trace of method being tracked with
+      # the correspondent time it took to run it.
+      # We make use of the start default argument
+      # on unit tests related to this method
+      #
+      def log_timed(log_message, start = Time.now)
+        check_timeout_reached
+
+        timed = true
+
+        yield
+
+        append_message(log_message + time_suffix_message(start: start))
+      rescue GRPC::DeadlineExceeded, TimeoutError
+        args = { cancelled: true }
+        args[:start] = start if timed
+
+        append_message(log_message + time_suffix_message(args))
+
+        raise TimeoutError
+      end
+
+      def check_timeout_reached
+        return unless time_expired?
+
+        raise TimeoutError
+      end
+
+      def time_left
+        (start_time + timeout.seconds) - Time.now
+      end
+
+      def full_message
+        header + log.join("\n")
+      end
+
+      # We always want to append in-place on the log
+      def append_message(message)
+        log << message
+      end
+
+      private
+
+      def time_expired?
+        time_left <= 0
+      end
+
+      def time_suffix_message(cancelled: false, start: nil)
+        return " (#{elapsed_time(start)}ms)" unless cancelled
+
+        if start
+          " (cancelled after #{elapsed_time(start)}ms)"
+        else
+          " (cancelled)"
+        end
+      end
+
+      def elapsed_time(start)
+        to_ms(Time.now - start)
+      end
+
+      def to_ms(elapsed)
+        (elapsed.to_f * 1000).round(2)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb
index 35eadf6fa93..4dcb3869d4f 100644
--- a/lib/gitlab/ci/ansi2html.rb
+++ b/lib/gitlab/ci/ansi2html.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # ANSI color library
 #
 # Implementation per http://en.wikipedia.org/wiki/ANSI_escape_code
@@ -29,105 +31,105 @@ module Gitlab
       end
 
       class Converter
-        def on_0(s) reset()                            end
+        def on_0(_) reset                              end
 
-        def on_1(s) enable(STYLE_SWITCHES[:bold])      end
+        def on_1(_) enable(STYLE_SWITCHES[:bold])      end
 
-        def on_3(s) enable(STYLE_SWITCHES[:italic])    end
+        def on_3(_) enable(STYLE_SWITCHES[:italic])    end
 
-        def on_4(s) enable(STYLE_SWITCHES[:underline]) end
+        def on_4(_) enable(STYLE_SWITCHES[:underline]) end
 
-        def on_8(s) enable(STYLE_SWITCHES[:conceal])   end
+        def on_8(_) enable(STYLE_SWITCHES[:conceal])   end
 
-        def on_9(s) enable(STYLE_SWITCHES[:cross])     end
+        def on_9(_) enable(STYLE_SWITCHES[:cross])     end
 
-        def on_21(s) disable(STYLE_SWITCHES[:bold])      end
+        def on_21(_) disable(STYLE_SWITCHES[:bold])      end
 
-        def on_22(s) disable(STYLE_SWITCHES[:bold])      end
+        def on_22(_) disable(STYLE_SWITCHES[:bold])      end
 
-        def on_23(s) disable(STYLE_SWITCHES[:italic])    end
+        def on_23(_) disable(STYLE_SWITCHES[:italic])    end
 
-        def on_24(s) disable(STYLE_SWITCHES[:underline]) end
+        def on_24(_) disable(STYLE_SWITCHES[:underline]) end
 
-        def on_28(s) disable(STYLE_SWITCHES[:conceal])   end
+        def on_28(_) disable(STYLE_SWITCHES[:conceal])   end
 
-        def on_29(s) disable(STYLE_SWITCHES[:cross])     end
+        def on_29(_) disable(STYLE_SWITCHES[:cross])     end
 
-        def on_30(s) set_fg_color(0) end
+        def on_30(_) set_fg_color(0) end
 
-        def on_31(s) set_fg_color(1) end
+        def on_31(_) set_fg_color(1) end
 
-        def on_32(s) set_fg_color(2) end
+        def on_32(_) set_fg_color(2) end
 
-        def on_33(s) set_fg_color(3) end
+        def on_33(_) set_fg_color(3) end
 
-        def on_34(s) set_fg_color(4) end
+        def on_34(_) set_fg_color(4) end
 
-        def on_35(s) set_fg_color(5) end
+        def on_35(_) set_fg_color(5) end
 
-        def on_36(s) set_fg_color(6) end
+        def on_36(_) set_fg_color(6) end
 
-        def on_37(s) set_fg_color(7) end
+        def on_37(_) set_fg_color(7) end
 
-        def on_38(s) set_fg_color_256(s) end
+        def on_38(stack) set_fg_color_256(stack) end
 
-        def on_39(s) set_fg_color(9) end
+        def on_39(_) set_fg_color(9) end
 
-        def on_40(s) set_bg_color(0) end
+        def on_40(_) set_bg_color(0) end
 
-        def on_41(s) set_bg_color(1) end
+        def on_41(_) set_bg_color(1) end
 
-        def on_42(s) set_bg_color(2) end
+        def on_42(_) set_bg_color(2) end
 
-        def on_43(s) set_bg_color(3) end
+        def on_43(_) set_bg_color(3) end
 
-        def on_44(s) set_bg_color(4) end
+        def on_44(_) set_bg_color(4) end
 
-        def on_45(s) set_bg_color(5) end
+        def on_45(_) set_bg_color(5) end
 
-        def on_46(s) set_bg_color(6) end
+        def on_46(_) set_bg_color(6) end
 
-        def on_47(s) set_bg_color(7) end
+        def on_47(_) set_bg_color(7) end
 
-        def on_48(s) set_bg_color_256(s) end
+        def on_48(stack) set_bg_color_256(stack) end
 
-        def on_49(s) set_bg_color(9) end
+        def on_49(_) set_bg_color(9) end
 
-        def on_90(s) set_fg_color(0, 'l') end
+        def on_90(_) set_fg_color(0, 'l') end
 
-        def on_91(s) set_fg_color(1, 'l') end
+        def on_91(_) set_fg_color(1, 'l') end
 
-        def on_92(s) set_fg_color(2, 'l') end
+        def on_92(_) set_fg_color(2, 'l') end
 
-        def on_93(s) set_fg_color(3, 'l') end
+        def on_93(_) set_fg_color(3, 'l') end
 
-        def on_94(s) set_fg_color(4, 'l') end
+        def on_94(_) set_fg_color(4, 'l') end
 
-        def on_95(s) set_fg_color(5, 'l') end
+        def on_95(_) set_fg_color(5, 'l') end
 
-        def on_96(s) set_fg_color(6, 'l') end
+        def on_96(_) set_fg_color(6, 'l') end
 
-        def on_97(s) set_fg_color(7, 'l') end
+        def on_97(_) set_fg_color(7, 'l') end
 
-        def on_99(s) set_fg_color(9, 'l') end
+        def on_99(_) set_fg_color(9, 'l') end
 
-        def on_100(s) set_bg_color(0, 'l') end
+        def on_100(_) set_bg_color(0, 'l') end
 
-        def on_101(s) set_bg_color(1, 'l') end
+        def on_101(_) set_bg_color(1, 'l') end
 
-        def on_102(s) set_bg_color(2, 'l') end
+        def on_102(_) set_bg_color(2, 'l') end
 
-        def on_103(s) set_bg_color(3, 'l') end
+        def on_103(_) set_bg_color(3, 'l') end
 
-        def on_104(s) set_bg_color(4, 'l') end
+        def on_104(_) set_bg_color(4, 'l') end
 
-        def on_105(s) set_bg_color(5, 'l') end
+        def on_105(_) set_bg_color(5, 'l') end
 
-        def on_106(s) set_bg_color(6, 'l') end
+        def on_106(_) set_bg_color(6, 'l') end
 
-        def on_107(s) set_bg_color(7, 'l') end
+        def on_107(_) set_bg_color(7, 'l') end
 
-        def on_109(s) set_bg_color(9, 'l') end
+        def on_109(_) set_bg_color(9, 'l') end
 
         attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask
 
@@ -175,7 +177,7 @@ module Gitlab
             end
           end
 
-          close_open_tags()
+          close_open_tags
 
           OpenStruct.new(
             html: @out.force_encoding(Encoding.default_external),
@@ -188,29 +190,29 @@ module Gitlab
           )
         end
 
-        def handle_section(s)
-          action = s[1]
-          timestamp = s[2]
-          section = s[3]
-          line = s.matched()[0...-5] # strips \r\033[0K
+        def handle_section(scanner)
+          action = scanner[1]
+          timestamp = scanner[2]
+          section = scanner[3]
+          line = scanner.matched[0...-5] # strips \r\033[0K
 
           @out << %{<div class="hidden" data-action="#{action}" data-timestamp="#{timestamp}" data-section="#{section}">#{line}</div>}
         end
 
-        def handle_sequence(s)
-          indicator = s[1]
-          commands = s[2].split ';'
-          terminator = s[3]
+        def handle_sequence(scanner)
+          indicator = scanner[1]
+          commands = scanner[2].split ';'
+          terminator = scanner[3]
 
           # We are only interested in color and text style changes - triggered by
           # sequences starting with '\e[' and ending with 'm'. Any other control
           # sequence gets stripped (including stuff like "delete last line")
           return unless indicator == '[' && terminator == 'm'
 
-          close_open_tags()
+          close_open_tags
 
-          if commands.empty?()
-            reset()
+          if commands.empty?
+            reset
             return
           end
 
@@ -220,7 +222,7 @@ module Gitlab
         end
 
         def evaluate_command_stack(stack)
-          return unless command = stack.shift()
+          return unless command = stack.shift
 
           if self.respond_to?("on_#{command}", true)
             self.__send__("on_#{command}", stack) # rubocop:disable GitlabSecurity/PublicSend
@@ -265,7 +267,7 @@ module Gitlab
         def reset_state
           @offset = 0
           @n_open_tags = 0
-          @out = ''
+          @out = +''
           reset
         end
 
@@ -331,8 +333,8 @@ module Gitlab
           return unless command_stack.length >= 2
           return unless command_stack[0] == "5"
 
-          command_stack.shift() # ignore the "5" command
-          color_index = command_stack.shift().to_i
+          command_stack.shift # ignore the "5" command
+          color_index = command_stack.shift.to_i
 
           return unless color_index >= 0
           return unless color_index <= 255
diff --git a/lib/gitlab/ci/build/artifacts/adapters/gzip_stream.rb b/lib/gitlab/ci/build/artifacts/adapters/gzip_stream.rb
new file mode 100644
index 00000000000..25a82086676
--- /dev/null
+++ b/lib/gitlab/ci/build/artifacts/adapters/gzip_stream.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Build
+      module Artifacts
+        module Adapters
+          class GzipStream
+            attr_reader :stream
+
+            InvalidStreamError = Class.new(StandardError)
+
+            def initialize(stream)
+              raise InvalidStreamError, "Stream is required" unless stream
+
+              @stream = stream
+            end
+
+            def each_blob
+              stream.seek(0)
+
+              until stream.eof?
+                gzip(stream) do |gz|
+                  yield gz.read, gz.orig_name
+                  unused = gz.unused&.length.to_i
+                  # pos has already reached to EOF at the moment
+                  # We rewind the pos to the top of unused files
+                  # to read next gzip stream, to support multistream archives
+                  # https://golang.org/src/compress/gzip/gunzip.go#L117
+                  stream.seek(-unused, IO::SEEK_CUR)
+                end
+              end
+            end
+
+            private
+
+            def gzip(stream, &block)
+              gz = Zlib::GzipReader.new(stream)
+              yield(gz)
+            rescue Zlib::Error => e
+              raise InvalidStreamError, e.message
+            ensure
+              gz&.finish
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/artifacts/adapters/raw_stream.rb b/lib/gitlab/ci/build/artifacts/adapters/raw_stream.rb
new file mode 100644
index 00000000000..cf37d700991
--- /dev/null
+++ b/lib/gitlab/ci/build/artifacts/adapters/raw_stream.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Build
+      module Artifacts
+        module Adapters
+          class RawStream
+            attr_reader :stream
+
+            InvalidStreamError = Class.new(StandardError)
+
+            def initialize(stream)
+              raise InvalidStreamError, "Stream is required" unless stream
+
+              @stream = stream
+            end
+
+            def each_blob
+              stream.seek(0)
+
+              yield(stream.read, 'raw') unless stream.eof?
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb
index 0bbd60d8ffe..08dac756cc1 100644
--- a/lib/gitlab/ci/build/artifacts/metadata.rb
+++ b/lib/gitlab/ci/build/artifacts/metadata.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'zlib'
 require 'json'
 
@@ -7,14 +9,15 @@ module Gitlab
       module Artifacts
         class Metadata
           ParserError = Class.new(StandardError)
+          InvalidStreamError = Class.new(StandardError)
 
           VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/
           INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)}
 
-          attr_reader :file, :path, :full_version
+          attr_reader :stream, :path, :full_version
 
-          def initialize(file, path, **opts)
-            @file, @path, @opts = file, path, opts
+          def initialize(stream, path, **opts)
+            @stream, @path, @opts = stream, path, opts
             @full_version = read_version
           end
 
@@ -58,9 +61,12 @@ module Gitlab
 
             until gz.eof?
               begin
-                path = read_string(gz).force_encoding('UTF-8')
-                meta = read_string(gz).force_encoding('UTF-8')
+                path = read_string(gz)&.force_encoding('UTF-8')
+                meta = read_string(gz)&.force_encoding('UTF-8')
 
+                # We might hit an EOF while reading either value, so we should
+                # abort if we don't get any data.
+                next unless path && meta
                 next unless path.valid_encoding? && meta.valid_encoding?
                 next unless path =~ match_pattern
                 next if path =~ INVALID_PATH_PATTERN
@@ -103,7 +109,17 @@ module Gitlab
           end
 
           def gzip(&block)
-            Zlib::GzipReader.open(@file, &block)
+            raise InvalidStreamError, "Invalid stream" unless @stream
+
+            # restart gzip reading
+            @stream.seek(0)
+
+            gz = Zlib::GzipReader.new(@stream)
+            yield(gz)
+          rescue Zlib::Error => e
+            raise InvalidStreamError, e.message
+          ensure
+            gz&.finish
           end
         end
       end
diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
index 428c0505808..d0a80518ae8 100644
--- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb
+++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Build
@@ -96,12 +98,14 @@ module Gitlab
               blank_node? || @entries.include?(@path.to_s)
             end
 
+            # rubocop: disable CodeReuse/ActiveRecord
             def total_size
               descendant_pattern = /^#{Regexp.escape(@path.to_s)}/
               entries.sum do |path, entry|
                 (entry[:size] if path =~ descendant_pattern).to_i
               end
             end
+            # rubocop: enable CodeReuse/ActiveRecord
 
             def path
               @path.to_s
diff --git a/lib/gitlab/ci/build/artifacts/path.rb b/lib/gitlab/ci/build/artifacts/path.rb
index 9cd9b36c5f8..65cd935afaa 100644
--- a/lib/gitlab/ci/build/artifacts/path.rb
+++ b/lib/gitlab/ci/build/artifacts/path.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Build
diff --git a/lib/gitlab/ci/build/credentials/base.rb b/lib/gitlab/ci/build/credentials/base.rb
index 29a7a27c963..58adf6e506d 100644
--- a/lib/gitlab/ci/build/credentials/base.rb
+++ b/lib/gitlab/ci/build/credentials/base.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Build
diff --git a/lib/gitlab/ci/build/credentials/factory.rb b/lib/gitlab/ci/build/credentials/factory.rb
index 2423aa8857d..fa805abb8bb 100644
--- a/lib/gitlab/ci/build/credentials/factory.rb
+++ b/lib/gitlab/ci/build/credentials/factory.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Build
diff --git a/lib/gitlab/ci/build/credentials/registry.rb b/lib/gitlab/ci/build/credentials/registry.rb
index 55eafcaed10..1c8588d9913 100644
--- a/lib/gitlab/ci/build/credentials/registry.rb
+++ b/lib/gitlab/ci/build/credentials/registry.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Build
diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb
index c811f88f483..4dd932f61d4 100644
--- a/lib/gitlab/ci/build/image.rb
+++ b/lib/gitlab/ci/build/image.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Build
diff --git a/lib/gitlab/ci/build/policy.rb b/lib/gitlab/ci/build/policy.rb
index d10cc7802d4..43c46ad74af 100644
--- a/lib/gitlab/ci/build/policy.rb
+++ b/lib/gitlab/ci/build/policy.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Build
diff --git a/lib/gitlab/ci/build/policy/changes.rb b/lib/gitlab/ci/build/policy/changes.rb
new file mode 100644
index 00000000000..1663c875426
--- /dev/null
+++ b/lib/gitlab/ci/build/policy/changes.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Build
+      module Policy
+        class Changes < Policy::Specification
+          def initialize(globs)
+            @globs = Array(globs)
+          end
+
+          def satisfied_by?(pipeline, seed)
+            return true unless pipeline.branch_updated?
+
+            pipeline.modified_paths.any? do |path|
+              @globs.any? do |glob|
+                File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/policy/kubernetes.rb b/lib/gitlab/ci/build/policy/kubernetes.rb
index 782f6c4c0af..4c7dc947cd0 100644
--- a/lib/gitlab/ci/build/policy/kubernetes.rb
+++ b/lib/gitlab/ci/build/policy/kubernetes.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Build
diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb
index 4aa5dc89f47..0e9bb5c94bb 100644
--- a/lib/gitlab/ci/build/policy/refs.rb
+++ b/lib/gitlab/ci/build/policy/refs.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Build
@@ -30,10 +32,14 @@ module Gitlab
             return true if pipeline.source == pattern
             return true if pipeline.source&.pluralize == pattern
 
-            if pattern.first == "/" && pattern.last == "/"
-              Regexp.new(pattern[1...-1]) =~ pipeline.ref
-            else
-              pattern == pipeline.ref
+            # patterns can be matched only when branch or tag is used
+            # the pattern matching does not work for merge requests pipelines
+            if pipeline.branch? || pipeline.tag?
+              if pattern.first == "/" && pattern.last == "/"
+                Regexp.new(pattern[1...-1]) =~ pipeline.ref
+              else
+                pattern == pipeline.ref
+              end
             end
           end
         end
diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb
index f09ba42c074..ceb5210cfb5 100644
--- a/lib/gitlab/ci/build/policy/specification.rb
+++ b/lib/gitlab/ci/build/policy/specification.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Build
diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb
index 9d2a362b7d4..0698136166a 100644
--- a/lib/gitlab/ci/build/policy/variables.rb
+++ b/lib/gitlab/ci/build/policy/variables.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Build
diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb
index 0b1ebe4e048..7fcabc035ac 100644
--- a/lib/gitlab/ci/build/step.rb
+++ b/lib/gitlab/ci/build/step.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Build
@@ -13,7 +15,6 @@ module Gitlab
           def from_commands(job)
             self.new(:script).tap do |step|
               step.script = job.options[:before_script].to_a + job.options[:script].to_a
-              step.script = job.commands.split("\n") if step.script.empty?
               step.timeout = job.metadata_timeout
               step.when = WHEN_ON_SUCCESS
             end
diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb
index 46ed330dbbf..7cabaadb122 100644
--- a/lib/gitlab/ci/charts.rb
+++ b/lib/gitlab/ci/charts.rb
@@ -1,13 +1,17 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Charts
       module DailyInterval
+        # rubocop: disable CodeReuse/ActiveRecord
         def grouped_count(query)
           query
             .group("DATE(#{::Ci::Pipeline.table_name}.created_at)")
             .count(:created_at)
             .transform_keys { |date| date.strftime(@format) } # rubocop:disable Gitlab/ModuleWithInstanceVariables
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         def interval_step
           @interval_step ||= 1.day
@@ -15,6 +19,7 @@ module Gitlab
       end
 
       module MonthlyInterval
+        # rubocop: disable CodeReuse/ActiveRecord
         def grouped_count(query)
           if Gitlab::Database.postgresql?
             query
@@ -27,6 +32,7 @@ module Gitlab
               .count(:created_at)
           end
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         def interval_step
           @interval_step ||= 1.month
@@ -46,8 +52,9 @@ module Gitlab
           collect
         end
 
+        # rubocop: disable CodeReuse/ActiveRecord
         def collect
-          query = project.pipelines
+          query = project.all_pipelines
             .where("? > #{::Ci::Pipeline.table_name}.created_at AND #{::Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection
 
           totals_count  = grouped_count(query)
@@ -64,6 +71,7 @@ module Gitlab
             current += interval_step
           end
         end
+        # rubocop: enable CodeReuse/ActiveRecord
       end
 
       class YearChart < Chart
@@ -107,7 +115,7 @@ module Gitlab
 
       class PipelineTime < Chart
         def collect
-          commits = project.pipelines.last(30)
+          commits = project.all_pipelines.last(30)
 
           commits.each do |commit|
             @labels << commit.short_sha
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 66ac4a40616..5875479183e 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -1,15 +1,24 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
-    ##
+    #
     # Base GitLab CI Configuration facade
     #
     class Config
-      # EE would override this and utilize opts argument
-      def initialize(config, opts = {})
-        @config = Loader.new(config).load!
+      ConfigError = Class.new(StandardError)
+
+      def initialize(config, project: nil, sha: nil, user: nil)
+        @config = Config::Extendable
+          .new(build_config(config, project: project, sha: sha, user: user))
+          .to_hash
 
         @global = Entry::Global.new(@config)
         @global.compose!
+      rescue Gitlab::Config::Loader::FormatError,
+             Extendable::ExtensionError,
+             External::Processor::IncludeError => e
+        raise Config::ConfigError, e.message
       end
 
       def valid?
@@ -58,6 +67,25 @@ module Gitlab
       def jobs
         @global.jobs_value
       end
+
+      private
+
+      def build_config(config, project:, sha:, user:)
+        initial_config = Gitlab::Config::Loader::Yaml.new(config).load!
+
+        if project
+          process_external_files(initial_config, project: project, sha: sha, user: user)
+        else
+          initial_config
+        end
+      end
+
+      def process_external_files(config, project:, sha:, user:)
+        Config::External::Processor.new(config,
+          project: project,
+          sha: sha || project.repository.root_ref_sha,
+          user: user).perform
+      end
     end
   end
 end
diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb
index 8275aacee9b..41613369ca2 100644
--- a/lib/gitlab/ci/config/entry/artifacts.rb
+++ b/lib/gitlab/ci/config/entry/artifacts.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -5,14 +7,17 @@ module Gitlab
         ##
         # Entry that represents a configuration of job artifacts.
         #
-        class Artifacts < Node
-          include Validatable
-          include Attributable
+        class Artifacts < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Configurable
+          include ::Gitlab::Config::Entry::Validatable
+          include ::Gitlab::Config::Entry::Attributable
 
-          ALLOWED_KEYS = %i[name untracked paths when expire_in].freeze
+          ALLOWED_KEYS = %i[name untracked paths reports when expire_in].freeze
 
           attributes ALLOWED_KEYS
 
+          entry :reports, Entry::Reports, description: 'Report-type artifacts.'
+
           validations do
             validates :config, type: Hash
             validates :config, allowed_keys: ALLOWED_KEYS
@@ -21,6 +26,7 @@ module Gitlab
               validates :name, type: String
               validates :untracked, boolean: true
               validates :paths, array_of_strings: true
+              validates :reports, type: Hash
               validates :when,
                 inclusion: { in: %w[on_success on_failure always],
                              message: 'should be on_success, on_failure ' \
@@ -28,6 +34,13 @@ module Gitlab
               validates :expire_in, duration: true
             end
           end
+
+          helpers :reports
+
+          def value
+            @config[:reports] = reports_value if @config.key?(:reports)
+            @config
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/config/entry/attributable.rb b/lib/gitlab/ci/config/entry/attributable.rb
deleted file mode 100644
index 3e87a09704e..00000000000
--- a/lib/gitlab/ci/config/entry/attributable.rb
+++ /dev/null
@@ -1,27 +0,0 @@
-module Gitlab
-  module Ci
-    class Config
-      module Entry
-        module Attributable
-          extend ActiveSupport::Concern
-
-          class_methods do
-            def attributes(*attributes)
-              attributes.flatten.each do |attribute|
-                if method_defined?(attribute)
-                  raise ArgumentError, 'Method already defined!'
-                end
-
-                define_method(attribute) do
-                  return unless config.is_a?(Hash)
-
-                  config[attribute]
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/entry/boolean.rb b/lib/gitlab/ci/config/entry/boolean.rb
deleted file mode 100644
index f3357f85b99..00000000000
--- a/lib/gitlab/ci/config/entry/boolean.rb
+++ /dev/null
@@ -1,18 +0,0 @@
-module Gitlab
-  module Ci
-    class Config
-      module Entry
-        ##
-        # Entry that represents a boolean value.
-        #
-        class Boolean < Node
-          include Validatable
-
-          validations do
-            validates :config, boolean: true
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb
index d7e09acbbf3..7b94af24c09 100644
--- a/lib/gitlab/ci/config/entry/cache.rb
+++ b/lib/gitlab/ci/config/entry/cache.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -5,9 +7,9 @@ module Gitlab
         ##
         # Entry that represents a cache configuration
         #
-        class Cache < Node
-          include Configurable
-          include Attributable
+        class Cache < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Configurable
+          include ::Gitlab::Config::Entry::Attributable
 
           ALLOWED_KEYS = %i[key untracked paths policy].freeze
           DEFAULT_POLICY = 'pull-push'.freeze
@@ -20,7 +22,7 @@ module Gitlab
           entry :key, Entry::Key,
             description: 'Cache key used to define a cache affinity.'
 
-          entry :untracked, Entry::Boolean,
+          entry :untracked, ::Gitlab::Config::Entry::Boolean,
             description: 'Cache all untracked files.'
 
           entry :paths, Entry::Paths,
diff --git a/lib/gitlab/ci/config/entry/commands.rb b/lib/gitlab/ci/config/entry/commands.rb
index 65d19db249c..02e368c1813 100644
--- a/lib/gitlab/ci/config/entry/commands.rb
+++ b/lib/gitlab/ci/config/entry/commands.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -5,22 +7,11 @@ module Gitlab
         ##
         # Entry that represents a job script.
         #
-        class Commands < Node
-          include Validatable
+        class Commands < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
 
           validations do
-            include LegacyValidationHelpers
-
-            validate do
-              unless string_or_array_of_strings?(config)
-                errors.add(:config,
-                           'should be a string or an array of strings')
-              end
-            end
-
-            def string_or_array_of_strings?(field)
-              validate_string(field) || validate_array_of_strings(field)
-            end
+            validates :config, array_of_strings_or_string: true
           end
 
           def value
diff --git a/lib/gitlab/ci/config/entry/configurable.rb b/lib/gitlab/ci/config/entry/configurable.rb
deleted file mode 100644
index db47c2f6185..00000000000
--- a/lib/gitlab/ci/config/entry/configurable.rb
+++ /dev/null
@@ -1,77 +0,0 @@
-module Gitlab
-  module Ci
-    class Config
-      module Entry
-        ##
-        # This mixin is responsible for adding DSL, which purpose is to
-        # simplifly process of adding child nodes.
-        #
-        # This can be used only if parent node is a configuration entry that
-        # holds a hash as a configuration value, for example:
-        #
-        # job:
-        #   script: ...
-        #   artifacts: ...
-        #
-        module Configurable
-          extend ActiveSupport::Concern
-
-          included do
-            include Validatable
-
-            validations do
-              validates :config, type: Hash
-            end
-          end
-
-          def compose!(deps = nil)
-            return unless valid?
-
-            self.class.nodes.each do |key, factory|
-              factory
-                .value(config[key])
-                .with(key: key, parent: self)
-
-              entries[key] = factory.create!
-            end
-
-            yield if block_given?
-
-            entries.each_value do |entry|
-              entry.compose!(deps)
-            end
-          end
-
-          class_methods do
-            def nodes
-              Hash[(@nodes || {}).map { |key, factory| [key, factory.dup] }]
-            end
-
-            private # rubocop:disable Lint/UselessAccessModifier
-
-            def entry(key, entry, metadata)
-              factory = Entry::Factory.new(entry)
-                .with(description: metadata[:description])
-
-              (@nodes ||= {}).merge!(key.to_sym => factory)
-            end
-
-            def helpers(*nodes)
-              nodes.each do |symbol|
-                define_method("#{symbol}_defined?") do
-                  entries[symbol]&.specified?
-                end
-
-                define_method("#{symbol}_value") do
-                  return unless entries[symbol] && entries[symbol].valid?
-
-                  entries[symbol].value
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/entry/coverage.rb b/lib/gitlab/ci/config/entry/coverage.rb
index 12a063059cb..89545158bed 100644
--- a/lib/gitlab/ci/config/entry/coverage.rb
+++ b/lib/gitlab/ci/config/entry/coverage.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -5,8 +7,8 @@ module Gitlab
         ##
         # Entry that represents Coverage settings.
         #
-        class Coverage < Node
-          include Validatable
+        class Coverage < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
 
           validations do
             validates :config, regexp: true
diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb
index 0c1f9eb7cbf..69a3a1aedef 100644
--- a/lib/gitlab/ci/config/entry/environment.rb
+++ b/lib/gitlab/ci/config/entry/environment.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -5,8 +7,8 @@ module Gitlab
         ##
         # Entry that represents an environment.
         #
-        class Environment < Node
-          include Validatable
+        class Environment < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
 
           ALLOWED_KEYS = %i[name url action on_stop].freeze
 
diff --git a/lib/gitlab/ci/config/entry/factory.rb b/lib/gitlab/ci/config/entry/factory.rb
deleted file mode 100644
index 6be8288748f..00000000000
--- a/lib/gitlab/ci/config/entry/factory.rb
+++ /dev/null
@@ -1,73 +0,0 @@
-module Gitlab
-  module Ci
-    class Config
-      module Entry
-        ##
-        # Factory class responsible for fabricating entry objects.
-        #
-        class Factory
-          InvalidFactory = Class.new(StandardError)
-
-          def initialize(entry)
-            @entry = entry
-            @metadata = {}
-            @attributes = {}
-          end
-
-          def value(value)
-            @value = value
-            self
-          end
-
-          def metadata(metadata)
-            @metadata.merge!(metadata)
-            self
-          end
-
-          def with(attributes)
-            @attributes.merge!(attributes)
-            self
-          end
-
-          def create!
-            raise InvalidFactory unless defined?(@value)
-
-            ##
-            # We assume that unspecified entry is undefined.
-            # See issue #18775.
-            #
-            if @value.nil?
-              Entry::Unspecified.new(
-                fabricate_unspecified
-              )
-            else
-              fabricate(@entry, @value)
-            end
-          end
-
-          private
-
-          def fabricate_unspecified
-            ##
-            # If entry has a default value we fabricate concrete node
-            # with default value.
-            #
-            if @entry.default.nil?
-              fabricate(Entry::Undefined)
-            else
-              fabricate(@entry, @entry.default)
-            end
-          end
-
-          def fabricate(entry, value = nil)
-            entry.new(value, @metadata).tap do |node|
-              node.key = @attributes[:key]
-              node.parent = @attributes[:parent]
-              node.description = @attributes[:description]
-            end
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/entry/global.rb b/lib/gitlab/ci/config/entry/global.rb
index a4ec8f0ff2f..09ecb5fdb99 100644
--- a/lib/gitlab/ci/config/entry/global.rb
+++ b/lib/gitlab/ci/config/entry/global.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -6,8 +8,8 @@ module Gitlab
         # This class represents a global entry - root Entry for entire
         # GitLab CI Configuration file.
         #
-        class Global < Node
-          include Configurable
+        class Global < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Configurable
 
           entry :before_script, Entry::Script,
             description: 'Script that will be executed before each job.'
@@ -45,14 +47,16 @@ module Gitlab
 
           private
 
+          # rubocop: disable CodeReuse/ActiveRecord
           def compose_jobs!
-            factory = Entry::Factory.new(Entry::Jobs)
+            factory = ::Gitlab::Config::Entry::Factory.new(Entry::Jobs)
               .value(@config.except(*self.class.nodes.keys))
               .with(key: :jobs, parent: self,
                     description: 'Jobs definition for this pipeline')
 
             @entries[:jobs] = factory.create!
           end
+          # rubocop: enable CodeReuse/ActiveRecord
 
           def compose_deprecated_entries!
             ##
diff --git a/lib/gitlab/ci/config/entry/hidden.rb b/lib/gitlab/ci/config/entry/hidden.rb
index 6fc3aa385bc..76e5d05639f 100644
--- a/lib/gitlab/ci/config/entry/hidden.rb
+++ b/lib/gitlab/ci/config/entry/hidden.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -5,8 +7,8 @@ module Gitlab
         ##
         # Entry that represents a hidden CI/CD key.
         #
-        class Hidden < Node
-          include Validatable
+        class Hidden < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
 
           validations do
             validates :config, presence: true
diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb
index 2844be80a84..a13a0625e90 100644
--- a/lib/gitlab/ci/config/entry/image.rb
+++ b/lib/gitlab/ci/config/entry/image.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -5,8 +7,8 @@ module Gitlab
         ##
         # Entry that represents a Docker image.
         #
-        class Image < Node
-          include Validatable
+        class Image < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
 
           ALLOWED_KEYS = %i[name entrypoint].freeze
 
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index 91aac6df4b1..290c9591b98 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -5,13 +7,14 @@ module Gitlab
         ##
         # Entry that represents a concrete CI/CD job.
         #
-        class Job < Node
-          include Configurable
-          include Attributable
+        class Job < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Configurable
+          include ::Gitlab::Config::Entry::Attributable
 
-          ALLOWED_KEYS = %i[tags script only except type image services allow_failure
-                            type stage when artifacts cache dependencies before_script
-                            after_script variables environment coverage retry].freeze
+          ALLOWED_KEYS = %i[tags script only except type image services
+                            allow_failure type stage when start_in artifacts cache
+                            dependencies before_script after_script variables
+                            environment coverage retry parallel extends].freeze
 
           validations do
             validates :config, allowed_keys: ALLOWED_KEYS
@@ -23,16 +26,19 @@ module Gitlab
             with_options allow_nil: true do
               validates :tags, array_of_strings: true
               validates :allow_failure, boolean: true
-              validates :retry, numericality: { only_integer: true,
-                                                greater_than_or_equal_to: 0,
-                                                less_than_or_equal_to: 2 }
+              validates :parallel, numericality: { only_integer: true,
+                                                   greater_than_or_equal_to: 2,
+                                                   less_than_or_equal_to: 50 }
               validates :when,
-                inclusion: { in: %w[on_success on_failure always manual],
+                inclusion: { in: %w[on_success on_failure always manual delayed],
                              message: 'should be on_success, on_failure, ' \
-                                      'always or manual' }
-
+                                      'always, manual or delayed' }
               validates :dependencies, array_of_strings: true
+              validates :extends, type: String
             end
+
+            validates :start_in, duration: { limit: '1 day' }, if: :delayed?
+            validates :start_in, absence: true, unless: :delayed?
           end
 
           entry :before_script, Entry::Script,
@@ -60,7 +66,8 @@ module Gitlab
             description: 'Services that will be used to execute this job.'
 
           entry :only, Entry::Policy,
-            description: 'Refs policy this job will be executed for.'
+            description: 'Refs policy this job will be executed for.',
+            default: Entry::Policy::DEFAULT_ONLY
 
           entry :except, Entry::Policy,
             description: 'Refs policy this job will be executed for.'
@@ -72,16 +79,21 @@ module Gitlab
             description: 'Artifacts configuration for this job.'
 
           entry :environment, Entry::Environment,
-               description: 'Environment configuration for this job.'
+            description: 'Environment configuration for this job.'
 
           entry :coverage, Entry::Coverage,
-               description: 'Coverage configuration for this job.'
+            description: 'Coverage configuration for this job.'
+
+          entry :retry, Entry::Retry,
+               description: 'Retry configuration for this job.'
 
           helpers :before_script, :script, :stage, :type, :after_script,
                   :cache, :image, :services, :only, :except, :variables,
-                  :artifacts, :commands, :environment, :coverage, :retry
+                  :artifacts, :environment, :coverage, :retry,
+                  :parallel
 
-          attributes :script, :tags, :allow_failure, :when, :dependencies, :retry
+          attributes :script, :tags, :allow_failure, :when, :dependencies,
+                     :retry, :parallel, :extends, :start_in
 
           def compose!(deps = nil)
             super do
@@ -103,14 +115,14 @@ module Gitlab
             @config.merge(to_hash.compact)
           end
 
-          def commands
-            (before_script_value.to_a + script_value.to_a).join("\n")
-          end
-
           def manual_action?
             self.when == 'manual'
           end
 
+          def delayed?
+            self.when == 'delayed'
+          end
+
           def ignored?
             allow_failure.nil? ? manual_action? : allow_failure
           end
@@ -134,7 +146,6 @@ module Gitlab
             { name: name,
               before_script: before_script_value,
               script: script_value,
-              commands: commands,
               image: image_value,
               services: services_value,
               stage: stage_value,
@@ -145,7 +156,8 @@ module Gitlab
               environment: environment_defined? ? environment_value : nil,
               environment_name: environment_defined? ? environment_value[:name] : nil,
               coverage: coverage_defined? ? coverage_value : nil,
-              retry: retry_defined? ? retry_value.to_i : nil,
+              retry: retry_defined? ? retry_value : nil,
+              parallel: parallel_defined? ? parallel_value.to_i : nil,
               artifacts: artifacts_value,
               after_script: after_script_value,
               ignore: ignored? }
diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb
index 5671a09480b..9845c4af655 100644
--- a/lib/gitlab/ci/config/entry/jobs.rb
+++ b/lib/gitlab/ci/config/entry/jobs.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -5,8 +7,8 @@ module Gitlab
         ##
         # Entry that represents a set of jobs.
         #
-        class Jobs < Node
-          include Validatable
+        class Jobs < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
 
           validations do
             validates :config, type: Hash
@@ -26,12 +28,17 @@ module Gitlab
             name.to_s.start_with?('.')
           end
 
+          def node_type(name)
+            hidden?(name) ? Entry::Hidden : Entry::Job
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
           def compose!(deps = nil)
             super do
               @config.each do |name, config|
-                node = hidden?(name) ? Entry::Hidden : Entry::Job
+                node = node_type(name)
 
-                factory = Entry::Factory.new(node)
+                factory = ::Gitlab::Config::Entry::Factory.new(node)
                   .value(config || {})
                   .metadata(name: name)
                   .with(key: name, parent: self,
@@ -45,6 +52,7 @@ module Gitlab
               end
             end
           end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/ci/config/entry/key.rb b/lib/gitlab/ci/config/entry/key.rb
index f27ad0a7759..0c10967e629 100644
--- a/lib/gitlab/ci/config/entry/key.rb
+++ b/lib/gitlab/ci/config/entry/key.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -5,8 +7,8 @@ module Gitlab
         ##
         # Entry that represents a key.
         #
-        class Key < Node
-          include Validatable
+        class Key < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
 
           validations do
             validates :config, key: true
diff --git a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb
deleted file mode 100644
index a78a85397bd..00000000000
--- a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb
+++ /dev/null
@@ -1,61 +0,0 @@
-module Gitlab
-  module Ci
-    class Config
-      module Entry
-        module LegacyValidationHelpers
-          private
-
-          def validate_duration(value)
-            value.is_a?(String) && ChronicDuration.parse(value)
-          rescue ChronicDuration::DurationParseError
-            false
-          end
-
-          def validate_array_of_strings(values)
-            values.is_a?(Array) && values.all? { |value| validate_string(value) }
-          end
-
-          def validate_array_of_strings_or_regexps(values)
-            values.is_a?(Array) && values.all? { |value| validate_string_or_regexp(value) }
-          end
-
-          def validate_variables(variables)
-            variables.is_a?(Hash) &&
-              variables.flatten.all? do |value|
-                validate_string(value) || validate_integer(value)
-              end
-          end
-
-          def validate_integer(value)
-            value.is_a?(Integer)
-          end
-
-          def validate_string(value)
-            value.is_a?(String) || value.is_a?(Symbol)
-          end
-
-          def validate_regexp(value)
-            !value.nil? && Regexp.new(value.to_s) && true
-          rescue RegexpError, TypeError
-            false
-          end
-
-          def validate_string_or_regexp(value)
-            return true if value.is_a?(Symbol)
-            return false unless value.is_a?(String)
-
-            if value.first == '/' && value.last == '/'
-              validate_regexp(value[1...-1])
-            else
-              true
-            end
-          end
-
-          def validate_boolean(value)
-            value.in?([true, false])
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/entry/node.rb b/lib/gitlab/ci/config/entry/node.rb
deleted file mode 100644
index 26505c91be3..00000000000
--- a/lib/gitlab/ci/config/entry/node.rb
+++ /dev/null
@@ -1,101 +0,0 @@
-module Gitlab
-  module Ci
-    class Config
-      module Entry
-        ##
-        # Base abstract class for each configuration entry node.
-        #
-        class Node
-          InvalidError = Class.new(StandardError)
-
-          attr_reader :config, :metadata
-          attr_accessor :key, :parent, :description
-
-          def initialize(config, **metadata)
-            @config = config
-            @metadata = metadata
-            @entries = {}
-
-            self.class.aspects.to_a.each do |aspect|
-              instance_exec(&aspect)
-            end
-          end
-
-          def [](key)
-            @entries[key] || Entry::Undefined.new
-          end
-
-          def compose!(deps = nil)
-            return unless valid?
-
-            yield if block_given?
-          end
-
-          def leaf?
-            @entries.none?
-          end
-
-          def descendants
-            @entries.values
-          end
-
-          def ancestors
-            @parent ? @parent.ancestors + [@parent] : []
-          end
-
-          def valid?
-            errors.none?
-          end
-
-          def errors
-            []
-          end
-
-          def value
-            if leaf?
-              @config
-            else
-              meaningful = @entries.select do |_key, value|
-                value.specified? && value.relevant?
-              end
-
-              Hash[meaningful.map { |key, entry| [key, entry.value] }]
-            end
-          end
-
-          def specified?
-            true
-          end
-
-          def relevant?
-            true
-          end
-
-          def location
-            name = @key.presence || self.class.name.to_s.demodulize
-                                        .underscore.humanize.downcase
-
-            ancestors.map(&:key).append(name).compact.join(':')
-          end
-
-          def inspect
-            val = leaf? ? config : descendants
-            unspecified = specified? ? '' : '(unspecified) '
-            "#<#{self.class.name} #{unspecified}{#{key}: #{val.inspect}}>"
-          end
-
-          def self.default
-          end
-
-          def self.aspects
-            @aspects ||= []
-          end
-
-          private
-
-          attr_reader :entries
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/entry/paths.rb b/lib/gitlab/ci/config/entry/paths.rb
index 68dad161149..d6f287c6552 100644
--- a/lib/gitlab/ci/config/entry/paths.rb
+++ b/lib/gitlab/ci/config/entry/paths.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -5,8 +7,8 @@ module Gitlab
         ##
         # Entry that represents an array of paths.
         #
-        class Paths < Node
-          include Validatable
+        class Paths < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
 
           validations do
             validates :config, array_of_strings: true
diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb
index 09e8e52b60f..adc3660d950 100644
--- a/lib/gitlab/ci/config/entry/policy.rb
+++ b/lib/gitlab/ci/config/entry/policy.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -5,12 +7,14 @@ module Gitlab
         ##
         # Entry that represents an only/except trigger policy for the job.
         #
-        class Policy < Simplifiable
+        class Policy < ::Gitlab::Config::Entry::Simplifiable
           strategy :RefsPolicy, if: -> (config) { config.is_a?(Array) }
           strategy :ComplexPolicy, if: -> (config) { config.is_a?(Hash) }
 
-          class RefsPolicy < Entry::Node
-            include Entry::Validatable
+          DEFAULT_ONLY = { refs: %w[branches tags] }.freeze
+
+          class RefsPolicy < ::Gitlab::Config::Entry::Node
+            include ::Gitlab::Config::Entry::Validatable
 
             validations do
               validates :config, array_of_strings_or_regexps: true
@@ -21,21 +25,23 @@ module Gitlab
             end
           end
 
-          class ComplexPolicy < Entry::Node
-            include Entry::Validatable
-            include Entry::Attributable
+          class ComplexPolicy < ::Gitlab::Config::Entry::Node
+            include ::Gitlab::Config::Entry::Validatable
+            include ::Gitlab::Config::Entry::Attributable
 
-            attributes :refs, :kubernetes, :variables
+            ALLOWED_KEYS = %i[refs kubernetes variables changes].freeze
+            attributes :refs, :kubernetes, :variables, :changes
 
             validations do
               validates :config, presence: true
-              validates :config, allowed_keys: %i[refs kubernetes variables]
+              validates :config, allowed_keys: ALLOWED_KEYS
               validate :variables_expressions_syntax
 
               with_options allow_nil: true do
                 validates :refs, array_of_strings_or_regexps: true
                 validates :kubernetes, allowed_values: %w[active]
                 validates :variables, array_of_strings: true
+                validates :changes, array_of_strings: true
               end
 
               def variables_expressions_syntax
@@ -54,13 +60,14 @@ module Gitlab
             end
           end
 
-          class UnknownStrategy < Entry::Node
+          class UnknownStrategy < ::Gitlab::Config::Entry::Node
             def errors
               ["#{location} has to be either an array of conditions or a hash"]
             end
           end
 
-          def self.default
+          def value
+            default.to_h.deep_merge(subject.value.to_h)
           end
         end
       end
diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb
new file mode 100644
index 00000000000..a3f6cc31321
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/reports.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a configuration of job artifacts.
+        #
+        class Reports < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
+          include ::Gitlab::Config::Entry::Attributable
+
+          ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast performance license_management].freeze
+
+          attributes ALLOWED_KEYS
+
+          validations do
+            validates :config, type: Hash
+            validates :config, allowed_keys: ALLOWED_KEYS
+
+            with_options allow_nil: true do
+              validates :junit, array_of_strings_or_string: true
+              validates :codequality, array_of_strings_or_string: true
+              validates :sast, array_of_strings_or_string: true
+              validates :dependency_scanning, array_of_strings_or_string: true
+              validates :container_scanning, array_of_strings_or_string: true
+              validates :dast, array_of_strings_or_string: true
+              validates :performance, array_of_strings_or_string: true
+              validates :license_management, array_of_strings_or_string: true
+            end
+          end
+
+          def value
+            @config.transform_values { |v| Array(v) }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/retry.rb b/lib/gitlab/ci/config/entry/retry.rb
new file mode 100644
index 00000000000..e9cbcb31e21
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/retry.rb
@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a retry config for a job.
+        #
+        class Retry < ::Gitlab::Config::Entry::Simplifiable
+          strategy :SimpleRetry, if: -> (config) { config.is_a?(Integer) }
+          strategy :FullRetry, if: -> (config) { config.is_a?(Hash) }
+
+          class SimpleRetry < ::Gitlab::Config::Entry::Node
+            include ::Gitlab::Config::Entry::Validatable
+
+            validations do
+              validates :config, numericality: { only_integer: true,
+                                                 greater_than_or_equal_to: 0,
+                                                 less_than_or_equal_to: 2 }
+            end
+
+            def value
+              {
+                max: config
+              }
+            end
+
+            def location
+              'retry'
+            end
+          end
+
+          class FullRetry < ::Gitlab::Config::Entry::Node
+            include ::Gitlab::Config::Entry::Validatable
+            include ::Gitlab::Config::Entry::Attributable
+
+            ALLOWED_KEYS = %i[max when].freeze
+            attributes :max, :when
+
+            validations do
+              validates :config, allowed_keys: ALLOWED_KEYS
+
+              with_options allow_nil: true do
+                validates :max, numericality: { only_integer: true,
+                                                greater_than_or_equal_to: 0,
+                                                less_than_or_equal_to: 2 }
+
+                validates :when, array_of_strings_or_string: true
+                validates :when,
+                          allowed_array_values: { in: FullRetry.possible_retry_when_values },
+                          if: -> (config) { config.when.is_a?(Array) }
+                validates :when,
+                          inclusion: { in: FullRetry.possible_retry_when_values },
+                          if: -> (config) { config.when.is_a?(String) }
+              end
+            end
+
+            def self.possible_retry_when_values
+              @possible_retry_when_values ||= ::Ci::Build.failure_reasons.keys.map(&:to_s) + ['always']
+            end
+
+            def value
+              super.tap do |config|
+                # make sure that `when` is an array, because we allow it to
+                # be passed as a String in config for simplicity
+                config[:when] = Array.wrap(config[:when]) if config[:when]
+              end
+            end
+
+            def location
+              'retry'
+            end
+          end
+
+          class UnknownStrategy < ::Gitlab::Config::Entry::Node
+            def errors
+              ["#{location} has to be either an integer or a hash"]
+            end
+
+            def location
+              'retry config'
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/script.rb b/lib/gitlab/ci/config/entry/script.rb
index 29ecd9995ca..9d25a82b521 100644
--- a/lib/gitlab/ci/config/entry/script.rb
+++ b/lib/gitlab/ci/config/entry/script.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -5,8 +7,8 @@ module Gitlab
         ##
         # Entry that represents a script.
         #
-        class Script < Node
-          include Validatable
+        class Script < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
 
           validations do
             validates :config, array_of_strings: true
diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb
index 3e2ebcff31a..6df67083310 100644
--- a/lib/gitlab/ci/config/entry/service.rb
+++ b/lib/gitlab/ci/config/entry/service.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -6,7 +8,7 @@ module Gitlab
         # Entry that represents a configuration of Docker service.
         #
         class Service < Image
-          include Validatable
+          include ::Gitlab::Config::Entry::Validatable
 
           ALLOWED_KEYS = %i[name entrypoint command alias].freeze
 
diff --git a/lib/gitlab/ci/config/entry/services.rb b/lib/gitlab/ci/config/entry/services.rb
index 0066894e069..71475f69218 100644
--- a/lib/gitlab/ci/config/entry/services.rb
+++ b/lib/gitlab/ci/config/entry/services.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -5,8 +7,8 @@ module Gitlab
         ##
         # Entry that represents a configuration of Docker services.
         #
-        class Services < Node
-          include Validatable
+        class Services < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
 
           validations do
             validates :config, type: Array
@@ -16,7 +18,7 @@ module Gitlab
             super do
               @entries = []
               @config.each do |config|
-                @entries << Entry::Factory.new(Entry::Service)
+                @entries << ::Gitlab::Config::Entry::Factory.new(Entry::Service)
                   .value(config || {})
                   .create!
               end
diff --git a/lib/gitlab/ci/config/entry/simplifiable.rb b/lib/gitlab/ci/config/entry/simplifiable.rb
deleted file mode 100644
index 12764629686..00000000000
--- a/lib/gitlab/ci/config/entry/simplifiable.rb
+++ /dev/null
@@ -1,43 +0,0 @@
-module Gitlab
-  module Ci
-    class Config
-      module Entry
-        class Simplifiable < SimpleDelegator
-          EntryStrategy = Struct.new(:name, :condition)
-
-          def initialize(config, **metadata)
-            unless self.class.const_defined?(:UnknownStrategy)
-              raise ArgumentError, 'UndefinedStrategy not available!'
-            end
-
-            strategy = self.class.strategies.find do |variant|
-              variant.condition.call(config)
-            end
-
-            entry = self.class.entry_class(strategy)
-
-            super(entry.new(config, metadata))
-          end
-
-          def self.strategy(name, **opts)
-            EntryStrategy.new(name, opts.fetch(:if)).tap do |strategy|
-              strategies.append(strategy)
-            end
-          end
-
-          def self.strategies
-            @strategies ||= []
-          end
-
-          def self.entry_class(strategy)
-            if strategy.present?
-              self.const_get(strategy.name)
-            else
-              self::UnknownStrategy
-            end
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/entry/stage.rb b/lib/gitlab/ci/config/entry/stage.rb
index b7afaba1de8..d6d576a3139 100644
--- a/lib/gitlab/ci/config/entry/stage.rb
+++ b/lib/gitlab/ci/config/entry/stage.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -5,8 +7,8 @@ module Gitlab
         ##
         # Entry that represents a stage for a job.
         #
-        class Stage < Node
-          include Validatable
+        class Stage < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
 
           validations do
             validates :config, type: String
diff --git a/lib/gitlab/ci/config/entry/stages.rb b/lib/gitlab/ci/config/entry/stages.rb
index ec187bd3732..2d715cbc6bb 100644
--- a/lib/gitlab/ci/config/entry/stages.rb
+++ b/lib/gitlab/ci/config/entry/stages.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -5,8 +7,8 @@ module Gitlab
         ##
         # Entry that represents a configuration for pipeline stages.
         #
-        class Stages < Node
-          include Validatable
+        class Stages < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
 
           validations do
             validates :config, array_of_strings: true
diff --git a/lib/gitlab/ci/config/entry/undefined.rb b/lib/gitlab/ci/config/entry/undefined.rb
deleted file mode 100644
index 1171ac10f22..00000000000
--- a/lib/gitlab/ci/config/entry/undefined.rb
+++ /dev/null
@@ -1,40 +0,0 @@
-module Gitlab
-  module Ci
-    class Config
-      module Entry
-        ##
-        # This class represents an undefined entry.
-        #
-        class Undefined < Node
-          def initialize(*)
-            super(nil)
-          end
-
-          def value
-            nil
-          end
-
-          def valid?
-            true
-          end
-
-          def errors
-            []
-          end
-
-          def specified?
-            false
-          end
-
-          def relevant?
-            false
-          end
-
-          def inspect
-            "#<#{self.class.name}>"
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/entry/unspecified.rb b/lib/gitlab/ci/config/entry/unspecified.rb
deleted file mode 100644
index fbb2551e870..00000000000
--- a/lib/gitlab/ci/config/entry/unspecified.rb
+++ /dev/null
@@ -1,19 +0,0 @@
-module Gitlab
-  module Ci
-    class Config
-      module Entry
-        ##
-        # This class represents an unspecified entry.
-        #
-        # It decorates original entry adding method that indicates it is
-        # unspecified.
-        #
-        class Unspecified < SimpleDelegator
-          def specified?
-            false
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/entry/validatable.rb b/lib/gitlab/ci/config/entry/validatable.rb
deleted file mode 100644
index e45787773a8..00000000000
--- a/lib/gitlab/ci/config/entry/validatable.rb
+++ /dev/null
@@ -1,38 +0,0 @@
-module Gitlab
-  module Ci
-    class Config
-      module Entry
-        module Validatable
-          extend ActiveSupport::Concern
-
-          def self.included(node)
-            node.aspects.append -> do
-              @validator = self.class.validator.new(self)
-              @validator.validate(:new)
-            end
-          end
-
-          def errors
-            @validator.messages + descendants.flat_map(&:errors) # rubocop:disable Gitlab/ModuleWithInstanceVariables
-          end
-
-          class_methods do
-            def validator
-              @validator ||= Class.new(Entry::Validator).tap do |validator|
-                if defined?(@validations)
-                  @validations.each { |rules| validator.class_eval(&rules) }
-                end
-              end
-            end
-
-            private
-
-            def validations(&block)
-              (@validations ||= []).append(block)
-            end
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/entry/validator.rb b/lib/gitlab/ci/config/entry/validator.rb
deleted file mode 100644
index 2df23a3edcd..00000000000
--- a/lib/gitlab/ci/config/entry/validator.rb
+++ /dev/null
@@ -1,26 +0,0 @@
-module Gitlab
-  module Ci
-    class Config
-      module Entry
-        class Validator < SimpleDelegator
-          include ActiveModel::Validations
-          include Entry::Validators
-
-          def initialize(entry)
-            super(entry)
-          end
-
-          def messages
-            errors.full_messages.map do |error|
-              "#{location} #{error}".downcase
-            end
-          end
-
-          def self.name
-            'Validator'
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb
deleted file mode 100644
index 55658900628..00000000000
--- a/lib/gitlab/ci/config/entry/validators.rb
+++ /dev/null
@@ -1,158 +0,0 @@
-module Gitlab
-  module Ci
-    class Config
-      module Entry
-        module Validators
-          class AllowedKeysValidator < ActiveModel::EachValidator
-            def validate_each(record, attribute, value)
-              unknown_keys = record.config.try(:keys).to_a - options[:in]
-
-              if unknown_keys.any?
-                record.errors.add(:config, 'contains unknown keys: ' +
-                                            unknown_keys.join(', '))
-              end
-            end
-          end
-
-          class AllowedValuesValidator < ActiveModel::EachValidator
-            def validate_each(record, attribute, value)
-              unless options[:in].include?(value.to_s)
-                record.errors.add(attribute, "unknown value: #{value}")
-              end
-            end
-          end
-
-          class ArrayOfStringsValidator < ActiveModel::EachValidator
-            include LegacyValidationHelpers
-
-            def validate_each(record, attribute, value)
-              unless validate_array_of_strings(value)
-                record.errors.add(attribute, 'should be an array of strings')
-              end
-            end
-          end
-
-          class BooleanValidator < ActiveModel::EachValidator
-            include LegacyValidationHelpers
-
-            def validate_each(record, attribute, value)
-              unless validate_boolean(value)
-                record.errors.add(attribute, 'should be a boolean value')
-              end
-            end
-          end
-
-          class DurationValidator < ActiveModel::EachValidator
-            include LegacyValidationHelpers
-
-            def validate_each(record, attribute, value)
-              unless validate_duration(value)
-                record.errors.add(attribute, 'should be a duration')
-              end
-            end
-          end
-
-          class HashOrStringValidator < ActiveModel::EachValidator
-            def validate_each(record, attribute, value)
-              unless value.is_a?(Hash) || value.is_a?(String)
-                record.errors.add(attribute, 'should be a hash or a string')
-              end
-            end
-          end
-
-          class KeyValidator < ActiveModel::EachValidator
-            include LegacyValidationHelpers
-
-            def validate_each(record, attribute, value)
-              if validate_string(value)
-                validate_path(record, attribute, value)
-              else
-                record.errors.add(attribute, 'should be a string or symbol')
-              end
-            end
-
-            private
-
-            def validate_path(record, attribute, value)
-              path = CGI.unescape(value.to_s)
-
-              if path.include?('/')
-                record.errors.add(attribute, 'cannot contain the "/" character')
-              elsif path == '.' || path == '..'
-                record.errors.add(attribute, 'cannot be "." or ".."')
-              end
-            end
-          end
-
-          class RegexpValidator < ActiveModel::EachValidator
-            include LegacyValidationHelpers
-
-            def validate_each(record, attribute, value)
-              unless validate_regexp(value)
-                record.errors.add(attribute, 'must be a regular expression')
-              end
-            end
-
-            private
-
-            def look_like_regexp?(value)
-              value.is_a?(String) && value.start_with?('/') &&
-                value.end_with?('/')
-            end
-
-            def validate_regexp(value)
-              look_like_regexp?(value) &&
-                Regexp.new(value.to_s[1...-1]) &&
-                true
-            rescue RegexpError
-              false
-            end
-          end
-
-          class ArrayOfStringsOrRegexpsValidator < RegexpValidator
-            def validate_each(record, attribute, value)
-              unless validate_array_of_strings_or_regexps(value)
-                record.errors.add(attribute, 'should be an array of strings or regexps')
-              end
-            end
-
-            private
-
-            def validate_array_of_strings_or_regexps(values)
-              values.is_a?(Array) && values.all?(&method(:validate_string_or_regexp))
-            end
-
-            def validate_string_or_regexp(value)
-              return false unless value.is_a?(String)
-              return validate_regexp(value) if look_like_regexp?(value)
-
-              true
-            end
-          end
-
-          class TypeValidator < ActiveModel::EachValidator
-            def validate_each(record, attribute, value)
-              type = options[:with]
-              raise unless type.is_a?(Class)
-
-              unless value.is_a?(type)
-                message = options[:message] || "should be a #{type.name}"
-                record.errors.add(attribute, message)
-              end
-            end
-          end
-
-          class VariablesValidator < ActiveModel::EachValidator
-            include LegacyValidationHelpers
-
-            def validate_each(record, attribute, value)
-              unless validate_variables(value)
-                record.errors.add(attribute, 'should be a hash of key value pairs')
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb
index 8acab605c91..c9d0c7cb568 100644
--- a/lib/gitlab/ci/config/entry/variables.rb
+++ b/lib/gitlab/ci/config/entry/variables.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Config
@@ -5,14 +7,14 @@ module Gitlab
         ##
         # Entry that represents environment variables.
         #
-        class Variables < Node
-          include Validatable
+        class Variables < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
 
           validations do
             validates :config, variables: true
           end
 
-          def self.default
+          def self.default(**)
             {}
           end
 
diff --git a/lib/gitlab/ci/config/extendable.rb b/lib/gitlab/ci/config/extendable.rb
new file mode 100644
index 00000000000..a43901c69fe
--- /dev/null
+++ b/lib/gitlab/ci/config/extendable.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      class Extendable
+        include Enumerable
+
+        ExtensionError = Class.new(StandardError)
+
+        def initialize(hash)
+          @hash = hash.to_h.deep_dup
+
+          each { |entry| entry.extend! if entry.extensible? }
+        end
+
+        def each
+          @hash.each_key do |key|
+            yield Extendable::Entry.new(key, @hash)
+          end
+        end
+
+        def to_hash
+          @hash.to_h
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/extendable/entry.rb b/lib/gitlab/ci/config/extendable/entry.rb
new file mode 100644
index 00000000000..7793db09d33
--- /dev/null
+++ b/lib/gitlab/ci/config/extendable/entry.rb
@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      class Extendable
+        class Entry
+          InvalidExtensionError = Class.new(Extendable::ExtensionError)
+          CircularDependencyError = Class.new(Extendable::ExtensionError)
+          NestingTooDeepError = Class.new(Extendable::ExtensionError)
+
+          MAX_NESTING_LEVELS = 10
+
+          attr_reader :key
+
+          def initialize(key, context, parent = nil)
+            @key = key
+            @context = context
+            @parent = parent
+
+            unless @context.key?(@key)
+              raise StandardError, 'Invalid entry key!'
+            end
+          end
+
+          def extensible?
+            value.is_a?(Hash) && value.key?(:extends)
+          end
+
+          def value
+            @value ||= @context.fetch(@key)
+          end
+
+          def base_hash!
+            @base ||= Extendable::Entry
+              .new(extends_key, @context, self)
+              .extend!
+          end
+
+          def extends_key
+            value.fetch(:extends).to_s.to_sym if extensible?
+          end
+
+          def ancestors
+            @ancestors ||= Array(@parent&.ancestors) + Array(@parent&.key)
+          end
+
+          def extend!
+            return value unless extensible?
+
+            if unknown_extension?
+              raise Entry::InvalidExtensionError,
+                    "#{key}: unknown key in `extends`"
+            end
+
+            if invalid_base?
+              raise Entry::InvalidExtensionError,
+                    "#{key}: invalid base hash in `extends`"
+            end
+
+            if nesting_too_deep?
+              raise Entry::NestingTooDeepError,
+                    "#{key}: nesting too deep in `extends`"
+            end
+
+            if circular_dependency?
+              raise Entry::CircularDependencyError,
+                    "#{key}: circular dependency detected in `extends`"
+            end
+
+            @context[key] = base_hash!.deep_merge(value)
+          end
+
+          private
+
+          def nesting_too_deep?
+            ancestors.count > MAX_NESTING_LEVELS
+          end
+
+          def circular_dependency?
+            ancestors.include?(key)
+          end
+
+          def unknown_extension?
+            !@context.key?(extends_key)
+          end
+
+          def invalid_base?
+            !@context[extends_key].is_a?(Hash)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb
new file mode 100644
index 00000000000..a747886093c
--- /dev/null
+++ b/lib/gitlab/ci/config/external/file/base.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module External
+        module File
+          class Base
+            include Gitlab::Utils::StrongMemoize
+
+            attr_reader :location, :params, :context, :errors
+
+            YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i.freeze
+
+            Context = Struct.new(:project, :sha, :user)
+
+            def initialize(params, context)
+              @params = params
+              @context = context
+              @errors = []
+
+              validate!
+            end
+
+            def matching?
+              location.present?
+            end
+
+            def invalid_extension?
+              location.nil? || !::File.basename(location).match?(YAML_WHITELIST_EXTENSION)
+            end
+
+            def valid?
+              errors.none?
+            end
+
+            def error_message
+              errors.first
+            end
+
+            def content
+              raise NotImplementedError, 'subclass must implement fetching raw content'
+            end
+
+            def to_hash
+              @hash ||= Gitlab::Config::Loader::Yaml.new(content).load!
+            rescue Gitlab::Config::Loader::FormatError
+              nil
+            end
+
+            protected
+
+            def validate!
+              validate_location!
+              validate_content! if errors.none?
+              validate_hash! if errors.none?
+            end
+
+            def validate_location!
+              if invalid_extension?
+                errors.push("Included file `#{location}` does not have YAML extension!")
+              end
+            end
+
+            def validate_content!
+              if content.blank?
+                errors.push("Included file `#{location}` is empty or does not exist!")
+              end
+            end
+
+            def validate_hash!
+              if to_hash.blank?
+                errors.push("Included file `#{location}` does not have valid YAML syntax!")
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb
new file mode 100644
index 00000000000..2535d178ba8
--- /dev/null
+++ b/lib/gitlab/ci/config/external/file/local.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module External
+        module File
+          class Local < Base
+            include Gitlab::Utils::StrongMemoize
+
+            def initialize(params, context)
+              @location = params[:local]
+
+              super
+            end
+
+            def content
+              strong_memoize(:content) { fetch_local_content }
+            end
+
+            private
+
+            def validate_content!
+              if content.nil?
+                errors.push("Local file `#{location}` does not exist!")
+              elsif content.blank?
+                errors.push("Local file `#{location}` is empty!")
+              end
+            end
+
+            def fetch_local_content
+              context.project.repository.blob_data_at(context.sha, location)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb
new file mode 100644
index 00000000000..e75540dbe5a
--- /dev/null
+++ b/lib/gitlab/ci/config/external/file/project.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module External
+        module File
+          class Project < Base
+            include Gitlab::Utils::StrongMemoize
+
+            attr_reader :project_name, :ref_name
+
+            def initialize(params, context = {})
+              @location = params[:file]
+              @project_name = params[:project]
+              @ref_name = params[:ref] || 'HEAD'
+
+              super
+            end
+
+            def matching?
+              super && project_name.present?
+            end
+
+            def content
+              strong_memoize(:content) { fetch_local_content }
+            end
+
+            private
+
+            def validate_content!
+              if !can_access_local_content?
+                errors.push("Project `#{project_name}` not found or access denied!")
+              elsif sha.nil?
+                errors.push("Project `#{project_name}` reference `#{ref_name}` does not exist!")
+              elsif content.nil?
+                errors.push("Project `#{project_name}` file `#{location}` does not exist!")
+              elsif content.blank?
+                errors.push("Project `#{project_name}` file `#{location}` is empty!")
+              end
+            end
+
+            def project
+              strong_memoize(:project) do
+                ::Project.find_by_full_path(project_name)
+              end
+            end
+
+            def can_access_local_content?
+              Ability.allowed?(context.user, :download_code, project)
+            end
+
+            def fetch_local_content
+              return unless can_access_local_content?
+              return unless sha
+
+              project.repository.blob_data_at(sha, location)
+            rescue GRPC::NotFound, GRPC::Internal
+              nil
+            end
+
+            def sha
+              strong_memoize(:sha) do
+                project.commit(ref_name).try(:sha)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb
new file mode 100644
index 00000000000..567a86c47e5
--- /dev/null
+++ b/lib/gitlab/ci/config/external/file/remote.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module External
+        module File
+          class Remote < Base
+            include Gitlab::Utils::StrongMemoize
+
+            def initialize(params, context)
+              @location = params[:remote]
+
+              super
+            end
+
+            def content
+              strong_memoize(:content) { fetch_remote_content }
+            end
+
+            private
+
+            def validate_location!
+              super
+
+              unless ::Gitlab::UrlSanitizer.valid?(location)
+                errors.push("Remote file `#{location}` does not have a valid address!")
+              end
+            end
+
+            def fetch_remote_content
+              begin
+                response = Gitlab::HTTP.get(location)
+              rescue SocketError
+                errors.push("Remote file `#{location}` could not be fetched because of a socket error!")
+              rescue Timeout::Error
+                errors.push("Remote file `#{location}` could not be fetched because of a timeout error!")
+              rescue Gitlab::HTTP::Error
+                errors.push("Remote file `#{location}` could not be fetched because of HTTP error!")
+              rescue Gitlab::HTTP::BlockedUrlError => e
+                errors.push("Remote file could not be fetched because #{e}!")
+              end
+
+              if response&.code.to_i >= 400
+                errors.push("Remote file `#{location}` could not be fetched because of HTTP code `#{response.code}` error!")
+              end
+
+              response.to_s if errors.none?
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb
new file mode 100644
index 00000000000..54f4cf74c4d
--- /dev/null
+++ b/lib/gitlab/ci/config/external/file/template.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module External
+        module File
+          class Template < Base
+            attr_reader :location, :project
+
+            SUFFIX = '.gitlab-ci.yml'.freeze
+
+            def initialize(params, context)
+              @location = params[:template]
+
+              super
+            end
+
+            def content
+              strong_memoize(:content) { fetch_template_content }
+            end
+
+            private
+
+            def validate_location!
+              super
+
+              unless template_name_valid?
+                errors.push("Template file `#{location}` is not a valid location!")
+              end
+            end
+
+            def template_name
+              return unless template_name_valid?
+
+              location.first(-SUFFIX.length)
+            end
+
+            def template_name_valid?
+              location.to_s.end_with?(SUFFIX)
+            end
+
+            def fetch_template_content
+              Gitlab::Template::GitlabCiYmlTemplate.find(template_name, project)&.content
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb
new file mode 100644
index 00000000000..108bfd5eb43
--- /dev/null
+++ b/lib/gitlab/ci/config/external/mapper.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module External
+        class Mapper
+          include Gitlab::Utils::StrongMemoize
+
+          FILE_CLASSES = [
+            External::File::Remote,
+            External::File::Template,
+            External::File::Local,
+            External::File::Project
+          ].freeze
+
+          AmbigiousSpecificationError = Class.new(StandardError)
+
+          def initialize(values, project:, sha:, user:)
+            @locations = Array.wrap(values.fetch(:include, []))
+            @project = project
+            @sha = sha
+            @user = user
+          end
+
+          def process
+            locations
+              .compact
+              .map(&method(:normalize_location))
+              .map(&method(:select_first_matching))
+          end
+
+          private
+
+          attr_reader :locations, :project, :sha, :user
+
+          # convert location if String to canonical form
+          def normalize_location(location)
+            if location.is_a?(String)
+              normalize_location_string(location)
+            else
+              location.deep_symbolize_keys
+            end
+          end
+
+          def normalize_location_string(location)
+            if ::Gitlab::UrlSanitizer.valid?(location)
+              { remote: location }
+            else
+              { local: location }
+            end
+          end
+
+          def select_first_matching(location)
+            matching = FILE_CLASSES.map do |file_class|
+              file_class.new(location, context)
+            end.select(&:matching?)
+
+            raise AmbigiousSpecificationError, "Include `#{location.to_json}` needs to match exactly one accessor!" unless matching.one?
+
+            matching.first
+          end
+
+          def context
+            strong_memoize(:context) do
+              External::File::Base::Context.new(project, sha, user)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb
new file mode 100644
index 00000000000..69bc164a039
--- /dev/null
+++ b/lib/gitlab/ci/config/external/processor.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module External
+        class Processor
+          IncludeError = Class.new(StandardError)
+
+          def initialize(values, project:, sha:, user:)
+            @values = values
+            @external_files = External::Mapper.new(values, project: project, sha: sha, user: user).process
+            @content = {}
+          rescue External::Mapper::AmbigiousSpecificationError => e
+            raise IncludeError, e.message
+          end
+
+          def perform
+            return @values if @external_files.empty?
+
+            validate_external_files!
+            merge_external_files!
+            append_inline_content!
+            remove_include_keyword!
+          end
+
+          private
+
+          def validate_external_files!
+            @external_files.each do |file|
+              raise IncludeError, file.error_message unless file.valid?
+            end
+          end
+
+          def merge_external_files!
+            @external_files.each do |file|
+              @content.deep_merge!(file.to_hash)
+            end
+          end
+
+          def append_inline_content!
+            @content.deep_merge!(@values)
+          end
+
+          def remove_include_keyword!
+            @content.tap { @content.delete(:include) }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb
new file mode 100644
index 00000000000..191f5d09645
--- /dev/null
+++ b/lib/gitlab/ci/config/normalizer.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      class Normalizer
+        def initialize(jobs_config)
+          @jobs_config = jobs_config
+        end
+
+        def normalize_jobs
+          extract_parallelized_jobs!
+          return @jobs_config if @parallelized_jobs.empty?
+
+          parallelized_config = parallelize_jobs
+          parallelize_dependencies(parallelized_config)
+        end
+
+        private
+
+        def extract_parallelized_jobs!
+          @parallelized_jobs = {}
+
+          @jobs_config.each do |job_name, config|
+            if config[:parallel]
+              @parallelized_jobs[job_name] = self.class.parallelize_job_names(job_name, config[:parallel])
+            end
+          end
+
+          @parallelized_jobs
+        end
+
+        def parallelize_jobs
+          @jobs_config.each_with_object({}) do |(job_name, config), hash|
+            if @parallelized_jobs.key?(job_name)
+              @parallelized_jobs[job_name].each { |name, index| hash[name.to_sym] = config.merge(name: name, instance: index) }
+            else
+              hash[job_name] = config
+            end
+
+            hash
+          end
+        end
+
+        def parallelize_dependencies(parallelized_config)
+          parallelized_job_names = @parallelized_jobs.keys.map(&:to_s)
+          parallelized_config.each_with_object({}) do |(job_name, config), hash|
+            if config[:dependencies] && (intersection = config[:dependencies] & parallelized_job_names).any?
+              parallelized_deps = intersection.map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) }.flatten
+              deps = config[:dependencies] - intersection + parallelized_deps
+              hash[job_name] = config.merge(dependencies: deps)
+            else
+              hash[job_name] = config
+            end
+
+            hash
+          end
+        end
+
+        def self.parallelize_job_names(name, total)
+          Array.new(total) { |index| ["#{name} #{index + 1}/#{total}", index + 1] }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb
index 73f36735e35..94f4a4e36c9 100644
--- a/lib/gitlab/ci/cron_parser.rb
+++ b/lib/gitlab/ci/cron_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class CronParser
@@ -33,7 +35,7 @@ module Gitlab
       # NOTE:
       # cron_timezone can only accept timezones listed in TZInfo::Timezone.
       # Aliases of Timezones from ActiveSupport::TimeZone are NOT accepted,
-      # because Rufus::Scheduler only supports TZInfo::Timezone.
+      # because Fugit::Cron only supports TZInfo::Timezone.
       #
       # For example, those codes have the same effect.
       # Time.zone = 'Pacific Time (US & Canada)' (ActiveSupport::TimeZone)
@@ -45,10 +47,7 @@ module Gitlab
       # If you want to know more, please take a look
       # https://github.com/rails/rails/blob/master/activesupport/lib/active_support/values/time_zone.rb
       def try_parse_cron(cron, cron_timezone)
-        cron_line = Rufus::Scheduler.parse("#{cron} #{cron_timezone}")
-        cron_line if cron_line.is_a?(Rufus::Scheduler::CronLine)
-      rescue
-        # noop
+        Fugit::Cron.parse("#{cron} #{cron_timezone}")
       end
     end
   end
diff --git a/lib/gitlab/ci/mask_secret.rb b/lib/gitlab/ci/mask_secret.rb
index 0daddaa638c..58d55b1bd6f 100644
--- a/lib/gitlab/ci/mask_secret.rb
+++ b/lib/gitlab/ci/mask_secret.rb
@@ -1,9 +1,13 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci::MaskSecret
     class << self
       def mask!(value, token)
         return value unless value.present? && token.present?
 
+        # We assume 'value' must be mutable, given
+        # that frozen string is enabled.
         value.gsub!(token, 'x' * token.length)
         value
       end
diff --git a/lib/gitlab/ci/model.rb b/lib/gitlab/ci/model.rb
index 3994a50772b..fbdb84c0522 100644
--- a/lib/gitlab/ci/model.rb
+++ b/lib/gitlab/ci/model.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Model
diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb
new file mode 100644
index 00000000000..eb63e6c8363
--- /dev/null
+++ b/lib/gitlab/ci/parsers.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Parsers
+      ParserNotFoundError = Class.new(ParserError)
+
+      def self.parsers
+        {
+          junit: ::Gitlab::Ci::Parsers::Test::Junit
+        }
+      end
+
+      def self.fabricate!(file_type)
+        parsers.fetch(file_type.to_sym).new
+      rescue KeyError
+        raise ParserNotFoundError, "Cannot find any parser matching file type '#{file_type}'"
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/parsers/parser_error.rb b/lib/gitlab/ci/parsers/parser_error.rb
new file mode 100644
index 00000000000..ef327737cdb
--- /dev/null
+++ b/lib/gitlab/ci/parsers/parser_error.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Parsers
+      ParserError = Class.new(StandardError)
+    end
+  end
+end
diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb
new file mode 100644
index 00000000000..dca60eabc1c
--- /dev/null
+++ b/lib/gitlab/ci/parsers/test/junit.rb
@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Parsers
+      module Test
+        class Junit
+          JunitParserError = Class.new(Gitlab::Ci::Parsers::ParserError)
+
+          def parse!(xml_data, test_suite)
+            root = Hash.from_xml(xml_data)
+
+            all_cases(root) do |test_case|
+              test_case = create_test_case(test_case)
+              test_suite.add_test_case(test_case)
+            end
+          rescue Nokogiri::XML::SyntaxError
+            raise JunitParserError, "XML parsing failed"
+          rescue
+            raise JunitParserError, "JUnit parsing failed"
+          end
+
+          private
+
+          def all_cases(root, parent = nil, &blk)
+            return unless root.present?
+
+            [root].flatten.compact.map do |node|
+              next unless node.is_a?(Hash)
+
+              # we allow only one top-level 'testsuites'
+              all_cases(node['testsuites'], root, &blk) unless parent
+
+              # we require at least one level of testsuites or testsuite
+              each_case(node['testcase'], &blk) if parent
+
+              # we allow multiple nested 'testsuite' (eg. PHPUnit)
+              all_cases(node['testsuite'], root, &blk)
+            end
+          end
+
+          def each_case(testcase, &blk)
+            return unless testcase.present?
+
+            [testcase].flatten.compact.map(&blk)
+          end
+
+          def create_test_case(data)
+            if data['failure']
+              status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED
+              system_output = data['failure']
+            else
+              status = ::Gitlab::Ci::Reports::TestCase::STATUS_SUCCESS
+              system_output = nil
+            end
+
+            ::Gitlab::Ci::Reports::TestCase.new(
+              classname: data['classname'],
+              name: data['name'],
+              file: data['file'],
+              execution_time: data['time'],
+              status: status,
+              system_output: system_output
+            )
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb
index efed19da21c..bab1c73e2f1 100644
--- a/lib/gitlab/ci/pipeline/chain/base.rb
+++ b/lib/gitlab/ci/pipeline/chain/base.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb
index b5eb0cfa2f0..41632211374 100644
--- a/lib/gitlab/ci/pipeline/chain/build.rb
+++ b/lib/gitlab/ci/pipeline/chain/build.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
@@ -14,7 +16,7 @@ module Gitlab
               trigger_requests: Array(@command.trigger_request),
               user: @command.current_user,
               pipeline_schedule: @command.schedule,
-              protected: @command.protected_ref?,
+              merge_request: @command.merge_request,
               variables_attributes: Array(@command.variables_attributes)
             )
 
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index a53c80d34f7..e62d547d862 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -1,13 +1,16 @@
-module Gitlab # rubocop:disable Naming/FileName
+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+module Gitlab
   module Ci
     module Pipeline
       module Chain
         Command = Struct.new(
           :source, :project, :current_user,
           :origin_ref, :checkout_sha, :after_sha, :before_sha,
-          :trigger_request, :schedule,
+          :trigger_request, :schedule, :merge_request,
           :ignore_skip_ci, :save_incompleted,
-          :seeds_block, :variables_attributes
+          :seeds_block, :variables_attributes, :push_options
         ) do
           include Gitlab::Utils::StrongMemoize
 
@@ -51,7 +54,13 @@ module Gitlab # rubocop:disable Naming/FileName
 
           def protected_ref?
             strong_memoize(:protected_ref) do
-              project.protected_for?(ref)
+              project.protected_for?(origin_ref)
+            end
+          end
+
+          def ambiguous_ref?
+            strong_memoize(:ambiguous_ref) do
+              project.repository.ambiguous_ref?(origin_ref)
             end
           end
         end
diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb
index f4c8d5342c1..aa627bdb009 100644
--- a/lib/gitlab/ci/pipeline/chain/create.rb
+++ b/lib/gitlab/ci/pipeline/chain/create.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
@@ -6,20 +8,7 @@ module Gitlab
           include Chain::Helpers
 
           def perform!
-            ::Ci::Pipeline.transaction do
-              pipeline.save!
-
-              ##
-              # Create environments before the pipeline starts.
-              #
-              pipeline.builds.each do |build|
-                if build.has_environment?
-                  project.environments.find_or_create_by(
-                    name: build.expanded_environment_name
-                  )
-                end
-              end
-            end
+            pipeline.save!
           rescue ActiveRecord::RecordInvalid => e
             error("Failed to persist the pipeline: #{e}")
           end
diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb
index bf1380a1da9..6bb3a75291b 100644
--- a/lib/gitlab/ci/pipeline/chain/helpers.rb
+++ b/lib/gitlab/ci/pipeline/chain/helpers.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb
index 69b8a8fc68f..0405292a25b 100644
--- a/lib/gitlab/ci/pipeline/chain/populate.rb
+++ b/lib/gitlab/ci/pipeline/chain/populate.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
@@ -8,6 +10,13 @@ module Gitlab
           PopulateError = Class.new(StandardError)
 
           def perform!
+            # Allocate next IID. This operation must be outside of transactions of pipeline creations.
+            pipeline.ensure_project_iid!
+
+            # Protect the pipeline. This is assigned in Populate instead of
+            # Build to prevent erroring out on ambiguous refs.
+            pipeline.protected = @command.protected_ref?
+
             ##
             # Populate pipeline with block argument of CreatePipelineService#execute.
             #
diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb
index e24630656d3..99780409085 100644
--- a/lib/gitlab/ci/pipeline/chain/sequence.rb
+++ b/lib/gitlab/ci/pipeline/chain/sequence.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb
index 32cbb7ca6af..79bbcc1ed1e 100644
--- a/lib/gitlab/ci/pipeline/chain/skip.rb
+++ b/lib/gitlab/ci/pipeline/chain/skip.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
@@ -6,6 +8,7 @@ module Gitlab
           include ::Gitlab::Utils::StrongMemoize
 
           SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i
+          SKIP_PUSH_OPTION = 'ci.skip'
 
           def perform!
             if skipped?
@@ -14,7 +17,7 @@ module Gitlab
           end
 
           def skipped?
-            !@command.ignore_skip_ci && commit_message_skips_ci?
+            !@command.ignore_skip_ci && (commit_message_skips_ci? || push_option_skips_ci?)
           end
 
           def break?
@@ -30,6 +33,10 @@ module Gitlab
               !!(@pipeline.git_commit_message =~ SKIP_PATTERN)
             end
           end
+
+          def push_option_skips_ci?
+            !!(@command.push_options&.include?(SKIP_PUSH_OPTION))
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
index 13c6fedd831..ebd7e6e8289 100644
--- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
+++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb
index a3bd2a5a23a..28c38cc3d18 100644
--- a/lib/gitlab/ci/pipeline/chain/validate/config.rb
+++ b/lib/gitlab/ci/pipeline/chain/validate/config.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/chain/validate/repository.rb b/lib/gitlab/ci/pipeline/chain/validate/repository.rb
index 9699c24e5b6..9c6c2bc8e25 100644
--- a/lib/gitlab/ci/pipeline/chain/validate/repository.rb
+++ b/lib/gitlab/ci/pipeline/chain/validate/repository.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
@@ -14,6 +16,10 @@ module Gitlab
               unless @command.sha
                 return error('Commit not found')
               end
+
+              if @command.ambiguous_ref?
+                return error('Ref is ambiguous')
+              end
             end
 
             def break?
diff --git a/lib/gitlab/ci/pipeline/duration.rb b/lib/gitlab/ci/pipeline/duration.rb
index 469fc094cc8..de24bbf688b 100644
--- a/lib/gitlab/ci/pipeline/duration.rb
+++ b/lib/gitlab/ci/pipeline/duration.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
@@ -86,6 +88,7 @@ module Gitlab
           end
         end
 
+        # rubocop: disable CodeReuse/ActiveRecord
         def from_pipeline(pipeline)
           status = %w[success failed running canceled]
           builds = pipeline.builds.latest
@@ -93,6 +96,7 @@ module Gitlab
 
           from_builds(builds)
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         def from_builds(builds)
           now = Time.now
@@ -134,9 +138,11 @@ module Gitlab
           Period.new(previous.first, [previous.last, current.last].max)
         end
 
+        # rubocop: disable CodeReuse/ActiveRecord
         def process_duration(periods)
           periods.sum(&:duration)
         end
+        # rubocop: enable CodeReuse/ActiveRecord
       end
     end
   end
diff --git a/lib/gitlab/ci/pipeline/expression.rb b/lib/gitlab/ci/pipeline/expression.rb
index f57df7c5637..61d392121d8 100644
--- a/lib/gitlab/ci/pipeline/expression.rb
+++ b/lib/gitlab/ci/pipeline/expression.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb
index 047ab66e9b3..70c774416f6 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb
index 3a2f0c6924e..668e85f5b9e 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb
index 10957598f76..cd17bc4d78b 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb
index a2778716924..be7258c201a 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb
index f640d0b5855..3ebceb92eb7 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb
index 9b239c29ea4..d7e6dacf068 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb
index 346c92dc51e..2db2bf011f1 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb
index f2611d65faf..ef9ddb6cae9 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb
index 37643c8ef53..85c0899e4f6 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb
index 4cacb1e62c9..f26542361a2 100644
--- a/lib/gitlab/ci/pipeline/expression/lexer.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb
index 90f94d0b763..ed184309ab4 100644
--- a/lib/gitlab/ci/pipeline/expression/parser.rb
+++ b/lib/gitlab/ci/pipeline/expression/parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb
index b36f1e0f865..b03611f756e 100644
--- a/lib/gitlab/ci/pipeline/expression/statement.rb
+++ b/lib/gitlab/ci/pipeline/expression/statement.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/expression/token.rb b/lib/gitlab/ci/pipeline/expression/token.rb
index 58211800b88..513d43f6fca 100644
--- a/lib/gitlab/ci/pipeline/expression/token.rb
+++ b/lib/gitlab/ci/pipeline/expression/token.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/preloader.rb b/lib/gitlab/ci/pipeline/preloader.rb
index e7a2e5511cf..db0a1ea4dab 100644
--- a/lib/gitlab/ci/pipeline/preloader.rb
+++ b/lib/gitlab/ci/pipeline/preloader.rb
@@ -5,23 +5,47 @@ module Gitlab
     module Pipeline
       # Class for preloading data associated with pipelines such as commit
       # authors.
-      module Preloader
-        def self.preload(pipelines)
-          # This ensures that all the pipeline commits are eager loaded before we
-          # start using them.
+      class Preloader
+        def self.preload!(pipelines)
+          ##
+          # This preloads all commits at once, because `Ci::Pipeline#commit` is
+          # using a lazy batch loading, what results in only one batched Gitaly
+          # call.
+          #
           pipelines.each(&:commit)
 
           pipelines.each do |pipeline|
-            # This preloads the author of every commit. We're using "lazy_author"
-            # here since "author" immediately loads the data on the first call.
-            pipeline.commit.try(:lazy_author)
-
-            # This preloads the number of warnings for every pipeline, ensuring
-            # that Ci::Pipeline#has_warnings? doesn't execute any additional
-            # queries.
-            pipeline.number_of_warnings
+            self.new(pipeline).tap do |preloader|
+              preloader.preload_commit_authors
+              preloader.preload_pipeline_warnings
+              preloader.preload_stages_warnings
+            end
           end
         end
+
+        def initialize(pipeline)
+          @pipeline = pipeline
+        end
+
+        def preload_commit_authors
+          # This also preloads the author of every commit. We're using "lazy_author"
+          # here since "author" immediately loads the data on the first call.
+          @pipeline.commit.try(:lazy_author)
+        end
+
+        def preload_pipeline_warnings
+          # This preloads the number of warnings for every pipeline, ensuring
+          # that Ci::Pipeline#has_warnings? doesn't execute any additional
+          # queries.
+          @pipeline.number_of_warnings
+        end
+
+        def preload_stages_warnings
+          # This preloads the number of warnings for every stage, ensuring
+          # that Ci::Stage#has_warnings? doesn't execute any additional
+          # queries.
+          @pipeline.stages.each { |stage| stage.number_of_warnings }
+        end
       end
     end
   end
diff --git a/lib/gitlab/ci/pipeline/seed/base.rb b/lib/gitlab/ci/pipeline/seed/base.rb
index db9706924bb..1fd3a61017f 100644
--- a/lib/gitlab/ci/pipeline/seed/base.rb
+++ b/lib/gitlab/ci/pipeline/seed/base.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index 6980b0b7aff..d8296940a04 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
@@ -36,9 +38,17 @@ module Gitlab
             )
           end
 
+          def bridge?
+            @attributes.to_h.dig(:options, :trigger).present?
+          end
+
           def to_resource
             strong_memoize(:resource) do
-              ::Ci::Build.new(attributes)
+              if bridge?
+                ::Ci::Bridge.new(attributes)
+              else
+                ::Ci::Build.new(attributes)
+              end
             end
           end
         end
diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb
index 2b58d9863a0..9c15064756a 100644
--- a/lib/gitlab/ci/pipeline/seed/stage.rb
+++ b/lib/gitlab/ci/pipeline/seed/stage.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Pipeline
@@ -37,7 +39,13 @@ module Gitlab
           def to_resource
             strong_memoize(:stage) do
               ::Ci::Stage.new(attributes).tap do |stage|
-                seeds.each { |seed| stage.builds << seed.to_resource }
+                seeds.each do |seed|
+                  if seed.bridge?
+                    stage.bridges << seed.to_resource
+                  else
+                    stage.builds << seed.to_resource
+                  end
+                end
               end
             end
           end
diff --git a/lib/gitlab/ci/reports/test_case.rb b/lib/gitlab/ci/reports/test_case.rb
new file mode 100644
index 00000000000..292e273a03a
--- /dev/null
+++ b/lib/gitlab/ci/reports/test_case.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Reports
+      class TestCase
+        STATUS_SUCCESS = 'success'.freeze
+        STATUS_FAILED = 'failed'.freeze
+        STATUS_SKIPPED = 'skipped'.freeze
+        STATUS_ERROR = 'error'.freeze
+        STATUS_TYPES = [STATUS_SUCCESS, STATUS_FAILED, STATUS_SKIPPED, STATUS_ERROR].freeze
+
+        attr_reader :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key
+
+        def initialize(name:, classname:, execution_time:, status:, file: nil, system_output: nil, stack_trace: nil)
+          @name = name
+          @classname = classname
+          @file = file
+          @execution_time = execution_time.to_f
+          @status = status
+          @system_output = system_output
+          @stack_trace = stack_trace
+          @key = sanitize_key_name("#{classname}_#{name}")
+        end
+
+        private
+
+        def sanitize_key_name(key)
+          key.gsub(/[^0-9A-Za-z]/, '-')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/reports/test_reports.rb b/lib/gitlab/ci/reports/test_reports.rb
new file mode 100644
index 00000000000..7397ff35d46
--- /dev/null
+++ b/lib/gitlab/ci/reports/test_reports.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Reports
+      class TestReports
+        attr_reader :test_suites
+
+        def initialize
+          @test_suites = {}
+        end
+
+        def get_suite(suite_name)
+          test_suites[suite_name] ||= TestSuite.new(suite_name)
+        end
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def total_time
+          test_suites.values.sum(&:total_time)
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def total_count
+          test_suites.values.sum(&:total_count)
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        def total_status
+          if failed_count > 0 || error_count > 0
+            TestCase::STATUS_FAILED
+          else
+            TestCase::STATUS_SUCCESS
+          end
+        end
+
+        TestCase::STATUS_TYPES.each do |status_type|
+          define_method("#{status_type}_count") do
+            # rubocop: disable CodeReuse/ActiveRecord
+            test_suites.values.sum { |suite| suite.public_send("#{status_type}_count") } # rubocop:disable GitlabSecurity/PublicSend
+            # rubocop: enable CodeReuse/ActiveRecord
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/reports/test_reports_comparer.rb b/lib/gitlab/ci/reports/test_reports_comparer.rb
new file mode 100644
index 00000000000..11810bdc0a8
--- /dev/null
+++ b/lib/gitlab/ci/reports/test_reports_comparer.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Reports
+      class TestReportsComparer
+        include Gitlab::Utils::StrongMemoize
+
+        attr_reader :base_reports, :head_reports
+
+        def initialize(base_reports, head_reports)
+          @base_reports = base_reports || TestReports.new
+          @head_reports = head_reports
+        end
+
+        def suite_comparers
+          strong_memoize(:suite_comparers) do
+            head_reports.test_suites.map do |name, test_suite|
+              TestSuiteComparer.new(name, base_reports.get_suite(name), test_suite)
+            end
+          end
+        end
+
+        def total_status
+          if suite_comparers.any? { |suite| suite.total_status == TestCase::STATUS_FAILED }
+            TestCase::STATUS_FAILED
+          else
+            TestCase::STATUS_SUCCESS
+          end
+        end
+
+        %w(total_count resolved_count failed_count).each do |method|
+          define_method(method) do
+            # rubocop: disable CodeReuse/ActiveRecord
+            suite_comparers.sum { |suite| suite.public_send(method) } # rubocop:disable GitlabSecurity/PublicSend
+            # rubocop: enable CodeReuse/ActiveRecord
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb
new file mode 100644
index 00000000000..b0391160c15
--- /dev/null
+++ b/lib/gitlab/ci/reports/test_suite.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Reports
+      class TestSuite
+        attr_reader :name
+        attr_reader :test_cases
+        attr_reader :total_time
+
+        def initialize(name = nil)
+          @name = name
+          @test_cases = {}
+          @total_time = 0.0
+          @duplicate_cases = []
+        end
+
+        def add_test_case(test_case)
+          @duplicate_cases << test_case if existing_key?(test_case)
+
+          @test_cases[test_case.status] ||= {}
+          @test_cases[test_case.status][test_case.key] = test_case
+          @total_time += test_case.execution_time
+        end
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def total_count
+          test_cases.values.sum(&:count)
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        def total_status
+          if failed_count > 0 || error_count > 0
+            TestCase::STATUS_FAILED
+          else
+            TestCase::STATUS_SUCCESS
+          end
+        end
+
+        TestCase::STATUS_TYPES.each do |status_type|
+          define_method("#{status_type}") do
+            test_cases[status_type] || {}
+          end
+
+          define_method("#{status_type}_count") do
+            test_cases[status_type]&.length.to_i
+          end
+        end
+
+        private
+
+        def existing_key?(test_case)
+          @test_cases[test_case.status]&.key?(test_case.key)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/reports/test_suite_comparer.rb b/lib/gitlab/ci/reports/test_suite_comparer.rb
new file mode 100644
index 00000000000..9cb7db5934c
--- /dev/null
+++ b/lib/gitlab/ci/reports/test_suite_comparer.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Reports
+      class TestSuiteComparer
+        include Gitlab::Utils::StrongMemoize
+
+        attr_reader :name, :base_suite, :head_suite
+
+        def initialize(name, base_suite, head_suite)
+          @name = name
+          @base_suite = base_suite || TestSuite.new
+          @head_suite = head_suite
+        end
+
+        def new_failures
+          strong_memoize(:new_failures) do
+            head_suite.failed.reject do |key, _|
+              base_suite.failed.include?(key)
+            end.values
+          end
+        end
+
+        def existing_failures
+          strong_memoize(:existing_failures) do
+            head_suite.failed.select do |key, _|
+              base_suite.failed.include?(key)
+            end.values
+          end
+        end
+
+        def resolved_failures
+          strong_memoize(:resolved_failures) do
+            head_suite.success.select do |key, _|
+              base_suite.failed.include?(key)
+            end.values
+          end
+        end
+
+        def total_count
+          head_suite.total_count
+        end
+
+        def total_status
+          head_suite.total_status
+        end
+
+        def resolved_count
+          resolved_failures.count
+        end
+
+        def failed_count
+          new_failures.count + existing_failures.count
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/bridge/common.rb b/lib/gitlab/ci/status/bridge/common.rb
new file mode 100644
index 00000000000..4746195c618
--- /dev/null
+++ b/lib/gitlab/ci/status/bridge/common.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Status
+      module Bridge
+        module Common
+          def label
+            subject.description
+          end
+
+          def has_details?
+            false
+          end
+
+          def has_action?
+            false
+          end
+
+          def details_path
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/bridge/factory.rb b/lib/gitlab/ci/status/bridge/factory.rb
new file mode 100644
index 00000000000..910de865483
--- /dev/null
+++ b/lib/gitlab/ci/status/bridge/factory.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Status
+      module Bridge
+        class Factory < Status::Factory
+          def self.common_helpers
+            Status::Bridge::Common
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/action.rb b/lib/gitlab/ci/status/build/action.rb
index 6c9125647ad..45d9ba41e92 100644
--- a/lib/gitlab/ci/status/build/action.rb
+++ b/lib/gitlab/ci/status/build/action.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb
index 024047d4983..43fb5cdbbe6 100644
--- a/lib/gitlab/ci/status/build/cancelable.rb
+++ b/lib/gitlab/ci/status/build/cancelable.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/build/canceled.rb b/lib/gitlab/ci/status/build/canceled.rb
index c83e2734a73..0518b9e673d 100644
--- a/lib/gitlab/ci/status/build/canceled.rb
+++ b/lib/gitlab/ci/status/build/canceled.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb
index c1fc70ac266..6a75ec5c37f 100644
--- a/lib/gitlab/ci/status/build/common.rb
+++ b/lib/gitlab/ci/status/build/common.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/build/created.rb b/lib/gitlab/ci/status/build/created.rb
index 5be8e9de425..780fea23123 100644
--- a/lib/gitlab/ci/status/build/created.rb
+++ b/lib/gitlab/ci/status/build/created.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/build/erased.rb b/lib/gitlab/ci/status/build/erased.rb
index 495227c2ffb..d74cfc1ee77 100644
--- a/lib/gitlab/ci/status/build/erased.rb
+++ b/lib/gitlab/ci/status/build/erased.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb
index 2b26ebb45a1..6e4bfe23f2b 100644
--- a/lib/gitlab/ci/status/build/factory.rb
+++ b/lib/gitlab/ci/status/build/factory.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
@@ -5,6 +7,7 @@ module Gitlab
         class Factory < Status::Factory
           def self.extended_statuses
             [[Status::Build::Erased,
+              Status::Build::Scheduled,
               Status::Build::Manual,
               Status::Build::Canceled,
               Status::Build::Created,
@@ -14,6 +17,7 @@ module Gitlab
               Status::Build::Retryable],
              [Status::Build::Failed],
              [Status::Build::FailedAllowed,
+              Status::Build::Unschedule,
               Status::Build::Play,
               Status::Build::Stop],
              [Status::Build::Action],
diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb
index 155f4fc1343..d40454df737 100644
--- a/lib/gitlab/ci/status/build/failed.rb
+++ b/lib/gitlab/ci/status/build/failed.rb
@@ -1,17 +1,25 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
       module Build
         class Failed < Status::Extended
           REASONS = {
-            'unknown_failure' => 'unknown failure',
-            'script_failure' => 'script failure',
-            'api_failure' => 'API failure',
-            'stuck_or_timeout_failure' => 'stuck or timeout failure',
-            'runner_system_failure' => 'runner system failure',
-            'missing_dependency_failure' => 'missing dependency failure'
+            unknown_failure: 'unknown failure',
+            script_failure: 'script failure',
+            api_failure: 'API failure',
+            stuck_or_timeout_failure: 'stuck or timeout failure',
+            runner_system_failure: 'runner system failure',
+            missing_dependency_failure: 'missing dependency failure',
+            runner_unsupported: 'unsupported runner',
+            stale_schedule: 'stale schedule',
+            job_execution_timeout: 'job execution timeout',
+            archived_failure: 'archived failure'
           }.freeze
 
+          private_constant :REASONS
+
           def status_tooltip
             base_message
           end
@@ -24,6 +32,10 @@ module Gitlab
             build.failed?
           end
 
+          def self.reasons
+            REASONS
+          end
+
           private
 
           def base_message
@@ -31,7 +43,11 @@ module Gitlab
           end
 
           def description
-            "<br> (#{REASONS[subject.failure_reason]})"
+            "- (#{failure_reason_message})"
+          end
+
+          def failure_reason_message
+            self.class.reasons.fetch(subject.failure_reason.to_sym)
           end
         end
       end
diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb
index ca0046fb1f7..d7570fdd3e2 100644
--- a/lib/gitlab/ci/status/build/failed_allowed.rb
+++ b/lib/gitlab/ci/status/build/failed_allowed.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb
index 042da6392d3..d01b09f1398 100644
--- a/lib/gitlab/ci/status/build/manual.rb
+++ b/lib/gitlab/ci/status/build/manual.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/build/pending.rb b/lib/gitlab/ci/status/build/pending.rb
index 9dd9a27ad57..95f668295dd 100644
--- a/lib/gitlab/ci/status/build/pending.rb
+++ b/lib/gitlab/ci/status/build/pending.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb
index a8b9ebf0803..c66b8ca5654 100644
--- a/lib/gitlab/ci/status/build/play.rb
+++ b/lib/gitlab/ci/status/build/play.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/build/retried.rb b/lib/gitlab/ci/status/build/retried.rb
index 6e190e4ee3c..b489dc68733 100644
--- a/lib/gitlab/ci/status/build/retried.rb
+++ b/lib/gitlab/ci/status/build/retried.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb
index 5aeb8e51480..eb6b3f21604 100644
--- a/lib/gitlab/ci/status/build/retryable.rb
+++ b/lib/gitlab/ci/status/build/retryable.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/build/scheduled.rb b/lib/gitlab/ci/status/build/scheduled.rb
new file mode 100644
index 00000000000..0a09dbe5f42
--- /dev/null
+++ b/lib/gitlab/ci/status/build/scheduled.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class Scheduled < Status::Extended
+          def illustration
+            {
+              image: 'illustrations/illustrations_scheduled-job_countdown.svg',
+              size: 'svg-394',
+              title: _("This is a delayed job to run in %{remainingTime}"),
+              content: _("This job will automatically run after its timer finishes. " \
+                         "Often they are used for incremental roll-out deploys " \
+                         "to production environments. When unscheduled it converts " \
+                         "into a manual action.")
+            }
+          end
+
+          def status_tooltip
+            "delayed manual action (%{remainingTime})"
+          end
+
+          def self.matches?(build, user)
+            build.scheduled? && build.scheduled_at
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/skipped.rb b/lib/gitlab/ci/status/build/skipped.rb
index 3e678d0baee..4fe2f7b3114 100644
--- a/lib/gitlab/ci/status/build/skipped.rb
+++ b/lib/gitlab/ci/status/build/skipped.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb
index dea838bfa39..a620e7ad126 100644
--- a/lib/gitlab/ci/status/build/stop.rb
+++ b/lib/gitlab/ci/status/build/stop.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/build/unschedule.rb b/lib/gitlab/ci/status/build/unschedule.rb
new file mode 100644
index 00000000000..9110839cb55
--- /dev/null
+++ b/lib/gitlab/ci/status/build/unschedule.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class Unschedule < Status::Extended
+          def label
+            'unschedule action'
+          end
+
+          def has_action?
+            can?(user, :update_build, subject)
+          end
+
+          def action_icon
+            'time-out'
+          end
+
+          def action_title
+            'Unschedule'
+          end
+
+          def action_button_title
+            _('Unschedule job')
+          end
+
+          def action_path
+            unschedule_project_job_path(subject.project, subject)
+          end
+
+          def action_method
+            :post
+          end
+
+          def self.matches?(build, user)
+            build.scheduled?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/canceled.rb b/lib/gitlab/ci/status/canceled.rb
index e6195a60d4f..07f37732023 100644
--- a/lib/gitlab/ci/status/canceled.rb
+++ b/lib/gitlab/ci/status/canceled.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb
index 9d6a2f51c11..ea773ee9944 100644
--- a/lib/gitlab/ci/status/core.rb
+++ b/lib/gitlab/ci/status/core.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/created.rb b/lib/gitlab/ci/status/created.rb
index 846f00b83dd..fface4bb97b 100644
--- a/lib/gitlab/ci/status/created.rb
+++ b/lib/gitlab/ci/status/created.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/extended.rb b/lib/gitlab/ci/status/extended.rb
index 1e8101f8949..b72a28ed0b6 100644
--- a/lib/gitlab/ci/status/extended.rb
+++ b/lib/gitlab/ci/status/extended.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/external/common.rb b/lib/gitlab/ci/status/external/common.rb
index 9307545b5b1..cd772819293 100644
--- a/lib/gitlab/ci/status/external/common.rb
+++ b/lib/gitlab/ci/status/external/common.rb
@@ -1,10 +1,12 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
       module External
         module Common
           def label
-            subject.description
+            subject.description.presence || super
           end
 
           def has_details?
diff --git a/lib/gitlab/ci/status/external/factory.rb b/lib/gitlab/ci/status/external/factory.rb
index 07b15bd8d97..91fafb940a8 100644
--- a/lib/gitlab/ci/status/external/factory.rb
+++ b/lib/gitlab/ci/status/external/factory.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb
index 15836c699c7..3446644eff8 100644
--- a/lib/gitlab/ci/status/factory.rb
+++ b/lib/gitlab/ci/status/factory.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/failed.rb b/lib/gitlab/ci/status/failed.rb
index 27ce85bd3ed..770ed7d4d5a 100644
--- a/lib/gitlab/ci/status/failed.rb
+++ b/lib/gitlab/ci/status/failed.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/group/common.rb b/lib/gitlab/ci/status/group/common.rb
index cfd4329a923..0b5ea0712ca 100644
--- a/lib/gitlab/ci/status/group/common.rb
+++ b/lib/gitlab/ci/status/group/common.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/group/factory.rb b/lib/gitlab/ci/status/group/factory.rb
index d118116cfc3..ee785856fdd 100644
--- a/lib/gitlab/ci/status/group/factory.rb
+++ b/lib/gitlab/ci/status/group/factory.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/manual.rb b/lib/gitlab/ci/status/manual.rb
index fc387e2fd25..50c92add400 100644
--- a/lib/gitlab/ci/status/manual.rb
+++ b/lib/gitlab/ci/status/manual.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/pending.rb b/lib/gitlab/ci/status/pending.rb
index 6780780db32..cea7e6ed938 100644
--- a/lib/gitlab/ci/status/pending.rb
+++ b/lib/gitlab/ci/status/pending.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/pipeline/blocked.rb b/lib/gitlab/ci/status/pipeline/blocked.rb
index bf7e484ee9b..ed13a439be0 100644
--- a/lib/gitlab/ci/status/pipeline/blocked.rb
+++ b/lib/gitlab/ci/status/pipeline/blocked.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/pipeline/common.rb b/lib/gitlab/ci/status/pipeline/common.rb
index 61bb07beb0f..7b34a2ea858 100644
--- a/lib/gitlab/ci/status/pipeline/common.rb
+++ b/lib/gitlab/ci/status/pipeline/common.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/pipeline/delayed.rb b/lib/gitlab/ci/status/pipeline/delayed.rb
new file mode 100644
index 00000000000..e61acdcd167
--- /dev/null
+++ b/lib/gitlab/ci/status/pipeline/delayed.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Status
+      module Pipeline
+        class Delayed < Status::Extended
+          def text
+            s_('CiStatusText|delayed')
+          end
+
+          def label
+            s_('CiStatusLabel|waiting for delayed job')
+          end
+
+          def self.matches?(pipeline, user)
+            pipeline.scheduled?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/pipeline/factory.rb b/lib/gitlab/ci/status/pipeline/factory.rb
index 17f9a75f436..5d1a8bbd924 100644
--- a/lib/gitlab/ci/status/pipeline/factory.rb
+++ b/lib/gitlab/ci/status/pipeline/factory.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
@@ -5,6 +7,7 @@ module Gitlab
         class Factory < Status::Factory
           def self.extended_statuses
             [[Status::SuccessWarning,
+              Status::Pipeline::Delayed,
               Status::Pipeline::Blocked]]
           end
 
diff --git a/lib/gitlab/ci/status/running.rb b/lib/gitlab/ci/status/running.rb
index ee13905e46d..ac7dd74cdce 100644
--- a/lib/gitlab/ci/status/running.rb
+++ b/lib/gitlab/ci/status/running.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/scheduled.rb b/lib/gitlab/ci/status/scheduled.rb
new file mode 100644
index 00000000000..16ad1da89e3
--- /dev/null
+++ b/lib/gitlab/ci/status/scheduled.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Status
+      class Scheduled < Status::Core
+        def text
+          s_('CiStatusText|delayed')
+        end
+
+        def label
+          s_('CiStatusLabel|delayed')
+        end
+
+        def icon
+          'status_scheduled'
+        end
+
+        def favicon
+          'favicon_status_scheduled'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/skipped.rb b/lib/gitlab/ci/status/skipped.rb
index 0dbdc4de426..aaec1e1d201 100644
--- a/lib/gitlab/ci/status/skipped.rb
+++ b/lib/gitlab/ci/status/skipped.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/stage/common.rb b/lib/gitlab/ci/status/stage/common.rb
index bc99d925347..f12daaa9676 100644
--- a/lib/gitlab/ci/status/stage/common.rb
+++ b/lib/gitlab/ci/status/stage/common.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
@@ -8,7 +10,9 @@ module Gitlab
           end
 
           def details_path
-            project_pipeline_path(subject.project, subject.pipeline, anchor: subject.name)
+            project_pipeline_path(subject.pipeline.project,
+                                  subject.pipeline,
+                                  anchor: subject.name)
           end
 
           def has_action?
diff --git a/lib/gitlab/ci/status/stage/factory.rb b/lib/gitlab/ci/status/stage/factory.rb
index 4c37f084d07..58f4642510b 100644
--- a/lib/gitlab/ci/status/stage/factory.rb
+++ b/lib/gitlab/ci/status/stage/factory.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/success.rb b/lib/gitlab/ci/status/success.rb
index 731013ec017..020f2c5b89f 100644
--- a/lib/gitlab/ci/status/success.rb
+++ b/lib/gitlab/ci/status/success.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb
index 32b4cf43e48..6632cd9b143 100644
--- a/lib/gitlab/ci/status/success_warning.rb
+++ b/lib/gitlab/ci/status/success_warning.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Status
diff --git a/lib/gitlab/ci/templates/Android.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.gitlab-ci.yml
new file mode 100644
index 00000000000..6e138639b71
--- /dev/null
+++ b/lib/gitlab/ci/templates/Android.gitlab-ci.yml
@@ -0,0 +1,45 @@
+# Read more about this script on this blog post https://about.gitlab.com/2018/10/24/setting-up-gitlab-ci-for-android-projects/, by Jason Lenny
+image: openjdk:8-jdk
+
+variables:
+  ANDROID_COMPILE_SDK: "28"
+  ANDROID_BUILD_TOOLS: "28.0.2"
+  ANDROID_SDK_TOOLS:   "4333796"
+
+before_script:
+  - apt-get --quiet update --yes
+  - apt-get --quiet install --yes wget tar unzip lib32stdc++6 lib32z1
+  - wget --quiet --output-document=android-sdk.zip https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_TOOLS}.zip
+  - unzip -d android-sdk-linux android-sdk.zip
+  - echo y | android-sdk-linux/tools/bin/sdkmanager "platforms;android-${ANDROID_COMPILE_SDK}" >/dev/null
+  - echo y | android-sdk-linux/tools/bin/sdkmanager "platform-tools" >/dev/null
+  - echo y | android-sdk-linux/tools/bin/sdkmanager "build-tools;${ANDROID_BUILD_TOOLS}" >/dev/null
+  - export ANDROID_HOME=$PWD/android-sdk-linux
+  - export PATH=$PATH:$PWD/android-sdk-linux/platform-tools/
+  - chmod +x ./gradlew
+  # temporarily disable checking for EPIPE error and use yes to accept all licenses
+  - set +o pipefail
+  - yes | android-sdk-linux/tools/bin/sdkmanager --licenses
+  - set -o pipefail
+
+stages:
+  - build
+  - test
+
+lintDebug:
+  stage: build
+  script:
+    - ./gradlew -Pci --console=plain :app:lintDebug -PbuildDir=lint
+
+assembleDebug:
+  stage: build
+  script:
+    - ./gradlew assembleDebug
+  artifacts:
+    paths:
+    - app/build/outputs/
+
+debugTests:
+  stage: test
+  script:
+    - ./gradlew -Pci --console=plain :app:testDebug
diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
new file mode 100644
index 00000000000..75a5bf142d2
--- /dev/null
+++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
@@ -0,0 +1,954 @@
+# Auto DevOps
+# This CI/CD configuration provides a standard pipeline for
+# * building a Docker image (using a buildpack if necessary),
+# * storing the image in the container registry,
+# * running tests from a buildpack,
+# * running code quality analysis,
+# * creating a review app for each topic branch,
+# * and continuous deployment to production
+#
+# Test jobs may be disabled by setting environment variables:
+# * test: TEST_DISABLED
+# * code_quality: CODE_QUALITY_DISABLED
+# * license_management: LICENSE_MANAGEMENT_DISABLED
+# * performance: PERFORMANCE_DISABLED
+# * sast: SAST_DISABLED
+# * dependency_scanning: DEPENDENCY_SCANNING_DISABLED
+# * container_scanning: CONTAINER_SCANNING_DISABLED
+# * dast: DAST_DISABLED
+# * review: REVIEW_DISABLED
+# * stop_review: REVIEW_DISABLED
+#
+# In order to deploy, you must have a Kubernetes cluster configured either
+# via a project integration, or via group/project variables.
+# AUTO_DEVOPS_DOMAIN must also be set as a variable at the group or project
+# level, or manually added below.
+#
+# Continuous deployment to production is enabled by default.
+# If you want to deploy to staging first, set STAGING_ENABLED environment variable.
+# If you want to enable incremental rollout, either manual or time based,
+# set INCREMENTAL_ROLLOUT_MODE environment variable to "manual" or "timed".
+# If you want to use canary deployments, set CANARY_ENABLED environment variable.
+#
+# If Auto DevOps fails to detect the proper buildpack, or if you want to
+# specify a custom buildpack, set a project variable `BUILDPACK_URL` to the
+# repository URL of the buildpack.
+# e.g. BUILDPACK_URL=https://github.com/heroku/heroku-buildpack-ruby.git#v142
+# If you need multiple buildpacks, add a file to your project called
+# `.buildpacks` that contains the URLs, one on each line, in order.
+# Note: Auto CI does not work with multiple buildpacks yet
+
+image: alpine:latest
+
+variables:
+  # AUTO_DEVOPS_DOMAIN is the application deployment domain and should be set as a variable at the group or project level.
+  # AUTO_DEVOPS_DOMAIN: domain.example.com
+
+  POSTGRES_USER: user
+  POSTGRES_PASSWORD: testing-password
+  POSTGRES_ENABLED: "true"
+  POSTGRES_DB: $CI_ENVIRONMENT_SLUG
+
+  KUBERNETES_VERSION: 1.11.6
+  HELM_VERSION: 2.12.2
+
+  DOCKER_DRIVER: overlay2
+
+stages:
+  - build
+  - test
+  - review
+  - dast
+  - staging
+  - canary
+  - production
+  - incremental rollout 10%
+  - incremental rollout 25%
+  - incremental rollout 50%
+  - incremental rollout 100%
+  - performance
+  - cleanup
+
+build:
+  stage: build
+  image: docker:stable-git
+  services:
+  - docker:stable-dind
+  script:
+    - setup_docker
+    - build
+  only:
+    - branches
+
+test:
+  services:
+    - postgres:latest
+  variables:
+    POSTGRES_DB: test
+  stage: test
+  image: gliderlabs/herokuish:latest
+  script:
+    - setup_test_db
+    - cp -R . /tmp/app
+    - /bin/herokuish buildpack test
+  only:
+    - branches
+  except:
+    variables:
+      - $TEST_DISABLED
+
+code_quality:
+  stage: test
+  image: docker:stable
+  allow_failure: true
+  services:
+    - docker:stable-dind
+  script:
+    - setup_docker
+    - code_quality
+  artifacts:
+    paths: [gl-code-quality-report.json]
+  only:
+    - branches
+  except:
+    variables:
+      - $CODE_QUALITY_DISABLED
+
+license_management:
+  stage: test
+  image:
+    name: "registry.gitlab.com/gitlab-org/security-products/license-management:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable"
+    entrypoint: [""]
+  allow_failure: true
+  script:
+    - license_management
+  artifacts:
+    paths: [gl-license-management-report.json]
+  only:
+    refs:
+      - branches
+    variables:
+      - $GITLAB_FEATURES =~ /\blicense_management\b/
+  except:
+    variables:
+      - $LICENSE_MANAGEMENT_DISABLED
+
+performance:
+  stage: performance
+  image: docker:stable
+  allow_failure: true
+  services:
+    - docker:stable-dind
+  script:
+    - setup_docker
+    - performance
+  artifacts:
+    paths:
+    - performance.json
+    - sitespeed-results/
+  only:
+    refs:
+      - branches
+    kubernetes: active
+  except:
+    variables:
+      - $PERFORMANCE_DISABLED
+
+sast:
+  stage: test
+  image: docker:stable
+  allow_failure: true
+  services:
+    - docker:stable-dind
+  script:
+    - setup_docker
+    - sast
+  artifacts:
+    reports:
+      sast: gl-sast-report.json
+  only:
+    refs:
+      - branches
+    variables:
+      - $GITLAB_FEATURES =~ /\bsast\b/
+  except:
+    variables:
+      - $SAST_DISABLED
+
+dependency_scanning:
+  stage: test
+  image: docker:stable
+  allow_failure: true
+  services:
+    - docker:stable-dind
+  script:
+    - setup_docker
+    - dependency_scanning
+  artifacts:
+    reports:
+      dependency_scanning: gl-dependency-scanning-report.json
+  only:
+    refs:
+      - branches
+    variables:
+      - $GITLAB_FEATURES =~ /\bdependency_scanning\b/
+  except:
+    variables:
+      - $DEPENDENCY_SCANNING_DISABLED
+
+container_scanning:
+  stage: test
+  image: docker:stable
+  allow_failure: true
+  services:
+    - docker:stable-dind
+  script:
+    - setup_docker
+    - container_scanning
+  artifacts:
+    paths: [gl-container-scanning-report.json]
+  only:
+    refs:
+      - branches
+    variables:
+      - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/
+  except:
+    variables:
+      - $CONTAINER_SCANNING_DISABLED
+
+dast:
+  stage: dast
+  allow_failure: true
+  image: registry.gitlab.com/gitlab-org/security-products/zaproxy
+  variables:
+    POSTGRES_DB: "false"
+  script:
+    - dast
+  artifacts:
+    paths: [gl-dast-report.json]
+  only:
+    refs:
+      - branches
+    kubernetes: active
+    variables:
+      - $GITLAB_FEATURES =~ /\bdast\b/
+  except:
+    refs:
+      - master
+    variables:
+      - $DAST_DISABLED
+
+review:
+  stage: review
+  script:
+    - check_kube_domain
+    - install_dependencies
+    - download_chart
+    - ensure_namespace
+    - initialize_tiller
+    - create_secret
+    - deploy
+    - persist_environment_url
+  environment:
+    name: review/$CI_COMMIT_REF_NAME
+    url: http://$CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN
+    on_stop: stop_review
+  artifacts:
+    paths: [environment_url.txt]
+  only:
+    refs:
+      - branches
+    kubernetes: active
+  except:
+    refs:
+      - master
+    variables:
+      - $REVIEW_DISABLED
+
+stop_review:
+  stage: cleanup
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - install_dependencies
+    - initialize_tiller
+    - delete
+  environment:
+    name: review/$CI_COMMIT_REF_NAME
+    action: stop
+  when: manual
+  allow_failure: true
+  only:
+    refs:
+      - branches
+    kubernetes: active
+  except:
+    refs:
+      - master
+    variables:
+      - $REVIEW_DISABLED
+
+# Staging deploys are disabled by default since
+# continuous deployment to production is enabled by default
+# If you prefer to automatically deploy to staging and
+# only manually promote to production, enable this job by setting
+# STAGING_ENABLED.
+
+staging:
+  stage: staging
+  script:
+    - check_kube_domain
+    - install_dependencies
+    - download_chart
+    - ensure_namespace
+    - initialize_tiller
+    - create_secret
+    - deploy
+  environment:
+    name: staging
+    url: http://$CI_PROJECT_PATH_SLUG-staging.$AUTO_DEVOPS_DOMAIN
+  only:
+    refs:
+      - master
+    kubernetes: active
+    variables:
+      - $STAGING_ENABLED
+
+# Canaries are also disabled by default, but if you want them,
+# and know what the downsides are, you can enable this by setting
+# CANARY_ENABLED.
+
+canary:
+  stage: canary
+  script:
+    - check_kube_domain
+    - install_dependencies
+    - download_chart
+    - ensure_namespace
+    - initialize_tiller
+    - create_secret
+    - deploy canary
+  environment:
+    name: production
+    url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN
+  when: manual
+  only:
+    refs:
+      - master
+    kubernetes: active
+    variables:
+      - $CANARY_ENABLED
+
+.production: &production_template
+  stage: production
+  script:
+    - check_kube_domain
+    - install_dependencies
+    - download_chart
+    - ensure_namespace
+    - initialize_tiller
+    - create_secret
+    - deploy
+    - delete canary
+    - delete rollout
+    - persist_environment_url
+  environment:
+    name: production
+    url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN
+  artifacts:
+    paths: [environment_url.txt]
+
+production:
+  <<: *production_template
+  only:
+    refs:
+      - master
+    kubernetes: active
+  except:
+    variables:
+      - $STAGING_ENABLED
+      - $CANARY_ENABLED
+      - $INCREMENTAL_ROLLOUT_ENABLED
+      - $INCREMENTAL_ROLLOUT_MODE
+
+production_manual:
+  <<: *production_template
+  when: manual
+  allow_failure: false
+  only:
+    refs:
+      - master
+    kubernetes: active
+    variables:
+      - $STAGING_ENABLED
+      - $CANARY_ENABLED
+  except:
+    variables:
+      - $INCREMENTAL_ROLLOUT_ENABLED
+      - $INCREMENTAL_ROLLOUT_MODE
+
+# This job implements incremental rollout on for every push to `master`.
+
+.rollout: &rollout_template
+  script:
+    - check_kube_domain
+    - install_dependencies
+    - download_chart
+    - ensure_namespace
+    - initialize_tiller
+    - create_secret
+    - deploy rollout $ROLLOUT_PERCENTAGE
+    - scale stable $((100-ROLLOUT_PERCENTAGE))
+    - delete canary
+    - persist_environment_url
+  environment:
+    name: production
+    url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN
+  artifacts:
+    paths: [environment_url.txt]
+
+.manual_rollout_template: &manual_rollout_template
+  <<: *rollout_template
+  stage: production
+  when: manual
+  # This selectors are backward compatible mode with $INCREMENTAL_ROLLOUT_ENABLED (before 11.4)
+  only:
+    refs:
+      - master
+    kubernetes: active
+    variables:
+      - $INCREMENTAL_ROLLOUT_MODE == "manual"
+      - $INCREMENTAL_ROLLOUT_ENABLED
+  except:
+    variables:
+      - $INCREMENTAL_ROLLOUT_MODE == "timed"
+
+.timed_rollout_template: &timed_rollout_template
+  <<: *rollout_template
+  when: delayed
+  start_in: 5 minutes
+  only:
+    refs:
+      - master
+    kubernetes: active
+    variables:
+      - $INCREMENTAL_ROLLOUT_MODE == "timed"
+
+timed rollout 10%:
+  <<: *timed_rollout_template
+  stage: incremental rollout 10%
+  variables:
+    ROLLOUT_PERCENTAGE: 10
+
+timed rollout 25%:
+  <<: *timed_rollout_template
+  stage: incremental rollout 25%
+  variables:
+    ROLLOUT_PERCENTAGE: 25
+
+timed rollout 50%:
+  <<: *timed_rollout_template
+  stage: incremental rollout 50%
+  variables:
+    ROLLOUT_PERCENTAGE: 50
+
+timed rollout 100%:
+  <<: *timed_rollout_template
+  <<: *production_template
+  stage: incremental rollout 100%
+  variables:
+    ROLLOUT_PERCENTAGE: 100
+
+rollout 10%:
+  <<: *manual_rollout_template
+  variables:
+    ROLLOUT_PERCENTAGE: 10
+
+rollout 25%:
+  <<: *manual_rollout_template
+  variables:
+    ROLLOUT_PERCENTAGE: 25
+
+rollout 50%:
+  <<: *manual_rollout_template
+  variables:
+    ROLLOUT_PERCENTAGE: 50
+
+rollout 100%:
+  <<: *manual_rollout_template
+  <<: *production_template
+  allow_failure: false
+
+# ---------------------------------------------------------------------------
+
+.auto_devops: &auto_devops |
+  # Auto DevOps variables and functions
+  [[ "$TRACE" ]] && set -x
+  auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB}
+  export DATABASE_URL=${DATABASE_URL-$auto_database_url}
+  export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG
+  export CI_APPLICATION_TAG=$CI_COMMIT_SHA
+  export CI_CONTAINER_NAME=ci_job_build_${CI_JOB_ID}
+  export TILLER_NAMESPACE=$KUBE_NAMESPACE
+  # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products
+  export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
+
+  function registry_login() {
+    if [[ -n "$CI_REGISTRY_USER" ]]; then
+      echo "Logging to GitLab Container Registry with CI credentials..."
+      docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
+      echo ""
+    fi
+  }
+
+  function container_scanning() {
+    registry_login
+
+    docker run -d --name db arminc/clair-db:latest
+    docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:v2.0.1
+    apk add -U wget ca-certificates
+    docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG}
+    wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64
+    mv clair-scanner_linux_amd64 clair-scanner
+    chmod +x clair-scanner
+    touch clair-whitelist.yml
+    retries=0
+    echo "Waiting for clair daemon to start"
+    while( ! wget -T 10 -q -O /dev/null http://docker:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done
+    ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-container-scanning-report.json -l clair.log -w clair-whitelist.yml ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} || true
+  }
+
+  function code_quality() {
+    docker run --env SOURCE_CODE="$PWD" \
+               --volume "$PWD":/code \
+               --volume /var/run/docker.sock:/var/run/docker.sock \
+               "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code
+  }
+
+  function license_management() {
+    /run.sh analyze .
+  }
+
+  function sast() {
+    case "$CI_SERVER_VERSION" in
+      *-ee)
+
+        # Deprecation notice for CONFIDENCE_LEVEL variable
+        if [ -z "$SAST_CONFIDENCE_LEVEL" -a "$CONFIDENCE_LEVEL" ]; then
+          SAST_CONFIDENCE_LEVEL="$CONFIDENCE_LEVEL"
+          echo "WARNING: CONFIDENCE_LEVEL is deprecated and MUST be replaced with SAST_CONFIDENCE_LEVEL"
+        fi
+
+        docker run --env SAST_CONFIDENCE_LEVEL="${SAST_CONFIDENCE_LEVEL:-3}" \
+                   --volume "$PWD:/code" \
+                   --volume /var/run/docker.sock:/var/run/docker.sock \
+                   "registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code
+        ;;
+      *)
+        echo "GitLab EE is required"
+        ;;
+    esac
+  }
+
+  function dependency_scanning() {
+    case "$CI_SERVER_VERSION" in
+      *-ee)
+        docker run --env DEP_SCAN_DISABLE_REMOTE_CHECKS="${DEP_SCAN_DISABLE_REMOTE_CHECKS:-false}" \
+                   --volume "$PWD:/code" \
+                   --volume /var/run/docker.sock:/var/run/docker.sock \
+                   "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code
+        ;;
+      *)
+        echo "GitLab EE is required"
+        ;;
+    esac
+  }
+
+  function get_replicas() {
+    track="${1:-stable}"
+    percentage="${2:-100}"
+
+    env_track=$( echo $track | tr -s  '[:lower:]'  '[:upper:]' )
+    env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s  '[:lower:]'  '[:upper:]' )
+
+    if [[ "$track" == "stable" ]] || [[ "$track" == "rollout" ]]; then
+      # for stable track get number of replicas from `PRODUCTION_REPLICAS`
+      eval new_replicas=\$${env_slug}_REPLICAS
+      if [[ -z "$new_replicas" ]]; then
+        new_replicas=$REPLICAS
+      fi
+    else
+      # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS`
+      eval new_replicas=\$${env_track}_${env_slug}_REPLICAS
+      if [[ -z "$new_replicas" ]]; then
+        eval new_replicas=\${env_track}_REPLICAS
+      fi
+    fi
+
+    replicas="${new_replicas:-1}"
+    replicas="$(($replicas * $percentage / 100))"
+
+    # always return at least one replicas
+    if [[ $replicas -gt 0 ]]; then
+      echo "$replicas"
+    else
+      echo 1
+    fi
+  }
+
+  # Extracts variables prefixed with K8S_SECRET_
+  # and creates a Kubernetes secret.
+  #
+  # e.g. If we have the following environment variables:
+  #   K8S_SECRET_A=value1
+  #   K8S_SECRET_B=multi\ word\ value
+  #
+  # Then we will create a secret with the following key-value pairs:
+  #   data:
+  #     A: dmFsdWUxCg==
+  #     B: bXVsdGkgd29yZCB2YWx1ZQo=
+  function create_application_secret() {
+    track="${1-stable}"
+    export APPLICATION_SECRET_NAME=$(application_secret_name "$track")
+
+    env | sed -n "s/^K8S_SECRET_\(.*\)$/\1/p" > k8s_prefixed_variables
+
+    kubectl create secret \
+      -n "$KUBE_NAMESPACE" generic "$APPLICATION_SECRET_NAME" \
+      --from-env-file k8s_prefixed_variables -o yaml --dry-run |
+      kubectl replace -n "$KUBE_NAMESPACE" --force -f -
+
+    export APPLICATION_SECRET_CHECKSUM=$(cat k8s_prefixed_variables | sha256sum | cut -d ' ' -f 1)
+
+    rm k8s_prefixed_variables
+  }
+
+  function deploy_name() {
+    name="$CI_ENVIRONMENT_SLUG"
+    track="${1-stable}"
+
+    if [[ "$track" != "stable" ]]; then
+      name="$name-$track"
+    fi
+
+    echo $name
+  }
+
+  function application_secret_name() {
+    track="${1-stable}"
+    name=$(deploy_name "$track")
+
+    echo "${name}-secret"
+  }
+
+  function deploy() {
+    track="${1-stable}"
+    percentage="${2:-100}"
+    name=$(deploy_name "$track")
+
+    replicas="1"
+    service_enabled="true"
+    postgres_enabled="$POSTGRES_ENABLED"
+
+    # if track is different than stable,
+    # re-use all attached resources
+    if [[ "$track" != "stable" ]]; then
+      service_enabled="false"
+      postgres_enabled="false"
+    fi
+
+    replicas=$(get_replicas "$track" "$percentage")
+
+    if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then
+      secret_name='gitlab-registry'
+    else
+      secret_name=''
+    fi
+
+    create_application_secret "$track"
+
+    env_slug=$(echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]')
+    eval env_ADDITIONAL_HOSTS=\$${env_slug}_ADDITIONAL_HOSTS
+    if [ -n "$env_ADDITIONAL_HOSTS" ]; then
+      additional_hosts="{$env_ADDITIONAL_HOSTS}"
+    elif [ -n "$ADDITIONAL_HOSTS" ]; then
+      additional_hosts="{$ADDITIONAL_HOSTS}"
+    fi
+
+    if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then
+      echo "Deploying first release with database initialization..."
+      helm upgrade --install \
+        --wait \
+        --set service.enabled="$service_enabled" \
+        --set releaseOverride="$CI_ENVIRONMENT_SLUG" \
+        --set image.repository="$CI_APPLICATION_REPOSITORY" \
+        --set image.tag="$CI_APPLICATION_TAG" \
+        --set image.pullPolicy=IfNotPresent \
+        --set image.secrets[0].name="$secret_name" \
+        --set application.track="$track" \
+        --set application.database_url="$DATABASE_URL" \
+        --set application.secretName="$APPLICATION_SECRET_NAME" \
+        --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \
+        --set service.commonName="le.$AUTO_DEVOPS_DOMAIN" \
+        --set service.url="$CI_ENVIRONMENT_URL" \
+        --set service.additionalHosts="$additional_hosts" \
+        --set replicaCount="$replicas" \
+        --set postgresql.enabled="$postgres_enabled" \
+        --set postgresql.nameOverride="postgres" \
+        --set postgresql.postgresUser="$POSTGRES_USER" \
+        --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \
+        --set postgresql.postgresDatabase="$POSTGRES_DB" \
+        --set application.initializeCommand="$DB_INITIALIZE" \
+        --namespace="$KUBE_NAMESPACE" \
+        "$name" \
+        chart/
+
+      echo "Deploying second release..."
+      helm upgrade --reuse-values \
+        --wait \
+        --set application.initializeCommand="" \
+        --set application.migrateCommand="$DB_MIGRATE" \
+        --namespace="$KUBE_NAMESPACE" \
+        "$name" \
+        chart/
+    else
+      echo "Deploying new release..."
+      helm upgrade --install \
+        --wait \
+        --set service.enabled="$service_enabled" \
+        --set releaseOverride="$CI_ENVIRONMENT_SLUG" \
+        --set image.repository="$CI_APPLICATION_REPOSITORY" \
+        --set image.tag="$CI_APPLICATION_TAG" \
+        --set image.pullPolicy=IfNotPresent \
+        --set image.secrets[0].name="$secret_name" \
+        --set application.track="$track" \
+        --set application.database_url="$DATABASE_URL" \
+        --set application.secretName="$APPLICATION_SECRET_NAME" \
+        --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \
+        --set service.commonName="le.$AUTO_DEVOPS_DOMAIN" \
+        --set service.url="$CI_ENVIRONMENT_URL" \
+        --set service.additionalHosts="$additional_hosts" \
+        --set replicaCount="$replicas" \
+        --set postgresql.enabled="$postgres_enabled" \
+        --set postgresql.nameOverride="postgres" \
+        --set postgresql.postgresUser="$POSTGRES_USER" \
+        --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \
+        --set postgresql.postgresDatabase="$POSTGRES_DB" \
+        --set application.migrateCommand="$DB_MIGRATE" \
+        --namespace="$KUBE_NAMESPACE" \
+        "$name" \
+        chart/
+    fi
+
+    kubectl rollout status -n "$KUBE_NAMESPACE" -w "deployment/$name"
+  }
+
+  function scale() {
+    track="${1-stable}"
+    percentage="${2-100}"
+    name=$(deploy_name "$track")
+
+    replicas=$(get_replicas "$track" "$percentage")
+
+    if [[ -n "$(helm ls -q "^$name$")" ]]; then
+      helm upgrade --reuse-values \
+        --wait \
+        --set replicaCount="$replicas" \
+        --namespace="$KUBE_NAMESPACE" \
+        "$name" \
+        chart/
+    fi
+  }
+
+  function install_dependencies() {
+    apk add -U openssl curl tar gzip bash ca-certificates git
+    curl -L -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
+    curl -L -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.28-r0/glibc-2.28-r0.apk
+    apk add glibc-2.28-r0.apk
+    rm glibc-2.28-r0.apk
+
+    curl "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx
+    mv linux-amd64/helm /usr/bin/
+    mv linux-amd64/tiller /usr/bin/
+    helm version --client
+    tiller -version
+
+    curl -L -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl"
+    chmod +x /usr/bin/kubectl
+    kubectl version --client
+  }
+
+  function setup_docker() {
+    if ! docker info &>/dev/null; then
+      if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
+        export DOCKER_HOST='tcp://localhost:2375'
+      fi
+    fi
+  }
+
+  function setup_test_db() {
+    if [ -z ${KUBERNETES_PORT+x} ]; then
+      DB_HOST=postgres
+    else
+      DB_HOST=localhost
+    fi
+    export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:5432/${POSTGRES_DB}"
+  }
+
+  function download_chart() {
+    if [[ ! -d chart ]]; then
+      auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app}
+      auto_chart_name=$(basename $auto_chart)
+      auto_chart_name=${auto_chart_name%.tgz}
+      auto_chart_name=${auto_chart_name%.tar.gz}
+    else
+      auto_chart="chart"
+      auto_chart_name="chart"
+    fi
+
+    helm init --client-only
+    helm repo add gitlab ${AUTO_DEVOPS_CHART_REPOSITORY:-https://charts.gitlab.io}
+    if [[ ! -d "$auto_chart" ]]; then
+      helm fetch ${auto_chart} --untar
+    fi
+    if [ "$auto_chart_name" != "chart" ]; then
+      mv ${auto_chart_name} chart
+    fi
+
+    helm dependency update chart/
+    helm dependency build chart/
+  }
+
+  function ensure_namespace() {
+    kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE"
+  }
+
+  function check_kube_domain() {
+    if [ -z ${AUTO_DEVOPS_DOMAIN+x} ]; then
+      echo "In order to deploy or use Review Apps, AUTO_DEVOPS_DOMAIN variable must be set"
+      echo "You can do it in Auto DevOps project settings or defining a variable at group or project level"
+      echo "You can also manually add it in .gitlab-ci.yml"
+      false
+    else
+      true
+    fi
+  }
+
+  function build() {
+    registry_login
+
+    if [[ -f Dockerfile ]]; then
+      echo "Building Dockerfile-based application..."
+      docker build \
+        --build-arg HTTP_PROXY="$HTTP_PROXY" \
+        --build-arg http_proxy="$http_proxy" \
+        --build-arg HTTPS_PROXY="$HTTPS_PROXY" \
+        --build-arg https_proxy="$https_proxy" \
+        --build-arg FTP_PROXY="$FTP_PROXY" \
+        --build-arg ftp_proxy="$ftp_proxy" \
+        --build-arg NO_PROXY="$NO_PROXY" \
+        --build-arg no_proxy="$no_proxy" \
+        -t "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" .
+    else
+      echo "Building Heroku-based application using gliderlabs/herokuish docker image..."
+      docker run -i \
+        -e BUILDPACK_URL \
+        -e HTTP_PROXY \
+        -e http_proxy \
+        -e HTTPS_PROXY \
+        -e https_proxy \
+        -e FTP_PROXY \
+        -e ftp_proxy \
+        -e NO_PROXY \
+        -e no_proxy \
+        --name="$CI_CONTAINER_NAME" -v "$(pwd):/tmp/app:ro" gliderlabs/herokuish /bin/herokuish buildpack build
+      docker commit "$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG"
+      docker rm "$CI_CONTAINER_NAME" >/dev/null
+      echo ""
+
+      echo "Configuring $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG docker image..."
+      docker create --expose 5000 --env PORT=5000 --name="$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" /bin/herokuish procfile start web
+      docker commit "$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG"
+      docker rm "$CI_CONTAINER_NAME" >/dev/null
+      echo ""
+    fi
+
+    echo "Pushing to GitLab Container Registry..."
+    docker push "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG"
+    echo ""
+  }
+
+  function initialize_tiller() {
+    echo "Checking Tiller..."
+
+    export HELM_HOST="localhost:44134"
+    tiller -listen ${HELM_HOST} -alsologtostderr > /dev/null 2>&1 &
+    echo "Tiller is listening on ${HELM_HOST}"
+
+    if ! helm version --debug; then
+      echo "Failed to init Tiller."
+      return 1
+    fi
+    echo ""
+  }
+
+  function create_secret() {
+    echo "Create secret..."
+    if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then
+      return
+    fi
+
+    kubectl create secret -n "$KUBE_NAMESPACE" \
+      docker-registry gitlab-registry \
+      --docker-server="$CI_REGISTRY" \
+      --docker-username="${CI_DEPLOY_USER:-$CI_REGISTRY_USER}" \
+      --docker-password="${CI_DEPLOY_PASSWORD:-$CI_REGISTRY_PASSWORD}" \
+      --docker-email="$GITLAB_USER_EMAIL" \
+      -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f -
+  }
+
+  function dast() {
+    export CI_ENVIRONMENT_URL=$(cat environment_url.txt)
+
+    mkdir /zap/wrk/
+    /zap/zap-baseline.py -J gl-dast-report.json -t "$CI_ENVIRONMENT_URL" || true
+    cp /zap/wrk/gl-dast-report.json .
+  }
+
+  function performance() {
+    export CI_ENVIRONMENT_URL=$(cat environment_url.txt)
+
+    mkdir gitlab-exporter
+    wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-5/index.js
+
+    mkdir sitespeed-results
+
+    if [ -f .gitlab-urls.txt ]
+    then
+      sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt
+      docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt
+    else
+      docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL"
+    fi
+
+    mv sitespeed-results/data/performance.json performance.json
+  }
+
+  function persist_environment_url() {
+      echo $CI_ENVIRONMENT_URL > environment_url.txt
+  }
+
+  function delete() {
+    track="${1-stable}"
+    name=$(deploy_name "$track")
+
+    if [[ -n "$(helm ls -q "^$name$")" ]]; then
+      helm delete --purge "$name"
+    fi
+
+    secret_name=$(application_secret_name "$track")
+    kubectl delete secret --ignore-not-found -n "$KUBE_NAMESPACE" "$secret_name"
+  }
+
+before_script:
+  - *auto_devops
diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml
new file mode 100644
index 00000000000..2d218b2e164
--- /dev/null
+++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml
@@ -0,0 +1,35 @@
+# see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options
+
+# you can delete this line if you're not using Docker
+image: busybox:latest
+
+before_script:
+  - echo "Before script section"
+  - echo "For example you might run an update here or install a build dependency"
+  - echo "Or perhaps you might print out some debugging details"
+   
+after_script:
+  - echo "After script section"
+  - echo "For example you might do some cleanup here"
+   
+build1:
+  stage: build
+  script:
+    - echo "Do your build here"
+   
+test1:
+  stage: test
+  script: 
+    - echo "Do a test here"
+    - echo "For example run a test suite"
+   
+test2:
+  stage: test
+  script: 
+    - echo "Do another parallel test here"
+    - echo "For example run a lint test"
+   
+deploy1:
+  stage: deploy
+  script:
+    - echo "Do your deploy here"
diff --git a/lib/gitlab/ci/templates/C++.gitlab-ci.yml b/lib/gitlab/ci/templates/C++.gitlab-ci.yml
new file mode 100644
index 00000000000..c83c49d8c95
--- /dev/null
+++ b/lib/gitlab/ci/templates/C++.gitlab-ci.yml
@@ -0,0 +1,26 @@
+# use the official gcc image, based on debian
+# can use verions as well, like gcc:5.2
+# see https://hub.docker.com/_/gcc/
+image: gcc
+
+build:
+  stage: build
+  # instead of calling g++ directly you can also use some build toolkit like make
+  # install the necessary build tools when needed
+  # before_script: 
+  #   - apt update && apt -y install make autoconf 
+  script: 
+    - g++ helloworld.cpp -o mybinary
+  artifacts:
+    paths:
+      - mybinary
+  # depending on your build setup it's most likely a good idea to cache outputs to reduce the build time
+  # cache:
+  #   paths:
+  #     - "*.o"
+
+# run tests using the binary built before
+test:
+  stage: test
+  script:
+    - ./runmytests.sh
diff --git a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml
new file mode 100644
index 00000000000..4d5b6484d6e
--- /dev/null
+++ b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml
@@ -0,0 +1,51 @@
+# This file uses Test Kitchen with the kitchen-dokken driver to
+# perform functional testing. Doing so requires that your runner be a
+# Docker runner configured for privileged mode. Please see
+# https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode
+# for help configuring your runner properly, or, if you want to switch
+# to a different driver, see http://kitchen.ci/docs/drivers
+
+image: "chef/chefdk"
+services:
+  - docker:dind
+
+variables:
+  DOCKER_HOST: "tcp://docker:2375"
+  KITCHEN_LOCAL_YAML: ".kitchen.dokken.yml"
+
+stages:
+  - lint
+  - unit
+  - functional
+
+foodcritic:
+  stage: lint
+  script:
+  - chef exec foodcritic .
+
+cookstyle:
+  stage: lint
+  script:
+  - chef exec cookstyle .
+
+chefspec:
+  stage: unit
+  script:
+  - chef exec rspec spec
+
+# Set up your test matrix here. Example:
+#verify-centos-6:
+#  stage: functional
+#  before_script:
+#  - apt-get update
+#  - apt-get -y install rsync
+#  script:
+#  - kitchen verify default-centos-6 --destroy=always
+#  
+#verify-centos-7:
+#  stage: functional
+#  before_script:
+#  - apt-get update
+#  - apt-get -y install rsync
+#  script:
+#  - kitchen verify default-centos-7 --destroy=always
diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml
new file mode 100644
index 00000000000..f066285b1ad
--- /dev/null
+++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml
@@ -0,0 +1,22 @@
+# Based on openjdk:8, already includes lein
+image: clojure:lein-2.7.0
+# If you need to configure a database, add a `services` section here
+# See https://docs.gitlab.com/ce/ci/services/postgres.html
+# Make sure you configure the connection as well
+
+before_script:
+  # If you need to install any external applications, like a 
+  # postgres client, you may want to uncomment the line below:
+  # 
+  #- apt-get update -y
+  #
+  # Retrieve project dependencies
+  # Do this on before_script since it'll be shared between both test and
+  # any production sections a user adds
+  - lein deps
+
+test:
+  script:
+  # If you need to run any migrations or configure the database, this 
+  # would be the point to do it.  
+  - lein test
diff --git a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml
new file mode 100644
index 00000000000..36386a19fdc
--- /dev/null
+++ b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml
@@ -0,0 +1,36 @@
+# Official language image. Look for the different tagged releases at:
+# https://hub.docker.com/r/crystallang/crystal/
+image: "crystallang/crystal:latest"
+
+# Pick zero or more services to be used on all builds.
+# Only needed when using a docker container to run your tests in.
+# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service
+# services:
+#   - mysql:latest
+#   - redis:latest
+#   - postgres:latest
+
+# variables:
+#   POSTGRES_DB: database_name
+
+# Cache shards in between builds
+cache:
+  paths:
+    - lib
+
+# This is a basic example for a shard or script which doesn't use
+# services such as redis or postgres
+before_script:
+  - apt-get update -qq && apt-get install -y -qq libxml2-dev
+  - crystal -v # Print out Crystal version for debugging
+  - shards
+
+# If you are using built-in Crystal Spec.
+spec:
+  script:
+  - crystal spec
+
+# If you are using minitest.cr
+minitest:
+  script:
+  - crystal test/spec_test.cr # change to the file(s) you execute for tests
diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml
new file mode 100644
index 00000000000..57afcbbe8b5
--- /dev/null
+++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml
@@ -0,0 +1,49 @@
+# Official framework image. Look for the different tagged releases at:
+# https://hub.docker.com/r/library/python
+image: python:latest
+
+# Pick zero or more services to be used on all builds.
+# Only needed when using a docker container to run your tests in.
+# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service
+services:
+  - mysql:latest
+  - postgres:latest
+
+variables:
+  POSTGRES_DB: database_name
+
+# This folder is cached between builds
+# http://docs.gitlab.com/ce/ci/yaml/README.html#cache
+cache:
+  paths:
+  - ~/.cache/pip/
+
+# This is a basic example for a gem or script which doesn't use
+# services such as redis or postgres
+before_script:
+  - python -V                                   # Print out python version for debugging
+  # Uncomment next line if your Django app needs a JS runtime:
+  # - apt-get update -q && apt-get install nodejs -yqq
+  - pip install -r requirements.txt
+
+# To get Django tests to work you may need to create a settings file using
+# the following DATABASES:
+# 
+# DATABASES = {
+#     'default': {
+#        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+#        'NAME': 'ci',
+#        'USER': 'postgres',
+#        'PASSWORD': 'postgres',
+#        'HOST': 'postgres',
+#        'PORT': '5432',
+#    },
+# }
+#
+# and then adding `--settings app.settings.ci` (or similar) to the test command
+
+test:
+  variables:
+    DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/$POSTGRES_DB"
+  script:
+  - python manage.py test
diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml
new file mode 100644
index 00000000000..eeefadaa019
--- /dev/null
+++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml
@@ -0,0 +1,24 @@
+# Official docker image.
+image: docker:latest
+
+services:
+  - docker:dind
+
+before_script:
+  - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+
+build-master:
+  stage: build
+  script:
+    - docker build --pull -t "$CI_REGISTRY_IMAGE" .
+    - docker push "$CI_REGISTRY_IMAGE"
+  only:
+    - master
+
+build:
+  stage: build
+  script:
+    - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" .
+    - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
+  except:
+    - master
diff --git a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml
new file mode 100644
index 00000000000..cf9c731637c
--- /dev/null
+++ b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml
@@ -0,0 +1,18 @@
+image: elixir:latest
+
+# Pick zero or more services to be used on all builds.
+# Only needed when using a docker container to run your tests in.
+# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service
+services:
+  - mysql:latest
+  - redis:latest
+  - postgres:latest
+
+before_script:
+  - mix local.rebar --force
+  - mix local.hex --force
+  - mix deps.get
+
+mix:
+  script:
+  - mix test
diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml
new file mode 100644
index 00000000000..d572d7a1edc
--- /dev/null
+++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml
@@ -0,0 +1,35 @@
+image: golang:latest
+
+variables:
+  # Please edit to your GitLab project
+  REPO_NAME: gitlab.com/namespace/project
+
+# The problem is that to be able to use go get, one needs to put
+# the repository in the $GOPATH. So for example if your gitlab domain
+# is gitlab.com, and that your repository is namespace/project, and
+# the default GOPATH being /go, then you'd need to have your
+# repository in /go/src/gitlab.com/namespace/project
+# Thus, making a symbolic link corrects this.
+before_script:
+  - mkdir -p $GOPATH/src/$(dirname $REPO_NAME)
+  - ln -svf $CI_PROJECT_DIR $GOPATH/src/$REPO_NAME
+  - cd $GOPATH/src/$REPO_NAME
+
+stages:
+    - test
+    - build
+
+format:
+    stage: test
+    script:
+      - go fmt $(go list ./... | grep -v /vendor/)
+      - go vet $(go list ./... | grep -v /vendor/)
+      - go test -race $(go list ./... | grep -v /vendor/)
+
+compile:
+    stage: build
+    script:
+      - go build -race -ldflags "-extldflags '-static'" -o $CI_PROJECT_DIR/mybinary
+    artifacts:
+      paths:
+        - mybinary
diff --git a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml
new file mode 100644
index 00000000000..48d98dddfad
--- /dev/null
+++ b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml
@@ -0,0 +1,36 @@
+# This is the Gradle build system for JVM applications
+# https://gradle.org/
+# https://github.com/gradle/gradle
+image: gradle:alpine
+
+# Disable the Gradle daemon for Continuous Integration servers as correctness
+# is usually a priority over speed in CI environments. Using a fresh
+# runtime for each build is more reliable since the runtime is completely
+# isolated from any previous builds.
+variables:
+  GRADLE_OPTS: "-Dorg.gradle.daemon=false"
+
+before_script:
+  - export GRADLE_USER_HOME=`pwd`/.gradle
+
+build:
+  stage: build
+  script: gradle --build-cache assemble
+  cache:
+    key: "$CI_COMMIT_REF_NAME"
+    policy: push
+    paths:
+      - build
+      - .gradle
+
+
+test:
+  stage: test
+  script: gradle check
+  cache:
+    key: "$CI_COMMIT_REF_NAME"
+    policy: pull
+    paths:
+      - build
+      - .gradle
+
diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml
new file mode 100644
index 00000000000..7fc698d50cf
--- /dev/null
+++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml
@@ -0,0 +1,40 @@
+# This template uses the java:8 docker image because there isn't any
+# official Grails image at this moment
+#
+# Grails Framework https://grails.org/ is a powerful Groovy-based web application framework for the JVM
+#
+# This yml works with Grails 3.x only
+# Feel free to change GRAILS_VERSION version with your project version (3.0.1, 3.1.1,...)
+# Feel free to change GRADLE_VERSION version with your gradle project version (2.13, 2.14,...)
+# If you use Angular profile, this yml it's prepared to work with it
+
+image: java:8
+
+variables:
+  GRAILS_VERSION: "3.1.9"
+  GRADLE_VERSION: "2.13"
+  
+# We use SDKMan as tool for managing versions
+before_script:
+   - apt-get update -qq && apt-get install -y -qq unzip
+   - curl -sSL https://get.sdkman.io | bash
+   - echo sdkman_auto_answer=true > /root/.sdkman/etc/config
+   - source /root/.sdkman/bin/sdkman-init.sh
+   - sdk install gradle $GRADLE_VERSION < /dev/null
+   - sdk use gradle $GRADLE_VERSION
+# As it's not a good idea to version gradle.properties feel free to add your
+# environments variable here   
+   - echo grailsVersion=$GRAILS_VERSION > gradle.properties
+   - echo gradleWrapperVersion=2.14 >> gradle.properties
+# refresh dependencies from your project   
+   - ./gradlew --refresh-dependencies
+# Be aware that if you are using Angular profile,
+# Bower cannot be run as root if you don't allow it before.
+# Feel free to remove next line if you are not using Bower
+   - echo {\"allow_root\":true} > /root/.bowerrc
+
+# This build job does the full grails pipeline
+# (compile, test, integrationTest, war, assemble).
+build:
+ script: 
+   - ./gradlew build
\ No newline at end of file
diff --git a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml
new file mode 100644
index 00000000000..04c21b4725d
--- /dev/null
+++ b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml
@@ -0,0 +1,76 @@
+# This is an example .gitlab-ci.yml file to test (and optionally report the coverage
+# results of) your [Julia][1] packages. Please refer to the [documentation][2]
+# for more information about package development in Julia.
+#
+# Here, it is assumed that your Julia package is named `MyPackage`. Change it to
+# whatever name you have given to your package.
+#
+# [1]: http://julialang.org/
+# [2]: https://docs.julialang.org/en/v1/manual/documentation/index.html
+
+# Below is the template to run your tests in Julia
+.test_template: &test_definition
+  # Uncomment below if you would like to run the tests on specific references
+  # only, such as the branches `master`, `development`, etc.
+  # only:
+  #   - master
+  #   - development
+  script:
+    # Let's run the tests. Substitute `coverage = false` below, if you do not
+    # want coverage results.
+    - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("MyPackage"); Pkg.test("MyPackage"; coverage = true)'
+    # Comment out below if you do not want coverage results.
+    - julia -e 'using Pkg; Pkg.add("Coverage"); 
+      import MyPackage; cd(joinpath(dirname(pathof(MyPackage)), ".."));
+      using Coverage; cl, tl = get_summary(process_folder());
+      println("(", cl/tl*100, "%) covered")'
+
+# Name a test and select an appropriate image.
+# images comes from Docker hub
+test:0.7:
+  image: julia:0.7
+  <<: *test_definition
+  
+test:1.0:
+  image: julia:1.0
+  <<: *test_definition
+
+# Maybe you would like to test your package against the development branch:
+# test:1.1-dev (not sure there is such an image in docker, so not tested yet):
+#  image: julia:v1.1-dev
+#  # ... allowing for failures, since we are testing against the development
+#  # branch:
+#  allow_failure: true
+#  <<: *test_definition
+
+# REMARK: Do not forget to enable the coverage feature for your project, if you
+# are using code coverage reporting above. This can be done by
+#
+# - Navigating to the `CI/CD Pipelines` settings of your project,
+# - Copying and pasting the default `Simplecov` regex example provided, i.e.,
+#   `\(\d+.\d+\%\) covered` in the `test coverage parsing` textfield.
+
+# Example documentation deployment
+pages:
+  image: julia:0.7
+  stage: deploy
+  script:
+    - apt-get update -qq && apt-get install -y git # needed by Documenter
+    - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("MyPackage");' # rebuild Julia (can be put somewhere else I'm sure
+    - julia -e 'using Pkg; import MyPackage; Pkg.add("Documenter")' # install Documenter
+    - julia --color=yes docs/make.jl # make documentation
+    - mv docs/build public # move to the directory picked up by Gitlab pages
+  artifacts:
+    paths:
+      - public
+  only:
+  - master
+
+
+# WARNING: This template is using the `julia` images from [Docker
+# Hub][3]. One can use custom Julia images and/or the official ones found
+# in the same place. However, care must be taken to correctly locate the binary
+# file (`/opt/julia/bin/julia` above), which is usually given on the image's
+# description page.
+#
+# [3]: https://hub.docker.com/_/julia/
diff --git a/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml
new file mode 100644
index 00000000000..a4aed36889e
--- /dev/null
+++ b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml
@@ -0,0 +1,11 @@
+# use docker image with latex preinstalled
+# since there is no official latex image, use https://github.com/blang/latex-docker
+# possible alternative: https://github.com/natlownes/docker-latex
+image: blang/latex
+
+build:
+  script:
+    - latexmk -pdf
+  artifacts:
+    paths:
+      - "*.pdf"
diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml
new file mode 100644
index 00000000000..d0cad285572
--- /dev/null
+++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml
@@ -0,0 +1,85 @@
+# Official framework image. Look for the different tagged releases at:
+# https://hub.docker.com/r/library/php
+image: php:latest
+
+# Pick zero or more services to be used on all builds.
+# Only needed when using a docker container to run your tests in.
+# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service
+services:
+  - mysql:latest
+
+variables:
+  MYSQL_DATABASE: project_name
+  MYSQL_ROOT_PASSWORD: secret
+
+# This folder is cached between builds
+# http://docs.gitlab.com/ce/ci/yaml/README.html#cache
+cache:
+  paths:
+  - vendor/
+  - node_modules/
+
+# This is a basic example for a gem or script which doesn't use
+# services such as redis or postgres
+before_script:
+  # Update packages 
+  - apt-get update -yqq
+  
+  # Prep for Node
+  - apt-get install gnupg -yqq
+
+  # Upgrade to Node 8
+  - curl -sL https://deb.nodesource.com/setup_8.x | bash -
+
+  # Install dependencies
+  - apt-get install git nodejs libcurl4-gnutls-dev libicu-dev libmcrypt-dev libvpx-dev libjpeg-dev libpng-dev libxpm-dev zlib1g-dev libfreetype6-dev libxml2-dev libexpat1-dev libbz2-dev libgmp3-dev libldap2-dev unixodbc-dev libpq-dev libsqlite3-dev libaspell-dev libsnmp-dev libpcre3-dev libtidy-dev -yqq
+
+  # Install php extensions
+  - docker-php-ext-install mbstring pdo_mysql curl json intl gd xml zip bz2 opcache
+
+  # Install & enable Xdebug for code coverage reports
+  - pecl install xdebug
+  - docker-php-ext-enable xdebug
+
+  # Install Composer and project dependencies.
+  - curl -sS https://getcomposer.org/installer | php
+  - php composer.phar install 
+
+  # Install Node dependencies.
+  # comment this out if you don't have a node dependency
+  - npm install
+
+  # Copy over testing configuration.
+  # Don't forget to set the database config in .env.testing correctly
+  # DB_HOST=mysql
+  # DB_DATABASE=project_name
+  # DB_USERNAME=root
+  # DB_PASSWORD=secret
+  - cp .env.testing .env
+
+  # Run npm build
+  # comment this out if you don't have a frontend build
+  # you can change this to to your frontend building script like
+  # npm run build
+  - npm run dev
+
+  # Generate an application key. Re-cache.
+  - php artisan key:generate
+  - php artisan config:cache
+
+  # Run database migrations.
+  - php artisan migrate
+
+  # Run database seed
+  - php artisan db:seed
+
+test:
+  script:
+  # run laravel tests
+  - php vendor/bin/phpunit --coverage-text --colors=never 
+
+  # run frontend tests
+  # if you have any task for testing frontend
+  # set it in your package.json script
+  # comment this out if you don't have a frontend test
+  - npm test
diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml
new file mode 100644
index 00000000000..492b3d03db2
--- /dev/null
+++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml
@@ -0,0 +1,102 @@
+---
+# Build JAVA applications using Apache Maven (http://maven.apache.org)
+# For docker image tags see https://hub.docker.com/_/maven/
+#
+# For general lifecycle information see https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html
+#
+# This template will build and test your projects as well as create the documentation.
+#
+# * Caches downloaded dependencies and plugins between invocation.
+# * Verify but don't deploy merge requests.
+# * Deploy built artifacts from master branch only.
+# * Shows how to use multiple jobs in test stage for verifying functionality
+#   with multiple JDKs.
+# * Uses site:stage to collect the documentation for multi-module projects.
+# * Publishes the documentation for `master` branch.
+
+variables:
+  # This will suppress any download for dependencies and plugins or upload messages which would clutter the console log.
+  # `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work.
+  MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true"
+  # As of Maven 3.3.0 instead of this you may define these options in `.mvn/maven.config` so the same config is used
+  # when running from the command line.
+  # `installAtEnd` and `deployAtEnd` are only effective with recent version of the corresponding plugins.
+  MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true"
+
+# Cache downloaded dependencies and plugins between builds.
+# To keep cache across branches add 'key: "$CI_JOB_NAME"'
+cache:
+  paths:
+    - .m2/repository
+
+# This will only validate and compile stuff and run e.g. maven-enforcer-plugin.
+# Because some enforcer rules might check dependency convergence and class duplications
+# we use `test-compile` here instead of `validate`, so the correct classpath is picked up.
+.validate: &validate
+  stage: build
+  script:
+    - 'mvn $MAVEN_CLI_OPTS test-compile'
+
+# For merge requests do not `deploy` but only run `verify`.
+# See https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html
+.verify: &verify
+  stage: test
+  script:
+    - 'mvn $MAVEN_CLI_OPTS verify site site:stage'
+  except:
+    - master
+
+# Validate merge requests using JDK7
+validate:jdk7:
+  <<: *validate
+  image: maven:3.3.9-jdk-7
+
+# Validate merge requests using JDK8
+validate:jdk8:
+  <<: *validate
+  image: maven:3.3.9-jdk-8
+
+# Verify merge requests using JDK7
+verify:jdk7:
+  <<: *verify
+  image: maven:3.3.9-jdk-7
+
+# Verify merge requests using JDK8
+verify:jdk8:
+  <<: *verify
+  image: maven:3.3.9-jdk-8
+
+
+# For `master` branch run `mvn deploy` automatically.
+# Here you need to decide whether you want to use JDK7 or 8.
+# To get this working you need to define a volume while configuring your gitlab-ci-multi-runner.
+# Mount your `settings.xml` as `/root/.m2/settings.xml` which holds your secrets.
+# See https://maven.apache.org/settings.html
+deploy:jdk8:
+  # Use stage test here, so the pages job may later pickup the created site.
+  stage: test
+  script:
+    - 'mvn $MAVEN_CLI_OPTS deploy site site:stage'
+  only:
+    - master
+  # Archive up the built documentation site.
+  artifacts:
+    paths:
+    - target/staging
+  image: maven:3.3.9-jdk-8
+
+
+pages:
+  image: busybox:latest
+  stage: deploy
+  script:
+    # Because Maven appends the artifactId automatically to the staging path if you did define a parent pom,
+    # you might need to use `mv target/staging/YOUR_ARTIFACT_ID public` instead.
+    - mv target/staging public
+  dependencies:
+    - deploy:jdk8
+  artifacts:
+    paths:
+    - public
+  only:
+    - master
diff --git a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml
new file mode 100644
index 00000000000..3585f99760f
--- /dev/null
+++ b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml
@@ -0,0 +1,42 @@
+# This is a simple gitlab continuous integration template (compatible with the shared runner provided on gitlab.com)
+# using the official mono docker image to build a visual studio project.
+#
+# MyProject.sln
+#   MyProject\
+#     MyProject\
+#        MyProject.csproj (console application)
+#     MyProject.Test\
+#        MyProject.Test.csproj (test library using nuget packages "NUnit" and "NUnit.ConsoleRunner")
+#
+# Please find the full example project here:
+# https://gitlab.com/tobiaskoch/gitlab-ci-example-mono
+
+# see https://hub.docker.com/_/mono/
+image: mono:latest
+
+stages:
+  - test
+  - deploy
+
+before_script:
+  - nuget restore -NonInteractive
+
+release:
+  stage: deploy
+  only:
+    - master
+  artifacts:
+    paths:
+      - build/release/MyProject.exe
+  script:
+    # The output path is relative to the position of the csproj-file
+    - msbuild /p:Configuration="Release" /p:Platform="Any CPU"
+      /p:OutputPath="./../../build/release/" "MyProject.sln"
+    
+debug:
+  stage: test
+  script:
+    # The output path is relative to the position of the csproj-file
+    - msbuild /p:Configuration="Debug" /p:Platform="Any CPU"
+      /p:OutputPath="./../../build/debug/" "MyProject.sln"
+    - mono packages/NUnit.ConsoleRunner.3.6.0/tools/nunit3-console.exe build/debug/MyProject.Test.dll
\ No newline at end of file
diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml
new file mode 100644
index 00000000000..41de1458582
--- /dev/null
+++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml
@@ -0,0 +1,27 @@
+# Official framework image. Look for the different tagged releases at:
+# https://hub.docker.com/r/library/node/tags/
+image: node:latest
+
+# Pick zero or more services to be used on all builds.
+# Only needed when using a docker container to run your tests in.
+# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service
+services:
+  - mysql:latest
+  - redis:latest
+  - postgres:latest
+
+# This folder is cached between builds
+# http://docs.gitlab.com/ce/ci/yaml/README.html#cache
+cache:
+  paths:
+  - node_modules/
+
+test_async:
+  script:
+   - npm install
+   - node ./specs/start.js ./specs/async.spec.js
+
+test_db:
+  script:
+   - npm install
+   - node ./specs/start.js ./specs/db-postgres.spec.js
diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml
new file mode 100644
index 00000000000..290b9997084
--- /dev/null
+++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml
@@ -0,0 +1,92 @@
+image: ayufan/openshift-cli
+
+stages:
+  - test
+  - review
+  - staging
+  - production
+  - cleanup
+
+variables:
+  OPENSHIFT_SERVER: openshift.default.svc.cluster.local
+  # OPENSHIFT_DOMAIN: apps.example.com
+  # Configure this variable in Secure Variables:
+  # OPENSHIFT_TOKEN: my.openshift.token
+
+test1:
+  stage: test
+  before_script: []
+  script:
+    - echo run tests
+
+test2:
+  stage: test
+  before_script: []
+  script:
+    - echo run tests
+
+.deploy: &deploy
+  before_script:
+    - oc login "$OPENSHIFT_SERVER" --token="$OPENSHIFT_TOKEN" --insecure-skip-tls-verify
+    - oc project "$CI_PROJECT_NAME-$CI_PROJECT_ID" 2> /dev/null || oc new-project "$CI_PROJECT_NAME-$CI_PROJECT_ID"
+  script:
+    - "oc get services $APP 2> /dev/null || oc new-app . --name=$APP --strategy=docker"
+    - "oc start-build $APP --from-dir=. --follow || sleep 3s && oc start-build $APP --from-dir=. --follow"
+    - "oc get routes $APP 2> /dev/null || oc expose service $APP --hostname=$APP_HOST"
+
+review:
+  <<: *deploy
+  stage: review
+  variables:
+    APP: review-$CI_COMMIT_REF_NAME
+    APP_HOST: $CI_PROJECT_NAME-$CI_ENVIRONMENT_SLUG.$OPENSHIFT_DOMAIN
+  environment:
+    name: review/$CI_COMMIT_REF_NAME
+    url: http://$CI_PROJECT_NAME-$CI_ENVIRONMENT_SLUG.$OPENSHIFT_DOMAIN
+    on_stop: stop-review
+  only:
+    - branches
+  except:
+    - master
+
+stop-review:
+  <<: *deploy
+  stage: cleanup
+  script:
+    - oc delete all -l "app=$APP"
+  when: manual
+  variables:
+    APP: review-$CI_COMMIT_REF_NAME
+    GIT_STRATEGY: none
+  environment:
+    name: review/$CI_COMMIT_REF_NAME
+    action: stop
+  only:
+    - branches
+  except:
+    - master
+
+staging:
+  <<: *deploy
+  stage: staging
+  variables:
+    APP: staging
+    APP_HOST: $CI_PROJECT_NAME-staging.$OPENSHIFT_DOMAIN
+  environment:
+    name: staging
+    url: http://$CI_PROJECT_NAME-staging.$OPENSHIFT_DOMAIN
+  only:
+    - master
+
+production:
+  <<: *deploy
+  stage: production
+  variables:
+    APP: production
+    APP_HOST: $CI_PROJECT_NAME.$OPENSHIFT_DOMAIN
+  when: manual
+  environment:
+    name: production
+    url: http://$CI_PROJECT_NAME.$OPENSHIFT_DOMAIN
+  only:
+    - master
diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml
new file mode 100644
index 00000000000..33f44ee9222
--- /dev/null
+++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml
@@ -0,0 +1,36 @@
+# Select image from https://hub.docker.com/_/php/
+image: php:7.1.1
+
+# Select what we should cache between builds
+cache:
+  paths:
+  - vendor/
+
+before_script:
+- apt-get update -yqq
+- apt-get install -yqq git libmcrypt-dev libpq-dev libcurl4-gnutls-dev libicu-dev libvpx-dev libjpeg-dev libpng-dev libxpm-dev zlib1g-dev libfreetype6-dev libxml2-dev libexpat1-dev libbz2-dev libgmp3-dev libldap2-dev unixodbc-dev libsqlite3-dev libaspell-dev libsnmp-dev libpcre3-dev libtidy-dev
+# Install PHP extensions
+- docker-php-ext-install mbstring mcrypt pdo_pgsql curl json intl gd xml zip bz2 opcache
+# Install & enable Xdebug for code coverage reports
+- pecl install xdebug
+- docker-php-ext-enable xdebug
+# Install and run Composer
+- curl -sS https://getcomposer.org/installer | php
+- php composer.phar install
+
+# Bring in any services we need http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service
+# See http://docs.gitlab.com/ce/ci/services/README.html for examples.
+services:
+  - mysql:5.7
+
+# Set any variables we need
+variables:
+  # Configure mysql environment variables (https://hub.docker.com/r/_/mysql/)
+  MYSQL_DATABASE: mysql_database
+  MYSQL_ROOT_PASSWORD: mysql_strong_password
+
+# Run our tests
+# If Xdebug was installed you can generate a coverage report and see code coverage metrics.
+test:
+  script:
+  - vendor/bin/phpunit --configuration phpunit.xml --coverage-text --colors=never
\ No newline at end of file
diff --git a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml
new file mode 100644
index 00000000000..fa296057c72
--- /dev/null
+++ b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml
@@ -0,0 +1,26 @@
+image:
+  name: hashicorp/packer:1.0.4
+  entrypoint:
+    - '/usr/bin/env'
+    - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
+
+before_script:
+  - packer --version
+
+stages:
+  - validate
+  - deploy
+
+validate:
+  stage: validate
+  script:
+    - find . -maxdepth 1 -name '*.json' -print0 | xargs -t0n1 packer validate
+
+build:
+  stage: deploy
+  environment: production
+  script:
+    - find . -maxdepth 1 -name '*.json' -print0 | xargs -t0n1 packer build
+  when: manual
+  only:
+    - master
diff --git a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml
new file mode 100644
index 00000000000..7fcc0b436b5
--- /dev/null
+++ b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml
@@ -0,0 +1,16 @@
+# Full project: https://gitlab.com/pages/brunch
+image: node:4.2.2
+
+pages:
+  cache:
+    paths:
+    - node_modules/
+
+  script:
+  - npm install -g brunch
+  - brunch build --production
+  artifacts:
+    paths:
+    - public
+  only:
+  - master
diff --git a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml
new file mode 100644
index 00000000000..791afdd23f1
--- /dev/null
+++ b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml
@@ -0,0 +1,13 @@
+# Full project: https://gitlab.com/pages/doxygen
+image: alpine
+
+pages:
+  script:
+  - apk update && apk add doxygen
+  - doxygen doxygen/Doxyfile
+  - mv doxygen/documentation/html/ public/
+  artifacts:
+    paths:
+    - public
+  only:
+  - master
diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml
new file mode 100644
index 00000000000..9df2a4797b2
--- /dev/null
+++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml
@@ -0,0 +1,17 @@
+image: node:latest
+
+# This folder is cached between builds
+# http://docs.gitlab.com/ce/ci/yaml/README.html#cache
+cache:
+  paths:
+  - node_modules/
+
+pages:
+  script:
+  - yarn install
+  - ./node_modules/.bin/gatsby build --prefix-paths
+  artifacts:
+    paths:
+    - public
+  only:
+  - master
diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml
new file mode 100644
index 00000000000..249a168aa33
--- /dev/null
+++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml
@@ -0,0 +1,12 @@
+# Full project: https://gitlab.com/pages/plain-html
+pages:
+  stage: deploy
+  script:
+  - mkdir .public
+  - cp -r * .public
+  - mv .public public
+  artifacts:
+    paths:
+    - public
+  only:
+  - master
diff --git a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml
new file mode 100644
index 00000000000..dd3ef149668
--- /dev/null
+++ b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml
@@ -0,0 +1,16 @@
+# Full project: https://gitlab.com/pages/harp
+image: node:4.2.2
+
+pages:
+  cache:
+    paths:
+    - node_modules
+
+  script:
+  - npm install -g harp
+  - harp compile ./ public
+  artifacts:
+    paths:
+    - public
+  only:
+  - master
diff --git a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml
new file mode 100644
index 00000000000..02d02250bbf
--- /dev/null
+++ b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml
@@ -0,0 +1,16 @@
+# Full project: https://gitlab.com/pages/hexo
+image: node:6.10.0
+
+pages:
+  script:
+  - npm install
+  - ./node_modules/hexo/bin/hexo generate
+  artifacts:
+    paths:
+    - public
+  cache:
+    paths:
+      - node_modules
+    key: project
+  only:
+  - master
diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml
new file mode 100644
index 00000000000..b8cfb0f56f6
--- /dev/null
+++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml
@@ -0,0 +1,17 @@
+# Full project: https://gitlab.com/pages/hugo
+image: dettmering/hugo-build
+
+pages:
+  script:
+  - hugo
+  artifacts:
+    paths:
+    - public
+  only:
+  - master
+  
+test:
+  script:
+  - hugo
+  except:
+  - master
diff --git a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml
new file mode 100644
index 00000000000..f5b40f2b9f1
--- /dev/null
+++ b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml
@@ -0,0 +1,25 @@
+# Full project: https://gitlab.com/pages/hyde
+image: python:2.7
+
+cache:
+  paths:
+  - vendor/
+
+test:
+  stage: test
+  script:
+    - pip install hyde
+    - hyde gen
+  except:
+    - master
+
+pages:
+  stage: deploy
+  script:
+    - pip install hyde
+    - hyde gen -d public
+  artifacts:
+    paths:
+    - public
+  only:
+    - master
diff --git a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml
new file mode 100644
index 00000000000..7abfaf53e8e
--- /dev/null
+++ b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml
@@ -0,0 +1,32 @@
+# This template uses the java:8 docker image because there isn't any
+# official JBake image at this moment
+#
+# JBake https://jbake.org/ is a Java based, open source, static site/blog generator for developers & designers
+#
+# This yml works with jBake 2.5.1
+# Feel free to change JBAKE_VERSION version 
+#
+# HowTo at: https://jorge.aguilera.gitlab.io/howtojbake/
+
+image: java:8
+
+variables:
+    JBAKE_VERSION: 2.5.1
+
+
+# We use SDKMan as tool for managing versions
+before_script:
+   - apt-get update -qq && apt-get install -y -qq unzip zip
+   - curl -sSL https://get.sdkman.io | bash
+   - echo sdkman_auto_answer=true > /root/.sdkman/etc/config
+   - source /root/.sdkman/bin/sdkman-init.sh
+   - sdk install jbake $JBAKE_VERSION < /dev/null
+   - sdk use jbake $JBAKE_VERSION
+
+# This build job produced the output directory of your site
+pages:
+ script:
+ - jbake . public
+ artifacts:
+   paths:
+   - public
\ No newline at end of file
diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml
new file mode 100644
index 00000000000..37f50554036
--- /dev/null
+++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml
@@ -0,0 +1,30 @@
+# Template project: https://gitlab.com/pages/jekyll
+# Docs: https://docs.gitlab.com/ce/pages/
+image: ruby:2.3
+
+variables:
+  JEKYLL_ENV: production
+
+before_script:
+- bundle install
+
+test:
+  stage: test
+  script:
+  - bundle exec jekyll build -d test
+  artifacts:
+    paths:
+    - test
+  except:
+  - master
+
+pages:
+  stage: deploy
+  script:
+  - bundle exec jekyll build -d public
+  artifacts:
+    paths:
+    - public
+  only:
+  - master
+
diff --git a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml
new file mode 100644
index 00000000000..0e5fb410a4e
--- /dev/null
+++ b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml
@@ -0,0 +1,42 @@
+# Jigsaw is a simple static sites generator with Laravel's Blade.
+#
+# Full project: https://github.com/tightenco/jigsaw
+
+image: php:7.2
+
+# These folders are cached between builds
+cache:
+  paths:
+  - vendor/
+  - node_modules/
+
+before_script:
+  # Update packages 
+  - apt-get update -yqq
+
+  # Install dependencies
+  - apt-get install -yqq gnupg zlib1g-dev libpng-dev
+  
+  # Install Node 8
+  - curl -sL https://deb.nodesource.com/setup_8.x | bash -
+  - apt-get install -yqq nodejs
+
+  # Install php extensions
+  - docker-php-ext-install zip
+
+  # Install Composer and project dependencies.
+  - curl -sS https://getcomposer.org/installer | php
+  - php composer.phar install 
+  
+  # Install Node dependencies.
+  - npm install
+
+pages:
+  script:
+  - npm run production
+  - mv build_production public
+  artifacts:
+    paths:
+    - public
+  only:
+  - master
diff --git a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml
new file mode 100644
index 00000000000..c5c44a5d86c
--- /dev/null
+++ b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml
@@ -0,0 +1,12 @@
+# Full project: https://gitlab.com/pages/hyde
+image: python:2.7
+
+pages:
+  script:
+  - pip install lektor
+  - lektor build --output-path public
+  artifacts:
+    paths:
+    - public
+  only:
+  - master
diff --git a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml
new file mode 100644
index 00000000000..50e8b7ccd46
--- /dev/null
+++ b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml
@@ -0,0 +1,17 @@
+# Full project: https://gitlab.com/pages/metalsmith
+image: node:4.2.2
+
+pages:
+  cache:
+    paths:
+    - node_modules/
+
+  script:
+  - npm install -g metalsmith
+  - npm install
+  - make build
+  artifacts:
+    paths:
+    - public
+  only:
+  - master
diff --git a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml
new file mode 100644
index 00000000000..9f4cc0574d6
--- /dev/null
+++ b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml
@@ -0,0 +1,27 @@
+# Full project: https://gitlab.com/pages/middleman
+image: ruby:2.3
+
+cache:
+  paths:
+  - vendor
+
+test:
+  script:
+  - apt-get update -yqqq
+  - apt-get install -y nodejs
+  - bundle install --path vendor
+  - bundle exec middleman build
+  except:
+    - master
+
+pages:
+  script:
+  - apt-get update -yqqq
+  - apt-get install -y nodejs
+  - bundle install --path vendor
+  - bundle exec middleman build
+  artifacts:
+    paths:
+    - public
+  only:
+  - master
diff --git a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml
new file mode 100644
index 00000000000..b469b316ba5
--- /dev/null
+++ b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml
@@ -0,0 +1,12 @@
+# Full project: https://gitlab.com/pages/nanoc
+image: ruby:2.3
+
+pages:
+  script:
+  - bundle install -j4
+  - nanoc
+  artifacts:
+    paths:
+    - public
+  only:
+  - master
diff --git a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml
new file mode 100644
index 00000000000..4762ec9acfd
--- /dev/null
+++ b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml
@@ -0,0 +1,15 @@
+# Full project: https://gitlab.com/pages/octopress
+image: ruby:2.3
+
+pages:
+  script:
+  - apt-get update -qq && apt-get install -qq nodejs
+  - bundle install -j4
+  - bundle exec rake generate
+  - mv public .public
+  - mv .public/octopress public
+  artifacts:
+    paths:
+    - public
+  only:
+  - master
diff --git a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml
new file mode 100644
index 00000000000..c5f3154f587
--- /dev/null
+++ b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml
@@ -0,0 +1,10 @@
+# Full project: https://gitlab.com/pages/pelican
+image: python:2.7-alpine
+
+pages:
+  script:
+  - pip install -r requirements.txt
+  - pelican -s publishconf.py
+  artifacts:
+    paths:
+    - public/
diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml
new file mode 100644
index 00000000000..098abe4daf5
--- /dev/null
+++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml
@@ -0,0 +1,51 @@
+# Official language image. Look for the different tagged releases at:
+# https://hub.docker.com/r/library/python/tags/
+image: python:latest
+
+# Change pip's cache directory to be inside the project directory since we can
+# only cache local items.
+variables:
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+
+# Pip's cache doesn't store the python packages
+# https://pip.pypa.io/en/stable/reference/pip_install/#caching
+#
+# If you want to also cache the installed packages, you have to install
+# them in a virtualenv and cache it as well.
+cache:
+  paths:
+    - .cache/pip
+    - venv/
+
+before_script:
+  - python -V               # Print out python version for debugging
+  - pip install virtualenv
+  - virtualenv venv
+  - source venv/bin/activate
+
+test:
+  script:
+  - python setup.py test
+  - pip install tox flake8  # you can also use tox
+  - tox -e py36,flake8
+
+run:
+  script:
+  - python setup.py bdist_wheel
+  # an alternative approach is to install and run:
+  - pip install dist/*
+  # run the command here
+  artifacts:
+    paths:
+    - dist/*.whl
+
+pages:
+  script:
+  - pip install sphinx sphinx-rtd-theme
+  - cd doc ; make html
+  - mv build/html/ ../public/
+  artifacts:
+    paths:
+    - public
+  only:
+  - master
diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml
new file mode 100644
index 00000000000..0d12cbc6460
--- /dev/null
+++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml
@@ -0,0 +1,53 @@
+# Official language image. Look for the different tagged releases at:
+# https://hub.docker.com/r/library/ruby/tags/
+image: "ruby:2.5"
+
+# Pick zero or more services to be used on all builds.
+# Only needed when using a docker container to run your tests in.
+# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service
+services:
+  - mysql:latest
+  - redis:latest
+  - postgres:latest
+
+variables:
+  POSTGRES_DB: database_name
+
+# Cache gems in between builds
+cache:
+  paths:
+    - vendor/ruby
+
+# This is a basic example for a gem or script which doesn't use
+# services such as redis or postgres
+before_script:
+  - ruby -v                                   # Print out ruby version for debugging
+  # Uncomment next line if your rails app needs a JS runtime:
+  # - apt-get update -q && apt-get install nodejs -yqq
+  - bundle install -j $(nproc) --path vendor  # Install dependencies into ./vendor/ruby
+
+# Optional - Delete if not using `rubocop`
+rubocop:
+  script:
+  - rubocop
+
+rspec:
+  script:
+  - rspec spec
+
+rails:
+  variables:
+    DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/$POSTGRES_DB"
+  script:
+  - rails db:migrate
+  - rails db:seed
+  - rails test
+
+# This deploy job uses a simple deploy flow to Heroku, other providers, e.g. AWS Elastic Beanstalk
+# are supported too: https://github.com/travis-ci/dpl
+deploy:
+  type: deploy
+  environment: production
+  script:
+  - gem install dpl
+  - dpl --provider=heroku --app=$HEROKU_APP_NAME --api-key=$HEROKU_PRODUCTION_KEY
diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml
new file mode 100644
index 00000000000..cab087c48c7
--- /dev/null
+++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml
@@ -0,0 +1,23 @@
+# Official language image. Look for the different tagged releases at:
+# https://hub.docker.com/r/library/rust/tags/
+image: "rust:latest"
+
+# Optional: Pick zero or more services to be used on all builds.
+# Only needed when using a docker container to run your tests in.
+# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service
+#services:
+#  - mysql:latest
+#  - redis:latest
+#  - postgres:latest
+
+# Optional: Install a C compiler, cmake and git into the container.
+# You will often need this when you (or any of your dependencies) depends on C code.
+#before_script:
+#- apt-get update -yqq
+#- apt-get install -yqq --no-install-recommends build-essential
+
+# Use cargo to test the project
+test:cargo:
+  script:
+  - rustc --version && cargo --version      # Print version info for debugging
+  - cargo test --all --verbose
diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml
new file mode 100644
index 00000000000..b4208ed9d7d
--- /dev/null
+++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml
@@ -0,0 +1,22 @@
+# Official Java image. Look for the different tagged releases at
+# https://hub.docker.com/r/library/java/tags/ . A Java image is not required
+# but an image with a JVM speeds up the build a bit.
+image: java:8
+
+before_script:
+  # Enable the usage of sources over https
+  - apt-get update -yqq
+  - apt-get install apt-transport-https -yqq
+  # Add keyserver for SBT
+  - echo "deb http://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
+  - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823
+  # Install SBT
+  - apt-get update -yqq
+  - apt-get install sbt -yqq
+  # Log the sbt version
+  - sbt sbt-version
+
+test:
+  script:
+    # Execute your project's tests
+    - sbt clean test
diff --git a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml
new file mode 100644
index 00000000000..ba8a802ba4f
--- /dev/null
+++ b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml
@@ -0,0 +1,30 @@
+# Lifted from: https://about.gitlab.com/2016/03/10/setting-up-gitlab-ci-for-ios-projects/
+# This file assumes an own GitLab CI runner, setup on a macOS system.
+stages:
+  - build
+  - archive
+
+build_project:
+  stage: build
+  script:
+    - xcodebuild clean -project ProjectName.xcodeproj -scheme SchemeName | xcpretty
+    - xcodebuild test -project ProjectName.xcodeproj -scheme SchemeName -destination 'platform=iOS Simulator,name=iPhone 8,OS=11.3' | xcpretty -s
+  tags:
+    - ios_11-3
+    - xcode_9-3
+    - macos_10-13
+
+archive_project:
+  stage: archive
+  script:
+    - xcodebuild clean archive -archivePath build/ProjectName -scheme SchemeName
+    - xcodebuild -exportArchive -exportFormat ipa -archivePath "build/ProjectName.xcarchive" -exportPath "build/ProjectName.ipa" -exportProvisioningProfile "ProvisioningProfileName"
+  only:
+    - master
+  artifacts:
+    paths:
+    - build/ProjectName.ipa
+  tags:
+    - ios_11-3
+    - xcode_9-3
+    - macos_10-13
diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
new file mode 100644
index 00000000000..7160fce26a8
--- /dev/null
+++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
@@ -0,0 +1,55 @@
+# Official image for Hashicorp's Terraform. It uses light image which is Alpine
+# based as it is much lighter.
+#
+# Entrypoint is also needed as image by default set `terraform` binary as an
+# entrypoint.
+image:
+  name: hashicorp/terraform:light
+  entrypoint:
+    - '/usr/bin/env'
+    - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
+
+# Default output file for Terraform plan
+variables:
+  PLAN: plan.tfplan
+
+cache:
+  paths:
+    - .terraform
+
+before_script:
+  - terraform --version
+  - terraform init
+
+stages:
+  - validate
+  - build
+  - deploy
+
+validate:
+  stage: validate
+  script:
+    - terraform validate
+
+plan:
+  stage: build
+  script:
+    - terraform plan -out=$PLAN
+  artifacts:
+    name: plan
+    paths:
+      - $PLAN
+
+# Separate apply job for manual launching Terraform as it can be destructive
+# action.
+apply:
+  stage: deploy
+  environment:
+    name: production
+  script:
+    - terraform apply -input=false $PLAN
+  dependencies:
+    - plan
+  when: manual
+  only:
+    - master
diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml
new file mode 100644
index 00000000000..fc3d4ecdbba
--- /dev/null
+++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml
@@ -0,0 +1,86 @@
+# The following script will work for any project that can be built from command line by msbuild
+# It uses powershell shell executor, so you need to add the following line to your config.toml file
+# (located in gitlab-runner.exe directory):
+#   shell = "powershell"
+#
+# The script is composed of 3 stages: build, test and deploy.  
+#
+# The build stage restores NuGet packages and uses msbuild to build the exe and msi
+# One major issue you'll find is that you can't build msi projects from command line
+# if you use vdproj.  There are workarounds building msi via devenv, but they rarely work
+# The best solution is migrating your vdproj projects to WiX, as it can be build directly 
+# by msbuild.
+#
+# The test stage runs nunit from command line against Test project inside your solution
+# It also saves the resulting TestResult.xml file
+#
+# The deploy stage copies the exe and msi from build stage to a network drive
+# You need to have the network drive mapped as Local System user for gitlab-runner service to see it
+# The best way to persist the mapping is via a scheduled task (see: https://stackoverflow.com/a/7867064/1288473),
+# running the following batch command: net use P: \\x.x.x.x\Projects /u:your_user your_pass /persistent:yes
+
+
+# place project specific paths in variables to make the rest of the script more generic
+variables:
+  EXE_RELEASE_FOLDER: 'YourApp\bin\Release'
+  MSI_RELEASE_FOLDER: 'Setup\bin\Release'
+  TEST_FOLDER: 'Tests\bin\Release'
+  DEPLOY_FOLDER: 'P:\Projects\YourApp\Builds'
+  
+  NUGET_PATH: 'C:\NuGet\nuget.exe'
+  MSBUILD_PATH: 'C:\Program Files (x86)\MSBuild\14.0\Bin\msbuild.exe'
+  NUNIT_PATH: 'C:\Program Files (x86)\NUnit.org\nunit-console\nunit3-console.exe'
+
+stages:
+  - build
+  - test
+  - deploy
+
+build_job:
+  stage: build
+  only:
+    - tags # the build process will only be started by git tag commits
+  script:
+    - '& "$env:NUGET_PATH" restore' # restore Nuget dependencies
+    - '& "$env:MSBUILD_PATH" /p:Configuration=Release' # build the project
+  artifacts:
+    expire_in: 1 week # save gitlab server space, we copy the files we need to deploy folder later on
+    paths:
+      - '$env:EXE_RELEASE_FOLDER\YourApp.exe' # saving exe to copy to deploy folder
+      - '$env:MSI_RELEASE_FOLDER\YourApp Setup.msi' # saving msi to copy to deploy folder
+      - '$env:TEST_FOLDER\' # saving entire Test project so NUnit can run tests 
+
+test_job:
+  stage: test
+  only:
+    - tags
+  script:
+    - '& "$env:NUNIT_PATH" ".\$env:TEST_FOLDER\Tests.dll"' # running NUnit tests
+  artifacts:
+    expire_in: 1 week # save gitlab server space, we copy the files we need to deploy folder later on
+    paths:
+      - '.\TestResult.xml' # saving NUnit results to copy to deploy folder
+  dependencies:
+    - build_job
+
+deploy_job:
+  stage: deploy
+  only:
+    - tags
+  script:
+    # Compose a folder for each release based on commit tag.
+    # Assuming your tag is Rev1.0.0.1, and your last commit message is 'First commit'
+    # the artifact files will be copied to:
+    # P:\Projects\YourApp\Builds\Rev1.0.0.1 - First commit\
+    - '$commitSubject = git log -1 --pretty=%s'
+    - '$deployFolder = $($env:DEPLOY_FOLDER) + "\" + $($env:CI_BUILD_TAG) + " - " + $commitSubject + "\"'
+
+    # xcopy takes care of recursively creating required folders
+    - 'xcopy /y ".\$env:EXE_RELEASE_FOLDER\YourApp.exe" "$deployFolder"'
+    - 'xcopy /y ".\$env:MSI_RELEASE_FOLDER\YourApp Setup.msi" "$deployFolder"'
+    - 'xcopy /y ".\TestResult.xml" "$deployFolder"'
+
+  dependencies:
+    - build_job
+    - test_job
+  
\ No newline at end of file
diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
index fe15fabc2e8..bf5f2a31f0e 100644
--- a/lib/gitlab/ci/trace.rb
+++ b/lib/gitlab/ci/trace.rb
@@ -1,7 +1,16 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Trace
+      include ::Gitlab::ExclusiveLeaseHelpers
+
+      LOCK_TTL = 1.minute
+      LOCK_RETRIES = 2
+      LOCK_SLEEP = 0.001.seconds
+
       ArchiveError = Class.new(StandardError)
+      AlreadyArchivedError = Class.new(StandardError)
 
       attr_reader :job
 
@@ -75,9 +84,39 @@ module Gitlab
         stream&.close
       end
 
-      def write(mode)
+      def write(mode, &blk)
+        in_write_lock do
+          unsafe_write!(mode, &blk)
+        end
+      end
+
+      def erase!
+        ##
+        # Erase the archived trace
+        trace_artifact&.destroy!
+
+        ##
+        # Erase the live trace
+        job.trace_chunks.fast_destroy_all # Destroy chunks of a live trace
+        FileUtils.rm_f(current_path) if current_path # Remove a trace file of a live trace
+        job.erase_old_trace! if job.has_old_trace? # Remove a trace in database of a live trace
+      ensure
+        @current_path = nil
+      end
+
+      def archive!
+        in_write_lock do
+          unsafe_archive!
+        end
+      end
+
+      private
+
+      def unsafe_write!(mode, &blk)
         stream = Gitlab::Ci::Trace::Stream.new do
-          if current_path
+          if trace_artifact
+            raise AlreadyArchivedError, 'Could not write to the archived trace'
+          elsif current_path
             File.open(current_path, mode)
           elsif Feature.enabled?('ci_enable_live_trace')
             Gitlab::Ci::Trace::ChunkedIO.new(job)
@@ -93,19 +132,8 @@ module Gitlab
         stream&.close
       end
 
-      def erase!
-        trace_artifact&.destroy
-
-        paths.each do |trace_path|
-          FileUtils.rm(trace_path, force: true)
-        end
-
-        job.trace_chunks.fast_destroy_all
-        job.erase_old_trace!
-      end
-
-      def archive!
-        raise ArchiveError, 'Already archived' if trace_artifact
+      def unsafe_archive!
+        raise AlreadyArchivedError, 'Could not archive again' if trace_artifact
         raise ArchiveError, 'Job is not finished yet' unless job.complete?
 
         if job.trace_chunks.any?
@@ -126,7 +154,10 @@ module Gitlab
         end
       end
 
-      private
+      def in_write_lock(&blk)
+        lock_key = "trace:write:lock:#{job.id}"
+        in_lock(lock_key, ttl: LOCK_TTL, retries: LOCK_RETRIES, sleep_sec: LOCK_SLEEP, &blk)
+      end
 
       def archive_stream!(stream)
         clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path|
@@ -148,6 +179,8 @@ module Gitlab
 
       def create_build_trace!(job, path)
         File.open(path) do |stream|
+          # TODO: Set `file_format: :raw` after we've cleaned up legacy traces migration
+          # https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/20307
           job.create_job_artifacts_trace!(
             project: job.project,
             file_type: :trace,
diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb
index bfe0c2a2c26..8c6fd56493f 100644
--- a/lib/gitlab/ci/trace/chunked_io.rb
+++ b/lib/gitlab/ci/trace/chunked_io.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 ##
 # This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html)
 # source: https://gitlab.com/snippets/1685610
@@ -11,7 +13,7 @@ module Gitlab
 
         attr_reader :build
         attr_reader :tell, :size
-        attr_reader :chunk, :chunk_range
+        attr_reader :chunk_data, :chunk_range
 
         alias_method :pos, :tell
 
@@ -66,42 +68,46 @@ module Gitlab
           end
         end
 
-        def read(length = nil, outbuf = "")
-          out = ""
+        def read(length = nil, outbuf = nil)
+          out = []
 
           length ||= size - tell
 
           until length <= 0 || eof?
             data = chunk_slice_from_offset
-            break if data.empty?
+            raise FailedToGetChunkError if data.empty?
 
             chunk_bytes = [CHUNK_SIZE - chunk_offset, length].min
-            chunk_data = data.byteslice(0, chunk_bytes)
+            chunk_data_slice = data.byteslice(0, chunk_bytes)
 
-            out << chunk_data
-            @tell += chunk_data.bytesize
-            length -= chunk_data.bytesize
+            out << chunk_data_slice
+            @tell += chunk_data_slice.bytesize
+            length -= chunk_data_slice.bytesize
           end
 
+          out = out.join
+
           # If outbuf is passed, we put the output into the buffer. This supports IO.copy_stream functionality
           if outbuf
-            outbuf.slice!(0, outbuf.bytesize)
-            outbuf << out
+            outbuf.replace(out)
           end
 
           out
         end
 
         def readline
-          out = ""
+          out = []
 
           until eof?
             data = chunk_slice_from_offset
+            raise FailedToGetChunkError if data.empty?
+
             new_line = data.index("\n")
 
             if !new_line.nil?
-              out << data[0..new_line]
-              @tell += new_line + 1
+              raw_data = data[0..new_line]
+              out << raw_data
+              @tell += raw_data.bytesize
               break
             else
               out << data
@@ -109,7 +115,7 @@ module Gitlab
             end
           end
 
-          out
+          out.join
         end
 
         def write(data)
@@ -118,13 +124,13 @@ module Gitlab
           while tell < start_pos + data.bytesize
             # get slice from current offset till the end where it falls into chunk
             chunk_bytes = CHUNK_SIZE - chunk_offset
-            chunk_data = data.byteslice(tell - start_pos, chunk_bytes)
+            data_slice = data.byteslice(tell - start_pos, chunk_bytes)
 
             # append data to chunk, overwriting from that point
-            ensure_chunk.append(chunk_data, chunk_offset)
+            ensure_chunk.append(data_slice, chunk_offset)
 
             # move offsets within buffer
-            @tell += chunk_data.bytesize
+            @tell += data_slice.bytesize
             @size = [size, tell].max
           end
 
@@ -133,6 +139,7 @@ module Gitlab
           invalidate_chunk_cache
         end
 
+        # rubocop: disable CodeReuse/ActiveRecord
         def truncate(offset)
           raise ArgumentError, 'Outside of file' if offset > size || offset < 0
           return if offset == size # Skip the following process as it doesn't affect anything
@@ -148,6 +155,7 @@ module Gitlab
         ensure
           invalidate_chunk_cache
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         def flush
           # no-op
@@ -178,12 +186,12 @@ module Gitlab
             current_chunk.tap do |chunk|
               raise FailedToGetChunkError unless chunk
 
-              @chunk = chunk.data
+              @chunk_data = chunk.data
               @chunk_range = chunk.range
             end
           end
 
-          @chunk[chunk_offset..CHUNK_SIZE]
+          @chunk_data.byteslice(chunk_offset, CHUNK_SIZE)
         end
 
         def chunk_offset
@@ -206,9 +214,11 @@ module Gitlab
           @chunks_cache = []
         end
 
+        # rubocop: disable CodeReuse/ActiveRecord
         def current_chunk
           @chunks_cache[chunk_index] ||= trace_chunks.find_by(chunk_index: chunk_index)
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         def build_chunk
           @chunks_cache[chunk_index] = ::Ci::BuildTraceChunk.new(build: build, chunk_index: chunk_index)
@@ -218,13 +228,17 @@ module Gitlab
           current_chunk || build_chunk
         end
 
+        # rubocop: disable CodeReuse/ActiveRecord
         def trace_chunks
           ::Ci::BuildTraceChunk.where(build: build)
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
+        # rubocop: disable CodeReuse/ActiveRecord
         def calculate_size
           trace_chunks.order(chunk_index: :desc).first.try(&:end_offset).to_i
         end
+        # rubocop: enable CodeReuse/ActiveRecord
       end
     end
   end
diff --git a/lib/gitlab/ci/trace/http_io.rb b/lib/gitlab/ci/trace/http_io.rb
deleted file mode 100644
index 8788af57a67..00000000000
--- a/lib/gitlab/ci/trace/http_io.rb
+++ /dev/null
@@ -1,197 +0,0 @@
-##
-# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html)
-# source: https://gitlab.com/snippets/1685610
-module Gitlab
-  module Ci
-    class Trace
-      class HttpIO
-        BUFFER_SIZE = 128.kilobytes
-
-        InvalidURLError = Class.new(StandardError)
-        FailedToGetChunkError = Class.new(StandardError)
-
-        attr_reader :uri, :size
-        attr_reader :tell
-        attr_reader :chunk, :chunk_range
-
-        alias_method :pos, :tell
-
-        def initialize(url, size)
-          raise InvalidURLError unless ::Gitlab::UrlSanitizer.valid?(url)
-
-          @uri = URI(url)
-          @size = size
-          @tell = 0
-        end
-
-        def close
-          # no-op
-        end
-
-        def binmode
-          # no-op
-        end
-
-        def binmode?
-          true
-        end
-
-        def path
-          nil
-        end
-
-        def url
-          @uri.to_s
-        end
-
-        def seek(pos, where = IO::SEEK_SET)
-          new_pos =
-            case where
-            when IO::SEEK_END
-              size + pos
-            when IO::SEEK_SET
-              pos
-            when IO::SEEK_CUR
-              tell + pos
-            else
-              -1
-            end
-
-          raise 'new position is outside of file' if new_pos < 0 || new_pos > size
-
-          @tell = new_pos
-        end
-
-        def eof?
-          tell == size
-        end
-
-        def each_line
-          until eof?
-            line = readline
-            break if line.nil?
-
-            yield(line)
-          end
-        end
-
-        def read(length = nil, outbuf = "")
-          out = ""
-
-          length ||= size - tell
-
-          until length <= 0 || eof?
-            data = get_chunk
-            break if data.empty?
-
-            chunk_bytes = [BUFFER_SIZE - chunk_offset, length].min
-            chunk_data = data.byteslice(0, chunk_bytes)
-
-            out << chunk_data
-            @tell += chunk_data.bytesize
-            length -= chunk_data.bytesize
-          end
-
-          # If outbuf is passed, we put the output into the buffer. This supports IO.copy_stream functionality
-          if outbuf
-            outbuf.slice!(0, outbuf.bytesize)
-            outbuf << out
-          end
-
-          out
-        end
-
-        def readline
-          out = ""
-
-          until eof?
-            data = get_chunk
-            new_line = data.index("\n")
-
-            if !new_line.nil?
-              out << data[0..new_line]
-              @tell += new_line + 1
-              break
-            else
-              out << data
-              @tell += data.bytesize
-            end
-          end
-
-          out
-        end
-
-        def write(data)
-          raise NotImplementedError
-        end
-
-        def truncate(offset)
-          raise NotImplementedError
-        end
-
-        def flush
-          raise NotImplementedError
-        end
-
-        def present?
-          true
-        end
-
-        private
-
-        ##
-        # The below methods are not implemented in IO class
-        #
-        def in_range?
-          @chunk_range&.include?(tell)
-        end
-
-        def get_chunk
-          unless in_range?
-            response = Net::HTTP.start(uri.hostname, uri.port, proxy_from_env: true, use_ssl: uri.scheme == 'https') do |http|
-              http.request(request)
-            end
-
-            raise FailedToGetChunkError unless response.code == '200' || response.code == '206'
-
-            @chunk = response.body.force_encoding(Encoding::BINARY)
-            @chunk_range = response.content_range
-
-            ##
-            # Note: If provider does not return content_range, then we set it as we requested
-            # Provider: minio
-            # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
-            # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
-            # Provider: AWS
-            # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
-            # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
-            # Provider: GCS
-            # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
-            # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPOK 200
-            @chunk_range ||= (chunk_start...(chunk_start + @chunk.bytesize))
-          end
-
-          @chunk[chunk_offset..BUFFER_SIZE]
-        end
-
-        def request
-          Net::HTTP::Get.new(uri).tap do |request|
-            request.set_range(chunk_start, BUFFER_SIZE)
-          end
-        end
-
-        def chunk_offset
-          tell % BUFFER_SIZE
-        end
-
-        def chunk_start
-          (tell / BUFFER_SIZE) * BUFFER_SIZE
-        end
-
-        def chunk_end
-          [chunk_start + BUFFER_SIZE, size].min
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/trace/section_parser.rb b/lib/gitlab/ci/trace/section_parser.rb
index 9bb0166c9e3..f33f8cc56c1 100644
--- a/lib/gitlab/ci/trace/section_parser.rb
+++ b/lib/gitlab/ci/trace/section_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Trace
@@ -75,19 +77,19 @@ module Gitlab
           @beginning_of_section_regex ||= /section_/.freeze
         end
 
-        def find_next_marker(s)
+        def find_next_marker(scanner)
           beginning_of_section_len = 8
-          maybe_marker = s.exist?(beginning_of_section_regex)
+          maybe_marker = scanner.exist?(beginning_of_section_regex)
 
           if maybe_marker.nil?
-            s.terminate
+            scanner.terminate
           else
             # repositioning at the beginning of the match
-            s.pos += maybe_marker - beginning_of_section_len
+            scanner.pos += maybe_marker - beginning_of_section_len
             if block_given?
-              good_marker = yield(s)
+              good_marker = yield(scanner)
               # if not a good marker: Consuming the matched beginning_of_section_regex
-              s.pos += beginning_of_section_len unless good_marker
+              scanner.pos += beginning_of_section_len unless good_marker
             end
           end
         end
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
index a71040e5e56..e61fb50a303 100644
--- a/lib/gitlab/ci/trace/stream.rb
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class Trace
@@ -41,19 +43,14 @@ module Gitlab
         def append(data, offset)
           data = data.force_encoding(Encoding::BINARY)
 
-          stream.truncate(offset)
-          stream.seek(0, IO::SEEK_END)
+          stream.seek(offset, IO::SEEK_SET)
           stream.write(data)
-          stream.flush()
+          stream.truncate(offset + data.bytesize)
+          stream.flush
         end
 
         def set(data)
-          data = data.force_encoding(Encoding::BINARY)
-
-          stream.seek(0, IO::SEEK_SET)
-          stream.write(data)
-          stream.truncate(data.bytesize)
-          stream.flush()
+          append(data, 0)
         end
 
         def raw(last_lines: nil)
@@ -129,8 +126,7 @@ module Gitlab
           debris = ''
 
           until (buf = read_backward(BUFFER_SIZE)).empty?
-            buf += debris
-            debris, *lines = buf.each_line.to_a
+            debris, *lines = (buf + debris).each_line.to_a
             lines.reverse_each do |line|
               yield(line.force_encoding(Encoding.default_external))
             end
diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb
index ad30b3f427c..a7b4e0348c2 100644
--- a/lib/gitlab/ci/variables/collection.rb
+++ b/lib/gitlab/ci/variables/collection.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Variables
diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb
index d00e5b07f95..e3e4e62cc02 100644
--- a/lib/gitlab/ci/variables/collection/item.rb
+++ b/lib/gitlab/ci/variables/collection/item.rb
@@ -1,9 +1,14 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     module Variables
       class Collection
         class Item
           def initialize(key:, value:, public: true, file: false)
+            raise ArgumentError, "`#{key}` must be of type String or nil value, while it was: #{value.class}" unless
+              value.is_a?(String) || value.nil?
+
             @variable = {
               key: key, value: value, public: public, file: file
             }
@@ -31,7 +36,7 @@ module Gitlab
           def self.fabricate(resource)
             case resource
             when Hash
-              self.new(resource)
+              self.new(resource.symbolize_keys)
             when ::HasVariable
               self.new(resource.to_runner_variable)
             when self
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index e829f2a95f8..07ba6f83d47 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -1,14 +1,16 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Ci
     class YamlProcessor
       ValidationError = Class.new(StandardError)
 
-      include Gitlab::Ci::Config::Entry::LegacyValidationHelpers
+      include Gitlab::Config::Entry::LegacyValidationHelpers
 
       attr_reader :cache, :stages, :jobs
 
       def initialize(config, opts = {})
-        @ci_config = Gitlab::Ci::Config.new(config, opts)
+        @ci_config = Gitlab::Ci::Config.new(config, **opts)
         @config = @ci_config.to_hash
 
         unless @ci_config.valid?
@@ -16,7 +18,7 @@ module Gitlab
         end
 
         initial_parsing
-      rescue Gitlab::Ci::Config::Loader::FormatError => e
+      rescue Gitlab::Ci::Config::ConfigError => e
         raise ValidationError, e.message
       end
 
@@ -31,8 +33,7 @@ module Gitlab
 
         { stage_idx: @stages.index(job[:stage]),
           stage: job[:stage],
-          commands: job[:commands],
-          tag_list: job[:tags] || [],
+          tag_list: job[:tags],
           name: job[:name].to_s,
           allow_failure: job[:ignore],
           when: job[:when] || 'on_success',
@@ -49,8 +50,12 @@ module Gitlab
             script: job[:script],
             after_script: job[:after_script],
             environment: job[:environment],
-            retry: job[:retry]
-          }.compact }
+            retry: job[:retry],
+            parallel: job[:parallel],
+            instance: job[:instance],
+            start_in: job[:start_in],
+            trigger: job[:trigger]
+          }.compact }.compact
       end
 
       def stage_builds_attributes(stage)
@@ -101,7 +106,7 @@ module Gitlab
         ##
         # Jobs
         #
-        @jobs = @ci_config.jobs
+        @jobs = Ci::Config::Normalizer.new(@ci_config.jobs).normalize_jobs
 
         @jobs.each do |name, job|
           # logical validation for job
diff --git a/lib/gitlab/ci_access.rb b/lib/gitlab/ci_access.rb
index def1373d8cf..d5d3eb804ae 100644
--- a/lib/gitlab/ci_access.rb
+++ b/lib/gitlab/ci_access.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   # For backwards compatibility, generic CI (which is a build without a user) is
   # allowed to :build_download_code without any other checks.
diff --git a/lib/gitlab/cleanup/project_upload_file_finder.rb b/lib/gitlab/cleanup/project_upload_file_finder.rb
new file mode 100644
index 00000000000..2ee8b60e76a
--- /dev/null
+++ b/lib/gitlab/cleanup/project_upload_file_finder.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Cleanup
+    class ProjectUploadFileFinder
+      FIND_BATCH_SIZE = 500
+      ABSOLUTE_UPLOAD_DIR = FileUploader.root.freeze
+      EXCLUDED_SYSTEM_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/-/*".freeze
+      EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze
+      EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze
+
+      # Paths are relative to the upload directory
+      def each_file_batch(batch_size: FIND_BATCH_SIZE, &block)
+        cmd = build_find_command(ABSOLUTE_UPLOAD_DIR)
+
+        Open3.popen2(*cmd) do |stdin, stdout, status_thread|
+          yield_paths_in_batches(stdout, batch_size, &block)
+
+          raise "Find command failed" unless status_thread.value.success?
+        end
+      end
+
+      private
+
+      def yield_paths_in_batches(stdout, batch_size, &block)
+        paths = []
+
+        stdout.each_line("\0") do |line|
+          paths << line.chomp("\0")
+
+          if paths.size >= batch_size
+            yield(paths)
+            paths = []
+          end
+        end
+
+        yield(paths) if paths.any?
+      end
+
+      def build_find_command(search_dir)
+        cmd = %W[find -L #{search_dir}
+                 -type f
+                 ! ( -path #{EXCLUDED_SYSTEM_UPLOADS_PATH} -prune )
+                 ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune )
+                 ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune )
+                 -print0]
+
+        ionice = which_ionice
+        cmd = %W[#{ionice} -c Idle] + cmd if ionice
+
+        log_msg = "find command: \"#{cmd.join(' ')}\""
+        Rails.logger.info log_msg
+
+        cmd
+      end
+
+      def which_ionice
+        Gitlab::Utils.which('ionice')
+      rescue StandardError
+        # In this case, returning false is relatively safe,
+        # even though it isn't very nice
+        false
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb
new file mode 100644
index 00000000000..82a405362c2
--- /dev/null
+++ b/lib/gitlab/cleanup/project_uploads.rb
@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Cleanup
+    class ProjectUploads
+      LOST_AND_FOUND = File.join(ProjectUploadFileFinder::ABSOLUTE_UPLOAD_DIR, '-', 'project-lost-found')
+
+      attr_reader :logger
+
+      def initialize(logger: nil)
+        @logger = logger || Rails.logger
+      end
+
+      def run!(dry_run: true)
+        logger.info "Looking for orphaned project uploads to clean up#{'. Dry run' if dry_run}..."
+
+        each_orphan_file do |path, upload_path|
+          result = cleanup(path, upload_path, dry_run)
+
+          logger.info result
+        end
+      end
+
+      private
+
+      def cleanup(path, upload_path, dry_run)
+        # This happened in staging:
+        # `find` returned a path on which `File.delete` raised `Errno::ENOENT`
+        return "Cannot find file: #{path}" unless File.exist?(path)
+
+        correct_path = upload_path && find_correct_path(upload_path)
+
+        if correct_path
+          move(path, correct_path, 'fix', dry_run)
+        else
+          move_to_lost_and_found(path, dry_run)
+        end
+      end
+
+      # Accepts a path in the form of "#{hex_secret}/#{filename}"
+      # rubocop: disable CodeReuse/ActiveRecord
+      def find_correct_path(upload_path)
+        upload = Upload.find_by(uploader: 'FileUploader', path: upload_path)
+        return unless upload && upload.local? && upload.model
+
+        upload.absolute_path
+      rescue => e
+        logger.error e.message
+
+        # absolute_path depends on a lot of code. If it doesn't work, then it
+        # it doesn't matter if the upload file is in the right place. Treat it
+        # as uncorrectable.
+        # I.e. the project record might be missing, which raises an exception.
+        nil
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      def move_to_lost_and_found(path, dry_run)
+        new_path = path.sub(/\A#{ProjectUploadFileFinder::ABSOLUTE_UPLOAD_DIR}/, LOST_AND_FOUND)
+
+        move(path, new_path, 'move to lost and found', dry_run)
+      end
+
+      def move(path, new_path, prefix, dry_run)
+        action = "#{prefix} #{path} -> #{new_path}"
+
+        if dry_run
+          "Can #{action}"
+        else
+          begin
+            FileUtils.mkdir_p(File.dirname(new_path))
+            FileUtils.mv(path, new_path)
+
+            "Did #{action}"
+          rescue => e
+            "Error during #{action}: #{e.inspect}"
+          end
+        end
+      end
+
+      # Yields absolute paths of project upload files that are not in the
+      # uploads table
+      def each_orphan_file
+        ProjectUploadFileFinder.new.each_file_batch do |file_paths|
+          logger.debug "Processing batch of #{file_paths.size} project upload file paths, starting with #{file_paths.first}"
+
+          file_paths.each do |path|
+            pup = ProjectUploadPath.from_path(path)
+
+            yield(path, pup.upload_path) if pup.orphan?
+          end
+        end
+      end
+
+      class ProjectUploadPath
+        PROJECT_FULL_PATH_REGEX = %r{\A#{FileUploader.root}/(.+)/(\h+/[^/]+)\z}.freeze
+
+        attr_reader :full_path, :upload_path
+
+        def initialize(full_path, upload_path)
+          @full_path = full_path
+          @upload_path = upload_path
+        end
+
+        def self.from_path(path)
+          path_matched = path.match(PROJECT_FULL_PATH_REGEX)
+          return new(nil, nil) unless path_matched
+
+          new(path_matched[1], path_matched[2])
+        end
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def orphan?
+          return true if full_path.nil? || upload_path.nil?
+
+          # It's possible to reduce to one query, but `where_full_path_in` is complex
+          !Upload.exists?(path: upload_path, model_id: project_id, model_type: 'Project', uploader: 'FileUploader')
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        private
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def project_id
+          @project_id ||= Project.where_full_path_in([full_path]).pluck(:id)
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cleanup/remote_uploads.rb b/lib/gitlab/cleanup/remote_uploads.rb
new file mode 100644
index 00000000000..03298d960a4
--- /dev/null
+++ b/lib/gitlab/cleanup/remote_uploads.rb
@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+module Gitlab
+  module Cleanup
+    class RemoteUploads
+      attr_reader :logger
+
+      BATCH_SIZE = 100
+
+      def initialize(logger: nil)
+        @logger = logger || Rails.logger
+      end
+
+      def run!(dry_run: false)
+        unless configuration.enabled
+          logger.warn "Object storage not enabled. Exit".color(:yellow)
+
+          return
+        end
+
+        logger.info "Looking for orphaned remote uploads to remove#{'. Dry run' if dry_run}..."
+
+        each_orphan_file do |file|
+          info = if dry_run
+                   "Can be moved to lost and found: #{file.key}"
+                 else
+                   new_path = move_to_lost_and_found(file)
+                   "Moved to lost and found: #{file.key} -> #{new_path}"
+                 end
+
+          logger.info(info)
+        end
+      end
+
+      private
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def each_orphan_file
+        # we want to skip files already moved to lost_and_found directory
+        lost_dir_match = "^#{lost_and_found_dir}\/"
+
+        remote_directory.files.each_slice(BATCH_SIZE) do |remote_files|
+          remote_files.reject! { |file| file.key.match(/#{lost_dir_match}/) }
+          file_paths = remote_files.map(&:key)
+          tracked_paths = Upload
+            .where(store: ObjectStorage::Store::REMOTE, path: file_paths)
+            .pluck(:path)
+
+          remote_files.reject! { |file| tracked_paths.include?(file.key) }
+          remote_files.each do |file|
+            yield file
+          end
+        end
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      def move_to_lost_and_found(file)
+        new_path = "#{lost_and_found_dir}/#{file.key}"
+
+        file.copy(configuration['remote_directory'], new_path)
+        file.destroy
+
+        new_path
+      end
+
+      def lost_and_found_dir
+        'lost_and_found'
+      end
+
+      def remote_directory
+        connection.directories.new(key: configuration['remote_directory'])
+      end
+
+      def connection
+        ::Fog::Storage.new(configuration['connection'].symbolize_keys)
+      end
+
+      def configuration
+        Gitlab.config.uploads.object_store
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb
index 7e7aaeeaa17..4ba921569ad 100644
--- a/lib/gitlab/closing_issue_extractor.rb
+++ b/lib/gitlab/closing_issue_extractor.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class ClosingIssueExtractor
     ISSUE_CLOSING_REGEX = begin
diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb
new file mode 100644
index 00000000000..b05dca409d1
--- /dev/null
+++ b/lib/gitlab/cluster/lifecycle_events.rb
@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Cluster
+    #
+    # LifecycleEvents lets Rails initializers register application startup hooks
+    # that are sensitive to forking. For example, to defer the creation of
+    # watchdog threads. This lets us abstract away the Unix process
+    # lifecycles of Unicorn, Sidekiq, Puma, Puma Cluster, etc.
+    #
+    # We have three lifecycle events.
+    #
+    # - before_fork (only in forking processes)
+    # - worker_start
+    # - before_master_restart (only in forking processes)
+    #
+    # Blocks will be executed in the order in which they are registered.
+    #
+    class LifecycleEvents
+      class << self
+        #
+        # Hook registration methods (called from initializers)
+        #
+        def on_worker_start(&block)
+          if in_clustered_environment?
+            # Defer block execution
+            (@worker_start_hooks ||= []) << block
+          else
+            yield
+          end
+        end
+
+        def on_before_fork(&block)
+          return unless in_clustered_environment?
+
+          # Defer block execution
+          (@before_fork_hooks ||= []) << block
+        end
+
+        def on_master_restart(&block)
+          return unless in_clustered_environment?
+
+          # Defer block execution
+          (@master_restart_hooks ||= []) << block
+        end
+
+        #
+        # Lifecycle integration methods (called from unicorn.rb, puma.rb, etc.)
+        #
+        def do_worker_start
+          @worker_start_hooks&.each do |block|
+            block.call
+          end
+        end
+
+        def do_before_fork
+          @before_fork_hooks&.each do |block|
+            block.call
+          end
+        end
+
+        def do_master_restart
+          @master_restart_hooks && @master_restart_hooks.each do |block|
+            block.call
+          end
+        end
+
+        # Puma doesn't use singletons (which is good) but
+        # this means we need to pass through whether the
+        # puma server is running in single mode or cluster mode
+        def set_puma_options(options)
+          @puma_options = options
+        end
+
+        private
+
+        def in_clustered_environment?
+          # Sidekiq doesn't fork
+          return false if Sidekiq.server?
+
+          # Unicorn always forks
+          return true if defined?(::Unicorn)
+
+          # Puma sometimes forks
+          return true if in_clustered_puma?
+
+          # Default assumption is that we don't fork
+          false
+        end
+
+        def in_clustered_puma?
+          return false unless defined?(::Puma)
+
+          @puma_options && @puma_options[:workers] && @puma_options[:workers] > 0
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb
new file mode 100644
index 00000000000..4ed9a9a02ab
--- /dev/null
+++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Cluster
+    class PumaWorkerKillerInitializer
+      def self.start(puma_options, puma_per_worker_max_memory_mb: 650)
+        require 'puma_worker_killer'
+
+        PumaWorkerKiller.config do |config|
+          # Note! ram is expressed in megabytes (whereas GITLAB_UNICORN_MEMORY_MAX is in bytes)
+          # Importantly RAM is for _all_workers (ie, the cluster),
+          # not each worker as is the case with GITLAB_UNICORN_MEMORY_MAX
+          worker_count = puma_options[:workers] || 1
+          # The Puma Worker Killer checks the total RAM used by both the master
+          # and worker processes. Bump the limits to N+1 instead of N workers
+          # to account for this:
+          # https://github.com/schneems/puma_worker_killer/blob/v0.1.0/lib/puma_worker_killer/puma_memory.rb#L57
+          config.ram = (worker_count + 1) * puma_per_worker_max_memory_mb
+
+          config.frequency = 20 # seconds
+
+          # We just want to limit to a fixed maximum, unrelated to the total amount
+          # of available RAM.
+          config.percent_usage = 0.98
+
+          # Ideally we'll never hit the maximum amount of memory. If so the worker
+          # is restarted already, thus periodically restarting workers shouldn't be
+          # needed.
+          config.rolling_restart_frequency = false
+        end
+
+        PumaWorkerKiller.start
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/color_schemes.rb b/lib/gitlab/color_schemes.rb
index 9c4664df903..881e5dbc923 100644
--- a/lib/gitlab/color_schemes.rb
+++ b/lib/gitlab/color_schemes.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   # Module containing GitLab's syntax color scheme definitions and helper
   # methods for accessing them.
@@ -10,7 +12,8 @@ module Gitlab
       Scheme.new(2, 'Dark',            'dark'),
       Scheme.new(3, 'Solarized Light', 'solarized-light'),
       Scheme.new(4, 'Solarized Dark',  'solarized-dark'),
-      Scheme.new(5, 'Monokai',         'monokai')
+      Scheme.new(5, 'Monokai',         'monokai'),
+      Scheme.new(6, 'None',            'none')
     ].freeze
 
     # Convenience method to get a space-separated String of all the color scheme
diff --git a/lib/gitlab/config/entry/attributable.rb b/lib/gitlab/config/entry/attributable.rb
new file mode 100644
index 00000000000..560fe63df0e
--- /dev/null
+++ b/lib/gitlab/config/entry/attributable.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Config
+    module Entry
+      module Attributable
+        extend ActiveSupport::Concern
+
+        class_methods do
+          def attributes(*attributes)
+            attributes.flatten.each do |attribute|
+              if method_defined?(attribute)
+                raise ArgumentError, 'Method already defined!'
+              end
+
+              define_method(attribute) do
+                return unless config.is_a?(Hash)
+
+                config[attribute]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/config/entry/boolean.rb b/lib/gitlab/config/entry/boolean.rb
new file mode 100644
index 00000000000..1e8a57356e3
--- /dev/null
+++ b/lib/gitlab/config/entry/boolean.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Config
+    module Entry
+      ##
+      # Entry that represents a boolean value.
+      #
+      class Boolean < Node
+        include Validatable
+
+        validations do
+          validates :config, boolean: true
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/config/entry/configurable.rb b/lib/gitlab/config/entry/configurable.rb
new file mode 100644
index 00000000000..37ba16dba25
--- /dev/null
+++ b/lib/gitlab/config/entry/configurable.rb
@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Config
+    module Entry
+      ##
+      # This mixin is responsible for adding DSL, which purpose is to
+      # simplifly process of adding child nodes.
+      #
+      # This can be used only if parent node is a configuration entry that
+      # holds a hash as a configuration value, for example:
+      #
+      # job:
+      #   script: ...
+      #   artifacts: ...
+      #
+      module Configurable
+        extend ActiveSupport::Concern
+
+        included do
+          include Validatable
+
+          validations do
+            validates :config, type: Hash
+          end
+        end
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def compose!(deps = nil)
+          return unless valid?
+
+          self.class.nodes.each do |key, factory|
+            factory
+              .value(config[key])
+              .with(key: key, parent: self)
+
+            entries[key] = factory.create!
+          end
+
+          yield if block_given?
+
+          entries.each_value do |entry|
+            entry.compose!(deps)
+          end
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        class_methods do
+          def nodes
+            Hash[(@nodes || {}).map { |key, factory| [key, factory.dup] }]
+          end
+
+          private
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def entry(key, entry, metadata)
+            factory = ::Gitlab::Config::Entry::Factory.new(entry)
+              .with(description: metadata[:description])
+              .with(default: metadata[:default])
+
+            (@nodes ||= {}).merge!(key.to_sym => factory)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+
+          def helpers(*nodes)
+            nodes.each do |symbol|
+              define_method("#{symbol}_defined?") do
+                entries[symbol]&.specified?
+              end
+
+              define_method("#{symbol}_value") do
+                return unless entries[symbol] && entries[symbol].valid?
+
+                entries[symbol].value
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/config/entry/factory.rb b/lib/gitlab/config/entry/factory.rb
new file mode 100644
index 00000000000..79f9ff32514
--- /dev/null
+++ b/lib/gitlab/config/entry/factory.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Config
+    module Entry
+      ##
+      # Factory class responsible for fabricating entry objects.
+      #
+      class Factory
+        InvalidFactory = Class.new(StandardError)
+
+        def initialize(entry)
+          @entry = entry
+          @metadata = {}
+          @attributes = { default: entry.default }
+        end
+
+        def value(value)
+          @value = value
+          self
+        end
+
+        def metadata(metadata)
+          @metadata.merge!(metadata.compact)
+          self
+        end
+
+        def with(attributes)
+          @attributes.merge!(attributes.compact)
+          self
+        end
+
+        def create!
+          raise InvalidFactory unless defined?(@value)
+
+          ##
+          # We assume that unspecified entry is undefined.
+          # See issue #18775.
+          #
+          if @value.nil?
+            Entry::Unspecified.new(fabricate_unspecified)
+          else
+            fabricate(@entry, @value)
+          end
+        end
+
+        private
+
+        def fabricate_unspecified
+          ##
+          # If entry has a default value we fabricate concrete node
+          # with default value.
+          #
+          default = @attributes.fetch(:default)
+
+          if default.nil?
+            fabricate(Entry::Undefined)
+          else
+            fabricate(@entry, default)
+          end
+        end
+
+        def fabricate(entry, value = nil)
+          entry.new(value, @metadata).tap do |node|
+            node.key = @attributes[:key]
+            node.parent = @attributes[:parent]
+            node.default = @attributes[:default]
+            node.description = @attributes[:description]
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/config/entry/legacy_validation_helpers.rb b/lib/gitlab/config/entry/legacy_validation_helpers.rb
new file mode 100644
index 00000000000..d3ab5625743
--- /dev/null
+++ b/lib/gitlab/config/entry/legacy_validation_helpers.rb
@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Config
+    module Entry
+      module LegacyValidationHelpers
+        private
+
+        def validate_duration(value)
+          value.is_a?(String) && ChronicDuration.parse(value)
+        rescue ChronicDuration::DurationParseError
+          false
+        end
+
+        def validate_duration_limit(value, limit)
+          return false unless value.is_a?(String)
+
+          ChronicDuration.parse(value).second.from_now <
+            ChronicDuration.parse(limit).second.from_now
+        rescue ChronicDuration::DurationParseError
+          false
+        end
+
+        def validate_array_of_strings(values)
+          values.is_a?(Array) && values.all? { |value| validate_string(value) }
+        end
+
+        def validate_array_of_strings_or_regexps(values)
+          values.is_a?(Array) && values.all? { |value| validate_string_or_regexp(value) }
+        end
+
+        def validate_variables(variables)
+          variables.is_a?(Hash) &&
+            variables.flatten.all? do |value|
+              validate_string(value) || validate_integer(value)
+            end
+        end
+
+        def validate_integer(value)
+          value.is_a?(Integer)
+        end
+
+        def validate_string(value)
+          value.is_a?(String) || value.is_a?(Symbol)
+        end
+
+        def validate_regexp(value)
+          !value.nil? && Regexp.new(value.to_s) && true
+        rescue RegexpError, TypeError
+          false
+        end
+
+        def validate_string_or_regexp(value)
+          return true if value.is_a?(Symbol)
+          return false unless value.is_a?(String)
+
+          if value.first == '/' && value.last == '/'
+            validate_regexp(value[1...-1])
+          else
+            true
+          end
+        end
+
+        def validate_boolean(value)
+          value.in?([true, false])
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/config/entry/node.rb b/lib/gitlab/config/entry/node.rb
new file mode 100644
index 00000000000..9999ab4ff95
--- /dev/null
+++ b/lib/gitlab/config/entry/node.rb
@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Config
+    module Entry
+      ##
+      # Base abstract class for each configuration entry node.
+      #
+      class Node
+        InvalidError = Class.new(StandardError)
+
+        attr_reader :config, :metadata
+        attr_accessor :key, :parent, :default, :description
+
+        def initialize(config, **metadata)
+          @config = config
+          @metadata = metadata
+          @entries = {}
+
+          self.class.aspects.to_a.each do |aspect|
+            instance_exec(&aspect)
+          end
+        end
+
+        def [](key)
+          @entries[key] || Entry::Undefined.new
+        end
+
+        def compose!(deps = nil)
+          return unless valid?
+
+          yield if block_given?
+        end
+
+        def leaf?
+          @entries.none?
+        end
+
+        def descendants
+          @entries.values
+        end
+
+        def ancestors
+          @parent ? @parent.ancestors + [@parent] : []
+        end
+
+        def valid?
+          errors.none?
+        end
+
+        def errors
+          []
+        end
+
+        def value
+          if leaf?
+            @config
+          else
+            meaningful = @entries.select do |_key, value|
+              value.specified? && value.relevant?
+            end
+
+            Hash[meaningful.map { |key, entry| [key, entry.value] }]
+          end
+        end
+
+        def specified?
+          true
+        end
+
+        def relevant?
+          true
+        end
+
+        def location
+          name = @key.presence || self.class.name.to_s.demodulize
+                                      .underscore.humanize.downcase
+
+          ancestors.map(&:key).append(name).compact.join(':')
+        end
+
+        def inspect
+          val = leaf? ? config : descendants
+          unspecified = specified? ? '' : '(unspecified) '
+          "#<#{self.class.name} #{unspecified}{#{key}: #{val.inspect}}>"
+        end
+
+        def self.default(**)
+        end
+
+        def self.aspects
+          @aspects ||= []
+        end
+
+        private
+
+        attr_reader :entries
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/config/entry/simplifiable.rb b/lib/gitlab/config/entry/simplifiable.rb
new file mode 100644
index 00000000000..5fbf7565e2a
--- /dev/null
+++ b/lib/gitlab/config/entry/simplifiable.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Config
+    module Entry
+      class Simplifiable < SimpleDelegator
+        EntryStrategy = Struct.new(:name, :condition)
+
+        attr_reader :subject
+
+        def initialize(config, **metadata)
+          unless self.class.const_defined?(:UnknownStrategy)
+            raise ArgumentError, 'UndefinedStrategy not available!'
+          end
+
+          strategy = self.class.strategies.find do |variant|
+            variant.condition.call(config)
+          end
+
+          entry = self.class.entry_class(strategy)
+
+          super(@subject = entry.new(config, metadata))
+        end
+
+        def self.strategy(name, **opts)
+          EntryStrategy.new(name, opts.fetch(:if)).tap do |strategy|
+            strategies.append(strategy)
+          end
+        end
+
+        def self.strategies
+          @strategies ||= []
+        end
+
+        def self.entry_class(strategy)
+          if strategy.present?
+            self.const_get(strategy.name)
+          else
+            self::UnknownStrategy
+          end
+        end
+
+        def self.default
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/config/entry/undefined.rb b/lib/gitlab/config/entry/undefined.rb
new file mode 100644
index 00000000000..5f708abc80c
--- /dev/null
+++ b/lib/gitlab/config/entry/undefined.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Config
+    module Entry
+      ##
+      # This class represents an undefined entry.
+      #
+      class Undefined < Node
+        def initialize(*)
+          super(nil)
+        end
+
+        def value
+          nil
+        end
+
+        def valid?
+          true
+        end
+
+        def errors
+          []
+        end
+
+        def specified?
+          false
+        end
+
+        def relevant?
+          false
+        end
+
+        def inspect
+          "#<#{self.class.name}>"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/config/entry/unspecified.rb b/lib/gitlab/config/entry/unspecified.rb
new file mode 100644
index 00000000000..c096180d0f8
--- /dev/null
+++ b/lib/gitlab/config/entry/unspecified.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Config
+    module Entry
+      ##
+      # This class represents an unspecified entry.
+      #
+      # It decorates original entry adding method that indicates it is
+      # unspecified.
+      #
+      class Unspecified < SimpleDelegator
+        def specified?
+          false
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/config/entry/validatable.rb b/lib/gitlab/config/entry/validatable.rb
new file mode 100644
index 00000000000..1c88c68c11c
--- /dev/null
+++ b/lib/gitlab/config/entry/validatable.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Config
+    module Entry
+      module Validatable
+        extend ActiveSupport::Concern
+
+        def self.included(node)
+          node.aspects.append -> do
+            @validator = self.class.validator.new(self)
+            @validator.validate(:new)
+          end
+        end
+
+        def errors
+          @validator.messages + descendants.flat_map(&:errors) # rubocop:disable Gitlab/ModuleWithInstanceVariables
+        end
+
+        class_methods do
+          def validator
+            @validator ||= Class.new(Entry::Validator).tap do |validator|
+              if defined?(@validations)
+                @validations.each { |rules| validator.class_eval(&rules) }
+              end
+            end
+          end
+
+          private
+
+          def validations(&block)
+            (@validations ||= []).append(block)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/config/entry/validator.rb b/lib/gitlab/config/entry/validator.rb
new file mode 100644
index 00000000000..e5efd4a7b0a
--- /dev/null
+++ b/lib/gitlab/config/entry/validator.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Config
+    module Entry
+      class Validator < SimpleDelegator
+        include ActiveModel::Validations
+        include Entry::Validators
+
+        def initialize(entry)
+          super(entry)
+        end
+
+        def messages
+          errors.full_messages.map do |error|
+            "#{location} #{error}".downcase
+          end
+        end
+
+        def self.name
+          'Validator'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb
new file mode 100644
index 00000000000..25bfa50f829
--- /dev/null
+++ b/lib/gitlab/config/entry/validators.rb
@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Config
+    module Entry
+      module Validators
+        class AllowedKeysValidator < ActiveModel::EachValidator
+          def validate_each(record, attribute, value)
+            unknown_keys = value.try(:keys).to_a - options[:in]
+
+            if unknown_keys.any?
+              record.errors.add(attribute, "contains unknown keys: " +
+                                            unknown_keys.join(', '))
+            end
+          end
+        end
+
+        class AllowedValuesValidator < ActiveModel::EachValidator
+          def validate_each(record, attribute, value)
+            unless options[:in].include?(value.to_s)
+              record.errors.add(attribute, "unknown value: #{value}")
+            end
+          end
+        end
+
+        class AllowedArrayValuesValidator < ActiveModel::EachValidator
+          def validate_each(record, attribute, value)
+            unkown_values = value - options[:in]
+            unless unkown_values.empty?
+              record.errors.add(attribute, "contains unknown values: " +
+                                            unkown_values.join(', '))
+            end
+          end
+        end
+
+        class ArrayOfStringsValidator < ActiveModel::EachValidator
+          include LegacyValidationHelpers
+
+          def validate_each(record, attribute, value)
+            unless validate_array_of_strings(value)
+              record.errors.add(attribute, 'should be an array of strings')
+            end
+          end
+        end
+
+        class BooleanValidator < ActiveModel::EachValidator
+          include LegacyValidationHelpers
+
+          def validate_each(record, attribute, value)
+            unless validate_boolean(value)
+              record.errors.add(attribute, 'should be a boolean value')
+            end
+          end
+        end
+
+        class DurationValidator < ActiveModel::EachValidator
+          include LegacyValidationHelpers
+
+          def validate_each(record, attribute, value)
+            unless validate_duration(value)
+              record.errors.add(attribute, 'should be a duration')
+            end
+
+            if options[:limit]
+              unless validate_duration_limit(value, options[:limit])
+                record.errors.add(attribute, 'should not exceed the limit')
+              end
+            end
+          end
+        end
+
+        class HashOrStringValidator < ActiveModel::EachValidator
+          def validate_each(record, attribute, value)
+            unless value.is_a?(Hash) || value.is_a?(String)
+              record.errors.add(attribute, 'should be a hash or a string')
+            end
+          end
+        end
+
+        class HashOrIntegerValidator < ActiveModel::EachValidator
+          def validate_each(record, attribute, value)
+            unless value.is_a?(Hash) || value.is_a?(Integer)
+              record.errors.add(attribute, 'should be a hash or an integer')
+            end
+          end
+        end
+
+        class KeyValidator < ActiveModel::EachValidator
+          include LegacyValidationHelpers
+
+          def validate_each(record, attribute, value)
+            if validate_string(value)
+              validate_path(record, attribute, value)
+            else
+              record.errors.add(attribute, 'should be a string or symbol')
+            end
+          end
+
+          private
+
+          def validate_path(record, attribute, value)
+            path = CGI.unescape(value.to_s)
+
+            if path.include?('/')
+              record.errors.add(attribute, 'cannot contain the "/" character')
+            elsif path == '.' || path == '..'
+              record.errors.add(attribute, 'cannot be "." or ".."')
+            end
+          end
+        end
+
+        class RegexpValidator < ActiveModel::EachValidator
+          include LegacyValidationHelpers
+
+          def validate_each(record, attribute, value)
+            unless validate_regexp(value)
+              record.errors.add(attribute, 'must be a regular expression')
+            end
+          end
+
+          private
+
+          def look_like_regexp?(value)
+            value.is_a?(String) && value.start_with?('/') &&
+              value.end_with?('/')
+          end
+
+          def validate_regexp(value)
+            look_like_regexp?(value) &&
+              Regexp.new(value.to_s[1...-1]) &&
+              true
+          rescue RegexpError
+            false
+          end
+        end
+
+        class ArrayOfStringsOrRegexpsValidator < RegexpValidator
+          def validate_each(record, attribute, value)
+            unless validate_array_of_strings_or_regexps(value)
+              record.errors.add(attribute, 'should be an array of strings or regexps')
+            end
+          end
+
+          private
+
+          def validate_array_of_strings_or_regexps(values)
+            values.is_a?(Array) && values.all?(&method(:validate_string_or_regexp))
+          end
+
+          def validate_string_or_regexp(value)
+            return false unless value.is_a?(String)
+            return validate_regexp(value) if look_like_regexp?(value)
+
+            true
+          end
+        end
+
+        class ArrayOfStringsOrStringValidator < RegexpValidator
+          def validate_each(record, attribute, value)
+            unless validate_array_of_strings_or_string(value)
+              record.errors.add(attribute, 'should be an array of strings or a string')
+            end
+          end
+
+          private
+
+          def validate_array_of_strings_or_string(values)
+            validate_array_of_strings(values) || validate_string(values)
+          end
+        end
+
+        class TypeValidator < ActiveModel::EachValidator
+          def validate_each(record, attribute, value)
+            type = options[:with]
+            raise unless type.is_a?(Class)
+
+            unless value.is_a?(type)
+              message = options[:message] || "should be a #{type.name}"
+              record.errors.add(attribute, message)
+            end
+          end
+        end
+
+        class VariablesValidator < ActiveModel::EachValidator
+          include LegacyValidationHelpers
+
+          def validate_each(record, attribute, value)
+            unless validate_variables(value)
+              record.errors.add(attribute, 'should be a hash of key value pairs')
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/config/loader/format_error.rb b/lib/gitlab/config/loader/format_error.rb
new file mode 100644
index 00000000000..848ff96d201
--- /dev/null
+++ b/lib/gitlab/config/loader/format_error.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Config
+    module Loader
+      FormatError = Class.new(StandardError)
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/loader.rb b/lib/gitlab/config/loader/yaml.rb
index 141d2714cb6..8159f8b8026 100644
--- a/lib/gitlab/ci/config/loader.rb
+++ b/lib/gitlab/config/loader/yaml.rb
@@ -1,13 +1,13 @@
-module Gitlab
-  module Ci
-    class Config
-      class Loader
-        FormatError = Class.new(StandardError)
+# frozen_string_literal: true
 
+module Gitlab
+  module Config
+    module Loader
+      class Yaml
         def initialize(config)
           @config = YAML.safe_load(config, [Symbol], [], true)
         rescue Psych::Exception => e
-          raise FormatError, e.message
+          raise Loader::FormatError, e.message
         end
 
         def valid?
@@ -16,7 +16,7 @@ module Gitlab
 
         def load!
           unless valid?
-            raise FormatError, 'Invalid configuration format'
+            raise Loader::FormatError, 'Invalid configuration format'
           end
 
           @config.deep_symbolize_keys
diff --git a/lib/gitlab/config_helper.rb b/lib/gitlab/config_helper.rb
index 41880069e4c..b7aa03384b7 100644
--- a/lib/gitlab/config_helper.rb
+++ b/lib/gitlab/config_helper.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab::ConfigHelper
   def gitlab_config_features
     Gitlab.config.gitlab.default_projects_features
diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb
index 2a0cb640a14..0ca99506311 100644
--- a/lib/gitlab/conflict/file.rb
+++ b/lib/gitlab/conflict/file.rb
@@ -1,8 +1,11 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Conflict
     class File
       include Gitlab::Routing
       include IconsHelper
+      include Gitlab::Utils::StrongMemoize
 
       CONTEXT_LINES = 3
 
@@ -30,11 +33,8 @@ module Gitlab
       end
 
       def highlight_lines!
-        their_file = lines.reject { |line| line.type == 'new' }.map(&:text).join("\n")
-        our_file = lines.reject { |line| line.type == 'old' }.map(&:text).join("\n")
-
-        their_highlight = Gitlab::Highlight.highlight(their_path, their_file, repository: repository).lines
-        our_highlight = Gitlab::Highlight.highlight(our_path, our_file, repository: repository).lines
+        their_highlight = Gitlab::Highlight.highlight(their_path, their_lines, language: their_language).lines
+        our_highlight = Gitlab::Highlight.highlight(our_path, our_lines, language: our_language).lines
 
         lines.each do |line|
           line.rich_text =
@@ -158,7 +158,6 @@ module Gitlab
         json_hash.tap do |json_hash|
           if opts[:full_content]
             json_hash[:content] = content
-            json_hash[:blob_ace_mode] = our_blob && our_blob.language.try(:ace_mode)
           else
             json_hash[:sections] = sections if type.text?
             json_hash[:type] = type
@@ -183,6 +182,34 @@ module Gitlab
             raw_line[:line_new], parent_file: self)
         end
       end
+
+      def their_language
+        strong_memoize(:their_language) do
+          repository.gitattribute(their_path, 'gitlab-language')
+        end
+      end
+
+      def our_language
+        strong_memoize(:our_language) do
+          if our_path == their_path
+            their_language
+          else
+            repository.gitattribute(our_path, 'gitlab-language')
+          end
+        end
+      end
+
+      def their_lines
+        strong_memoize(:their_lines) do
+          lines.reject { |line| line.type == 'new' }.map(&:text).join("\n")
+        end
+      end
+
+      def our_lines
+        strong_memoize(:our_lines) do
+          lines.reject { |line| line.type == 'old' }.map(&:text).join("\n")
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb
index 65a65b67975..53406af2c4e 100644
--- a/lib/gitlab/conflict/file_collection.rb
+++ b/lib/gitlab/conflict/file_collection.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Conflict
     class FileCollection
diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb
index 4c28489f45a..5ed6427072a 100644
--- a/lib/gitlab/contributions_calendar.rb
+++ b/lib/gitlab/contributions_calendar.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class ContributionsCalendar
     attr_reader :contributor
@@ -7,9 +9,14 @@ module Gitlab
     def initialize(contributor, current_user = nil)
       @contributor = contributor
       @current_user = current_user
-      @projects = ContributedProjectsFinder.new(contributor).execute(current_user)
+      @projects = if @contributor.include_private_contributions?
+                    ContributedProjectsFinder.new(@contributor).execute(@contributor)
+                  else
+                    ContributedProjectsFinder.new(contributor).execute(current_user)
+                  end
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def activity_dates
       return @activity_dates if @activity_dates.present?
 
@@ -25,25 +32,25 @@ module Gitlab
       note_events = event_counts(date_from, :merge_requests)
         .having(action: [Event::COMMENTED])
 
-      union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events, note_events])
-      events = Event.find_by_sql(union.to_sql).map(&:attributes)
+      events = Event
+        .from_union([repo_events, issue_events, mr_events, note_events])
+        .map(&:attributes)
 
       @activity_dates = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities|
         activities[event["date"]] += event["total_amount"]
       end
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def events_by_date(date)
       return Event.none unless can_read_cross_project?
 
-      events = Event.contributions.where(author_id: contributor.id)
+      Event.contributions.where(author_id: contributor.id)
         .where(created_at: date.beginning_of_day..date.end_of_day)
         .where(project_id: projects)
-
-      # Use visible_to_user? instead of the complicated logic in activity_dates
-      # because we're only viewing the events for a single day.
-      events.select { |event| event.visible_to_user?(current_user) }
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     def starting_year
       1.year.ago.year
@@ -59,13 +66,14 @@ module Gitlab
       Ability.allowed?(current_user, :read_cross_project)
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def event_counts(date_from, feature)
       t = Event.arel_table
 
       # re-running the contributed projects query in each union is expensive, so
       # use IN(project_ids...) instead. It's the intersection of two users so
       # the list will be (relatively) short
-      @contributed_project_ids ||= projects.uniq.pluck(:id)
+      @contributed_project_ids ||= projects.distinct.pluck(:id)
       authed_projects = Project.where(id: @contributed_project_ids)
         .with_feature_available_for_user(feature, current_user)
         .reorder(nil)
@@ -87,5 +95,6 @@ module Gitlab
         .where(conditions)
         .where("events.project_id in (#{authed_projects.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection
     end
+    # rubocop: enable CodeReuse/ActiveRecord
   end
 end
diff --git a/lib/gitlab/contributor.rb b/lib/gitlab/contributor.rb
index c41e92b620f..d74d5a86aa0 100644
--- a/lib/gitlab/contributor.rb
+++ b/lib/gitlab/contributor.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class Contributor
     attr_accessor :email, :name, :commits, :additions, :deletions
diff --git a/lib/gitlab/correlation_id.rb b/lib/gitlab/correlation_id.rb
new file mode 100644
index 00000000000..0f9bde4390e
--- /dev/null
+++ b/lib/gitlab/correlation_id.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CorrelationId
+    LOG_KEY = 'correlation_id'.freeze
+
+    class << self
+      def use_id(correlation_id, &blk)
+        # always generate a id if null is passed
+        correlation_id ||= new_id
+
+        ids.push(correlation_id || new_id)
+
+        begin
+          yield(current_id)
+        ensure
+          ids.pop
+        end
+      end
+
+      def current_id
+        ids.last
+      end
+
+      def current_or_new_id
+        current_id || new_id
+      end
+
+      private
+
+      def ids
+        Thread.current[:correlation_id] ||= []
+      end
+
+      def new_id
+        SecureRandom.uuid
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cross_project_access.rb b/lib/gitlab/cross_project_access.rb
index 6eaed51b64c..4ddc7e02d1b 100644
--- a/lib/gitlab/cross_project_access.rb
+++ b/lib/gitlab/cross_project_access.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class CrossProjectAccess
     class << self
diff --git a/lib/gitlab/cross_project_access/check_collection.rb b/lib/gitlab/cross_project_access/check_collection.rb
index 88376232065..55527ba5e87 100644
--- a/lib/gitlab/cross_project_access/check_collection.rb
+++ b/lib/gitlab/cross_project_access/check_collection.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class CrossProjectAccess
     class CheckCollection
diff --git a/lib/gitlab/cross_project_access/check_info.rb b/lib/gitlab/cross_project_access/check_info.rb
index e8a845c7f1e..2a9eacad680 100644
--- a/lib/gitlab/cross_project_access/check_info.rb
+++ b/lib/gitlab/cross_project_access/check_info.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class CrossProjectAccess
     class CheckInfo
diff --git a/lib/gitlab/cross_project_access/class_methods.rb b/lib/gitlab/cross_project_access/class_methods.rb
index 90eac94800c..64ad30794d3 100644
--- a/lib/gitlab/cross_project_access/class_methods.rb
+++ b/lib/gitlab/cross_project_access/class_methods.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class CrossProjectAccess
     module ClassMethods
diff --git a/lib/gitlab/crypto_helper.rb b/lib/gitlab/crypto_helper.rb
new file mode 100644
index 00000000000..87a03d9c58f
--- /dev/null
+++ b/lib/gitlab/crypto_helper.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CryptoHelper
+    extend self
+
+    AES256_GCM_OPTIONS = {
+      algorithm: 'aes-256-gcm',
+      key: Settings.attr_encrypted_db_key_base_32,
+      iv: Settings.attr_encrypted_db_key_base_12
+    }.freeze
+
+    def sha256(value)
+      salt = Settings.attr_encrypted_db_key_base_truncated
+      ::Digest::SHA256.base64digest("#{value}#{salt}")
+    end
+
+    def aes256_gcm_encrypt(value)
+      encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value))
+      Base64.strict_encode64(encrypted_token)
+    end
+
+    def aes256_gcm_decrypt(value)
+      return unless value
+
+      encrypted_token = Base64.decode64(value)
+      Encryptor.decrypt(AES256_GCM_OPTIONS.merge(value: encrypted_token))
+    end
+  end
+end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 6cf7aa1bf0d..552aad83dd4 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -1,16 +1,14 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CurrentSettings
     class << self
       def current_application_settings
-        if RequestStore.active?
-          RequestStore.fetch(:current_application_settings) { ensure_application_settings! }
-        else
-          ensure_application_settings!
-        end
+        Gitlab::SafeRequestStore.fetch(:current_application_settings) { ensure_application_settings! }
       end
 
-      def fake_application_settings(attributes = {})
-        Gitlab::FakeApplicationSettings.new(::ApplicationSetting.defaults.merge(attributes || {}))
+      def clear_in_memory_application_settings!
+        @in_memory_application_settings = nil
       end
 
       def method_missing(name, *args, &block)
@@ -24,7 +22,22 @@ module Gitlab
       private
 
       def ensure_application_settings!
+        cached_application_settings || uncached_application_settings
+      end
+
+      def cached_application_settings
         return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true'
+
+        begin
+          ::ApplicationSetting.cached
+        rescue
+          # In case Redis isn't running
+          # or the Redis UNIX socket file is not available
+          # or the DB is not running (we use migrations in the cache key)
+        end
+      end
+
+      def uncached_application_settings
         return fake_application_settings unless connect_to_db?
 
         current_settings = ::ApplicationSetting.current
@@ -33,28 +46,21 @@ module Gitlab
         # and other callers from failing, use any loaded settings and return
         # defaults for missing columns.
         if ActiveRecord::Migrator.needs_migration?
-          return fake_application_settings(current_settings&.attributes)
-        end
-
-        return current_settings if current_settings.present?
-
-        with_fallback_to_fake_application_settings do
-          ::ApplicationSetting.create_from_defaults || in_memory_application_settings
+          db_attributes = current_settings&.attributes || {}
+          ::ApplicationSetting.build_from_defaults(db_attributes)
+        elsif current_settings.present?
+          current_settings
+        else
+          ::ApplicationSetting.create_from_defaults
         end
       end
 
-      def in_memory_application_settings
-        with_fallback_to_fake_application_settings do
-          @in_memory_application_settings ||= ::ApplicationSetting.build_from_defaults # rubocop:disable Gitlab/ModuleWithInstanceVariables
-        end
+      def fake_application_settings(attributes = {})
+        Gitlab::FakeApplicationSettings.new(::ApplicationSetting.defaults.merge(attributes || {}))
       end
 
-      def with_fallback_to_fake_application_settings(&block)
-        yield
-      rescue
-        # In case the application_settings table is not created yet, or if a new
-        # ApplicationSetting column is not yet migrated we fallback to a simple OpenStruct
-        fake_application_settings
+      def in_memory_application_settings
+        @in_memory_application_settings ||= ::ApplicationSetting.build_from_defaults
       end
 
       def connect_to_db?
diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb
index e3e3767cc75..304d60996a6 100644
--- a/lib/gitlab/cycle_analytics/base_event_fetcher.rb
+++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class BaseEventFetcher
diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb
index 86d708be0d6..36231b187cd 100644
--- a/lib/gitlab/cycle_analytics/base_query.rb
+++ b/lib/gitlab/cycle_analytics/base_query.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     module BaseQuery
diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb
index 038d5a19bc4..e2d6a301734 100644
--- a/lib/gitlab/cycle_analytics/base_stage.rb
+++ b/lib/gitlab/cycle_analytics/base_stage.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class BaseStage
diff --git a/lib/gitlab/cycle_analytics/code_event_fetcher.rb b/lib/gitlab/cycle_analytics/code_event_fetcher.rb
index 06357c9b377..591db3c35e6 100644
--- a/lib/gitlab/cycle_analytics/code_event_fetcher.rb
+++ b/lib/gitlab/cycle_analytics/code_event_fetcher.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class CodeEventFetcher < BaseEventFetcher
diff --git a/lib/gitlab/cycle_analytics/code_stage.rb b/lib/gitlab/cycle_analytics/code_stage.rb
index 5f9dc9a4303..2e5f9ef5a40 100644
--- a/lib/gitlab/cycle_analytics/code_stage.rb
+++ b/lib/gitlab/cycle_analytics/code_stage.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class CodeStage < BaseStage
diff --git a/lib/gitlab/cycle_analytics/event_fetcher.rb b/lib/gitlab/cycle_analytics/event_fetcher.rb
index 50e126cf00b..98a30a8fc97 100644
--- a/lib/gitlab/cycle_analytics/event_fetcher.rb
+++ b/lib/gitlab/cycle_analytics/event_fetcher.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     module EventFetcher
diff --git a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb
index 1754f91dccb..30c6ead8968 100644
--- a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb
+++ b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class IssueEventFetcher < BaseEventFetcher
diff --git a/lib/gitlab/cycle_analytics/issue_stage.rb b/lib/gitlab/cycle_analytics/issue_stage.rb
index 7b03811efb2..4eae2da512c 100644
--- a/lib/gitlab/cycle_analytics/issue_stage.rb
+++ b/lib/gitlab/cycle_analytics/issue_stage.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class IssueStage < BaseStage
diff --git a/lib/gitlab/cycle_analytics/metrics_tables.rb b/lib/gitlab/cycle_analytics/metrics_tables.rb
index f5d08c0b658..3e0302d308d 100644
--- a/lib/gitlab/cycle_analytics/metrics_tables.rb
+++ b/lib/gitlab/cycle_analytics/metrics_tables.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     module MetricsTables
diff --git a/lib/gitlab/cycle_analytics/permissions.rb b/lib/gitlab/cycle_analytics/permissions.rb
index 1e11e84a9cb..afefd09b614 100644
--- a/lib/gitlab/cycle_analytics/permissions.rb
+++ b/lib/gitlab/cycle_analytics/permissions.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class Permissions
diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb
index 086203b9ccc..db8ac3becea 100644
--- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb
+++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class PlanEventFetcher < BaseEventFetcher
diff --git a/lib/gitlab/cycle_analytics/plan_stage.rb b/lib/gitlab/cycle_analytics/plan_stage.rb
index 1a0afb56b4f..513e4575be0 100644
--- a/lib/gitlab/cycle_analytics/plan_stage.rb
+++ b/lib/gitlab/cycle_analytics/plan_stage.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class PlanStage < BaseStage
diff --git a/lib/gitlab/cycle_analytics/production_event_fetcher.rb b/lib/gitlab/cycle_analytics/production_event_fetcher.rb
index 0fa2e87f673..6681cb42c90 100644
--- a/lib/gitlab/cycle_analytics/production_event_fetcher.rb
+++ b/lib/gitlab/cycle_analytics/production_event_fetcher.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class ProductionEventFetcher < IssueEventFetcher
diff --git a/lib/gitlab/cycle_analytics/production_helper.rb b/lib/gitlab/cycle_analytics/production_helper.rb
index d0ca62e46e4..aff65b150fb 100644
--- a/lib/gitlab/cycle_analytics/production_helper.rb
+++ b/lib/gitlab/cycle_analytics/production_helper.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     module ProductionHelper
diff --git a/lib/gitlab/cycle_analytics/production_stage.rb b/lib/gitlab/cycle_analytics/production_stage.rb
index 0fa8a65cb99..6fd7214dce7 100644
--- a/lib/gitlab/cycle_analytics/production_stage.rb
+++ b/lib/gitlab/cycle_analytics/production_stage.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class ProductionStage < BaseStage
diff --git a/lib/gitlab/cycle_analytics/review_event_fetcher.rb b/lib/gitlab/cycle_analytics/review_event_fetcher.rb
index dada819a2a8..de100295281 100644
--- a/lib/gitlab/cycle_analytics/review_event_fetcher.rb
+++ b/lib/gitlab/cycle_analytics/review_event_fetcher.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class ReviewEventFetcher < BaseEventFetcher
diff --git a/lib/gitlab/cycle_analytics/review_stage.rb b/lib/gitlab/cycle_analytics/review_stage.rb
index cfbbdc43fd9..294b656bc55 100644
--- a/lib/gitlab/cycle_analytics/review_stage.rb
+++ b/lib/gitlab/cycle_analytics/review_stage.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class ReviewStage < BaseStage
diff --git a/lib/gitlab/cycle_analytics/stage.rb b/lib/gitlab/cycle_analytics/stage.rb
index 28e0455df59..1bd40a7aa18 100644
--- a/lib/gitlab/cycle_analytics/stage.rb
+++ b/lib/gitlab/cycle_analytics/stage.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     module Stage
diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb
index fc77bd86097..5198dd5b4eb 100644
--- a/lib/gitlab/cycle_analytics/stage_summary.rb
+++ b/lib/gitlab/cycle_analytics/stage_summary.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class StageSummary
diff --git a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb
index 2f014153ca5..70ce82383b3 100644
--- a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb
+++ b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class StagingEventFetcher < BaseEventFetcher
diff --git a/lib/gitlab/cycle_analytics/staging_stage.rb b/lib/gitlab/cycle_analytics/staging_stage.rb
index d5684bb9201..dbc2414ff66 100644
--- a/lib/gitlab/cycle_analytics/staging_stage.rb
+++ b/lib/gitlab/cycle_analytics/staging_stage.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class StagingStage < BaseStage
diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb
index a917ddccac7..709221c648e 100644
--- a/lib/gitlab/cycle_analytics/summary/base.rb
+++ b/lib/gitlab/cycle_analytics/summary/base.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     module Summary
diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb
index 0a88e052f60..f0019b26fa2 100644
--- a/lib/gitlab/cycle_analytics/summary/commit.rb
+++ b/lib/gitlab/cycle_analytics/summary/commit.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     module Summary
@@ -7,9 +9,7 @@ module Gitlab
         end
 
         def value
-          Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-            @value ||= count_commits
-          end
+          @value ||= count_commits
         end
 
         private
@@ -21,19 +21,11 @@ module Gitlab
         def count_commits
           return unless ref
 
-          repository = @project.repository.raw_repository
-          sha = @project.repository.commit(ref).sha
-
-          cmd = %W(git --git-dir=#{repository.path} log)
-          cmd << '--format=%H'
-          cmd << "--after=#{@from.iso8601}"
-          cmd << sha
-
-          output, status = Gitlab::Popen.popen(cmd)
-
-          raise IOError, output unless status.zero?
+          gitaly_commit_client.commit_count(ref, after: @from)
+        end
 
-          output.lines.count
+        def gitaly_commit_client
+          Gitlab::GitalyClient::CommitService.new(@project.repository.raw_repository)
         end
 
         def ref
diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb
index 099d798aac6..3b56dc2a7bc 100644
--- a/lib/gitlab/cycle_analytics/summary/deploy.rb
+++ b/lib/gitlab/cycle_analytics/summary/deploy.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     module Summary
diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb
index 9bbf7a2685f..51695c86192 100644
--- a/lib/gitlab/cycle_analytics/summary/issue.rb
+++ b/lib/gitlab/cycle_analytics/summary/issue.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     module Summary
diff --git a/lib/gitlab/cycle_analytics/test_event_fetcher.rb b/lib/gitlab/cycle_analytics/test_event_fetcher.rb
index a2589c6601a..4d5ea5b7c34 100644
--- a/lib/gitlab/cycle_analytics/test_event_fetcher.rb
+++ b/lib/gitlab/cycle_analytics/test_event_fetcher.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class TestEventFetcher < StagingEventFetcher
diff --git a/lib/gitlab/cycle_analytics/test_stage.rb b/lib/gitlab/cycle_analytics/test_stage.rb
index 0e9d235ca79..c31b664148b 100644
--- a/lib/gitlab/cycle_analytics/test_stage.rb
+++ b/lib/gitlab/cycle_analytics/test_stage.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class TestStage < BaseStage
diff --git a/lib/gitlab/cycle_analytics/updater.rb b/lib/gitlab/cycle_analytics/updater.rb
index 953268ebd46..c642809a792 100644
--- a/lib/gitlab/cycle_analytics/updater.rb
+++ b/lib/gitlab/cycle_analytics/updater.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class Updater
diff --git a/lib/gitlab/cycle_analytics/usage_data.rb b/lib/gitlab/cycle_analytics/usage_data.rb
index 5122e3417ca..913ee373f54 100644
--- a/lib/gitlab/cycle_analytics/usage_data.rb
+++ b/lib/gitlab/cycle_analytics/usage_data.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module CycleAnalytics
     class UsageData
diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb
index bd14c7eece3..6d5fc4219fb 100644
--- a/lib/gitlab/daemon.rb
+++ b/lib/gitlab/daemon.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class Daemon
     def self.initialize_instance(*args)
diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb
index 2f1445a050a..3407380127e 100644
--- a/lib/gitlab/data_builder/build.rb
+++ b/lib/gitlab/data_builder/build.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DataBuilder
     module Build
@@ -28,6 +30,7 @@ module Gitlab
           build_finished_at: build.finished_at,
           build_duration: build.duration,
           build_allow_failure: build.allow_failure,
+          build_failure_reason: build.failure_reason,
 
           # TODO: do we still need it?
           project_id: project.id,
diff --git a/lib/gitlab/data_builder/note.rb b/lib/gitlab/data_builder/note.rb
index f573368e572..65601dcdf31 100644
--- a/lib/gitlab/data_builder/note.rb
+++ b/lib/gitlab/data_builder/note.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DataBuilder
     module Note
diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb
index 1e283cc092b..76c8b4ec5c2 100644
--- a/lib/gitlab/data_builder/pipeline.rb
+++ b/lib/gitlab/data_builder/pipeline.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DataBuilder
     module Pipeline
@@ -26,7 +28,8 @@ module Gitlab
           stages: pipeline.stages_names,
           created_at: pipeline.created_at,
           finished_at: pipeline.finished_at,
-          duration: pipeline.duration
+          duration: pipeline.duration,
+          variables: pipeline.variables.map(&:hook_attrs)
         }
       end
 
@@ -55,7 +58,7 @@ module Gitlab
           id: runner.id,
           description: runner.description,
           active: runner.active?,
-          is_shared: runner.is_shared?
+          is_shared: runner.instance_type?
         }
       end
     end
diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb
index c169c8fe135..ea08b5f7eae 100644
--- a/lib/gitlab/data_builder/push.rb
+++ b/lib/gitlab/data_builder/push.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DataBuilder
     module Push
@@ -29,7 +31,11 @@ module Gitlab
               }
             }
           ],
-          total_commits_count: 1
+          total_commits_count: 1,
+          push_options: [
+            "ci.skip",
+            "custom option"
+          ]
         }.freeze
 
       # Produce a hash of post-receive data
@@ -50,10 +56,12 @@ module Gitlab
       #     homepage: String,
       #   },
       #   commits: Array,
-      #   total_commits_count: Fixnum
+      #   total_commits_count: Fixnum,
+      #   push_options: Array
       # }
       #
-      def build(project, user, oldrev, newrev, ref, commits = [], message = nil, commits_count: nil)
+      # rubocop:disable Metrics/ParameterLists
+      def build(project, user, oldrev, newrev, ref, commits = [], message = nil, commits_count: nil, push_options: [])
         commits = Array(commits)
 
         # Total commits count
@@ -85,23 +93,28 @@ module Gitlab
           user_id: user.id,
           user_name: user.name,
           user_username: user.username,
-          user_email: user.email,
+          user_email: user.public_email,
           user_avatar: user.avatar_url(only_path: false),
           project_id: project.id,
           project: project.hook_attrs,
           commits: commit_attrs,
           total_commits_count: commits_count,
+          push_options: push_options,
           # DEPRECATED
           repository: project.hook_attrs.slice(:name, :url, :description, :homepage,
                                                :git_http_url, :git_ssh_url, :visibility_level)
         }
       end
 
-      # This method provide a sample data generated with
+      # This method provides a sample data generated with
       # existing project and commits to test webhooks
       def build_sample(project, user)
+        # Use sample data if repo has no commit
+        # (expect the case of test service configuration settings)
+        return sample_data if project.empty_repo?
+
         ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{project.default_branch}"
-        commits = project.repository.commits(project.default_branch.to_s, limit: 3) rescue []
+        commits = project.repository.commits(project.default_branch.to_s, limit: 3)
 
         build(project, user, commits.last&.id, commits.first&.id, ref, commits)
       end
diff --git a/lib/gitlab/data_builder/repository.rb b/lib/gitlab/data_builder/repository.rb
index c9c13ec6487..0e627fd623e 100644
--- a/lib/gitlab/data_builder/repository.rb
+++ b/lib/gitlab/data_builder/repository.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DataBuilder
     module Repository
diff --git a/lib/gitlab/data_builder/wiki_page.rb b/lib/gitlab/data_builder/wiki_page.rb
index 226974b698c..9368446fa59 100644
--- a/lib/gitlab/data_builder/wiki_page.rb
+++ b/lib/gitlab/data_builder/wiki_page.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DataBuilder
     module WikiPage
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index d49d055c3f2..b6ca777e029 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Database
     # The max value of INTEGER type is the same between MySQL and PostgreSQL:
@@ -33,7 +35,6 @@ module Gitlab
       adapter_name.casecmp('postgresql').zero?
     end
 
-    # Overridden in EE
     def self.read_only?
       false
     end
@@ -42,10 +43,31 @@ module Gitlab
       !self.read_only?
     end
 
+    # Check whether the underlying database is in read-only mode
+    def self.db_read_only?
+      if postgresql?
+        pg_is_in_recovery =
+          ActiveRecord::Base.connection.execute('SELECT pg_is_in_recovery()')
+            .first.fetch('pg_is_in_recovery')
+
+        Gitlab::Utils.to_boolean(pg_is_in_recovery)
+      else
+        false
+      end
+    end
+
+    def self.db_read_write?
+      !self.db_read_only?
+    end
+
     def self.version
       @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1]
     end
 
+    def self.postgresql_9_or_less?
+      postgresql? && version.to_f < 10
+    end
+
     def self.join_lateral_supported?
       postgresql? && version.to_f >= 9.3
     end
@@ -54,15 +76,37 @@ module Gitlab
       postgresql? && version.to_f >= 9.4
     end
 
+    def self.pg_stat_wal_receiver_supported?
+      postgresql? && version.to_f >= 9.6
+    end
+
+    # map some of the function names that changed between PostgreSQL 9 and 10
+    # https://wiki.postgresql.org/wiki/New_in_postgres_10
+    def self.pg_wal_lsn_diff
+      Gitlab::Database.postgresql_9_or_less? ? 'pg_xlog_location_diff' : 'pg_wal_lsn_diff'
+    end
+
+    def self.pg_current_wal_insert_lsn
+      Gitlab::Database.postgresql_9_or_less? ? 'pg_current_xlog_insert_location' : 'pg_current_wal_insert_lsn'
+    end
+
+    def self.pg_last_wal_receive_lsn
+      Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_receive_location' : 'pg_last_wal_receive_lsn'
+    end
+
+    def self.pg_last_wal_replay_lsn
+      Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_replay_location' : 'pg_last_wal_replay_lsn'
+    end
+
     def self.nulls_last_order(field, direction = 'ASC')
       order = "#{field} #{direction}"
 
       if postgresql?
-        order << ' NULLS LAST'
+        order = "#{order} NULLS LAST"
       else
         # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL
         # columns. In the (default) ascending order, `0` comes first.
-        order.prepend("#{field} IS NULL, ") if direction == 'ASC'
+        order = "#{field} IS NULL, #{order}" if direction == 'ASC'
       end
 
       order
@@ -72,11 +116,11 @@ module Gitlab
       order = "#{field} #{direction}"
 
       if postgresql?
-        order << ' NULLS FIRST'
+        order = "#{order} NULLS FIRST"
       else
         # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL
         # columns. In the (default) ascending order, `0` comes first.
-        order.prepend("#{field} IS NULL, ") if direction == 'DESC'
+        order = "#{field} IS NULL, #{order}" if direction == 'DESC'
       end
 
       order
@@ -143,7 +187,7 @@ module Gitlab
       EOF
 
       if return_ids
-        sql << 'RETURNING id'
+        sql = "#{sql}RETURNING id"
       end
 
       result = connection.execute(sql)
@@ -188,8 +232,7 @@ module Gitlab
     end
 
     def self.cached_table_exists?(table_name)
-      # Rails 5 uses data_source_exists? instead of table_exists?
-      connection.schema_cache.table_exists?(table_name)
+      connection.schema_cache.data_source_exists?(table_name)
     end
 
     private_class_method :connection
@@ -205,5 +248,21 @@ module Gitlab
     end
 
     private_class_method :database_version
+
+    def self.add_post_migrate_path_to_rails(force: false)
+      return if ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] && !force
+
+      Rails.application.config.paths['db'].each do |db_path|
+        path = Rails.root.join(db_path, 'post_migrate').to_s
+
+        unless Rails.application.config.paths['db/migrate'].include? path
+          Rails.application.config.paths['db/migrate'] << path
+
+          # Rails memoizes migrations at certain points where it won't read the above
+          # path just yet. As such we must also update the following list of paths.
+          ActiveRecord::Migrator.migrations_paths << path
+        end
+      end
+    end
   end
 end
diff --git a/lib/gitlab/database/arel_methods.rb b/lib/gitlab/database/arel_methods.rb
deleted file mode 100644
index d7e3ce08b32..00000000000
--- a/lib/gitlab/database/arel_methods.rb
+++ /dev/null
@@ -1,18 +0,0 @@
-module Gitlab
-  module Database
-    module ArelMethods
-      private
-
-      # In Arel 7.0.0 (Arel 7.1.4 is used in Rails 5.0) the `engine` parameter of `Arel::UpdateManager#initializer`
-      # was removed.
-      # Remove this file and inline this method when removing rails5? code.
-      def arel_update_manager
-        if Gitlab.rails5?
-          Arel::UpdateManager.new
-        else
-          Arel::UpdateManager.new(ActiveRecord::Base)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/database/count.rb b/lib/gitlab/database/count.rb
index 5f549ed2b3c..f3d37ccd72a 100644
--- a/lib/gitlab/database/count.rb
+++ b/lib/gitlab/database/count.rb
@@ -1,5 +1,12 @@
+# frozen_string_literal: true
+
 # For large tables, PostgreSQL can take a long time to count rows due to MVCC.
-# We can optimize this by using the reltuples count as described in https://wiki.postgresql.org/wiki/Slow_Counting.
+# We can optimize this by using various strategies for approximate counting.
+#
+# For example, we can use the reltuples count as described in https://wiki.postgresql.org/wiki/Slow_Counting.
+#
+# However, since statistics are not always up to date, we also implement a table sampling strategy
+# that performs an exact count but only on a sample of the table. See TablesampleCountStrategy.
 module Gitlab
   module Database
     module Count
@@ -18,68 +25,30 @@ module Gitlab
         end
 
       # Takes in an array of models and returns a Hash for the approximate
-      # counts for them.  If the model's table has not been vacuumed or
-      # analyzed recently, simply run the Model.count to get the data.
+      # counts for them.
+      #
+      # Various count strategies can be specified that are executed in
+      # sequence until all tables have an approximate count attached
+      # or we run out of strategies.
+      #
+      # Note that not all strategies are available on all supported RDBMS.
       #
       # @param [Array]
       # @return [Hash] of Model -> count mapping
-      def self.approximate_counts(models)
-        table_to_model_map = models.each_with_object({}) do |model, hash|
-          hash[model.table_name] = model
-        end
-
-        table_names = table_to_model_map.keys
-        counts_by_table_name = Gitlab::Database.postgresql? ? reltuples_from_recently_updated(table_names) : {}
-
-        # Convert table -> count to Model -> count
-        counts_by_model = counts_by_table_name.each_with_object({}) do |pair, hash|
-          model = table_to_model_map[pair.first]
-          hash[model] = pair.second
-        end
-
-        missing_tables = table_names - counts_by_table_name.keys
+      def self.approximate_counts(models, strategies: [TablesampleCountStrategy, ReltuplesCountStrategy, ExactCountStrategy])
+        strategies.each_with_object({}) do |strategy, counts_by_model|
+          if strategy.enabled?
+            models_with_missing_counts = models - counts_by_model.keys
 
-        missing_tables.each do |table|
-          model = table_to_model_map[table]
-          counts_by_model[model] = model.count
-        end
-
-        counts_by_model
-      end
+            break counts_by_model if models_with_missing_counts.empty?
 
-      # Returns a hash of the table names that have recently updated tuples.
-      #
-      # @param [Array] table names
-      # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 })
-      def self.reltuples_from_recently_updated(table_names)
-        query = postgresql_estimate_query(table_names)
-        rows = []
+            counts = strategy.new(models_with_missing_counts).count
 
-        # Querying tuple stats only works on the primary. Due to load
-        # balancing, we need to ensure this query hits the load balancer.  The
-        # easiest way to do this is to start a transaction.
-        ActiveRecord::Base.transaction do
-          rows = ActiveRecord::Base.connection.select_all(query)
+            counts.each do |model, count|
+              counts_by_model[model] = count
+            end
+          end
         end
-
-        rows.each_with_object({}) { |row, data| data[row['table_name']] = row['estimate'].to_i }
-      rescue *CONNECTION_ERRORS
-        {}
-      end
-
-      # Generates the PostgreSQL query to return the tuples for tables
-      # that have been vacuumed or analyzed in the last hour.
-      #
-      # @param [Array] table names
-      # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 })
-      def self.postgresql_estimate_query(table_names)
-        time = "to_timestamp(#{1.hour.ago.to_i})"
-        <<~SQL
-          SELECT pg_class.relname AS table_name, reltuples::bigint AS estimate FROM pg_class
-          LEFT JOIN pg_stat_user_tables ON pg_class.relname = pg_stat_user_tables.relname
-          WHERE pg_class.relname IN (#{table_names.map { |table| "'#{table}'" }.join(',')})
-          AND (last_vacuum > #{time} OR last_autovacuum > #{time} OR last_analyze > #{time} OR last_autoanalyze > #{time})
-        SQL
       end
     end
   end
diff --git a/lib/gitlab/database/count/exact_count_strategy.rb b/lib/gitlab/database/count/exact_count_strategy.rb
new file mode 100644
index 00000000000..fa6951eda22
--- /dev/null
+++ b/lib/gitlab/database/count/exact_count_strategy.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module Count
+      # This strategy performs an exact count on the model.
+      #
+      # This is guaranteed to be accurate, however it also scans the
+      # whole table. Hence, there are no guarantees with respect
+      # to runtime.
+      #
+      # Note that for very large tables, this may even timeout.
+      class ExactCountStrategy
+        attr_reader :models
+        def initialize(models)
+          @models = models
+        end
+
+        def count
+          models.each_with_object({}) do |model, data|
+            data[model] = model.count
+          end
+        rescue *CONNECTION_ERRORS
+          {}
+        end
+
+        def self.enabled?
+          true
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb
new file mode 100644
index 00000000000..c3a674aeb7e
--- /dev/null
+++ b/lib/gitlab/database/count/reltuples_count_strategy.rb
@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module Count
+      class PgClass < ActiveRecord::Base
+        self.table_name = 'pg_class'
+      end
+
+      # This strategy counts based on PostgreSQL's statistics in pg_stat_user_tables.
+      #
+      # Specifically, it relies on the column reltuples in said table. An additional
+      # check is performed to make sure statistics were updated within the last hour.
+      #
+      # Otherwise, this strategy skips tables with outdated statistics.
+      #
+      # There are no guarantees with respect to the accuracy of this strategy. Runtime
+      # however is guaranteed to be "fast", because it only looks up statistics.
+      class ReltuplesCountStrategy
+        attr_reader :models
+        def initialize(models)
+          @models = models
+        end
+
+        # Returns a hash of the table names that have recently updated tuples.
+        #
+        # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 })
+        def count
+          size_estimates
+        rescue *CONNECTION_ERRORS
+          {}
+        end
+
+        def self.enabled?
+          Gitlab::Database.postgresql?
+        end
+
+        private
+
+        def table_names
+          models.map(&:table_name)
+        end
+
+        def size_estimates(check_statistics: true)
+          table_to_model = models.each_with_object({}) { |model, h| h[model.table_name] = model }
+
+          # Querying tuple stats only works on the primary. Due to load balancing, the
+          # easiest way to do this is to start a transaction.
+          ActiveRecord::Base.transaction do
+            get_statistics(table_names, check_statistics: check_statistics).each_with_object({}) do |row, data|
+              model = table_to_model[row.table_name]
+              data[model] = row.estimate
+            end
+          end
+        end
+
+        # Generates the PostgreSQL query to return the tuples for tables
+        # that have been vacuumed or analyzed in the last hour.
+        #
+        # @param [Array] table names
+        # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 })
+        def get_statistics(table_names, check_statistics: true)
+          time = 1.hour.ago
+
+          query = PgClass.joins("LEFT JOIN pg_stat_user_tables USING (relname)")
+            .where(relname: table_names)
+            .select('pg_class.relname AS table_name, reltuples::bigint AS estimate')
+
+          if check_statistics
+            query = query.where('last_vacuum > ? OR last_autovacuum > ? OR last_analyze > ? OR last_autoanalyze > ?',
+                                time, time, time, time)
+          end
+
+          query
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb
new file mode 100644
index 00000000000..cf1cf054dbf
--- /dev/null
+++ b/lib/gitlab/database/count/tablesample_count_strategy.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module Count
+      # A tablesample count executes in two phases:
+      # * Estimate table sizes based on reltuples.
+      # * Based on the estimate:
+      #   * If the table is considered 'small', execute an exact relation count.
+      #   * Otherwise, count on a sample of the table using TABLESAMPLE.
+      #
+      # The size of the sample is chosen in a way that we always roughly scan
+      # the same amount of rows (see TABLESAMPLE_ROW_TARGET).
+      #
+      # There are no guarantees with respect to the accuracy of the result or runtime.
+      class TablesampleCountStrategy < ReltuplesCountStrategy
+        EXACT_COUNT_THRESHOLD = 10_000
+        TABLESAMPLE_ROW_TARGET = 10_000
+
+        def count
+          estimates = size_estimates(check_statistics: false)
+
+          models.each_with_object({}) do |model, count_by_model|
+            count = perform_count(model, estimates[model])
+            count_by_model[model] = count if count
+          end
+        rescue *CONNECTION_ERRORS
+          {}
+        end
+
+        def self.enabled?
+          Gitlab::Database.postgresql? && Feature.enabled?(:tablesample_counts)
+        end
+
+        private
+
+        def perform_count(model, estimate)
+          # If we estimate 0, we may not have statistics at all. Don't use them.
+          return nil unless estimate && estimate > 0
+
+          if estimate < EXACT_COUNT_THRESHOLD
+            # The table is considered small, the assumption here is that
+            # the exact count will be fast anyways.
+            model.count
+          else
+            # The table is considered large, let's only count on a sample.
+            tablesample_count(model, estimate)
+          end
+        end
+
+        def tablesample_count(model, estimate)
+          portion = (TABLESAMPLE_ROW_TARGET.to_f / estimate).round(4)
+          inverse = 1 / portion
+          query = <<~SQL
+            SELECT (COUNT(*)*#{inverse})::integer AS count
+            FROM #{model.table_name} TABLESAMPLE SYSTEM (#{portion * 100})
+          SQL
+
+          rows = ActiveRecord::Base.connection.select_all(query)
+
+          Integer(rows.first['count'])
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/date_time.rb b/lib/gitlab/database/date_time.rb
index 25e56998038..79d2caff151 100644
--- a/lib/gitlab/database/date_time.rb
+++ b/lib/gitlab/database/date_time.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Database
     module DateTime
diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb
index d32837f5793..862ab96c887 100644
--- a/lib/gitlab/database/grant.rb
+++ b/lib/gitlab/database/grant.rb
@@ -1,7 +1,11 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Database
     # Model that can be used for querying permissions of a SQL user.
     class Grant < ActiveRecord::Base
+      include FromUnion
+
       self.table_name =
         if Database.postgresql?
           'information_schema.role_table_grants'
@@ -42,9 +46,7 @@ module Gitlab
               .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')")
           ]
 
-          union = SQL::Union.new(queries).to_sql
-
-          Grant.from("(#{union}) privs").any?
+          Grant.from_union(queries, alias_as: 'privs').any?
         end
       end
     end
diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb
index 74fed447289..1455e410d4b 100644
--- a/lib/gitlab/database/median.rb
+++ b/lib/gitlab/database/median.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # https://www.periscopedata.com/blog/medians-in-sql.html
 module Gitlab
   module Database
@@ -33,7 +35,7 @@ module Gitlab
       end
 
       def mysql_median_datetime_sql(arel_table, query_so_far, column_sym)
-        query = arel_table
+        query = arel_table.from
                 .from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name))
                 .project(average([arel_table[column_sym]], 'median'))
                 .where(
@@ -143,7 +145,7 @@ module Gitlab
                 .order(arel_table[column_sym])
             ).as('row_id')
 
-          count = arel_table.from(arel_table.alias)
+          count = arel_table.from.from(arel_table.alias)
                     .project('COUNT(*)')
                     .where(arel_table[partition_column].eq(arel_table.alias[partition_column]))
                     .as('ct')
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index c21bae5e16b..3abd0600e9d 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -1,8 +1,8 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Database
     module MigrationHelpers
-      include Gitlab::Database::ArelMethods
-
       BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job
       BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time
 
@@ -58,7 +58,6 @@ module Gitlab
 
         if Database.postgresql?
           options = options.merge({ algorithm: :concurrently })
-          disable_statement_timeout
         end
 
         if index_exists?(table_name, column_name, options)
@@ -66,7 +65,9 @@ module Gitlab
           return
         end
 
-        add_index(table_name, column_name, options)
+        disable_statement_timeout do
+          add_index(table_name, column_name, options)
+        end
       end
 
       # Removes an existed index, concurrently when supported
@@ -87,7 +88,6 @@ module Gitlab
 
         if supports_drop_index_concurrently?
           options = options.merge({ algorithm: :concurrently })
-          disable_statement_timeout
         end
 
         unless index_exists?(table_name, column_name, options)
@@ -95,7 +95,9 @@ module Gitlab
           return
         end
 
-        remove_index(table_name, options.merge({ column: column_name }))
+        disable_statement_timeout do
+          remove_index(table_name, options.merge({ column: column_name }))
+        end
       end
 
       # Removes an existing index, concurrently when supported
@@ -116,7 +118,6 @@ module Gitlab
 
         if supports_drop_index_concurrently?
           options = options.merge({ algorithm: :concurrently })
-          disable_statement_timeout
         end
 
         unless index_exists_by_name?(table_name, index_name)
@@ -124,7 +125,9 @@ module Gitlab
           return
         end
 
-        remove_index(table_name, options.merge({ name: index_name }))
+        disable_statement_timeout do
+          remove_index(table_name, options.merge({ name: index_name }))
+        end
       end
 
       # Only available on Postgresql >= 9.2
@@ -171,8 +174,6 @@ module Gitlab
           on_delete = 'SET NULL' if on_delete == :nullify
         end
 
-        disable_statement_timeout
-
         key_name = concurrent_foreign_key_name(source, column)
 
         unless foreign_key_exists?(source, target, column: column)
@@ -199,7 +200,9 @@ module Gitlab
         # while running.
         #
         # Note this is a no-op in case the constraint is VALID already
-        execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{key_name};")
+        disable_statement_timeout do
+          execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{key_name};")
+        end
       end
 
       def foreign_key_exists?(source, target = nil, column: nil)
@@ -224,8 +227,48 @@ module Gitlab
       # Long-running migrations may take more than the timeout allowed by
       # the database. Disable the session's statement timeout to ensure
       # migrations don't get killed prematurely. (PostgreSQL only)
+      #
+      # There are two possible ways to disable the statement timeout:
+      #
+      # - Per transaction (this is the preferred and default mode)
+      # - Per connection (requires a cleanup after the execution)
+      #
+      # When using a per connection disable statement, code must be inside
+      # a block so we can automatically execute `RESET ALL` after block finishes
+      # otherwise the statement will still be disabled until connection is dropped
+      # or `RESET ALL` is executed
       def disable_statement_timeout
-        execute('SET statement_timeout TO 0') if Database.postgresql?
+        # bypass disabled_statement logic when not using postgres, but still execute block when one is given
+        unless Database.postgresql?
+          if block_given?
+            yield
+          end
+
+          return
+        end
+
+        if block_given?
+          begin
+            execute('SET statement_timeout TO 0')
+
+            yield
+          ensure
+            execute('RESET ALL')
+          end
+        else
+          unless transaction_open?
+            raise <<~ERROR
+              Cannot call disable_statement_timeout() without a transaction open or outside of a transaction block.
+              If you don't want to use a transaction wrap your code in a block call:
+
+              disable_statement_timeout { # code that requires disabled statement here }
+
+              This will make sure statement_timeout is disabled before and reset after the block execution is finished.
+            ERROR
+          end
+
+          execute('SET LOCAL statement_timeout TO 0')
+        end
       end
 
       def true_value
@@ -316,7 +359,7 @@ module Gitlab
           stop_arel = yield table, stop_arel if block_given?
           stop_row = exec_query(stop_arel.to_sql).to_hash.first
 
-          update_arel = arel_update_manager
+          update_arel = Arel::UpdateManager.new
             .table(table)
             .set([[table[column], value]])
             .where(table[:id].gteq(start_id))
@@ -367,30 +410,30 @@ module Gitlab
             'in the body of your migration class'
         end
 
-        disable_statement_timeout
-
-        transaction do
-          if limit
-            add_column(table, column, type, default: nil, limit: limit)
-          else
-            add_column(table, column, type, default: nil)
+        disable_statement_timeout do
+          transaction do
+            if limit
+              add_column(table, column, type, default: nil, limit: limit)
+            else
+              add_column(table, column, type, default: nil)
+            end
+
+            # Changing the default before the update ensures any newly inserted
+            # rows already use the proper default value.
+            change_column_default(table, column, default)
           end
 
-          # Changing the default before the update ensures any newly inserted
-          # rows already use the proper default value.
-          change_column_default(table, column, default)
-        end
-
-        begin
-          update_column_in_batches(table, column, default, &block)
+          begin
+            update_column_in_batches(table, column, default, &block)
 
-          change_column_null(table, column, false) unless allow_null
-        # We want to rescue _all_ exceptions here, even those that don't inherit
-        # from StandardError.
-        rescue Exception => error # rubocop: disable all
-          remove_column(table, column)
+            change_column_null(table, column, false) unless allow_null
+          # We want to rescue _all_ exceptions here, even those that don't inherit
+          # from StandardError.
+          rescue Exception => error # rubocop: disable all
+            remove_column(table, column)
 
-          raise error
+            raise error
+          end
         end
       end
 
@@ -596,6 +639,97 @@ module Gitlab
         end
       end
 
+      # Renames a column using a background migration.
+      #
+      # Because this method uses a background migration it's more suitable for
+      # large tables. For small tables it's better to use
+      # `rename_column_concurrently` since it can complete its work in a much
+      # shorter amount of time and doesn't rely on Sidekiq.
+      #
+      # Example usage:
+      #
+      #     rename_column_using_background_migration(
+      #       :users,
+      #       :feed_token,
+      #       :rss_token
+      #     )
+      #
+      # table - The name of the database table containing the column.
+      #
+      # old - The old column name.
+      #
+      # new - The new column name.
+      #
+      # type - The type of the new column. If no type is given the old column's
+      #        type is used.
+      #
+      # batch_size - The number of rows to schedule in a single background
+      #              migration.
+      #
+      # interval - The time interval between every background migration.
+      def rename_column_using_background_migration(
+        table,
+        old_column,
+        new_column,
+        type: nil,
+        batch_size: 10_000,
+        interval: 10.minutes
+      )
+
+        check_trigger_permissions!(table)
+
+        old_col = column_for(table, old_column)
+        new_type = type || old_col.type
+        max_index = 0
+
+        add_column(table, new_column, new_type,
+                   limit: old_col.limit,
+                   precision: old_col.precision,
+                   scale: old_col.scale)
+
+        # We set the default value _after_ adding the column so we don't end up
+        # updating any existing data with the default value. This isn't
+        # necessary since we copy over old values further down.
+        change_column_default(table, new_column, old_col.default) if old_col.default
+
+        install_rename_triggers(table, old_column, new_column)
+
+        model = Class.new(ActiveRecord::Base) do
+          self.table_name = table
+
+          include ::EachBatch
+        end
+
+        # Schedule the jobs that will copy the data from the old column to the
+        # new one. Rows with NULL values in our source column are skipped since
+        # the target column is already NULL at this point.
+        model.where.not(old_column => nil).each_batch(of: batch_size) do |batch, index|
+          start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
+          max_index = index
+
+          BackgroundMigrationWorker.perform_in(
+            index * interval,
+            'CopyColumn',
+            [table, old_column, new_column, start_id, end_id]
+          )
+        end
+
+        # Schedule the renaming of the column to happen (initially) 1 hour after
+        # the last batch finished.
+        BackgroundMigrationWorker.perform_in(
+          (max_index * interval) + 1.hour,
+          'CleanupConcurrentRename',
+          [table, old_column, new_column]
+        )
+
+        if perform_background_migration_inline?
+          # To ensure the schema is up to date immediately we perform the
+          # migration inline in dev / test environments.
+          Gitlab::BackgroundMigration.steal('CopyColumn')
+          Gitlab::BackgroundMigration.steal('CleanupConcurrentRename')
+        end
+      end
+
       def perform_background_migration_inline?
         Rails.env.test? || Rails.env.development?
       end
@@ -745,7 +879,7 @@ module Gitlab
         columns(table).find { |column| column.name == name }
       end
 
-      # This will replace the first occurance of a string in a column with
+      # This will replace the first occurrence of a string in a column with
       # the replacement
       # On postgresql we can use `regexp_replace` for that.
       # On mysql we find the location of the pattern, and overwrite it
@@ -803,7 +937,7 @@ database (#{dbname}) using a super user and running:
 
 For MySQL you instead need to run:
 
-    GRANT ALL PRIVILEGES ON *.* TO #{user}@'%'
+    GRANT ALL PRIVILEGES ON #{dbname}.* TO #{user}@'%'
 
 Both queries will grant the user super user permissions, ensuring you don't run
 into similar problems in the future (e.g. when new tables are created).
@@ -839,9 +973,10 @@ into similar problems in the future (e.g. when new tables are created).
         raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id')
 
         jobs = []
+        table_name = model_class.quoted_table_name
 
         model_class.each_batch(of: batch_size) do |relation|
-          start_id, end_id = relation.pluck('MIN(id), MAX(id)').first
+          start_id, end_id = relation.pluck("MIN(#{table_name}.id), MAX(#{table_name}.id)").first
 
           if jobs.length >= BACKGROUND_MIGRATION_JOB_BUFFER_SIZE
             # Note: This code path generally only helps with many millions of rows
@@ -888,8 +1023,8 @@ into similar problems in the future (e.g. when new tables are created).
 
         # To not overload the worker too much we enforce a minimum interval both
         # when scheduling and performing jobs.
-        if delay_interval < BackgroundMigrationWorker::MIN_INTERVAL
-          delay_interval = BackgroundMigrationWorker::MIN_INTERVAL
+        if delay_interval < BackgroundMigrationWorker.minimum_interval
+          delay_interval = BackgroundMigrationWorker.minimum_interval
         end
 
         model_class.each_batch(of: batch_size) do |relation, index|
@@ -939,6 +1074,10 @@ into similar problems in the future (e.g. when new tables are created).
 
         connection.select_value(index_sql).to_i > 0
       end
+
+      def mysql_compatible_index_length
+        Gitlab::Database.mysql? ? 20 : nil
+      end
     end
   end
 end
diff --git a/lib/gitlab/database/multi_threaded_migration.rb b/lib/gitlab/database/multi_threaded_migration.rb
index 7ae5a4c17c8..1d39a3d0b57 100644
--- a/lib/gitlab/database/multi_threaded_migration.rb
+++ b/lib/gitlab/database/multi_threaded_migration.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Database
     module MultiThreadedMigration
diff --git a/lib/gitlab/database/read_only_relation.rb b/lib/gitlab/database/read_only_relation.rb
index 4571ad122ce..2362208e5dd 100644
--- a/lib/gitlab/database/read_only_relation.rb
+++ b/lib/gitlab/database/read_only_relation.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Database
     # Module that can be injected into a ActiveRecord::Relation to make it
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb
index f333ff22300..2314246da55 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # This module can be included in migrations to make it easier to rename paths
 # of `Namespace` & `Project` models certain paths would become `reserved`.
 #
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb
index 62d4d0a92a6..f1dc3ed74fe 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Database
     module RenameReservedPathsMigration
@@ -37,6 +39,7 @@ module Gitlab
           class Namespace < ActiveRecord::Base
             include MigrationClasses::Routable
             self.table_name = 'namespaces'
+            self.inheritance_column = :_type_disabled
             belongs_to :parent,
                        class_name: "#{MigrationClasses.name}::Namespace"
             has_one :route, as: :source
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
index 14de28a1d08..60afa4bcd52 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
@@ -1,10 +1,10 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Database
     module RenameReservedPathsMigration
       module V1
         class RenameBase
-          include Gitlab::Database::ArelMethods
-
           attr_reader :paths, :migration
 
           delegate :update_column_in_batches,
@@ -64,7 +64,7 @@ module Gitlab
                                             old_full_path,
                                             new_full_path)
 
-            update = arel_update_manager
+            update = Arel::UpdateManager.new
               .table(routes)
               .set([[routes[:path], replace_statement]])
               .where(Arel::Nodes::SqlLiteral.new(filter))
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
index 73971af6a74..6bbad707f0f 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Database
     module RenameReservedPathsMigration
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
index 827aeb12a02..580be9fe267 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Database
     module RenameReservedPathsMigration
diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb
index b2d8ee81977..8d97adaff99 100644
--- a/lib/gitlab/database/sha_attribute.rb
+++ b/lib/gitlab/database/sha_attribute.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Database
     BINARY_TYPE =
@@ -6,14 +8,7 @@ module Gitlab
         # behaviour from the default Binary type.
         ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea
       else
-        # In Rails 5.0 `Type` has been moved from `ActiveRecord` to `ActiveModel`
-        # https://github.com/rails/rails/commit/9cc8c6f3730df3d94c81a55be9ee1b7b4ffd29f6#diff-f8ba7983a51d687976e115adcd95822b
-        # Remove this method and leave just `ActiveModel::Type::Binary` when removing Gitlab.rails5? code.
-        if Gitlab.rails5?
-          ActiveModel::Type::Binary
-        else
-          ActiveRecord::Type::Binary
-        end
+        ActiveModel::Type::Binary
       end
 
     # Class for casting binary data to hexadecimal SHA1 hashes (and vice-versa).
@@ -24,31 +19,9 @@ module Gitlab
     class ShaAttribute < BINARY_TYPE
       PACK_FORMAT = 'H*'.freeze
 
-      # It is called from activerecord-4.2.10/lib/active_record internal methods.
-      # Remove this method when removing Gitlab.rails5? code.
-      def type_cast_from_database(value)
-        unpack_sha(super)
-      end
-
-      # It is called from activerecord-4.2.10/lib/active_record internal methods.
-      # Remove this method when removing Gitlab.rails5? code.
-      def type_cast_for_database(value)
-        serialize(value)
-      end
-
-      # It is called from activerecord-5.0.6/lib/active_record/attribute.rb
-      # Remove this method when removing Gitlab.rails5? code..
-      def deserialize(value)
-        value = Gitlab.rails5? ? super : method(:type_cast_from_database).super_method.call(value)
-
-        unpack_sha(value)
-      end
-
-      # Rename this method to `deserialize(value)` removing Gitlab.rails5? code.
       # Casts binary data to a SHA1 in hexadecimal.
-      def unpack_sha(value)
-        # Uncomment this line when removing Gitlab.rails5? code.
-        # value = super
+      def deserialize(value)
+        value = super(value)
         value ? value.unpack(PACK_FORMAT)[0] : nil
       end
 
@@ -56,7 +29,7 @@ module Gitlab
       def serialize(value)
         arg = value ? [value].pack(PACK_FORMAT) : nil
 
-        Gitlab.rails5? ? super(arg) : method(:type_cast_for_database).super_method.call(arg)
+        super(arg)
       end
     end
   end
diff --git a/lib/gitlab/database/subquery.rb b/lib/gitlab/database/subquery.rb
new file mode 100644
index 00000000000..10971d2b274
--- /dev/null
+++ b/lib/gitlab/database/subquery.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module Subquery
+      class << self
+        def self_join(relation)
+          t = relation.arel_table
+          # Work around a bug in Rails 5, where LIMIT causes trouble
+          # See https://gitlab.com/gitlab-org/gitlab-ce/issues/51729
+          r = relation.limit(nil).arel
+          r.take(relation.limit_value) if relation.limit_value
+          t2 = r.as('t2')
+
+          relation.unscoped.joins(t.join(t2).on(t[:id].eq(t2[:id])).join_sources.first)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker.rb b/lib/gitlab/dependency_linker.rb
index 3192bf6f667..c63d9e5bb71 100644
--- a/lib/gitlab/dependency_linker.rb
+++ b/lib/gitlab/dependency_linker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DependencyLinker
     LINKERS = [
diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb
index d2360583741..ac2efe598b4 100644
--- a/lib/gitlab/dependency_linker/base_linker.rb
+++ b/lib/gitlab/dependency_linker/base_linker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DependencyLinker
     class BaseLinker
diff --git a/lib/gitlab/dependency_linker/cartfile_linker.rb b/lib/gitlab/dependency_linker/cartfile_linker.rb
index 4f69f2c4ab2..0e33f0956dd 100644
--- a/lib/gitlab/dependency_linker/cartfile_linker.rb
+++ b/lib/gitlab/dependency_linker/cartfile_linker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DependencyLinker
     class CartfileLinker < MethodLinker
diff --git a/lib/gitlab/dependency_linker/cocoapods.rb b/lib/gitlab/dependency_linker/cocoapods.rb
index 2fbde7da1b4..38eabe303de 100644
--- a/lib/gitlab/dependency_linker/cocoapods.rb
+++ b/lib/gitlab/dependency_linker/cocoapods.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DependencyLinker
     module Cocoapods
diff --git a/lib/gitlab/dependency_linker/composer_json_linker.rb b/lib/gitlab/dependency_linker/composer_json_linker.rb
index cfd4ec15125..22d2bead891 100644
--- a/lib/gitlab/dependency_linker/composer_json_linker.rb
+++ b/lib/gitlab/dependency_linker/composer_json_linker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DependencyLinker
     class ComposerJsonLinker < PackageJsonLinker
diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb
index bfea836bcb2..8ab219c4962 100644
--- a/lib/gitlab/dependency_linker/gemfile_linker.rb
+++ b/lib/gitlab/dependency_linker/gemfile_linker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DependencyLinker
     class GemfileLinker < MethodLinker
diff --git a/lib/gitlab/dependency_linker/gemspec_linker.rb b/lib/gitlab/dependency_linker/gemspec_linker.rb
index f1783ee2ab4..b924ea86d89 100644
--- a/lib/gitlab/dependency_linker/gemspec_linker.rb
+++ b/lib/gitlab/dependency_linker/gemspec_linker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DependencyLinker
     class GemspecLinker < MethodLinker
diff --git a/lib/gitlab/dependency_linker/godeps_json_linker.rb b/lib/gitlab/dependency_linker/godeps_json_linker.rb
index fe091baee6d..d24c137793e 100644
--- a/lib/gitlab/dependency_linker/godeps_json_linker.rb
+++ b/lib/gitlab/dependency_linker/godeps_json_linker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DependencyLinker
     class GodepsJsonLinker < JsonLinker
diff --git a/lib/gitlab/dependency_linker/json_linker.rb b/lib/gitlab/dependency_linker/json_linker.rb
index a8ef25233d8..298d214df61 100644
--- a/lib/gitlab/dependency_linker/json_linker.rb
+++ b/lib/gitlab/dependency_linker/json_linker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DependencyLinker
     class JsonLinker < BaseLinker
diff --git a/lib/gitlab/dependency_linker/method_linker.rb b/lib/gitlab/dependency_linker/method_linker.rb
index 0ffa2a83c93..d4d85bb3390 100644
--- a/lib/gitlab/dependency_linker/method_linker.rb
+++ b/lib/gitlab/dependency_linker/method_linker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DependencyLinker
     class MethodLinker < BaseLinker
diff --git a/lib/gitlab/dependency_linker/package_json_linker.rb b/lib/gitlab/dependency_linker/package_json_linker.rb
index 330c95f0880..578e25f806a 100644
--- a/lib/gitlab/dependency_linker/package_json_linker.rb
+++ b/lib/gitlab/dependency_linker/package_json_linker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DependencyLinker
     class PackageJsonLinker < JsonLinker
diff --git a/lib/gitlab/dependency_linker/podfile_linker.rb b/lib/gitlab/dependency_linker/podfile_linker.rb
index 60ad166ea17..def9b04cca9 100644
--- a/lib/gitlab/dependency_linker/podfile_linker.rb
+++ b/lib/gitlab/dependency_linker/podfile_linker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DependencyLinker
     class PodfileLinker < GemfileLinker
diff --git a/lib/gitlab/dependency_linker/podspec_json_linker.rb b/lib/gitlab/dependency_linker/podspec_json_linker.rb
index d82237ed3f1..1a2493e7cc0 100644
--- a/lib/gitlab/dependency_linker/podspec_json_linker.rb
+++ b/lib/gitlab/dependency_linker/podspec_json_linker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DependencyLinker
     class PodspecJsonLinker < JsonLinker
diff --git a/lib/gitlab/dependency_linker/podspec_linker.rb b/lib/gitlab/dependency_linker/podspec_linker.rb
index 924e55e9820..6b1758c5a43 100644
--- a/lib/gitlab/dependency_linker/podspec_linker.rb
+++ b/lib/gitlab/dependency_linker/podspec_linker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DependencyLinker
     class PodspecLinker < MethodLinker
diff --git a/lib/gitlab/dependency_linker/requirements_txt_linker.rb b/lib/gitlab/dependency_linker/requirements_txt_linker.rb
index 9c9620bc36a..f630c13b760 100644
--- a/lib/gitlab/dependency_linker/requirements_txt_linker.rb
+++ b/lib/gitlab/dependency_linker/requirements_txt_linker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module DependencyLinker
     class RequirementsTxtLinker < BaseLinker
diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb
index 81df47964be..dc245377ccc 100644
--- a/lib/gitlab/diff/diff_refs.rb
+++ b/lib/gitlab/diff/diff_refs.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Diff
     class DiffRefs
@@ -21,7 +23,7 @@ module Gitlab
       alias_method :eql?, :==
 
       def hash
-        [base_sha, start_sha, head_sha].hash
+        [self.class, base_sha, start_sha, head_sha].hash
       end
 
       # There is only one case in which we will have `start_sha` and `head_sha`,
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index 765fb0289a8..e410d5a8333 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -1,7 +1,11 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Diff
     class File
-      attr_reader :diff, :repository, :diff_refs, :fallback_diff_refs
+      include Gitlab::Utils::StrongMemoize
+
+      attr_reader :diff, :repository, :diff_refs, :fallback_diff_refs, :unique_identifier
 
       delegate :new_file?, :deleted_file?, :renamed_file?,
         :old_path, :new_path, :a_mode, :b_mode, :mode_changed?,
@@ -20,11 +24,21 @@ module Gitlab
         DiffViewer::Image
       ].sort_by { |v| v.binary? ? 0 : 1 }.freeze
 
-      def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil)
+      def initialize(
+        diff,
+        repository:,
+        diff_refs: nil,
+        fallback_diff_refs: nil,
+        stats: nil,
+        unique_identifier: nil)
+
         @diff = diff
+        @stats = stats
         @repository = repository
         @diff_refs = diff_refs
         @fallback_diff_refs = fallback_diff_refs
+        @unique_identifier = unique_identifier
+        @unfolded = false
 
         # Ensure items are collected in the the batch
         new_blob_lazy
@@ -63,7 +77,15 @@ module Gitlab
       def line_for_position(pos)
         return nil unless pos.position_type == 'text'
 
-        diff_lines.find { |line| line.old_line == pos.old_line && line.new_line == pos.new_line }
+        # This method is normally used to find which line the diff was
+        # commented on, and in this context, it's normally the raw diff persisted
+        # at `note_diff_files`, which is a fraction of the entire diff
+        # (it goes from the first line, to the commented line, or
+        # one line below). Therefore it's more performant to fetch
+        # from bottom to top instead of the other way around.
+        diff_lines
+          .reverse_each
+          .find { |line| line.old_line == pos.old_line && line.new_line == pos.new_line }
       end
 
       def position_for_line_code(code)
@@ -78,9 +100,12 @@ module Gitlab
 
       # Returns the raw diff content up to the given line index
       def diff_hunk(diff_line)
-        # Adding 2 because of the @@ diff header and Enum#take should consider
-        # an extra line, because we're passing an index.
-        raw_diff.each_line.take(diff_line.index + 2).join
+        diff_line_index = diff_line.index
+        # @@ (match) header is not kept if it's found in the top of the file,
+        # therefore we should keep an extra line on this scenario.
+        diff_line_index += 1 unless diff_lines.first.match?
+
+        diff_lines.select { |line| line.index <= diff_line_index }.map(&:text).join("\n")
       end
 
       def old_sha
@@ -115,6 +140,16 @@ module Gitlab
         old_blob_lazy&.itself
       end
 
+      def new_blob_lines_between(from_line, to_line)
+        return [] unless new_blob
+
+        from_index = from_line - 1
+        to_index = to_line - 1
+
+        new_blob.load_all_data!
+        new_blob.data.lines[from_index..to_index]
+      end
+
       def content_sha
         new_content_sha || old_content_sha
       end
@@ -127,11 +162,35 @@ module Gitlab
 
       # Array of Gitlab::Diff::Line objects
       def diff_lines
-        @diff_lines ||= Gitlab::Diff::Parser.new.parse(raw_diff.each_line).to_a
+        @diff_lines ||=
+          Gitlab::Diff::Parser.new.parse(raw_diff.each_line, diff_file: self).to_a
+      end
+
+      # Changes diff_lines according to the given position. That is,
+      # it checks whether the position requires blob lines into the diff
+      # in order to be presented.
+      def unfold_diff_lines(position)
+        return unless position
+
+        unfolder = Gitlab::Diff::LinesUnfolder.new(self, position)
+
+        if unfolder.unfold_required?
+          @diff_lines = unfolder.unfolded_diff_lines
+          @unfolded = true
+        end
+      end
+
+      def unfolded?
+        @unfolded
+      end
+
+      def highlight_loaded?
+        @highlighted_diff_lines.present?
       end
 
       def highlighted_diff_lines
-        @highlighted_diff_lines ||= Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight
+        @highlighted_diff_lines ||=
+          Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight
       end
 
       # Array[<Hash>] with right/left keys that contains Gitlab::Diff::Line objects which text is hightlighted
@@ -160,11 +219,11 @@ module Gitlab
       end
 
       def added_lines
-        diff_lines.count(&:added?)
+        @stats&.additions || diff_lines.count(&:added?)
       end
 
       def removed_lines
-        diff_lines.count(&:removed?)
+        @stats&.deletions || diff_lines.count(&:removed?)
       end
 
       def file_identifier
@@ -175,12 +234,12 @@ module Gitlab
         repository.attributes(file_path).fetch('diff') { true }
       end
 
-      def binary?
-        has_binary_notice? || try_blobs(:binary?)
+      def binary_in_repo?
+        has_binary_notice? || try_blobs(:binary_in_repo?)
       end
 
-      def text?
-        !binary?
+      def text_in_repo?
+        !binary_in_repo?
       end
 
       def external_storage_error?
@@ -206,20 +265,32 @@ module Gitlab
         old_blob && new_blob && old_blob.binary? != new_blob.binary?
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def size
         valid_blobs.map(&:size).sum
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def raw_size
         valid_blobs.map(&:raw_size).sum
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
-      def raw_binary?
-        try_blobs(:raw_binary?)
+      def empty?
+        valid_blobs.map(&:empty?).all?
       end
 
-      def raw_text?
-        !raw_binary? && !different_type?
+      def binary?
+        strong_memoize(:is_binary) do
+          try_blobs(:binary?)
+        end
+      end
+
+      def text?
+        strong_memoize(:is_text) do
+          !binary? && !different_type?
+        end
       end
 
       def simple_viewer
@@ -236,8 +307,34 @@ module Gitlab
         simple_viewer.is_a?(DiffViewer::Text) && (ignore_errors || simple_viewer.render_error.nil?)
       end
 
+      # This adds the bottom match line to the array if needed. It contains
+      # the data to load more context lines.
+      def diff_lines_for_serializer
+        lines = highlighted_diff_lines
+
+        return if lines.empty?
+        return if blob.nil?
+
+        last_line = lines.last
+
+        if last_line.new_pos < total_blob_lines(blob) && !deleted_file?
+          match_line = Gitlab::Diff::Line.new("", 'match', nil, last_line.old_pos, last_line.new_pos)
+          lines.push(match_line)
+        end
+
+        lines
+      end
+
       private
 
+      def total_blob_lines(blob)
+        @total_lines ||= begin
+          line_count = blob.lines.size
+          line_count -= 1 if line_count > 0 && blob.lines.last.blank?
+          line_count
+        end
+      end
+
       # We can't use Object#try because Blob doesn't inherit from Object, but
       # from BasicObject (via SimpleDelegator).
       def try_blobs(meth)
@@ -276,19 +373,19 @@ module Gitlab
         return DiffViewer::NotDiffable unless diffable?
 
         if content_changed?
-          if raw_text?
+          if text?
             DiffViewer::Text
           else
             DiffViewer::NoPreview
           end
         elsif new_file?
-          if raw_text?
+          if text?
             DiffViewer::Text
           else
             DiffViewer::Added
           end
         elsif deleted_file?
-          if raw_text?
+          if text?
             DiffViewer::Text
           else
             DiffViewer::Deleted
diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb
index c79d8d3cb21..c5bbf522f7c 100644
--- a/lib/gitlab/diff/file_collection/base.rb
+++ b/lib/gitlab/diff/file_collection/base.rb
@@ -1,28 +1,47 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Diff
     module FileCollection
       class Base
-        attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs
+        include Gitlab::Utils::StrongMemoize
+
+        attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs, :diffable
 
         delegate :count, :size, :real_size, to: :diff_files
 
         def self.default_options
-          ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false)
+          ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false, include_stats: true)
         end
 
         def initialize(diffable, project:, diff_options: nil, diff_refs: nil, fallback_diff_refs: nil)
           diff_options = self.class.default_options.merge(diff_options || {})
 
           @diffable = diffable
-          @diffs = diffable.raw_diffs(diff_options)
+          @include_stats = diff_options.delete(:include_stats)
           @project = project
           @diff_options = diff_options
           @diff_refs = diff_refs
           @fallback_diff_refs = fallback_diff_refs
+          @repository = project.repository
+        end
+
+        def diffs
+          @diffs ||= diffable.raw_diffs(diff_options)
         end
 
         def diff_files
-          @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) }
+          @diff_files ||= diffs.decorate! { |diff| decorate_diff!(diff) }
+        end
+
+        # This mutates `diff_files` lines.
+        def unfold_diff_files(positions)
+          positions_grouped_by_path = positions.group_by { |position| position.file_path }
+
+          diff_files.each do |diff_file|
+            positions = positions_grouped_by_path.fetch(diff_file.file_path, [])
+            positions.each { |position| diff_file.unfold_diff_lines(position) }
+          end
         end
 
         def diff_file_with_old_path(old_path)
@@ -33,12 +52,37 @@ module Gitlab
           diff_files.find { |diff_file| diff_file.new_path == new_path }
         end
 
+        def clear_cache
+          # No-op
+        end
+
+        def write_cache
+          # No-op
+        end
+
         private
 
+        def diff_stats_collection
+          strong_memoize(:diff_stats) do
+            # There are scenarios where we don't need to request Diff Stats,
+            # when caching for instance.
+            next unless @include_stats
+            next unless diff_refs
+
+            @repository.diff_stats(diff_refs.base_sha, diff_refs.head_sha)
+          end
+        end
+
         def decorate_diff!(diff)
           return diff if diff.is_a?(File)
 
-          Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs, fallback_diff_refs: fallback_diff_refs)
+          stats = diff_stats_collection&.find_by_path(diff.new_path)
+
+          Gitlab::Diff::File.new(diff,
+                                 repository: project.repository,
+                                 diff_refs: diff_refs,
+                                 fallback_diff_refs: fallback_diff_refs,
+                                 stats: stats)
         end
       end
     end
diff --git a/lib/gitlab/diff/file_collection/commit.rb b/lib/gitlab/diff/file_collection/commit.rb
index 4dc297ec036..7b1d6171e82 100644
--- a/lib/gitlab/diff/file_collection/commit.rb
+++ b/lib/gitlab/diff/file_collection/commit.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Diff
     module FileCollection
diff --git a/lib/gitlab/diff/file_collection/compare.rb b/lib/gitlab/diff/file_collection/compare.rb
index 20d8f891cc3..663bad95db7 100644
--- a/lib/gitlab/diff/file_collection/compare.rb
+++ b/lib/gitlab/diff/file_collection/compare.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Diff
     module FileCollection
@@ -8,6 +10,10 @@ module Gitlab
             diff_options: diff_options,
             diff_refs:    diff_refs)
         end
+
+        def unfold_diff_lines(positions)
+          # no-op
+        end
       end
     end
   end
diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb
index c358ae428cf..e29bf75f341 100644
--- a/lib/gitlab/diff/file_collection/merge_request_diff.rb
+++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb
@@ -1,7 +1,11 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Diff
     module FileCollection
       class MergeRequestDiff < Base
+        extend ::Gitlab::Utils::Override
+
         def initialize(merge_request_diff, diff_options:)
           @merge_request_diff = merge_request_diff
 
@@ -13,70 +17,35 @@ module Gitlab
         end
 
         def diff_files
-          # Make sure to _not_ send any method call to Gitlab::Diff::File
-          # _before_ all of them were collected (`super`). Premature method calls will
-          # trigger N+1 RPCs to Gitaly through BatchLoader records (Blob.lazy).
-          #
           diff_files = super
 
-          diff_files.each { |diff_file| cache_highlight!(diff_file) if cacheable?(diff_file) }
-          store_highlight_cache
+          diff_files.each { |diff_file| cache.decorate(diff_file) }
 
           diff_files
         end
 
-        def real_size
-          @merge_request_diff.real_size
+        override :write_cache
+        def write_cache
+          cache.write_if_empty
         end
 
-        def clear_cache!
-          Rails.cache.delete(cache_key)
+        override :clear_cache
+        def clear_cache
+          cache.clear
         end
 
         def cache_key
-          [@merge_request_diff, 'highlighted-diff-files', diff_options]
-        end
-
-        private
-
-        def highlight_diff_file_from_cache!(diff_file, cache_diff_lines)
-          diff_file.highlighted_diff_lines = cache_diff_lines.map do |line|
-            Gitlab::Diff::Line.init_from_hash(line)
-          end
+          cache.key
         end
 
-        #
-        # If we find the highlighted diff files lines on the cache we replace existing diff_files lines (no highlighted)
-        # for the highlighted ones, so we just skip their execution.
-        # If the highlighted diff files lines are not cached we calculate and cache them.
-        #
-        # The content of the cache is a Hash where the key identifies the file and the values are Arrays of
-        # hashes that represent serialized diff lines.
-        #
-        def cache_highlight!(diff_file)
-          item_key = diff_file.file_identifier
-
-          if highlight_cache[item_key]
-            highlight_diff_file_from_cache!(diff_file, highlight_cache[item_key])
-          else
-            highlight_cache[item_key] = diff_file.highlighted_diff_lines.map(&:to_hash)
-          end
-        end
-
-        def highlight_cache
-          return @highlight_cache if defined?(@highlight_cache)
-
-          @highlight_cache = Rails.cache.read(cache_key) || {}
-          @highlight_cache_was_empty = @highlight_cache.empty?
-          @highlight_cache
+        def real_size
+          @merge_request_diff.real_size
         end
 
-        def store_highlight_cache
-          Rails.cache.write(cache_key, highlight_cache, expires_in: 1.week) if @highlight_cache_was_empty
-        end
+        private
 
-        def cacheable?(diff_file)
-          @merge_request_diff.present? && diff_file.text? && diff_file.diffable?
+        def cache
+          @cache ||= Gitlab::Diff::HighlightCache.new(self)
         end
       end
     end
diff --git a/lib/gitlab/diff/formatters/base_formatter.rb b/lib/gitlab/diff/formatters/base_formatter.rb
index 5e923b9e602..9704aed82c1 100644
--- a/lib/gitlab/diff/formatters/base_formatter.rb
+++ b/lib/gitlab/diff/formatters/base_formatter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Diff
     module Formatters
diff --git a/lib/gitlab/diff/formatters/image_formatter.rb b/lib/gitlab/diff/formatters/image_formatter.rb
index ccd0d309972..5bc9f0c337f 100644
--- a/lib/gitlab/diff/formatters/image_formatter.rb
+++ b/lib/gitlab/diff/formatters/image_formatter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Diff
     module Formatters
diff --git a/lib/gitlab/diff/formatters/text_formatter.rb b/lib/gitlab/diff/formatters/text_formatter.rb
index 01c7e9f51ab..f6e247ef665 100644
--- a/lib/gitlab/diff/formatters/text_formatter.rb
+++ b/lib/gitlab/diff/formatters/text_formatter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Diff
     module Formatters
diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb
index 5c1baa19b66..d2484217ab9 100644
--- a/lib/gitlab/diff/highlight.rb
+++ b/lib/gitlab/diff/highlight.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Diff
     class Highlight
@@ -24,7 +26,7 @@ module Gitlab
           # ignore highlighting for "match" lines
           next diff_line if diff_line.meta?
 
-          rich_line = highlight_line(diff_line) || diff_line.text
+          rich_line = highlight_line(diff_line) || ERB::Util.html_escape(diff_line.text)
 
           if line_inline_diffs = inline_diffs[i]
             begin
@@ -37,7 +39,7 @@ module Gitlab
             end
           end
 
-          diff_line.text = rich_line
+          diff_line.rich_text = rich_line
 
           diff_line
         end
@@ -79,7 +81,7 @@ module Gitlab
         return [] unless blob
 
         blob.load_all_data!
-        Gitlab::Highlight.highlight(blob.path, blob.data, repository: repository).lines
+        blob.present.highlight.lines
       end
     end
   end
diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb
new file mode 100644
index 00000000000..e4390771db2
--- /dev/null
+++ b/lib/gitlab/diff/highlight_cache.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+#
+module Gitlab
+  module Diff
+    class HighlightCache
+      delegate :diffable, to: :@diff_collection
+      delegate :diff_options, to: :@diff_collection
+
+      def initialize(diff_collection, backend: Rails.cache)
+        @backend = backend
+        @diff_collection = diff_collection
+      end
+
+      # - Reads from cache
+      # - Assigns DiffFile#highlighted_diff_lines for cached files
+      def decorate(diff_file)
+        if content = read_file(diff_file)
+          diff_file.highlighted_diff_lines = content.map do |line|
+            Gitlab::Diff::Line.init_from_hash(line)
+          end
+        end
+      end
+
+      # It populates a Hash in order to submit a single write to the memory
+      # cache. This avoids excessive IO generated by N+1's (1 writing for
+      # each highlighted line or file).
+      def write_if_empty
+        return if cached_content.present?
+
+        @diff_collection.diff_files.each do |diff_file|
+          next unless cacheable?(diff_file)
+
+          diff_file_id = diff_file.file_identifier
+
+          cached_content[diff_file_id] = diff_file.highlighted_diff_lines.map(&:to_hash)
+        end
+
+        cache.write(key, cached_content, expires_in: 1.week)
+      end
+
+      def clear
+        cache.delete(key)
+      end
+
+      def key
+        [diffable, 'highlighted-diff-files', Gitlab::Diff::Line::SERIALIZE_KEYS, diff_options]
+      end
+
+      private
+
+      def read_file(diff_file)
+        cached_content[diff_file.file_identifier]
+      end
+
+      def cache
+        @backend
+      end
+
+      def cached_content
+        @cached_content ||= cache.read(key) || {}
+      end
+
+      def cacheable?(diff_file)
+        diffable.present? && diff_file.text? && diff_file.diffable?
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/image_point.rb b/lib/gitlab/diff/image_point.rb
index 65332dfd239..a3ce032f8e2 100644
--- a/lib/gitlab/diff/image_point.rb
+++ b/lib/gitlab/diff/image_point.rb
@@ -1,13 +1,15 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Diff
     class ImagePoint
       attr_reader :width, :height, :x, :y
 
-      def initialize(width, height, x, y)
+      def initialize(width, height, new_x, new_y)
         @width = width
         @height = height
-        @x = x
-        @y = y
+        @x = new_x
+        @y = new_y
       end
 
       def to_h
diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb
index 54783a07919..5815d1bae4a 100644
--- a/lib/gitlab/diff/inline_diff.rb
+++ b/lib/gitlab/diff/inline_diff.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Diff
     class InlineDiff
@@ -67,10 +69,11 @@ module Gitlab
         private
 
         # Finds pairs of old/new line pairs that represent the same line that changed
+        # rubocop: disable CodeReuse/ActiveRecord
         def find_changed_line_pairs(lines)
           # Prefixes of all diff lines, indicating their types
           # For example: `" - +  -+  ---+++ --+  -++"`
-          line_prefixes = lines.each_with_object("") { |line, s| s << (line[0] || ' ') }.gsub(/[^ +-]/, ' ')
+          line_prefixes = lines.each_with_object(+"") { |line, s| s << (line[0] || ' ') }.gsub(/[^ +-]/, ' ')
 
           changed_line_pairs = []
           line_prefixes.scan(LINE_PAIRS_PATTERN) do
@@ -89,11 +92,12 @@ module Gitlab
 
           changed_line_pairs
         end
+        # rubocop: enable CodeReuse/ActiveRecord
       end
 
       private
 
-      def longest_common_prefix(a, b)
+      def longest_common_prefix(a, b) # rubocop:disable Naming/UncommunicativeMethodParamName
         max_length = [a.length, b.length].max
 
         length = 0
@@ -109,7 +113,7 @@ module Gitlab
         length
       end
 
-      def longest_common_suffix(a, b)
+      def longest_common_suffix(a, b) # rubocop:disable Naming/UncommunicativeMethodParamName
         longest_common_prefix(a.reverse, b.reverse)
       end
     end
diff --git a/lib/gitlab/diff/inline_diff_markdown_marker.rb b/lib/gitlab/diff/inline_diff_markdown_marker.rb
index c2a2eb15931..3c536c43a9e 100644
--- a/lib/gitlab/diff/inline_diff_markdown_marker.rb
+++ b/lib/gitlab/diff/inline_diff_markdown_marker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Diff
     class InlineDiffMarkdownMarker < Gitlab::StringRangeMarker
diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb
index 81e91ea0ab7..1bbde1ffd2a 100644
--- a/lib/gitlab/diff/inline_diff_marker.rb
+++ b/lib/gitlab/diff/inline_diff_marker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Diff
     class InlineDiffMarker < Gitlab::StringRangeMarker
diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb
index 0603141e441..001748afb41 100644
--- a/lib/gitlab/diff/line.rb
+++ b/lib/gitlab/diff/line.rb
@@ -1,27 +1,39 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Diff
     class Line
-      attr_reader :type, :index, :old_pos, :new_pos
+      SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze
+
+      attr_reader :line_code, :type, :old_pos, :new_pos
       attr_writer :rich_text
-      attr_accessor :text
+      attr_accessor :text, :index
 
-      def initialize(text, type, index, old_pos, new_pos, parent_file: nil)
+      def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil)
         @text, @type, @index = text, type, index
         @old_pos, @new_pos = old_pos, new_pos
         @parent_file = parent_file
-      end
+        @rich_text = rich_text
 
-      def self.init_from_hash(hash)
-        new(hash[:text], hash[:type], hash[:index], hash[:old_pos], hash[:new_pos])
+        # When line code is not provided from cache store we build it
+        # using the parent_file(Diff::File or Conflict::File).
+        @line_code = line_code || calculate_line_code
       end
 
-      def serialize_keys
-        @serialize_keys ||= %i(text type index old_pos new_pos)
+      def self.init_from_hash(hash)
+        new(hash[:text],
+            hash[:type],
+            hash[:index],
+            hash[:old_pos],
+            hash[:new_pos],
+            parent_file: hash[:parent_file],
+            line_code: hash[:line_code],
+            rich_text: hash[:rich_text])
       end
 
       def to_hash
         hash = {}
-        serialize_keys.each { |key| hash[key] = send(key) } # rubocop:disable GitlabSecurity/PublicSend
+        SERIALIZE_KEYS.each { |key| hash[key] = send(key) } # rubocop:disable GitlabSecurity/PublicSend
         hash
       end
 
@@ -53,25 +65,44 @@ module Gitlab
         %w[match new-nonewline old-nonewline].include?(type)
       end
 
+      def match?
+        type == :match
+      end
+
       def discussable?
         !meta?
       end
 
+      def suggestible?
+        !removed?
+      end
+
       def rich_text
-        @parent_file.highlight_lines! if @parent_file && !@rich_text
+        @parent_file.try(:highlight_lines!) if @parent_file && !@rich_text
 
         @rich_text
       end
 
-      def as_json(opts = nil)
+      def meta_positions
+        return unless meta?
+
         {
-          type: type,
-          old_line: old_line,
-          new_line: new_line,
-          text: text,
-          rich_text: rich_text || text
+          old_pos: old_pos,
+          new_pos: new_pos
         }
       end
+
+      # We have to keep this here since it is still used for conflict resolution
+      # Conflict::File#as_json renders json diff lines in sections
+      def as_json(opts = nil)
+        DiffLineSerializer.new.represent(self)
+      end
+
+      private
+
+      def calculate_line_code
+        @parent_file&.line_code(self)
+      end
     end
   end
 end
diff --git a/lib/gitlab/diff/line_mapper.rb b/lib/gitlab/diff/line_mapper.rb
index cf71d47df8e..fba7bff4781 100644
--- a/lib/gitlab/diff/line_mapper.rb
+++ b/lib/gitlab/diff/line_mapper.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # When provided a diff for a specific file, maps old line numbers to new line
 # numbers and back, to find out where a specific line in a file was moved by the
 # changes.
diff --git a/lib/gitlab/diff/lines_unfolder.rb b/lib/gitlab/diff/lines_unfolder.rb
new file mode 100644
index 00000000000..6cf904b2b2a
--- /dev/null
+++ b/lib/gitlab/diff/lines_unfolder.rb
@@ -0,0 +1,240 @@
+# frozen_string_literal: true
+
+# Given a position, calculates which Blob lines should be extracted, treated and
+# injected in the current diff file lines in order to present a "unfolded" diff.
+module Gitlab
+  module Diff
+    class LinesUnfolder
+      include Gitlab::Utils::StrongMemoize
+
+      UNFOLD_CONTEXT_SIZE = 3
+
+      def initialize(diff_file, position)
+        @diff_file = diff_file
+        @blob = diff_file.old_blob
+        @position = position
+        @generate_top_match_line = true
+        @generate_bottom_match_line = true
+
+        # These methods update `@generate_top_match_line` and
+        # `@generate_bottom_match_line`.
+        @from_blob_line = calculate_from_blob_line!
+        @to_blob_line = calculate_to_blob_line!
+      end
+
+      # Returns merged diff lines with required blob lines with correct
+      # positions.
+      def unfolded_diff_lines
+        strong_memoize(:unfolded_diff_lines) do
+          next unless unfold_required?
+
+          merged_diff_with_blob_lines
+        end
+      end
+
+      # Returns the extracted lines from the old blob which should be merged
+      # with the current diff lines.
+      def blob_lines
+        strong_memoize(:blob_lines) do
+          # Blob lines, unlike diffs, doesn't start with an empty space for
+          # unchanged line, so the parsing and highlighting step can get fuzzy
+          # without the following change.
+          line_prefix = ' '
+          blob_as_diff_lines = @blob.data.each_line.map { |line| "#{line_prefix}#{line}" }
+
+          lines = Gitlab::Diff::Parser.new.parse(blob_as_diff_lines, diff_file: @diff_file).to_a
+
+          from = from_blob_line - 1
+          to = to_blob_line - 1
+
+          lines[from..to]
+        end
+      end
+
+      def unfold_required?
+        strong_memoize(:unfold_required) do
+          next false unless @diff_file.text?
+          next false unless @position.unchanged?
+          next false if @diff_file.new_file? || @diff_file.deleted_file?
+          next false unless @position.old_line
+          # Invalid position (MR import scenario)
+          next false if @position.old_line > @blob.lines.size
+          next false if @diff_file.diff_lines.empty?
+          next false if @diff_file.line_for_position(@position)
+          next false unless unfold_line
+
+          true
+        end
+      end
+
+      private
+
+      attr_reader :from_blob_line, :to_blob_line
+
+      def merged_diff_with_blob_lines
+        lines = @diff_file.diff_lines
+        match_line = unfold_line
+        insert_index = bottom? ? -1 : match_line.index
+
+        lines -= [match_line] unless bottom?
+
+        lines.insert(insert_index, *blob_lines_with_matches)
+
+        # The inserted blob lines have invalid indexes, so we need
+        # to reindex them.
+        reindex(lines)
+
+        lines
+      end
+
+      # Returns 'unchanged' blob lines with recalculated `old_pos` and
+      # `new_pos` and the recalculated new match line (needed if we for instance
+      # we unfolded once, but there are still folded lines).
+      def blob_lines_with_matches
+        old_pos = from_blob_line
+        new_pos = from_blob_line + offset
+
+        new_blob_lines = []
+
+        new_blob_lines.push(top_blob_match_line) if top_blob_match_line
+
+        blob_lines.each do |line|
+          new_blob_lines << Gitlab::Diff::Line.new(line.text, line.type, nil, old_pos, new_pos,
+                                                   parent_file: @diff_file)
+
+          old_pos += 1
+          new_pos += 1
+        end
+
+        new_blob_lines.push(bottom_blob_match_line) if bottom_blob_match_line
+
+        new_blob_lines
+      end
+
+      def reindex(lines)
+        lines.each_with_index { |line, i| line.index = i }
+      end
+
+      def top_blob_match_line
+        strong_memoize(:top_blob_match_line) do
+          next unless @generate_top_match_line
+
+          old_pos = from_blob_line
+          new_pos = from_blob_line + offset
+
+          build_match_line(old_pos, new_pos)
+        end
+      end
+
+      def bottom_blob_match_line
+        strong_memoize(:bottom_blob_match_line) do
+          # The bottom line match addition is already handled on
+          # Diff::File#diff_lines_for_serializer
+          next if bottom?
+          next unless @generate_bottom_match_line
+
+          position = line_after_unfold_position.old_pos
+
+          old_pos = position
+          new_pos = position + offset
+
+          build_match_line(old_pos, new_pos)
+        end
+      end
+
+      def build_match_line(old_pos, new_pos)
+        blob_lines_length = blob_lines.length
+        old_line_ref = [old_pos, blob_lines_length].join(',')
+        new_line_ref = [new_pos, blob_lines_length].join(',')
+        new_match_line_str = "@@ -#{old_line_ref}+#{new_line_ref} @@"
+
+        Gitlab::Diff::Line.new(new_match_line_str, 'match', nil, old_pos, new_pos)
+      end
+
+      # Returns the first line position that should be extracted
+      # from `blob_lines`.
+      def calculate_from_blob_line!
+        return unless unfold_required?
+
+        from = comment_position - UNFOLD_CONTEXT_SIZE
+
+        prev_line_number =
+          if bottom?
+            last_line.old_pos
+          else
+            # There's no line before the match if it's in the top-most
+            # position.
+            line_before_unfold_position&.old_pos || 0
+          end
+
+        if from <= prev_line_number + 1
+          @generate_top_match_line = false
+          from = prev_line_number + 1
+        end
+
+        from
+      end
+
+      # Returns the last line position that should be extracted
+      # from `blob_lines`.
+      def calculate_to_blob_line!
+        return unless unfold_required?
+
+        to = comment_position + UNFOLD_CONTEXT_SIZE
+
+        return to if bottom?
+
+        next_line_number = line_after_unfold_position.old_pos
+
+        if to >= next_line_number - 1
+          @generate_bottom_match_line = false
+          to = next_line_number - 1
+        end
+
+        to
+      end
+
+      def offset
+        unfold_line.new_pos - unfold_line.old_pos
+      end
+
+      def line_before_unfold_position
+        return unless index = unfold_line&.index
+
+        @diff_file.diff_lines[index - 1] if index > 0
+      end
+
+      def line_after_unfold_position
+        return unless index = unfold_line&.index
+
+        @diff_file.diff_lines[index + 1] if index >= 0
+      end
+
+      def bottom?
+        strong_memoize(:bottom) do
+          @position.old_line > last_line.old_pos
+        end
+      end
+
+      # Returns the line which needed to be expanded in order to send a comment
+      # in `@position`.
+      def unfold_line
+        strong_memoize(:unfold_line) do
+          next last_line if bottom?
+
+          @diff_file.diff_lines.find do |line|
+            line.old_pos > comment_position && line.type == 'match'
+          end
+        end
+      end
+
+      def comment_position
+        @position.old_line
+      end
+
+      def last_line
+        @diff_file.diff_lines.last
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb
index 0cb26fa45c8..77b65fea726 100644
--- a/lib/gitlab/diff/parallel_diff.rb
+++ b/lib/gitlab/diff/parallel_diff.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Diff
     class ParallelDiff
diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb
index 8302f30a0a2..4a47e4b80b6 100644
--- a/lib/gitlab/diff/parser.rb
+++ b/lib/gitlab/diff/parser.rb
@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Diff
     class Parser
       include Enumerable
 
-      def parse(lines)
+      def parse(lines, diff_file: nil)
         return [] if lines.blank?
 
         @lines = lines
@@ -31,17 +33,17 @@ module Gitlab
 
               next if line_old <= 1 && line_new <= 1 # top of file
 
-              yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new)
+              yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: diff_file)
               line_obj_index += 1
               next
             elsif line[0] == '\\'
               type = "#{context}-nonewline"
 
-              yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new)
+              yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: diff_file)
               line_obj_index += 1
             else
               type = identification_type(line)
-              yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new)
+              yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: diff_file)
               line_obj_index += 1
             end
 
diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb
index 978962ab2eb..e8f98f52111 100644
--- a/lib/gitlab/diff/position.rb
+++ b/lib/gitlab/diff/position.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Defines a specific location, identified by paths line numbers and image coordinates,
 # within a specific diff, identified by start, head and base commit ids.
 module Gitlab
@@ -69,6 +71,10 @@ module Gitlab
         JSON.generate(formatter.to_h, opts)
       end
 
+      def as_json(opts = nil)
+        to_h.as_json(opts)
+      end
+
       def type
         formatter.line_age
       end
@@ -97,25 +103,29 @@ module Gitlab
         @diff_refs ||= DiffRefs.new(base_sha: base_sha, start_sha: start_sha, head_sha: head_sha)
       end
 
+      def unfolded_diff?(repository)
+        diff_file(repository)&.unfolded?
+      end
+
       def diff_file(repository)
         return @diff_file if defined?(@diff_file)
 
         @diff_file = begin
-          if RequestStore.active?
-            key = {
-              project_id: repository.project.id,
-              start_sha: start_sha,
-              head_sha: head_sha,
-              path: file_path
-            }
-
-            RequestStore.fetch(key) { find_diff_file(repository) }
-          else
-            find_diff_file(repository)
-          end
+          key = {
+            project_id: repository.project.id,
+            start_sha: start_sha,
+            head_sha: head_sha,
+            path: file_path
+          }
+
+          Gitlab::SafeRequestStore.fetch(key) { find_diff_file(repository) }
         end
       end
 
+      def diff_options
+        { paths: paths, expanded: true, include_stats: false }
+      end
+
       def diff_line(repository)
         @diff_line ||= diff_file(repository)&.line_for_position(self)
       end
@@ -130,7 +140,13 @@ module Gitlab
         return unless diff_refs.complete?
         return unless comparison = diff_refs.compare_in(repository.project)
 
-        comparison.diffs(paths: paths, expanded: true).diff_files.first
+        file = comparison.diffs(diff_options).diff_files.first
+
+        # We need to unfold diff lines according to the position in order
+        # to correctly calculate the line code and trace position changes.
+        file&.unfold_diff_lines(self)
+
+        file
       end
 
       def get_formatter_class(type)
diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb
index b68a1636814..af3df820422 100644
--- a/lib/gitlab/diff/position_tracer.rb
+++ b/lib/gitlab/diff/position_tracer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Finds the diff position in the new diff that corresponds to the same location
 # specified by the provided position in the old diff.
 module Gitlab
@@ -24,7 +26,7 @@ module Gitlab
         # head of `feature` was commit B, resulting in the original diff A->B.
         # Since creation, `master` was updated to C.
         # Now `feature` is being updated to D, and the newly generated MR diff is C->D.
-        # It is possible that C and D are direct decendants of A and B respectively,
+        # It is possible that C and D are direct descendants of A and B respectively,
         # but this isn't necessarily the case as rebases and merges come into play.
         #
         # Suppose we have a diff note on the original diff A->B. Now that the MR
diff --git a/lib/gitlab/discussions_diff/file_collection.rb b/lib/gitlab/discussions_diff/file_collection.rb
new file mode 100644
index 00000000000..4ab7314f509
--- /dev/null
+++ b/lib/gitlab/discussions_diff/file_collection.rb
@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module DiscussionsDiff
+    class FileCollection
+      include Gitlab::Utils::StrongMemoize
+
+      def initialize(collection)
+        @collection = collection
+      end
+
+      # Returns a Gitlab::Diff::File with the given ID (`unique_identifier` in
+      # Gitlab::Diff::File).
+      def find_by_id(id)
+        diff_files_indexed_by_id[id]
+      end
+
+      # Writes cache and preloads highlighted diff lines for
+      # object IDs, in @collection.
+      #
+      # highlightable_ids - Diff file `Array` responding to ID. The ID will be used
+      # to generate the cache key.
+      #
+      # - Highlight cache is written just for uncached diff files
+      # - The cache content is not updated (there's no need to do so)
+      def load_highlight(highlightable_ids)
+        preload_highlighted_lines(highlightable_ids)
+      end
+
+      private
+
+      def preload_highlighted_lines(ids)
+        cached_content = read_cache(ids)
+
+        uncached_ids = ids.select.each_with_index { |_, i| cached_content[i].nil? }
+        mapping = highlighted_lines_by_ids(uncached_ids)
+
+        HighlightCache.write_multiple(mapping)
+
+        diffs = diff_files_indexed_by_id.values_at(*ids)
+
+        diffs.zip(cached_content).each do |diff, cached_lines|
+          next unless diff && cached_lines
+
+          diff.highlighted_diff_lines = cached_lines
+        end
+      end
+
+      def read_cache(ids)
+        HighlightCache.read_multiple(ids)
+      end
+
+      def diff_files_indexed_by_id
+        strong_memoize(:diff_files_indexed_by_id) do
+          diff_files.index_by(&:unique_identifier)
+        end
+      end
+
+      def diff_files
+        strong_memoize(:diff_files) do
+          @collection.map(&:raw_diff_file)
+        end
+      end
+
+      # Processes the diff lines highlighting for diff files matching the given
+      # IDs.
+      #
+      # Returns a Hash with { id => [Array of Gitlab::Diff::line], ...]
+      def highlighted_lines_by_ids(ids)
+        diff_files_indexed_by_id.slice(*ids).each_with_object({}) do |(id, file), hash|
+          hash[id] = file.highlighted_diff_lines.map(&:to_hash)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb
new file mode 100644
index 00000000000..270cfb89488
--- /dev/null
+++ b/lib/gitlab/discussions_diff/highlight_cache.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+#
+module Gitlab
+  module DiscussionsDiff
+    class HighlightCache
+      class << self
+        VERSION = 1
+        EXPIRATION = 1.week
+
+        # Sets multiple keys to a given value. The value
+        # is serialized as JSON.
+        #
+        # mapping - Write multiple cache values at once
+        def write_multiple(mapping)
+          Redis::Cache.with do |redis|
+            redis.multi do |multi|
+              mapping.each do |raw_key, value|
+                key = cache_key_for(raw_key)
+
+                multi.set(key, value.to_json, ex: EXPIRATION)
+              end
+            end
+          end
+        end
+
+        # Reads multiple cache keys at once.
+        #
+        # raw_keys - An Array of unique cache keys, without namespaces.
+        #
+        # It returns a list of deserialized diff lines. Ex.:
+        # [[Gitlab::Diff::Line, ...], [Gitlab::Diff::Line]]
+        def read_multiple(raw_keys)
+          return [] if raw_keys.empty?
+
+          keys = raw_keys.map { |id| cache_key_for(id) }
+
+          content =
+            Redis::Cache.with do |redis|
+              redis.mget(keys)
+            end
+
+          content.map! do |lines|
+            next unless lines
+
+            JSON.parse(lines).map! do |line|
+              line = line.with_indifferent_access
+              rich_text = line[:rich_text]
+              line[:rich_text] = rich_text&.html_safe
+
+              Gitlab::Diff::Line.init_from_hash(line)
+            end
+          end
+        end
+
+        def cache_key_for(raw_key)
+          "#{cache_key_prefix}:#{raw_key}"
+        end
+
+        private
+
+        def cache_key_prefix
+          "#{Redis::Cache::CACHE_NAMESPACE}:#{VERSION}:discussion-highlight"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/downtime_check.rb b/lib/gitlab/downtime_check.rb
index 941244694e2..31bb6810391 100644
--- a/lib/gitlab/downtime_check.rb
+++ b/lib/gitlab/downtime_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   # Checks if a set of migrations requires downtime or not.
   class DowntimeCheck
diff --git a/lib/gitlab/downtime_check/message.rb b/lib/gitlab/downtime_check/message.rb
index 543e62794c5..ec38bd769a3 100644
--- a/lib/gitlab/downtime_check/message.rb
+++ b/lib/gitlab/downtime_check/message.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class DowntimeCheck
     class Message
@@ -18,13 +20,13 @@ module Gitlab
       def to_s
         label = offline ? OFFLINE : ONLINE
 
-        message = "[#{label}]: #{path}"
+        message = ["[#{label}]: #{path}"]
 
         if reason?
-          message += ":\n\n#{reason}\n\n"
+          message << ":\n\n#{reason}\n\n"
         end
 
-        message
+        message.join
       end
 
       def reason?
diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb
index 8c72d00c1f3..01fd261404b 100644
--- a/lib/gitlab/ee_compat_check.rb
+++ b/lib/gitlab/ee_compat_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # rubocop: disable Rails/Output
 module Gitlab
   # Checks if a set of migrations requires downtime or not.
@@ -5,7 +7,7 @@ module Gitlab
     CANONICAL_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze
     CANONICAL_EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze
     CHECK_DIR = Rails.root.join('ee_compat_check')
-    IGNORED_FILES_REGEX = %r{VERSION|CHANGELOG\.md|db/schema\.rb|locale/gitlab\.pot}i.freeze
+    IGNORED_FILES_REGEX = /VERSION|CHANGELOG\.md/i.freeze
     PLEASE_READ_THIS_BANNER = %Q{
       ============================================================
       ===================== PLEASE READ THIS =====================
@@ -138,15 +140,23 @@ module Gitlab
 
     def ee_branch_presence_check!
       ee_remotes.keys.each do |remote|
-        [ce_branch, ee_branch_prefix, ee_branch_suffix].each do |branch|
-          _, status = step("Fetching #{remote}/#{branch}", %W[git fetch #{remote} #{branch}])
+        output, _ = step(
+          "Searching #{remote}",
+          %W[git ls-remote #{remote} *#{minimal_ee_branch_name}*])
 
-          if status.zero?
-            @ee_remote_with_branch = remote
-            @ee_branch_found = branch
-            return true
-          end
-        end
+        branches =
+          output.scan(%r{(?<=refs/heads/|refs/tags/).+}).sort_by(&:size)
+
+        next if branches.empty?
+
+        branch = branches.first
+
+        step("Fetching #{remote}/#{branch}", %W[git fetch #{remote} #{branch}])
+
+        @ee_remote_with_branch = remote
+        @ee_branch_found = branch
+
+        return true
       end
 
       puts
@@ -271,8 +281,12 @@ module Gitlab
       @ee_patch_full_path ||= patches_dir.join(ee_patch_name)
     end
 
+    def minimal_ee_branch_name
+      @minimal_ee_branch_name ||= ce_branch.sub(/(\Ace\-|\-ce\z)/, '')
+    end
+
     def patch_name_from_branch(branch_name)
-      branch_name.parameterize << '.patch'
+      "#{branch_name.parameterize}.patch"
     end
 
     def patch_url
@@ -420,9 +434,11 @@ module Gitlab
     end
 
     def conflicting_files_msg
-      failed_files.reduce("The conflicts detected were as follows:\n") do |memo, file|
-        memo << "\n        - #{file}"
-      end
+      header = "The conflicts detected were as follows:\n"
+      separator = "\n        - "
+      failed_items = failed_files.join(separator)
+
+      "#{header}#{separator}#{failed_items}"
     end
   end
 end
diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb
index 83440ae227d..3323ce60158 100644
--- a/lib/gitlab/email/attachment_uploader.rb
+++ b/lib/gitlab/email/attachment_uploader.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Email
     class AttachmentUploader
@@ -21,8 +23,8 @@ module Gitlab
               content_type: attachment.content_type
             }
 
-            link = UploadService.new(project, file).execute
-            attachments << link if link
+            uploader = UploadService.new(project, file).execute
+            attachments << uploader.to_h if uploader
           ensure
             tmp.close!
           end
diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb
index e08b5be8984..cebedb19dcc 100644
--- a/lib/gitlab/email/handler.rb
+++ b/lib/gitlab/email/handler.rb
@@ -1,20 +1,23 @@
-require 'gitlab/email/handler/create_merge_request_handler'
-require 'gitlab/email/handler/create_note_handler'
-require 'gitlab/email/handler/create_issue_handler'
-require 'gitlab/email/handler/unsubscribe_handler'
+# frozen_string_literal: true
 
 module Gitlab
   module Email
     module Handler
-      HANDLERS = [
-        UnsubscribeHandler,
-        CreateNoteHandler,
-        CreateMergeRequestHandler,
-        CreateIssueHandler
-      ].freeze
+      def self.handlers
+        @handlers ||= load_handlers
+      end
+
+      def self.load_handlers
+        [
+          UnsubscribeHandler,
+          CreateNoteHandler,
+          CreateMergeRequestHandler,
+          CreateIssueHandler
+        ]
+      end
 
       def self.for(mail, mail_key)
-        HANDLERS.find do |klass|
+        handlers.find do |klass|
           handler = klass.new(mail, mail_key)
           break handler if handler.can_handle?
         end
diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb
index 0bba433d04b..f89d1d15010 100644
--- a/lib/gitlab/email/handler/base_handler.rb
+++ b/lib/gitlab/email/handler/base_handler.rb
@@ -1,15 +1,19 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Email
     module Handler
       class BaseHandler
         attr_reader :mail, :mail_key
 
+        HANDLER_ACTION_BASE_REGEX ||= /(?<project_slug>.+)-(?<project_id>\d+)/.freeze
+
         def initialize(mail, mail_key)
           @mail = mail
           @mail_key = mail_key
         end
 
-        def can_execute?
+        def can_handle?
           raise NotImplementedError
         end
 
diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb
index 764f93f6d3d..78a3a9489ac 100644
--- a/lib/gitlab/email/handler/create_issue_handler.rb
+++ b/lib/gitlab/email/handler/create_issue_handler.rb
@@ -1,20 +1,34 @@
+# frozen_string_literal: true
+
 require 'gitlab/email/handler/base_handler'
 
+# handles issue creation emails with these formats:
+#   incoming+gitlab-org-gitlab-ce-20-Author_Token12345678-issue@incoming.gitlab.com
+#   incoming+gitlab-org/gitlab-ce+Author_Token12345678@incoming.gitlab.com (legacy)
 module Gitlab
   module Email
     module Handler
       class CreateIssueHandler < BaseHandler
         include ReplyProcessing
-        attr_reader :project_path, :incoming_email_token
+
+        HANDLER_REGEX        = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-issue\z/.freeze
+        HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\+(?<incoming_email_token>.*)\z/.freeze
 
         def initialize(mail, mail_key)
           super(mail, mail_key)
-          @project_path, @incoming_email_token =
-            mail_key && mail_key.split('+', 2)
+
+          if !mail_key&.include?('/') && (matched = HANDLER_REGEX.match(mail_key.to_s))
+            @project_slug         = matched[:project_slug]
+            @project_id           = matched[:project_id]&.to_i
+            @incoming_email_token = matched[:incoming_email_token]
+          elsif matched = HANDLER_REGEX_LEGACY.match(mail_key.to_s)
+            @project_path         = matched[:project_path]
+            @incoming_email_token = matched[:incoming_email_token]
+          end
         end
 
         def can_handle?
-          !incoming_email_token.nil? && !incoming_email_token.include?("+") && !mail_key.include?(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)
+          incoming_email_token && (project_id || can_handle_legacy_format?)
         end
 
         def execute
@@ -28,17 +42,11 @@ module Gitlab
             record_name: 'issue')
         end
 
+        # rubocop: disable CodeReuse/ActiveRecord
         def author
           @author ||= User.find_by(incoming_email_token: incoming_email_token)
         end
-
-        def project
-          @project ||= Project.find_by_full_path(project_path)
-        end
-
-        def metrics_params
-          super.merge(project: project&.full_path)
-        end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         private
 
@@ -50,6 +58,10 @@ module Gitlab
             description: message_including_reply
           ).execute
         end
+
+        def can_handle_legacy_format?
+          project_path && !incoming_email_token.include?('+') && !mail_key.include?(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX_LEGACY)
+        end
       end
     end
   end
diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb
index 2f864f2082b..b3b5063f2ca 100644
--- a/lib/gitlab/email/handler/create_merge_request_handler.rb
+++ b/lib/gitlab/email/handler/create_merge_request_handler.rb
@@ -1,23 +1,35 @@
+# frozen_string_literal: true
+
 require 'gitlab/email/handler/base_handler'
 require 'gitlab/email/handler/reply_processing'
 
+# handles merge request creation emails with these formats:
+#   incoming+gitlab-org-gitlab-ce-20-Author_Token12345678-merge-request@incoming.gitlab.com
+#   incoming+gitlab-org/gitlab-ce+merge-request+Author_Token12345678@incoming.gitlab.com (legacy)
 module Gitlab
   module Email
     module Handler
       class CreateMergeRequestHandler < BaseHandler
         include ReplyProcessing
-        attr_reader :project_path, :incoming_email_token
+
+        HANDLER_REGEX        = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-merge-request\z/.freeze
+        HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\+merge-request\+(?<incoming_email_token>.*)/.freeze
 
         def initialize(mail, mail_key)
           super(mail, mail_key)
 
-          if m = /\A([^\+]*)\+merge-request\+(.*)/.match(mail_key.to_s)
-            @project_path, @incoming_email_token = m.captures
+          if !mail_key&.include?('/') && (matched = HANDLER_REGEX.match(mail_key.to_s))
+            @project_slug         = matched[:project_slug]
+            @project_id           = matched[:project_id]&.to_i
+            @incoming_email_token = matched[:incoming_email_token]
+          elsif matched = HANDLER_REGEX_LEGACY.match(mail_key.to_s)
+            @project_path         = matched[:project_path]
+            @incoming_email_token = matched[:incoming_email_token]
           end
         end
 
         def can_handle?
-          @project_path && @incoming_email_token
+          incoming_email_token && (project_id || project_path)
         end
 
         def execute
@@ -32,22 +44,32 @@ module Gitlab
             record_name: 'merge_request')
         end
 
+        # rubocop: disable CodeReuse/ActiveRecord
         def author
           @author ||= User.find_by(incoming_email_token: incoming_email_token)
         end
-
-        def project
-          @project ||= Project.find_by_full_path(project_path)
-        end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         def metrics_params
-          super.merge(project: project&.full_path)
+          super.merge(includes_patches: patch_attachments.any?)
         end
 
         private
 
+        def build_merge_request
+          MergeRequests::BuildService.new(project, author, merge_request_params).execute
+        end
+
         def create_merge_request
-          merge_request = MergeRequests::BuildService.new(project, author, merge_request_params).execute
+          merge_request = build_merge_request
+
+          if patch_attachments.any?
+            apply_patches_to_source_branch(start_branch: merge_request.target_branch)
+            remove_patch_attachments
+            # Rebuild the merge request as the source branch might just have
+            # been created, so we should re-validate.
+            merge_request = build_merge_request
+          end
 
           if merge_request.errors.any?
             merge_request
@@ -59,12 +81,42 @@ module Gitlab
         def merge_request_params
           params = {
             source_project_id: project.id,
-            source_branch: mail.subject,
+            source_branch: source_branch,
             target_project_id: project.id
           }
           params[:description] = message if message.present?
           params
         end
+
+        def apply_patches_to_source_branch(start_branch:)
+          patches = patch_attachments.map { |patch| patch.body.decoded }
+
+          result = Commits::CommitPatchService
+                     .new(project, author, branch_name: source_branch, patches: patches, start_branch: start_branch)
+                     .execute
+
+          if result[:status] != :success
+            message = "Could not apply patches to #{source_branch}:\n#{result[:message]}"
+            raise InvalidAttachment, message
+          end
+        end
+
+        def remove_patch_attachments
+          patch_attachments.each { |patch| mail.parts.delete(patch) }
+          # reset the message, so it needs to be reprocessed when the attachments
+          # have been modified
+          @message = nil
+        end
+
+        def patch_attachments
+          @patches ||= mail.attachments
+                         .select { |attachment| attachment.filename.ends_with?('.patch') }
+                         .sort_by(&:filename)
+        end
+
+        def source_branch
+          @source_branch ||= mail.subject
+        end
       end
     end
   end
diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb
index 5791dbd0484..b00af15364d 100644
--- a/lib/gitlab/email/handler/create_note_handler.rb
+++ b/lib/gitlab/email/handler/create_note_handler.rb
@@ -1,6 +1,10 @@
+# frozen_string_literal: true
+
 require 'gitlab/email/handler/base_handler'
 require 'gitlab/email/handler/reply_processing'
 
+# handles note/reply creation emails with these formats:
+#   incoming+1234567890abcdef1234567890abcdef@incoming.gitlab.com
 module Gitlab
   module Email
     module Handler
@@ -28,10 +32,6 @@ module Gitlab
             record_name: 'comment')
         end
 
-        def metrics_params
-          super.merge(project: project&.full_path)
-        end
-
         private
 
         def author
diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb
index 38b1425364f..d8f4be8ada1 100644
--- a/lib/gitlab/email/handler/reply_processing.rb
+++ b/lib/gitlab/email/handler/reply_processing.rb
@@ -1,16 +1,31 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Email
     module Handler
       module ReplyProcessing
         private
 
+        attr_reader :project_id, :project_slug, :project_path, :incoming_email_token
+
         def author
           raise NotImplementedError
         end
 
+        # rubocop:disable Gitlab/ModuleWithInstanceVariables
         def project
-          raise NotImplementedError
+          return @project if instance_variable_defined?(:@project)
+
+          if project_id
+            @project = Project.find_by_id(project_id)
+            @project = nil unless valid_project_slug?(@project)
+          else
+            @project = Project.find_by_full_path(project_path)
+          end
+
+          @project
         end
+        # rubocop:enable Gitlab/ModuleWithInstanceVariables
 
         def message
           @message ||= process_message
@@ -41,7 +56,7 @@ module Gitlab
             raise ProjectNotFound unless author.can?(:read_project, project)
           end
 
-          raise UserNotAuthorizedError unless author.can?(permission, project || noteable)
+          raise UserNotAuthorizedError unless author.can?(permission, try(:noteable) || project)
         end
 
         def verify_record!(record:, invalid_exception:, record_name:)
@@ -56,6 +71,10 @@ module Gitlab
 
           raise invalid_exception, msg
         end
+
+        def valid_project_slug?(found_project)
+          project_slug == found_project.full_path_slug
+        end
       end
     end
   end
diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb
index ea80e21532e..20e4c125626 100644
--- a/lib/gitlab/email/handler/unsubscribe_handler.rb
+++ b/lib/gitlab/email/handler/unsubscribe_handler.rb
@@ -1,13 +1,29 @@
+# frozen_string_literal: true
+
 require 'gitlab/email/handler/base_handler'
 
+# handles unsubscribe emails with these formats:
+#   incoming+1234567890abcdef1234567890abcdef-unsubscribe@incoming.gitlab.com
+#   incoming+1234567890abcdef1234567890abcdef+unsubscribe@incoming.gitlab.com (legacy)
 module Gitlab
   module Email
     module Handler
       class UnsubscribeHandler < BaseHandler
         delegate :project, to: :sent_notification, allow_nil: true
 
+        HANDLER_REGEX_FOR    = -> (suffix) { /\A(?<reply_token>\w+)#{Regexp.escape(suffix)}\z/ }.freeze
+        HANDLER_REGEX        = HANDLER_REGEX_FOR.call(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX).freeze
+        HANDLER_REGEX_LEGACY = HANDLER_REGEX_FOR.call(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX_LEGACY).freeze
+
+        def initialize(mail, mail_key)
+          super(mail, mail_key)
+
+          matched = HANDLER_REGEX.match(mail_key.to_s) || HANDLER_REGEX_LEGACY.match(mail_key.to_s)
+          @reply_token = matched[:reply_token] if matched
+        end
+
         def can_handle?
-          mail_key =~ /\A\w+#{Regexp.escape(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)}\z/
+          reply_token.present?
         end
 
         def execute
@@ -20,18 +36,12 @@ module Gitlab
           noteable.unsubscribe(sent_notification.recipient)
         end
 
-        def metrics_params
-          super.merge(project: project&.full_path)
-        end
-
         private
 
-        def sent_notification
-          @sent_notification ||= SentNotification.for(reply_key)
-        end
+        attr_reader :reply_token
 
-        def reply_key
-          mail_key.sub(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX, '')
+        def sent_notification
+          @sent_notification ||= SentNotification.for(reply_token)
         end
       end
     end
diff --git a/lib/gitlab/email/hook/additional_headers_interceptor.rb b/lib/gitlab/email/hook/additional_headers_interceptor.rb
new file mode 100644
index 00000000000..aa2ef76069b
--- /dev/null
+++ b/lib/gitlab/email/hook/additional_headers_interceptor.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Email
+    module Hook
+      class AdditionalHeadersInterceptor
+        def self.delivering_email(message)
+          message.header['Auto-Submitted'] ||= 'auto-generated'
+          message.header['X-Auto-Response-Suppress'] ||= 'All'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/hook/delivery_metrics_observer.rb b/lib/gitlab/email/hook/delivery_metrics_observer.rb
new file mode 100644
index 00000000000..c7af485fcc5
--- /dev/null
+++ b/lib/gitlab/email/hook/delivery_metrics_observer.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Email
+    module Hook
+      class DeliveryMetricsObserver
+        extend Gitlab::Utils::StrongMemoize
+
+        def self.delivering_email(_message)
+          delivery_attempts_counter.increment
+        end
+
+        def self.delivered_email(_message)
+          delivered_emails_counter.increment
+        end
+
+        def self.delivery_attempts_counter
+          strong_memoize(:delivery_attempts_counter) do
+            Gitlab::Metrics.counter(:gitlab_emails_delivery_attempts_total,
+                                    'Counter of total emails delivery attempts')
+          end
+        end
+
+        def self.delivered_emails_counter
+          strong_memoize(:delivered_emails_counter) do
+            Gitlab::Metrics.counter(:gitlab_emails_delivered_total,
+                                    'Counter of total emails delievered')
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/hook/disable_email_interceptor.rb b/lib/gitlab/email/hook/disable_email_interceptor.rb
new file mode 100644
index 00000000000..6b6b1d85109
--- /dev/null
+++ b/lib/gitlab/email/hook/disable_email_interceptor.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Email
+    module Hook
+      class DisableEmailInterceptor
+        def self.delivering_email(message)
+          message.perform_deliveries = false
+
+          Rails.logger.info "Emails disabled! Interceptor prevented sending mail #{message.subject}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/hook/email_template_interceptor.rb b/lib/gitlab/email/hook/email_template_interceptor.rb
new file mode 100644
index 00000000000..13f8db2051d
--- /dev/null
+++ b/lib/gitlab/email/hook/email_template_interceptor.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Email
+    module Hook
+      class EmailTemplateInterceptor
+        ##
+        # Remove HTML part if HTML emails are disabled.
+        #
+        def self.delivering_email(message)
+          unless Gitlab::CurrentSettings.html_emails_enabled
+            message.parts.delete_if do |part|
+              part.content_type.start_with?('text/html')
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/html_parser.rb b/lib/gitlab/email/html_parser.rb
index 50559a48973..77f299bcade 100644
--- a/lib/gitlab/email/html_parser.rb
+++ b/lib/gitlab/email/html_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Email
     class HTMLParser
diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb
index cd9d3a6483f..ec412e7a8b1 100644
--- a/lib/gitlab/email/message/repository_push.rb
+++ b/lib/gitlab/email/message/repository_push.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Email
     module Message
@@ -116,7 +118,7 @@ module Gitlab
         end
 
         def subject
-          subject_text = '[Git]'
+          subject_text = ['[Git]']
           subject_text << "[#{project.full_path}]"
           subject_text << "[#{ref_name}]" if @action == :push
           subject_text << ' '
@@ -134,6 +136,8 @@ module Gitlab
             subject_action[0] = subject_action[0].capitalize
             subject_text << "#{subject_action} #{ref_type} #{ref_name}"
           end
+
+          subject_text.join
         end
       end
     end
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index d8c594ad0e7..d28f6b301fa 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_dependency 'gitlab/email/handler'
 
 # Inspired in great part by Discourse's Email::Receiver
@@ -18,6 +20,7 @@ module Gitlab
     InvalidIssueError = Class.new(InvalidRecordError)
     InvalidMergeRequestError = Class.new(InvalidRecordError)
     UnknownIncomingEmail = Class.new(ProcessingError)
+    InvalidAttachment = Class.new(ProcessingError)
 
     class Receiver
       def initialize(raw)
diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb
index ae6b84607d6..2743f011ca6 100644
--- a/lib/gitlab/email/reply_parser.rb
+++ b/lib/gitlab/email/reply_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Inspired in great part by Discourse's Email::Receiver
 module Gitlab
   module Email
diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb
index 89cf659bce4..ce1dfb0753c 100644
--- a/lib/gitlab/emoji.rb
+++ b/lib/gitlab/emoji.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Emoji
     extend self
diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb
index 0b8f6cfe3cb..a4a154c80f7 100644
--- a/lib/gitlab/encoding_helper.rb
+++ b/lib/gitlab/encoding_helper.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module EncodingHelper
     extend self
@@ -65,17 +67,17 @@ module Gitlab
         clean(message)
       end
     rescue ArgumentError
-      return nil
+      nil
     end
 
-    def encode_binary(s)
-      return "" if s.nil?
+    def encode_binary(str)
+      return "" if str.nil?
 
-      s.dup.force_encoding(Encoding::ASCII_8BIT)
+      str.dup.force_encoding(Encoding::ASCII_8BIT)
     end
 
-    def binary_stringio(s)
-      StringIO.new(s || '').tap { |io| io.set_encoding(Encoding::ASCII_8BIT) }
+    def binary_stringio(str)
+      StringIO.new(str.freeze || '').tap { |io| io.set_encoding(Encoding::ASCII_8BIT) }
     end
 
     private
diff --git a/lib/gitlab/environment.rb b/lib/gitlab/environment.rb
index 5e0dd6e7859..b1a9603d3a5 100644
--- a/lib/gitlab/environment.rb
+++ b/lib/gitlab/environment.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Environment
     def self.hostname
diff --git a/lib/gitlab/environment_logger.rb b/lib/gitlab/environment_logger.rb
index 407cc572656..862a516ca71 100644
--- a/lib/gitlab/environment_logger.rb
+++ b/lib/gitlab/environment_logger.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class EnvironmentLogger < Gitlab::Logger
     def self.file_name_noext
diff --git a/lib/gitlab/error_tracking/error.rb b/lib/gitlab/error_tracking/error.rb
new file mode 100644
index 00000000000..4af5192aa6a
--- /dev/null
+++ b/lib/gitlab/error_tracking/error.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ErrorTracking
+    class Error
+      include ActiveModel::Model
+
+      attr_accessor :id, :title, :type, :user_count, :count,
+        :first_seen, :last_seen, :message, :culprit,
+        :external_url, :project_id, :project_name, :project_slug,
+        :short_id, :status, :frequency
+    end
+  end
+end
diff --git a/lib/gitlab/error_tracking/project.rb b/lib/gitlab/error_tracking/project.rb
new file mode 100644
index 00000000000..93e81da5034
--- /dev/null
+++ b/lib/gitlab/error_tracking/project.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ErrorTracking
+    class Project
+      include ActiveModel::Model
+
+      ACCESSORS = [
+        :id, :name, :status, :slug, :organization_name,
+        :organization_id, :organization_slug
+      ].freeze
+
+      attr_accessor(*ACCESSORS)
+    end
+  end
+end
diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb
index d5d35dbd97f..a11d6b66409 100644
--- a/lib/gitlab/etag_caching/middleware.rb
+++ b/lib/gitlab/etag_caching/middleware.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module EtagCaching
     class Middleware
@@ -6,7 +8,7 @@ module Gitlab
       end
 
       def call(env)
-        request = Rack::Request.new(env)
+        request = ActionDispatch::Request.new(env)
         route = Gitlab::EtagCaching::Router.match(request.path_info)
         return @app.call(env) unless route
 
diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb
index 75167a6b088..08e30214b46 100644
--- a/lib/gitlab/etag_caching/router.rb
+++ b/lib/gitlab/etag_caching/router.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module EtagCaching
     class Router
diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb
index 21172ff8d93..2395e7be026 100644
--- a/lib/gitlab/etag_caching/store.rb
+++ b/lib/gitlab/etag_caching/store.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module EtagCaching
     class Store
diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb
index 12b5e240962..d466d2a514c 100644
--- a/lib/gitlab/exclusive_lease.rb
+++ b/lib/gitlab/exclusive_lease.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'securerandom'
 
 module Gitlab
diff --git a/lib/gitlab/exclusive_lease_helpers.rb b/lib/gitlab/exclusive_lease_helpers.rb
new file mode 100644
index 00000000000..7961d4bbd6e
--- /dev/null
+++ b/lib/gitlab/exclusive_lease_helpers.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  # This module provides helper methods which are intregrated with GitLab::ExclusiveLease
+  module ExclusiveLeaseHelpers
+    FailedToObtainLockError = Class.new(StandardError)
+
+    ##
+    # This helper method blocks a process/thread until the other process cancel the obrainted lease key.
+    #
+    # Note: It's basically discouraged to use this method in the unicorn's thread,
+    #       because it holds the connection until all `retries` is consumed.
+    #       This could potentially eat up all connection pools.
+    def in_lock(key, ttl: 1.minute, retries: 10, sleep_sec: 0.01.seconds)
+      raise ArgumentError, 'Key needs to be specified' unless key
+
+      lease = Gitlab::ExclusiveLease.new(key, timeout: ttl)
+
+      until uuid = lease.try_obtain
+        # Keep trying until we obtain the lease. To prevent hammering Redis too
+        # much we'll wait for a bit.
+        sleep(sleep_sec)
+        break if (retries -= 1) < 0
+      end
+
+      raise FailedToObtainLockError, 'Failed to obtain a lock' unless uuid
+
+      yield
+    ensure
+      Gitlab::ExclusiveLease.cancel(key, uuid)
+    end
+  end
+end
diff --git a/lib/gitlab/fake_application_settings.rb b/lib/gitlab/fake_application_settings.rb
index bb14a8cd9e7..bd806269bf0 100644
--- a/lib/gitlab/fake_application_settings.rb
+++ b/lib/gitlab/fake_application_settings.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # This class extends an OpenStruct object by adding predicate methods to mimic
 # ActiveRecord access. We rely on the initial values being true or false to
 # determine whether to define a predicate method because for a newly-added
@@ -5,12 +7,6 @@
 # column type without parsing db/schema.rb.
 module Gitlab
   class FakeApplicationSettings < OpenStruct
-    def initialize(options = {})
-      super
-
-      FakeApplicationSettings.define_predicate_methods(options)
-    end
-
     # Mimic ActiveRecord predicate methods for boolean values
     def self.define_predicate_methods(options)
       options.each do |key, value|
@@ -23,5 +19,27 @@ module Gitlab
         end
       end
     end
+
+    def initialize(options = {})
+      super
+
+      FakeApplicationSettings.define_predicate_methods(options)
+    end
+
+    def key_restriction_for(type)
+      0
+    end
+
+    def allowed_key_types
+      ApplicationSetting::SUPPORTED_KEY_TYPES
+    end
+
+    def pick_repository_storage
+      repository_storages.sample
+    end
+
+    def commit_email_hostname
+      super.presence || ApplicationSetting.default_commit_email_hostname
+    end
   end
 end
diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb
new file mode 100644
index 00000000000..1ae2f9dfd93
--- /dev/null
+++ b/lib/gitlab/favicon.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class Favicon
+    class << self
+      def main
+        image_name =
+          if appearance_favicon.exists?
+            appearance_favicon.url
+          elsif Gitlab::Utils.to_boolean(ENV['CANARY'])
+            'favicon-yellow.png'
+          elsif Rails.env.development?
+            'favicon-blue.png'
+          else
+            'favicon.png'
+          end
+
+        ActionController::Base.helpers.image_path(image_name, host: host)
+      end
+
+      def status_overlay(status_name)
+        path = File.join(
+          'ci_favicons',
+          "#{status_name}.png"
+        )
+
+        ActionController::Base.helpers.image_path(path, host: host)
+      end
+
+      def available_status_names
+        @available_status_names ||= begin
+          Dir.glob(Rails.root.join('app', 'assets', 'images', 'ci_favicons', '*.png'))
+            .map { |file| File.basename(file, '.png') }
+            .sort
+        end
+      end
+
+      private
+
+      # we only want to create full urls when there's a different asset_host
+      # configured.
+      def host
+        asset_host = ActionController::Base.asset_host
+        if asset_host.nil? || asset_host == Gitlab.config.gitlab.base_url
+          nil
+        else
+          Gitlab.config.gitlab.base_url
+        end
+      end
+
+      def appearance
+        Gitlab::SafeRequestStore[:appearance] ||= (Appearance.current || Appearance.new)
+      end
+
+      def appearance_favicon
+        appearance.favicon
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb
index 49bc9c0b671..2770469ca9f 100644
--- a/lib/gitlab/file_detector.rb
+++ b/lib/gitlab/file_detector.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'set'
 
 module Gitlab
@@ -6,9 +8,9 @@ module Gitlab
   module FileDetector
     PATTERNS = {
       # Project files
-      readme: %r{\Areadme[^/]*\z}i,
+      readme: /\A(#{Regexp.union(*Gitlab::MarkupHelper::PLAIN_FILENAMES).source})(\.(#{Regexp.union(*Gitlab::MarkupHelper::EXTENSIONS).source}))?\z/i,
       changelog: %r{\A(changelog|history|changes|news)[^/]*\z}i,
-      license: %r{\A(licen[sc]e|copying)(\.[^/]+)?\z}i,
+      license: %r{\A((un)?licen[sc]e|copying)(\.[^/]+)?\z}i,
       contributing: %r{\Acontributing[^/]*\z}i,
       version: 'version',
       avatar: /\Alogo\.(png|jpg|gif)\z/,
@@ -18,7 +20,6 @@ module Gitlab
 
       # Configuration files
       gitignore: '.gitignore',
-      koding: '.koding.yml',
       gitlab_ci: '.gitlab-ci.yml',
       route_map: '.gitlab/route-map.yml',
 
diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb
index 8c082c0c336..3958814208c 100644
--- a/lib/gitlab/file_finder.rb
+++ b/lib/gitlab/file_finder.rb
@@ -1,9 +1,9 @@
+# frozen_string_literal: true
+
 # This class finds files in a repository by name and content
 # the result is joined and sorted by file name
 module Gitlab
   class FileFinder
-    BATCH_SIZE = 100
-
     attr_reader :project, :ref
 
     delegate :repository, to: :project
@@ -14,41 +14,35 @@ module Gitlab
     end
 
     def find(query)
-      by_content = find_by_content(query)
+      query = Gitlab::Search::Query.new(query, encode_binary: true) do
+        filter :filename, matcher: ->(filter, blob) { blob.binary_filename =~ /#{filter[:regex_value]}$/i }
+        filter :path, matcher: ->(filter, blob) { blob.binary_filename =~ /#{filter[:regex_value]}/i }
+        filter :extension, matcher: ->(filter, blob) { blob.binary_filename =~ /\.#{filter[:regex_value]}$/i }
+      end
+
+      files = find_by_filename(query.term) + find_by_content(query.term)
 
-      already_found = Set.new(by_content.map(&:filename))
-      by_filename = find_by_filename(query, except: already_found)
+      files = query.filter_results(files) if query.filters.any?
 
-      (by_content + by_filename)
-        .sort_by(&:filename)
-        .map { |blob| [blob.filename, blob] }
+      files
     end
 
     private
 
     def find_by_content(query)
-      results = repository.search_files_by_content(query, ref).first(BATCH_SIZE)
-      results.map { |result| Gitlab::ProjectSearchResults.parse_search_result(result, project) }
+      repository.search_files_by_content(query, ref).map do |result|
+        Gitlab::Search::FoundBlob.new(content_match: result, project: project, ref: ref, repository: repository)
+      end
     end
 
-    def find_by_filename(query, except: [])
-      filenames = repository.search_files_by_name(query, ref).first(BATCH_SIZE)
-      filenames.delete_if { |filename| except.include?(filename) } unless except.empty?
-
-      blob_refs = filenames.map { |filename| [ref, filename] }
-      blobs = Gitlab::Git::Blob.batch(repository, blob_refs, blob_size_limit: 1024)
-
-      blobs.map do |blob|
-        Gitlab::SearchResults::FoundBlob.new(
-          id: blob.id,
-          filename: blob.path,
-          basename: File.basename(blob.path),
-          ref: ref,
-          startline: 1,
-          data: blob.data,
-          project: project
-        )
+    def find_by_filename(query)
+      search_filenames(query).map do |filename|
+        Gitlab::Search::FoundBlob.new(blob_filename: filename, project: project, ref: ref, repository: repository)
       end
     end
+
+    def search_filenames(query)
+      repository.search_files_by_name(query, ref)
+    end
   end
 end
diff --git a/lib/gitlab/file_markdown_link_builder.rb b/lib/gitlab/file_markdown_link_builder.rb
new file mode 100644
index 00000000000..180140e7da2
--- /dev/null
+++ b/lib/gitlab/file_markdown_link_builder.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# Builds the markdown link of a file
+# It needs the methods filename and secure_url (final destination url) to be defined.
+module Gitlab
+  module FileMarkdownLinkBuilder
+    include FileTypeDetection
+
+    def markdown_link
+      return unless name = markdown_name
+
+      markdown = "[#{name.gsub(']', '\\]')}](#{secure_url})"
+      markdown = "!#{markdown}" if image_or_video? || dangerous?
+      markdown
+    end
+
+    def markdown_name
+      return unless filename.present?
+
+      image_or_video? ? File.basename(filename, File.extname(filename)) : filename
+    end
+  end
+end
diff --git a/lib/gitlab/file_type_detection.rb b/lib/gitlab/file_type_detection.rb
new file mode 100644
index 00000000000..25ee07cf940
--- /dev/null
+++ b/lib/gitlab/file_type_detection.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+# File helpers methods.
+# It needs the method filename to be defined.
+module Gitlab
+  module FileTypeDetection
+    IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico].freeze
+    # We recommend using the .mp4 format over .mov. Videos in .mov format can
+    # still be used but you really need to make sure they are served with the
+    # proper MIME type video/mp4 and not video/quicktime or your videos won't play
+    # on IE >= 9.
+    # http://archive.sublimevideo.info/20150912/docs.sublimevideo.net/troubleshooting.html
+    VIDEO_EXT = %w[mp4 m4v mov webm ogv].freeze
+    # These extension types can contain dangerous code and should only be embedded inline with
+    # proper filtering. They should always be tagged as "Content-Disposition: attachment", not "inline".
+    DANGEROUS_EXT = %w[svg].freeze
+
+    def image?
+      extension_match?(IMAGE_EXT)
+    end
+
+    def video?
+      extension_match?(VIDEO_EXT)
+    end
+
+    def image_or_video?
+      image? || video?
+    end
+
+    def dangerous?
+      extension_match?(DANGEROUS_EXT)
+    end
+
+    private
+
+    def extension_match?(extensions)
+      return false unless filename
+
+      extension = File.extname(filename).delete('.')
+      extensions.include?(extension.downcase)
+    end
+  end
+end
diff --git a/lib/gitlab/fogbugz_import/client.rb b/lib/gitlab/fogbugz_import/client.rb
index acb000e3e23..dd747a79673 100644
--- a/lib/gitlab/fogbugz_import/client.rb
+++ b/lib/gitlab/fogbugz_import/client.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'fogbugz'
 
 module Gitlab
diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb
index 8953bc8c148..431911d1eee 100644
--- a/lib/gitlab/fogbugz_import/importer.rb
+++ b/lib/gitlab/fogbugz_import/importer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module FogbugzImport
     class Importer
@@ -79,6 +81,7 @@ module Gitlab
         ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true)
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def user_info(person_id)
         user_hash = user_map[person_id.to_s]
 
@@ -95,7 +98,9 @@ module Gitlab
 
         { name: user_name, gitlab_id: gitlab_id }
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def import_cases
         return unless @cases
 
@@ -141,6 +146,7 @@ module Gitlab
           import_issue_comments(issue, comments)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def opened_content(comments)
         while comment = comments.shift
@@ -191,19 +197,19 @@ module Gitlab
         end
       end
 
-      def linkify_issues(s)
-        s = s.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2')
-        s = s.gsub(/([Cc]ase) ([0-9]+)/, '\1 #\2')
-        s
+      def linkify_issues(str)
+        str = str.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2')
+        str = str.gsub(/([Cc]ase) ([0-9]+)/, '\1 #\2')
+        str
       end
 
-      def escape_for_markdown(s)
-        s = s.gsub(/^#/, "\\#")
-        s = s.gsub(/^-/, "\\-")
-        s = s.gsub("`", "\\~")
-        s = s.delete("\r")
-        s = s.gsub("\n", "  \n")
-        s
+      def escape_for_markdown(str)
+        str = str.gsub(/^#/, "\\#")
+        str = str.gsub(/^-/, "\\-")
+        str = str.gsub("`", "\\~")
+        str = str.delete("\r")
+        str = str.gsub("\n", "  \n")
+        str
       end
 
       def format_content(raw_content)
diff --git a/lib/gitlab/fogbugz_import/project_creator.rb b/lib/gitlab/fogbugz_import/project_creator.rb
index 1918d5b208d..3c71031a8d9 100644
--- a/lib/gitlab/fogbugz_import/project_creator.rb
+++ b/lib/gitlab/fogbugz_import/project_creator.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module FogbugzImport
     class ProjectCreator
diff --git a/lib/gitlab/fogbugz_import/repository.rb b/lib/gitlab/fogbugz_import/repository.rb
index d1dc63db2b2..b958dcf6cbf 100644
--- a/lib/gitlab/fogbugz_import/repository.rb
+++ b/lib/gitlab/fogbugz_import/repository.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module FogbugzImport
     class Repository
diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb
index 455814a9159..4d82acd9d87 100644
--- a/lib/gitlab/gfm/reference_rewriter.rb
+++ b/lib/gitlab/gfm/reference_rewriter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Gfm
     ##
@@ -31,19 +33,19 @@ module Gitlab
     class ReferenceRewriter
       RewriteError = Class.new(StandardError)
 
-      def initialize(text, source_project, current_user)
+      def initialize(text, source_parent, current_user)
         @text = text
-        @source_project = source_project
+        @source_parent = source_parent
         @current_user = current_user
         @original_html = markdown(text)
         @pattern = Gitlab::ReferenceExtractor.references_pattern
       end
 
-      def rewrite(target_project)
+      def rewrite(target_parent)
         return @text unless needs_rewrite?
 
         @text.gsub(@pattern) do |reference|
-          unfold_reference(reference, Regexp.last_match, target_project)
+          unfold_reference(reference, Regexp.last_match, target_parent)
         end
       end
 
@@ -53,14 +55,14 @@ module Gitlab
 
       private
 
-      def unfold_reference(reference, match, target_project)
+      def unfold_reference(reference, match, target_parent)
         before = @text[0...match.begin(0)]
         after = @text[match.end(0)..-1]
 
         referable = find_referable(reference)
         return reference unless referable
 
-        cross_reference = build_cross_reference(referable, target_project)
+        cross_reference = build_cross_reference(referable, target_parent)
         return reference if reference == cross_reference
 
         if cross_reference.nil?
@@ -72,17 +74,17 @@ module Gitlab
       end
 
       def find_referable(reference)
-        extractor = Gitlab::ReferenceExtractor.new(@source_project,
+        extractor = Gitlab::ReferenceExtractor.new(@source_parent,
                                                    @current_user)
         extractor.analyze(reference)
         extractor.all.first
       end
 
-      def build_cross_reference(referable, target_project)
+      def build_cross_reference(referable, target_parent)
         if referable.respond_to?(:project)
-          referable.to_reference(target_project)
+          referable.to_reference(target_parent)
         else
-          referable.to_reference(@source_project, target_project: target_project)
+          referable.to_reference(@source_parent, target_project: target_parent)
         end
       end
 
@@ -91,7 +93,7 @@ module Gitlab
       end
 
       def markdown(text)
-        Banzai.render(text, project: @source_project, no_original_data: true)
+        Banzai.render(text, project: @source_parent, no_original_data: true, no_sourcepos: true)
       end
     end
   end
diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb
index b6eeb5d9a2b..2d1c9ac40ae 100644
--- a/lib/gitlab/gfm/uploads_rewriter.rb
+++ b/lib/gitlab/gfm/uploads_rewriter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'fileutils'
 
 module Gitlab
@@ -6,7 +8,7 @@ module Gitlab
     # Class that rewrites markdown links for uploads
     #
     # Using a pattern defined in `FileUploader` it copies files to a new
-    # project and rewrites all links to uploads in in a given text.
+    # project and rewrites all links to uploads in a given text.
     #
     #
     class UploadsRewriter
@@ -16,18 +18,16 @@ module Gitlab
         @pattern = FileUploader::MARKDOWN_PATTERN
       end
 
-      def rewrite(target_project)
+      def rewrite(target_parent)
         return @text unless needs_rewrite?
 
         @text.gsub(@pattern) do |markdown|
           file = find_file(@source_project, $~[:secret], $~[:file])
           break markdown unless file.try(:exists?)
 
-          new_uploader = FileUploader.new(target_project)
-          with_link_in_tmp_dir(file.file) do |open_tmp_file|
-            new_uploader.store!(open_tmp_file)
-          end
-          new_uploader.markdown_link
+          klass = target_parent.is_a?(Namespace) ? NamespaceFileUploader : FileUploader
+          moved = klass.copy_to(file, target_parent)
+          moved.markdown_link
         end
       end
 
@@ -48,20 +48,7 @@ module Gitlab
       def find_file(project, secret, file)
         uploader = FileUploader.new(project, secret: secret)
         uploader.retrieve_from_store!(file)
-        uploader.file
-      end
-
-      # Because the uploaders use 'move_to_store' we must have a temporary
-      # file that is allowed to be (re)moved.
-      def with_link_in_tmp_dir(file)
-        dir = Dir.mktmpdir('UploadsRewriter', File.dirname(file))
-        # The filename matters to Carrierwave so we make sure to preserve it
-        tmp_file = File.join(dir, File.basename(file))
-        File.link(file, tmp_file)
-        # Open the file to placate Carrierwave
-        File.open(tmp_file) { |open_file| yield open_file }
-      ensure
-        FileUtils.rm_rf(dir)
+        uploader
       end
     end
   end
diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb
index e85e87a54af..44a62586a23 100644
--- a/lib/gitlab/git.rb
+++ b/lib/gitlab/git.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_dependency 'gitlab/encoding_helper'
 
 module Gitlab
@@ -10,9 +12,11 @@ module Gitlab
     TAG_REF_PREFIX = "refs/tags/".freeze
     BRANCH_REF_PREFIX = "refs/heads/".freeze
 
-    CommandError = Class.new(StandardError)
-    CommitError = Class.new(StandardError)
-    OSError = Class.new(StandardError)
+    BaseError = Class.new(StandardError)
+    CommandError = Class.new(BaseError)
+    CommitError = Class.new(BaseError)
+    OSError = Class.new(BaseError)
+    UnknownRef = Class.new(BaseError)
 
     class << self
       include Gitlab::EncodingHelper
@@ -50,11 +54,11 @@ module Gitlab
       end
 
       def tag_ref?(ref)
-        ref.start_with?(TAG_REF_PREFIX)
+        ref =~ /^#{TAG_REF_PREFIX}.+/
       end
 
       def branch_ref?(ref)
-        ref.start_with?(BRANCH_REF_PREFIX)
+        ref =~ /^#{BRANCH_REF_PREFIX}.+/
       end
 
       def blank_ref?(ref)
@@ -62,7 +66,7 @@ module Gitlab
       end
 
       def version
-        Gitlab::VersionInfo.parse(Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --version)).first)
+        Gitlab::Git::Version.git_version
       end
 
       def check_namespace!(*objects)
diff --git a/lib/gitlab/git/attributes_at_ref_parser.rb b/lib/gitlab/git/attributes_at_ref_parser.rb
index 26b5bd520d5..cbddf836ce8 100644
--- a/lib/gitlab/git/attributes_at_ref_parser.rb
+++ b/lib/gitlab/git/attributes_at_ref_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     # Parses root .gitattributes file at a given ref
diff --git a/lib/gitlab/git/attributes_parser.rb b/lib/gitlab/git/attributes_parser.rb
index 08f4d7d4f5c..8b9d74ae8e7 100644
--- a/lib/gitlab/git/attributes_parser.rb
+++ b/lib/gitlab/git/attributes_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     # Class for parsing Git attribute files and extracting the attributes for
diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb
index 4158d50cd9e..b118eda37f8 100644
--- a/lib/gitlab/git/blame.rb
+++ b/lib/gitlab/git/blame.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     class Blame
@@ -22,24 +24,9 @@ module Gitlab
       private
 
       def load_blame
-        raw_output = @repo.gitaly_migrate(:blame) do |is_enabled|
-          if is_enabled
-            load_blame_by_gitaly
-          else
-            load_blame_by_shelling_out
-          end
-        end
-
-        output = encode_utf8(raw_output)
-        process_raw_blame output
-      end
-
-      def load_blame_by_gitaly
-        @repo.gitaly_commit_client.raw_blame(@sha, @path)
-      end
+        output = encode_utf8(@repo.gitaly_commit_client.raw_blame(@sha, @path))
 
-      def load_blame_by_shelling_out
-        @repo.shell_blame(@sha, @path)
+        process_raw_blame(output)
       end
 
       def process_raw_blame(output)
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index 156d077a69c..259a2b7911a 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -1,15 +1,16 @@
-# Gitaly note: JV: seems to be completely migrated (behind feature flags).
+# frozen_string_literal: true
 
 module Gitlab
   module Git
     class Blob
-      include Linguist::BlobHelper
+      include Gitlab::BlobHelper
       include Gitlab::EncodingHelper
+      extend Gitlab::Git::WrapsGitalyErrors
 
       # This number is the maximum amount of data that we want to display to
-      # the user. We load as much as we can for encoding detection
-      # (Linguist) and LFS pointer parsing. All other cases where we need full
-      # blob data should use load_all_data!.
+      # the user. We load as much as we can for encoding detection and LFS
+      # pointer parsing. All other cases where we need full blob data should
+      # use load_all_data!.
       MAX_DATA_DISPLAY_SIZE = 10.megabytes
 
       # These limits are used as a heuristic to ignore files which can't be LFS
@@ -21,24 +22,36 @@ module Gitlab
       attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary
 
       class << self
-        def find(repository, sha, path)
-          Gitlab::GitalyClient.migrate(:project_raw_show) do |is_enabled|
-            if is_enabled
-              find_by_gitaly(repository, sha, path)
-            else
-              find_by_rugged(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE)
-            end
+        def find(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE)
+          return unless path
+
+          path = path.sub(%r{\A/*}, '')
+          path = '/' if path.empty?
+          name = File.basename(path)
+
+          # Gitaly will think that setting the limit to 0 means unlimited, while
+          # the client might only need the metadata and thus set the limit to 0.
+          # In this method we'll then set the limit to 1, but clear the byte of data
+          # that we got back so for the outside world it looks like the limit was
+          # actually 0.
+          req_limit = limit == 0 ? 1 : limit
+
+          entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit)
+          return unless entry
+
+          entry.data = "" if limit == 0
+
+          case entry.type
+          when :COMMIT
+            new(id: entry.oid, name: name, size: 0, data: '', path: path, commit_id: sha)
+          when :BLOB
+            new(id: entry.oid, name: name, size: entry.size, data: entry.data.dup, mode: entry.mode.to_s(8),
+                path: path, commit_id: sha, binary: binary?(entry.data))
           end
         end
 
         def raw(repository, sha)
-          Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled|
-            if is_enabled
-              repository.gitaly_blob_client.get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE)
-            else
-              rugged_raw(repository, sha, limit: MAX_DATA_DISPLAY_SIZE)
-            end
-          end
+          repository.gitaly_blob_client.get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE)
         end
 
         # Returns an array of Blob instances, specified in blob_references as
@@ -49,17 +62,8 @@ module Gitlab
         # Keep in mind that this method may allocate a lot of memory. It is up
         # to the caller to limit the number of blobs and blob_size_limit.
         #
-        # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/798
         def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE)
-          Gitlab::GitalyClient.migrate(:list_blobs_by_sha_path) do |is_enabled|
-            if is_enabled
-              repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a
-            else
-              blob_references.map do |sha, path|
-                find_by_rugged(repository, sha, path, limit: blob_size_limit)
-              end
-            end
-          end
+          repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a
         end
 
         # Returns an array of Blob instances just with the metadata, that means
@@ -72,16 +76,8 @@ module Gitlab
         # Returns array of Gitlab::Git::Blob
         # Does not guarantee blob data will be set
         def batch_lfs_pointers(repository, blob_ids)
-          repository.gitaly_migrate(:batch_lfs_pointers) do |is_enabled|
-            if is_enabled
-              repository.gitaly_blob_client.batch_lfs_pointers(blob_ids.to_a)
-            else
-              blob_ids.lazy
-                      .select { |sha| possible_lfs_blob?(repository, sha) }
-                      .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) }
-                      .select(&:lfs_pointer?)
-                      .force
-            end
+          wrapped_gitaly_errors do
+            repository.gitaly_blob_client.batch_lfs_pointers(blob_ids.to_a)
           end
         end
 
@@ -92,151 +88,6 @@ module Gitlab
         def size_could_be_lfs?(size)
           size.between?(LFS_POINTER_MIN_SIZE, LFS_POINTER_MAX_SIZE)
         end
-
-        private
-
-        # Recursive search of blob id by path
-        #
-        # Ex.
-        #   blog/            # oid: 1a
-        #     app/           # oid: 2a
-        #       models/      # oid: 3a
-        #       file.rb      # oid: 4a
-        #
-        #
-        # Blob.find_entry_by_path(repo, '1a', 'blog', 'app', 'file.rb') # => '4a'
-        #
-        def find_entry_by_path(repository, root_id, *path_parts)
-          root_tree = repository.lookup(root_id)
-
-          entry = root_tree.find do |entry|
-            entry[:name] == path_parts[0]
-          end
-
-          return nil unless entry
-
-          if path_parts.size > 1
-            return nil unless entry[:type] == :tree
-
-            path_parts.shift
-            find_entry_by_path(repository, entry[:oid], *path_parts)
-          else
-            [:blob, :commit].include?(entry[:type]) ? entry : nil
-          end
-        end
-
-        def submodule_blob(blob_entry, path, sha)
-          new(
-            id: blob_entry[:oid],
-            name: blob_entry[:name],
-            size: 0,
-            data: '',
-            path: path,
-            commit_id: sha
-          )
-        end
-
-        def find_by_gitaly(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE)
-          return unless path
-
-          path = path.sub(%r{\A/*}, '')
-          path = '/' if path.empty?
-          name = File.basename(path)
-
-          # Gitaly will think that setting the limit to 0 means unlimited, while
-          # the client might only need the metadata and thus set the limit to 0.
-          # In this method we'll then set the limit to 1, but clear the byte of data
-          # that we got back so for the outside world it looks like the limit was
-          # actually 0.
-          req_limit = limit == 0 ? 1 : limit
-
-          entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit)
-          return unless entry
-
-          entry.data = "" if limit == 0
-
-          case entry.type
-          when :COMMIT
-            new(
-              id: entry.oid,
-              name: name,
-              size: 0,
-              data: '',
-              path: path,
-              commit_id: sha
-            )
-          when :BLOB
-            new(
-              id: entry.oid,
-              name: name,
-              size: entry.size,
-              data: entry.data.dup,
-              mode: entry.mode.to_s(8),
-              path: path,
-              commit_id: sha,
-              binary: binary?(entry.data)
-            )
-          end
-        end
-
-        def find_by_rugged(repository, sha, path, limit:)
-          return unless path
-
-          # Strip any leading / characters from the path
-          path = path.sub(%r{\A/*}, '')
-
-          rugged_commit = repository.lookup(sha)
-          root_tree = rugged_commit.tree
-
-          blob_entry = find_entry_by_path(repository, root_tree.oid, *path.split('/'))
-
-          return nil unless blob_entry
-
-          if blob_entry[:type] == :commit
-            submodule_blob(blob_entry, path, sha)
-          else
-            blob = repository.lookup(blob_entry[:oid])
-
-            if blob
-              new(
-                id: blob.oid,
-                name: blob_entry[:name],
-                size: blob.size,
-                # Rugged::Blob#content is expensive; don't call it if we don't have to.
-                data: limit.zero? ? '' : blob.content(limit),
-                mode: blob_entry[:filemode].to_s(8),
-                path: path,
-                commit_id: sha,
-                binary: blob.binary?
-              )
-            end
-          end
-        rescue Rugged::ReferenceError
-          nil
-        end
-
-        def rugged_raw(repository, sha, limit:)
-          blob = repository.lookup(sha)
-
-          return unless blob.is_a?(Rugged::Blob)
-
-          new(
-            id: blob.oid,
-            size: blob.size,
-            data: blob.content(limit),
-            binary: blob.binary?
-          )
-        end
-
-        # Efficient lookup to determine if object size
-        # and type make it a possible LFS blob without loading
-        # blob content into memory with repository.lookup(sha)
-        def possible_lfs_blob?(repository, sha)
-          object_header = repository.rugged.read_header(sha)
-
-          object_header[:type] == :blob &&
-            size_could_be_lfs?(object_header[:len])
-        end
       end
 
       def initialize(options)
@@ -249,7 +100,7 @@ module Gitlab
         @loaded_all_data = @loaded_size == size
       end
 
-      def binary?
+      def binary_in_repo?
         @binary.nil? ? super : @binary == true
       end
 
@@ -268,16 +119,7 @@ module Gitlab
 
         return if @loaded_all_data
 
-        @data = Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled|
-          begin
-            if is_enabled
-              repository.gitaly_blob_client.get_blob(oid: id, limit: -1).data
-            else
-              repository.lookup(id).content
-            end
-          end
-        end
-
+        @data = repository.gitaly_blob_client.get_blob(oid: id, limit: -1).data
         @loaded_all_data = true
         @loaded_size = @data.bytesize
       end
@@ -332,7 +174,7 @@ module Gitlab
       private
 
       def has_lfs_version_key?
-        !empty? && text? && data.start_with?("version https://git-lfs.github.com/spec")
+        !empty? && text_in_repo? && data.start_with?("version https://git-lfs.github.com/spec")
       end
     end
   end
diff --git a/lib/gitlab/git/blob_snippet.rb b/lib/gitlab/git/blob_snippet.rb
deleted file mode 100644
index 68116e775c6..00000000000
--- a/lib/gitlab/git/blob_snippet.rb
+++ /dev/null
@@ -1,34 +0,0 @@
-# Gitaly note: JV: no RPC's here.
-
-module Gitlab
-  module Git
-    class BlobSnippet
-      include Linguist::BlobHelper
-
-      attr_accessor :ref
-      attr_accessor :lines
-      attr_accessor :filename
-      attr_accessor :startline
-
-      def initialize(ref, lines, startline, filename)
-        @ref, @lines, @startline, @filename = ref, lines, startline, filename
-      end
-
-      def data
-        lines&.join("\n")
-      end
-
-      def name
-        filename
-      end
-
-      def size
-        data.length
-      end
-
-      def mode
-        nil
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb
index 6351cfb83e3..9447cfa0fb6 100644
--- a/lib/gitlab/git/branch.rb
+++ b/lib/gitlab/git/branch.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     class Branch < Ref
diff --git a/lib/gitlab/git/bundle_file.rb b/lib/gitlab/git/bundle_file.rb
new file mode 100644
index 00000000000..8384a436fcc
--- /dev/null
+++ b/lib/gitlab/git/bundle_file.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Git
+    class BundleFile
+      # All git bundle files start with this string
+      #
+      # https://github.com/git/git/blob/v2.20.1/bundle.c#L15
+      MAGIC = "# v2 git bundle\n"
+
+      InvalidBundleError = Class.new(StandardError)
+
+      attr_reader :filename
+
+      def self.check!(filename)
+        new(filename).check!
+      end
+
+      def initialize(filename)
+        @filename = filename
+      end
+
+      def check!
+        data = File.open(filename, 'r') { |f| f.read(MAGIC.size) }
+
+        raise InvalidBundleError, 'Invalid bundle file' unless data == MAGIC
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index 89f761dd515..5863815ca85 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -1,8 +1,11 @@
-# Gitlab::Git::Commit is a wrapper around native Rugged::Commit object
+# frozen_string_literal: true
+
+# Gitlab::Git::Commit is a wrapper around Gitaly::GitCommit
 module Gitlab
   module Git
     class Commit
       include Gitlab::EncodingHelper
+      extend Gitlab::Git::WrapsGitalyErrors
 
       attr_accessor :raw_commit, :head
 
@@ -53,34 +56,21 @@ module Gitlab
           # Already a commit?
           return commit_id if commit_id.is_a?(Gitlab::Git::Commit)
 
-          # A rugged reference?
-          commit_id = Gitlab::Git::Ref.dereference_object(commit_id)
-          return decorate(repo, commit_id) if commit_id.is_a?(Rugged::Commit)
-
           # Some weird thing?
           return nil unless commit_id.is_a?(String)
 
-          commit = repo.gitaly_migrate(:find_commit) do |is_enabled|
-            if is_enabled
-              repo.gitaly_commit_client.find_commit(commit_id)
-            else
-              rugged_find(repo, commit_id)
-            end
+          # This saves us an RPC round trip.
+          return nil if commit_id.include?(':')
+
+          commit = wrapped_gitaly_errors do
+            repo.gitaly_commit_client.find_commit(commit_id)
           end
 
           decorate(repo, commit) if commit
-        rescue Rugged::ReferenceError, Rugged::InvalidError, Rugged::ObjectError,
-               Gitlab::Git::CommandError, Gitlab::Git::Repository::NoRepository,
-               Rugged::OdbError, Rugged::TreeError, ArgumentError
+        rescue Gitlab::Git::CommandError, Gitlab::Git::Repository::NoRepository, ArgumentError
           nil
         end
 
-        def rugged_find(repo, commit_id)
-          obj = repo.rev_parse_target(commit_id)
-
-          obj.is_a?(Rugged::Commit) ? obj : nil
-        end
-
         # Get last commit for HEAD
         #
         # Ex.
@@ -113,15 +103,9 @@ module Gitlab
         #   Commit.between(repo, '29eda46b', 'master')
         #
         def between(repo, base, head)
-          Gitlab::GitalyClient.migrate(:commits_between) do |is_enabled|
-            if is_enabled
-              repo.gitaly_commit_client.between(base, head)
-            else
-              repo.rugged_commits_between(base, head).map { |c| decorate(repo, c) }
-            end
+          wrapped_gitaly_errors do
+            repo.gitaly_commit_client.between(base, head)
           end
-        rescue Rugged::ReferenceError
-          []
         end
 
         # Returns commits collection
@@ -143,190 +127,55 @@ module Gitlab
         #        :topo, or any combination of them (in an array). Commit ordering types
         #        are documented here:
         #        http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant)
-        #
-        # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/326
         def find_all(repo, options = {})
-          Gitlab::GitalyClient.migrate(:find_all_commits) do |is_enabled|
-            if is_enabled
-              find_all_by_gitaly(repo, options)
-            else
-              find_all_by_rugged(repo, options)
-            end
-          end
-        end
-
-        def find_all_by_rugged(repo, options = {})
-          actual_options = options.dup
-
-          allowed_options = [:ref, :max_count, :skip, :order]
-
-          actual_options.keep_if do |key|
-            allowed_options.include?(key)
-          end
-
-          default_options = { skip: 0 }
-          actual_options = default_options.merge(actual_options)
-
-          rugged = repo.rugged
-          walker = Rugged::Walker.new(rugged)
-
-          if actual_options[:ref]
-            walker.push(rugged.rev_parse_oid(actual_options[:ref]))
-          else
-            rugged.references.each("refs/heads/*") do |ref|
-              walker.push(ref.target_id)
-            end
-          end
-
-          walker.sorting(rugged_sort_type(actual_options[:order]))
-
-          commits = []
-          offset = actual_options[:skip]
-          limit = actual_options[:max_count]
-          walker.each(offset: offset, limit: limit) do |commit|
-            commits.push(decorate(repo, commit))
+          wrapped_gitaly_errors do
+            Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options)
           end
-
-          walker.reset
-
-          commits
-        rescue Rugged::OdbError
-          []
-        end
-
-        def find_all_by_gitaly(repo, options = {})
-          Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options)
         end
 
         def decorate(repository, commit, ref = nil)
           Gitlab::Git::Commit.new(repository, commit, ref)
         end
 
-        # Returns the `Rugged` sorting type constant for one or more given
-        # sort types. Valid keys are `:none`, `:topo`, and `:date`, or an array
-        # containing more than one of them. `:date` uses a combination of date and
-        # topological sorting to closer mimic git's native ordering.
-        def rugged_sort_type(sort_type)
-          @rugged_sort_types ||= {
-            none: Rugged::SORT_NONE,
-            topo: Rugged::SORT_TOPO,
-            date: Rugged::SORT_DATE | Rugged::SORT_TOPO
-          }
-
-          @rugged_sort_types.fetch(sort_type, Rugged::SORT_NONE)
-        end
-
         def shas_with_signatures(repository, shas)
-          GitalyClient.migrate(:filter_shas_with_signatures) do |is_enabled|
-            if is_enabled
-              Gitlab::GitalyClient::CommitService.new(repository).filter_shas_with_signatures(shas)
-            else
-              shas.select do |sha|
-                begin
-                  Rugged::Commit.extract_signature(repository.rugged, sha)
-                rescue Rugged::OdbError
-                  false
-                end
-              end
-            end
-          end
+          Gitlab::GitalyClient::CommitService.new(repository).filter_shas_with_signatures(shas)
         end
 
         # Only to be used when the object ids will not necessarily have a
         # relation to each other. The last 10 commits for a branch for example,
         # should go through .where
         def batch_by_oid(repo, oids)
-          repo.gitaly_migrate(:list_commits_by_oid,
-                              status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-            if is_enabled
-              repo.gitaly_commit_client.list_commits_by_oid(oids)
-            else
-              oids.map { |oid| find(repo, oid) }.compact
-            end
+          wrapped_gitaly_errors do
+            repo.gitaly_commit_client.list_commits_by_oid(oids)
           end
         end
 
         def extract_signature(repository, commit_id)
-          repository.gitaly_migrate(:extract_commit_signature) do |is_enabled|
-            if is_enabled
-              repository.gitaly_commit_client.extract_signature(commit_id)
-            else
-              rugged_extract_signature(repository, commit_id)
-            end
-          end
+          repository.gitaly_commit_client.extract_signature(commit_id)
         end
 
         def extract_signature_lazily(repository, commit_id)
-          BatchLoader.for({ repository: repository, commit_id: commit_id }).batch do |items, loader|
-            items_by_repo = items.group_by { |i| i[:repository] }
-
-            items_by_repo.each do |repo, items|
-              commit_ids = items.map { |i| i[:commit_id] }
-
-              signatures = batch_signature_extraction(repository, commit_ids)
-
-              signatures.each do |commit_sha, signature_data|
-                loader.call({ repository: repository, commit_id: commit_sha }, signature_data)
-              end
+          BatchLoader.for(commit_id).batch(key: repository) do |commit_ids, loader, args|
+            batch_signature_extraction(args[:key], commit_ids).each do |commit_id, signature_data|
+              loader.call(commit_id, signature_data)
             end
           end
         end
 
         def batch_signature_extraction(repository, commit_ids)
-          repository.gitaly_migrate(:extract_commit_signature_in_batch) do |is_enabled|
-            if is_enabled
-              gitaly_batch_signature_extraction(repository, commit_ids)
-            else
-              rugged_batch_signature_extraction(repository, commit_ids)
-            end
-          end
-        end
-
-        def gitaly_batch_signature_extraction(repository, commit_ids)
           repository.gitaly_commit_client.get_commit_signatures(commit_ids)
         end
 
-        def rugged_batch_signature_extraction(repository, commit_ids)
-          commit_ids.each_with_object({}) do |commit_id, signatures|
-            signature_data = rugged_extract_signature(repository, commit_id)
-            next unless signature_data
-
-            signatures[commit_id] = signature_data
-          end
-        end
-
-        def rugged_extract_signature(repository, commit_id)
-          begin
-            Rugged::Commit.extract_signature(repository.rugged, commit_id)
-          rescue Rugged::OdbError
-            nil
-          end
-        end
-
         def get_message(repository, commit_id)
-          BatchLoader.for({ repository: repository, commit_id: commit_id }).batch do |items, loader|
-            items_by_repo = items.group_by { |i| i[:repository] }
-
-            items_by_repo.each do |repo, items|
-              commit_ids = items.map { |i| i[:commit_id] }
-
-              messages = get_messages(repository, commit_ids)
-
-              messages.each do |commit_sha, message|
-                loader.call({ repository: repository, commit_id: commit_sha }, message)
-              end
+          BatchLoader.for(commit_id).batch(key: repository) do |commit_ids, loader, args|
+            get_messages(args[:key], commit_ids).each do |commit_id, message|
+              loader.call(commit_id, message)
             end
           end
         end
 
         def get_messages(repository, commit_ids)
-          repository.gitaly_migrate(:commit_messages) do |is_enabled|
-            if is_enabled
-              repository.gitaly_commit_client.get_commit_messages(commit_ids)
-            else
-              commit_ids.map { |id| [id, rugged_find(repository, id).message] }.to_h
-            end
-          end
+          repository.gitaly_commit_client.get_commit_messages(commit_ids)
         end
       end
 
@@ -339,8 +188,6 @@ module Gitlab
         case raw_commit
         when Hash
           init_from_hash(raw_commit)
-        when Rugged::Commit
-          init_from_rugged(raw_commit)
         when Gitaly::GitCommit
           init_from_gitaly(raw_commit)
         else
@@ -378,40 +225,12 @@ module Gitlab
       # empty repo. See Repository#diff for keys allowed in the +options+
       # hash.
       def diff_from_parent(options = {})
-        Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled|
-          if is_enabled
-            @repository.gitaly_commit_client.diff_from_parent(self, options)
-          else
-            rugged_diff_from_parent(options)
-          end
-        end
-      end
-
-      def rugged_diff_from_parent(options = {})
-        options ||= {}
-        break_rewrites = options[:break_rewrites]
-        actual_options = Gitlab::Git::Diff.filter_diff_options(options)
-
-        diff = if rugged_commit.parents.empty?
-                 rugged_commit.diff(actual_options.merge(reverse: true))
-               else
-                 rugged_commit.parents[0].diff(rugged_commit, actual_options)
-               end
-
-        diff.find_similar!(break_rewrites: break_rewrites)
-        diff
+        @repository.gitaly_commit_client.diff_from_parent(self, options)
       end
 
       def deltas
         @deltas ||= begin
-          deltas = Gitlab::GitalyClient.migrate(:commit_deltas) do |is_enabled|
-            if is_enabled
-              @repository.gitaly_commit_client.commit_deltas(self)
-            else
-              rugged_diff_from_parent.each_delta
-            end
-          end
-
+          deltas = @repository.gitaly_commit_client.commit_deltas(self)
           deltas.map { |delta| Gitlab::Git::Diff.new(delta) }
         end
       end
@@ -479,14 +298,6 @@ module Gitlab
         encode! @committer_email
       end
 
-      def rugged_commit
-        @rugged_commit ||= if raw_commit.is_a?(Rugged::Commit)
-                             raw_commit
-                           else
-                             @repository.rev_parse_target(id)
-                           end
-      end
-
       def merge_commit?
         parent_ids.size > 1
       end
@@ -494,13 +305,17 @@ module Gitlab
       def tree_entry(path)
         return unless path.present?
 
-        @repository.gitaly_migrate(:commit_tree_entry) do |is_migrated|
-          if is_migrated
-            gitaly_tree_entry(path)
-          else
-            rugged_tree_entry(path)
-          end
-        end
+        # We're only interested in metadata, so limit actual data to 1 byte
+        # since Gitaly doesn't support "send no data" option.
+        entry = @repository.gitaly_commit_client.tree_entry(id, path, 1)
+        return unless entry
+
+        entry = entry.to_h
+        entry.delete(:data)
+        entry[:name] = File.basename(path)
+        entry[:type] = entry[:type].downcase
+
+        entry
       end
 
       def to_gitaly_commit
@@ -512,8 +327,8 @@ module Gitlab
           subject: message_split[0] ? message_split[0].chomp.b : "",
           body: raw_commit.message.b,
           parent_ids: raw_commit.parent_ids,
-          author: gitaly_commit_author_from_rugged(raw_commit.author),
-          committer: gitaly_commit_author_from_rugged(raw_commit.committer)
+          author: gitaly_commit_author_from_raw(raw_commit.author),
+          committer: gitaly_commit_author_from_raw(raw_commit.committer)
         )
       end
 
@@ -527,22 +342,6 @@ module Gitlab
         end
       end
 
-      def init_from_rugged(commit)
-        author = commit.author
-        committer = commit.committer
-
-        @raw_commit = commit
-        @id = commit.oid
-        @message = commit.message
-        @authored_date = author[:time]
-        @committed_date = committer[:time]
-        @author_name = author[:name]
-        @author_email = author[:email]
-        @committer_name = committer[:name]
-        @committer_email = committer[:email]
-        @parent_ids = commit.parents.map(&:oid)
-      end
-
       def init_from_gitaly(commit)
         @raw_commit = commit
         @id = commit.id
@@ -563,29 +362,7 @@ module Gitlab
         SERIALIZE_KEYS
       end
 
-      def gitaly_tree_entry(path)
-        # We're only interested in metadata, so limit actual data to 1 byte
-        # since Gitaly doesn't support "send no data" option.
-        entry = @repository.gitaly_commit_client.tree_entry(id, path, 1)
-        return unless entry
-
-        # To be compatible with the rugged format
-        entry = entry.to_h
-        entry.delete(:data)
-        entry[:name] = File.basename(path)
-        entry[:type] = entry[:type].downcase
-
-        entry
-      end
-
-      # Is this the same as Blob.find_entry_by_path ?
-      def rugged_tree_entry(path)
-        rugged_commit.tree.path(path)
-      rescue Rugged::TreeError
-        nil
-      end
-
-      def gitaly_commit_author_from_rugged(author_or_committer)
+      def gitaly_commit_author_from_raw(author_or_committer)
         Gitaly::CommitAuthor.new(
           name: author_or_committer[:name].b,
           email: author_or_committer[:email].b,
diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb
index 8463b1eb794..8815088d23c 100644
--- a/lib/gitlab/git/commit_stats.rb
+++ b/lib/gitlab/git/commit_stats.rb
@@ -1,10 +1,12 @@
-# Gitaly note: JV: 1 RPC, migration in progress.
+# frozen_string_literal: true
 
 # Gitlab::Git::CommitStats counts the additions, deletions, and total changes
 # in a commit.
 module Gitlab
   module Git
     class CommitStats
+      include Gitlab::Git::WrapsGitalyErrors
+
       attr_reader :id, :additions, :deletions, :total
 
       # Instantiate a CommitStats object
@@ -16,12 +18,8 @@ module Gitlab
         @deletions = 0
         @total = 0
 
-        repo.gitaly_migrate(:commit_stats) do |is_enabled|
-          if is_enabled
-            gitaly_stats(repo, commit)
-          else
-            rugged_stats(commit)
-          end
+        wrapped_gitaly_errors do
+          gitaly_stats(repo, commit)
         end
       end
 
@@ -31,12 +29,6 @@ module Gitlab
         @deletions = stats.deletions
         @total = @additions + @deletions
       end
-
-      def rugged_stats(commit)
-        diff = commit.rugged_diff_from_parent
-        _files_changed, @additions, @deletions = diff.stat
-        @total = @additions + @deletions
-      end
     end
   end
 end
diff --git a/lib/gitlab/git/committer_with_hooks.rb b/lib/gitlab/git/committer_with_hooks.rb
deleted file mode 100644
index a8a59f998cd..00000000000
--- a/lib/gitlab/git/committer_with_hooks.rb
+++ /dev/null
@@ -1,47 +0,0 @@
-module Gitlab
-  module Git
-    class CommitterWithHooks < Gollum::Committer
-      attr_reader :gl_wiki
-
-      def initialize(gl_wiki, options = {})
-        @gl_wiki = gl_wiki
-        super(gl_wiki.gollum_wiki, options)
-      end
-
-      def commit
-        # TODO: Remove after 10.8
-        return super unless allowed_to_run_hooks?
-
-        result = Gitlab::Git::OperationService.new(git_user, gl_wiki.repository).with_branch(
-          @wiki.ref,
-          start_branch_name: @wiki.ref
-        ) do |start_commit|
-          super(false)
-        end
-
-        result[:newrev]
-      rescue Gitlab::Git::HooksService::PreReceiveError => e
-        message = "Custom Hook failed: #{e.message}"
-        raise Gitlab::Git::Wiki::OperationError, message
-      end
-
-      private
-
-      # TODO: Remove after 10.8
-      def allowed_to_run_hooks?
-        @options[:user_id] != 0 && @options[:username].present?
-      end
-
-      def git_user
-        @git_user ||= Gitlab::Git::User.new(@options[:username],
-                                            @options[:name],
-                                            @options[:email],
-                                            gitlab_id)
-      end
-
-      def gitlab_id
-        Gitlab::GlId.gl_id_from_id_value(@options[:user_id])
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb
index 7cb842256d0..ab5245ba7cb 100644
--- a/lib/gitlab/git/compare.rb
+++ b/lib/gitlab/git/compare.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Gitaly note: JV: no RPC's here.
 
 module Gitlab
diff --git a/lib/gitlab/git/conflict/file.rb b/lib/gitlab/git/conflict/file.rb
index f08dab59ce4..7ffe4a7ae81 100644
--- a/lib/gitlab/git/conflict/file.rb
+++ b/lib/gitlab/git/conflict/file.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     module Conflict
diff --git a/lib/gitlab/git/conflict/parser.rb b/lib/gitlab/git/conflict/parser.rb
index fb5717dd556..20de8ebde4e 100644
--- a/lib/gitlab/git/conflict/parser.rb
+++ b/lib/gitlab/git/conflict/parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     module Conflict
diff --git a/lib/gitlab/git/conflict/resolution.rb b/lib/gitlab/git/conflict/resolution.rb
index ab9be683e15..04299a2d10c 100644
--- a/lib/gitlab/git/conflict/resolution.rb
+++ b/lib/gitlab/git/conflict/resolution.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     module Conflict
diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb
index c3cb0264112..26e82643a4c 100644
--- a/lib/gitlab/git/conflict/resolver.rb
+++ b/lib/gitlab/git/conflict/resolver.rb
@@ -1,7 +1,11 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     module Conflict
       class Resolver
+        include Gitlab::Git::WrapsGitalyErrors
+
         ConflictSideMissing = Class.new(StandardError)
         ResolutionError = Class.new(StandardError)
 
@@ -12,28 +16,18 @@ module Gitlab
         end
 
         def conflicts
-          @conflicts ||= begin
-            @target_repository.gitaly_migrate(:conflicts_list_conflict_files) do |is_enabled|
-              if is_enabled
-                gitaly_conflicts_client(@target_repository).list_conflict_files.to_a
-              else
-                rugged_list_conflict_files
-              end
-            end
+          @conflicts ||= wrapped_gitaly_errors do
+            gitaly_conflicts_client(@target_repository).list_conflict_files.to_a
           end
         rescue GRPC::FailedPrecondition => e
           raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing.new(e.message)
-        rescue Rugged::ReferenceError, Rugged::OdbError, GRPC::BadStatus => e
+        rescue GRPC::BadStatus => e
           raise Gitlab::Git::CommandError.new(e)
         end
 
         def resolve_conflicts(source_repository, resolution, source_branch:, target_branch:)
-          source_repository.gitaly_migrate(:conflicts_resolve_conflicts) do |is_enabled|
-            if is_enabled
-              gitaly_conflicts_client(source_repository).resolve_conflicts(@target_repository, resolution, source_branch, target_branch)
-            else
-              rugged_resolve_conflicts(source_repository, resolution, source_branch, target_branch)
-            end
+          wrapped_gitaly_errors do
+            gitaly_conflicts_client(source_repository).resolve_conflicts(@target_repository, resolution, source_branch, target_branch)
           end
         end
 
@@ -61,57 +55,6 @@ module Gitlab
         def gitaly_conflicts_client(repository)
           repository.gitaly_conflicts_client(@our_commit_oid, @their_commit_oid)
         end
-
-        def write_resolved_file_to_index(repository, index, file, params)
-          if params[:sections]
-            resolved_lines = file.resolve_lines(params[:sections])
-            new_file = resolved_lines.map { |line| line[:full_line] }.join("\n")
-
-            new_file << "\n" if file.our_blob.data.end_with?("\n")
-          elsif params[:content]
-            new_file = file.resolve_content(params[:content])
-          end
-
-          our_path = file.our_path
-
-          oid = repository.rugged.write(new_file, :blob)
-          index.add(path: our_path, oid: oid, mode: file.our_mode)
-          index.conflict_remove(our_path)
-        end
-
-        def rugged_list_conflict_files
-          target_index = @target_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid)
-
-          # We don't need to do `with_repo_branch_commit` here, because the target
-          # project always fetches source refs when creating merge request diffs.
-          conflict_files(@target_repository, target_index)
-        end
-
-        def rugged_resolve_conflicts(source_repository, resolution, source_branch, target_branch)
-          source_repository.with_repo_branch_commit(@target_repository, target_branch) do
-            index = source_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid)
-            conflicts = conflict_files(source_repository, index)
-
-            resolution.files.each do |file_params|
-              conflict_file = conflict_for_path(conflicts, file_params[:old_path], file_params[:new_path])
-
-              write_resolved_file_to_index(source_repository, index, conflict_file, file_params)
-            end
-
-            unless index.conflicts.empty?
-              missing_files = index.conflicts.map { |file| file[:ours][:path] }
-
-              raise ResolutionError, "Missing resolutions for the following files: #{missing_files.join(', ')}"
-            end
-
-            commit_params = {
-              message: resolution.commit_message,
-              parents: [@our_commit_oid, @their_commit_oid]
-            }
-
-            source_repository.commit_index(resolution.user, source_branch, index, commit_params)
-          end
-        end
       end
     end
   end
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index b58296375ef..74a4633424f 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -1,6 +1,5 @@
-# Gitaly note: JV: needs RPC for Gitlab::Git::Diff.between.
+# frozen_string_literal: true
 
-# Gitlab::Git::Diff is a wrapper around native Rugged::Diff object
 module Gitlab
   module Git
     class Diff
@@ -22,13 +21,17 @@ module Gitlab
 
       alias_method :expanded?, :expanded
 
-      SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze
+      # The default maximum content size to display a diff patch.
+      #
+      # If this value ever changes, make sure to create a migration to update
+      # current records, and default of `ApplicationSettings#diff_max_patch_bytes`.
+      DEFAULT_MAX_PATCH_BYTES = 100.kilobytes
 
-      # The maximum size of a diff to display.
-      SIZE_LIMIT = 100.kilobytes
+      # This is a limitation applied on the source (Gitaly), therefore we don't allow
+      # persisting limits over that.
+      MAX_PATCH_BYTES_UPPER_BOUND = 500.kilobytes
 
-      # The maximum size before a diff is collapsed.
-      COLLAPSE_LIMIT = 10.kilobytes
+      SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze
 
       class << self
         def between(repo, head, base, options = {}, *paths)
@@ -52,20 +55,31 @@ module Gitlab
           repo.diff(common_commit, head, actual_options, *paths)
         end
 
-        # Return a copy of the +options+ hash containing only keys that can be
-        # passed to Rugged.  Allowed options are:
+        # Return a copy of the +options+ hash containing only recognized keys.
+        # Allowed options are:
         #
         #  :ignore_whitespace_change ::
         #    If true, changes in amount of whitespace will be ignored.
         #
-        #  :disable_pathspec_match ::
-        #    If true, the given +*paths+ will be applied as exact matches,
-        #    instead of as fnmatch patterns.
+        #  :max_files ::
+        #    Limit how many files will patches be allowed for before collapsing
+        #
+        #  :max_lines ::
+        #    Limit how many patch lines (across all files) will be allowed for
+        #    before collapsing
         #
+        #  :limits ::
+        #    A hash with additional limits to check before collapsing patches.
+        #    Allowed keys are: `max_bytes`, `safe_max_files`, `safe_max_lines`
+        #    and `safe_max_bytes`
+        #
+        #  :expanded ::
+        #    If false, patch raw data will not be included in the diff after
+        #    `max_files`, `max_lines` or any of the limits in `limits` are
+        #    exceeded
         def filter_diff_options(options, default_options = {})
-          allowed_options = [:ignore_whitespace_change,
-                             :disable_pathspec_match, :paths,
-                             :max_files, :max_lines, :limits, :expanded]
+          allowed_options = [:ignore_whitespace_change, :max_files, :max_lines,
+                             :limits, :expanded]
 
           if default_options
             actual_defaults = default_options.dup
@@ -93,10 +107,30 @@ module Gitlab
         #
         # "Binary files a/file/path and b/file/path differ\n"
         # This is used when we detect that a diff is binary
-        # using CharlockHolmes when Rugged treats it as text.
+        # using CharlockHolmes.
         def binary_message(old_path, new_path)
           "Binary files #{old_path} and #{new_path} differ\n"
         end
+
+        # Returns the limit of bytes a single diff file can reach before it
+        # appears as 'collapsed' for end-users.
+        # By convention, it's 10% of the persisted `diff_max_patch_bytes`.
+        #
+        # Example: If we have 100k for the `diff_max_patch_bytes`, it will be 10k by
+        # default.
+        #
+        # Patches surpassing this limit should still be persisted in the database.
+        def patch_safe_limit_bytes
+          patch_hard_limit_bytes / 10
+        end
+
+        # Returns the limit for a single diff file (patch).
+        #
+        # Patches surpassing this limit shouldn't be persisted in the database
+        # and will be presented as 'too large' for end-users.
+        def patch_hard_limit_bytes
+          Gitlab::CurrentSettings.diff_max_patch_bytes
+        end
       end
 
       def initialize(raw_diff, expanded: true)
@@ -106,8 +140,6 @@ module Gitlab
         when Hash
           init_from_hash(raw_diff)
           prune_diff_if_eligible
-        when Rugged::Patch, Rugged::Diff::Delta
-          init_from_rugged(raw_diff)
         when Gitlab::GitalyClient::Diff
           init_from_gitaly(raw_diff)
           prune_diff_if_eligible
@@ -144,7 +176,7 @@ module Gitlab
 
       def too_large?
         if @too_large.nil?
-          @too_large = @diff.bytesize >= SIZE_LIMIT
+          @too_large = @diff.bytesize >= self.class.patch_hard_limit_bytes
         else
           @too_large
         end
@@ -162,7 +194,7 @@ module Gitlab
       def collapsed?
         return @collapsed if defined?(@collapsed)
 
-        @collapsed = !expanded && @diff.bytesize >= COLLAPSE_LIMIT
+        @collapsed = !expanded && @diff.bytesize >= self.class.patch_safe_limit_bytes
       end
 
       def collapse!
@@ -184,31 +216,6 @@ module Gitlab
 
       private
 
-      def init_from_rugged(rugged)
-        if rugged.is_a?(Rugged::Patch)
-          init_from_rugged_patch(rugged)
-          d = rugged.delta
-        else
-          d = rugged
-        end
-
-        @new_path = encode!(d.new_file[:path])
-        @old_path = encode!(d.old_file[:path])
-        @a_mode = d.old_file[:mode].to_s(8)
-        @b_mode = d.new_file[:mode].to_s(8)
-        @new_file = d.added?
-        @renamed_file = d.renamed?
-        @deleted_file = d.deleted?
-      end
-
-      def init_from_rugged_patch(patch)
-        # Don't bother initializing diffs that are too large. If a diff is
-        # binary we're not going to display anything so we skip the size check.
-        return if !patch.delta.binary? && prune_large_patch(patch)
-
-        @diff = encode!(strip_diff_headers(patch.to_s))
-      end
-
       def init_from_hash(hash)
         raw_diff = hash.symbolize_keys
 
@@ -226,6 +233,7 @@ module Gitlab
         @new_file = diff.from_id == BLANK_SHA
         @renamed_file = diff.from_path != diff.to_path
         @deleted_file = diff.to_id == BLANK_SHA
+        @too_large = diff.too_large if diff.respond_to?(:too_large)
 
         collapse! if diff.respond_to?(:collapsed) && diff.collapsed
       end
@@ -237,47 +245,6 @@ module Gitlab
           collapse!
         end
       end
-
-      # If the patch surpasses any of the diff limits it calls the appropiate
-      # prune method and returns true. Otherwise returns false.
-      def prune_large_patch(patch)
-        size = 0
-
-        patch.each_hunk do |hunk|
-          hunk.each_line do |line|
-            size += line.content.bytesize
-
-            if size >= SIZE_LIMIT
-              too_large!
-              return true # rubocop:disable Cop/AvoidReturnFromBlocks
-            end
-          end
-        end
-
-        if !expanded && size >= COLLAPSE_LIMIT
-          collapse!
-          return true
-        end
-
-        false
-      end
-
-      # Strip out the information at the beginning of the patch's text to match
-      # Grit's output
-      def strip_diff_headers(diff_text)
-        # Delete everything up to the first line that starts with '---' or
-        # 'Binary'
-        diff_text.sub!(/\A.*?^(---|Binary)/m, '\1')
-
-        if diff_text.start_with?('---', 'Binary')
-          diff_text
-        else
-          # If the diff_text did not contain a line starting with '---' or
-          # 'Binary', return the empty string. No idea why; we are just
-          # preserving behavior from before the refactor.
-          ''
-        end
-      end
     end
   end
 end
diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb
index 6a601561c2a..5c70cb6c66c 100644
--- a/lib/gitlab/git/diff_collection.rb
+++ b/lib/gitlab/git/diff_collection.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Gitaly note: JV: no RPC's here.
 
 module Gitlab
@@ -11,7 +13,7 @@ module Gitlab
 
       delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits
 
-      def self.collection_limits(options = {})
+      def self.limits(options = {})
         limits = {}
         limits[:max_files] = options.fetch(:max_files, DEFAULT_LIMITS[:max_files])
         limits[:max_lines] = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines])
@@ -19,13 +21,14 @@ module Gitlab
         limits[:safe_max_files] = [limits[:max_files], DEFAULT_LIMITS[:max_files]].min
         limits[:safe_max_lines] = [limits[:max_lines], DEFAULT_LIMITS[:max_lines]].min
         limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file
+        limits[:max_patch_bytes] = Gitlab::Git::Diff.patch_hard_limit_bytes
 
         OpenStruct.new(limits)
       end
 
       def initialize(iterator, options = {})
         @iterator = iterator
-        @limits = self.class.collection_limits(options)
+        @limits = self.class.limits(options)
         @enforce_limits = !!options.fetch(:limits, true)
         @expanded = !!options.fetch(:expanded, true)
 
@@ -42,12 +45,10 @@ module Gitlab
         return if @overflow
         return if @iterator.nil?
 
-        Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled|
-          if is_enabled && @iterator.is_a?(Gitlab::GitalyClient::DiffStitcher)
-            each_gitaly_patch(&block)
-          else
-            each_rugged_patch(&block)
-          end
+        if @iterator.is_a?(Gitlab::GitalyClient::DiffStitcher)
+          each_gitaly_patch(&block)
+        else
+          each_serialized_patch(&block)
         end
 
         @populated = true
@@ -118,7 +119,7 @@ module Gitlab
         end
       end
 
-      def each_rugged_patch
+      def each_serialized_patch
         i = @array.length
 
         @iterator.each do |raw|
diff --git a/lib/gitlab/git/diff_stats_collection.rb b/lib/gitlab/git/diff_stats_collection.rb
new file mode 100644
index 00000000000..998c41497a2
--- /dev/null
+++ b/lib/gitlab/git/diff_stats_collection.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Git
+    class DiffStatsCollection
+      include Gitlab::Utils::StrongMemoize
+      include Enumerable
+
+      def initialize(diff_stats)
+        @collection = diff_stats
+      end
+
+      def each(&block)
+        @collection.each(&block)
+      end
+
+      def find_by_path(path)
+        indexed_by_path[path]
+      end
+
+      def paths
+        @collection.map(&:path)
+      end
+
+      private
+
+      def indexed_by_path
+        strong_memoize(:indexed_by_path) do
+          index_by { |stats| stats.path }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb
deleted file mode 100644
index 00c943fdb25..00000000000
--- a/lib/gitlab/git/gitlab_projects.rb
+++ /dev/null
@@ -1,285 +0,0 @@
-module Gitlab
-  module Git
-    class GitlabProjects
-      include Gitlab::Git::Popen
-      include Gitlab::Utils::StrongMemoize
-
-      # Name of shard where repositories are stored.
-      # Example: nfs-file06
-      attr_reader :shard_name
-
-      # Relative path is a directory name for repository with .git at the end.
-      # Example: gitlab-org/gitlab-test.git
-      attr_reader :repository_relative_path
-
-      # This is the path at which the gitlab-shell hooks directory can be found.
-      # It's essential for integration between git and GitLab proper. All new
-      # repositories should have their hooks directory symlinked here.
-      attr_reader :global_hooks_path
-
-      attr_reader :logger
-
-      def initialize(shard_name, repository_relative_path, global_hooks_path:, logger:)
-        @shard_name = shard_name
-        @repository_relative_path = repository_relative_path
-
-        @logger = logger
-        @global_hooks_path = global_hooks_path
-        @output = StringIO.new
-      end
-
-      def output
-        io = @output.dup
-        io.rewind
-        io.read
-      end
-
-      # Absolute path to the repository.
-      # Example: /home/git/repositorities/gitlab-org/gitlab-test.git
-      # Probably will be removed when we fully migrate to Gitaly, part of
-      # https://gitlab.com/gitlab-org/gitaly/issues/1124.
-      def repository_absolute_path
-        strong_memoize(:repository_absolute_path) do
-          File.join(shard_path, repository_relative_path)
-        end
-      end
-
-      def shard_path
-        strong_memoize(:shard_path) do
-          Gitlab.config.repositories.storages.fetch(shard_name).legacy_disk_path
-        end
-      end
-
-      # Import project via git clone --bare
-      # URL must be publicly cloneable
-      def import_project(source, timeout)
-        Gitlab::GitalyClient.migrate(:import_repository, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_import_repository(source)
-          else
-            git_import_repository(source, timeout)
-          end
-        end
-      end
-
-      def fork_repository(new_shard_name, new_repository_relative_path)
-        Gitlab::GitalyClient.migrate(:fork_repository,
-                                     status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_fork_repository(new_shard_name, new_repository_relative_path)
-          else
-            git_fork_repository(new_shard_name, new_repository_relative_path)
-          end
-        end
-      end
-
-      def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil, prune: true)
-        tags_option = tags ? '--tags' : '--no-tags'
-
-        logger.info "Fetching remote #{name} for repository #{repository_absolute_path}."
-        cmd = %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet)
-        cmd << '--prune' if prune
-        cmd << '--force' if force
-        cmd << tags_option
-
-        setup_ssh_auth(ssh_key, known_hosts) do |env|
-          success = run_with_timeout(cmd, timeout, repository_absolute_path, env)
-
-          unless success
-            logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed."
-          end
-
-          success
-        end
-      end
-
-      def push_branches(remote_name, timeout, force, branch_names)
-        logger.info "Pushing branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}"
-        cmd = %W(#{Gitlab.config.git.bin_path} push)
-        cmd << '--force' if force
-        cmd += %W(-- #{remote_name}).concat(branch_names)
-
-        success = run_with_timeout(cmd, timeout, repository_absolute_path)
-
-        unless success
-          logger.error("Pushing branches to remote #{remote_name} failed.")
-        end
-
-        success
-      end
-
-      def delete_remote_branches(remote_name, branch_names)
-        branches = branch_names.map { |branch_name| ":#{branch_name}" }
-
-        logger.info "Pushing deleted branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}"
-        cmd = %W(#{Gitlab.config.git.bin_path} push -- #{remote_name}).concat(branches)
-
-        success = run(cmd, repository_absolute_path)
-
-        unless success
-          logger.error("Pushing deleted branches to remote #{remote_name} failed.")
-        end
-
-        success
-      end
-
-      protected
-
-      def run(*args)
-        output, exitstatus = popen(*args)
-        @output << output
-
-        exitstatus&.zero?
-      end
-
-      def run_with_timeout(*args)
-        output, exitstatus = popen_with_timeout(*args)
-        @output << output
-
-        exitstatus&.zero?
-      rescue Timeout::Error
-        @output.puts('Timed out')
-
-        false
-      end
-
-      def mask_password_in_url(url)
-        result = URI(url)
-        result.password = "*****" unless result.password.nil?
-        result.user = "*****" unless result.user.nil? # it's needed for oauth access_token
-        result
-      rescue
-        url
-      end
-
-      def remove_origin_in_repo
-        cmd = %W(#{Gitlab.config.git.bin_path} remote rm origin)
-        run(cmd, repository_absolute_path)
-      end
-
-      # Builds a small shell script that can be used to execute SSH with a set of
-      # custom options.
-      #
-      # Options are expanded as `'-oKey="Value"'`, so SSH will correctly interpret
-      # paths with spaces in them. We trust the user not to embed single or double
-      # quotes in the key or value.
-      def custom_ssh_script(options = {})
-        args = options.map { |k, v| %Q{'-o#{k}="#{v}"'} }.join(' ')
-
-        [
-          "#!/bin/sh",
-          "exec ssh #{args} \"$@\""
-        ].join("\n")
-      end
-
-      # Known hosts data and private keys can be passed to gitlab-shell in the
-      # environment. If present, this method puts them into temporary files, writes
-      # a script that can substitute as `ssh`, setting the options to respect those
-      # files, and yields: { "GIT_SSH" => "/tmp/myScript" }
-      def setup_ssh_auth(key, known_hosts)
-        options = {}
-
-        if key
-          key_file = Tempfile.new('gitlab-shell-key-file')
-          key_file.chmod(0o400)
-          key_file.write(key)
-          key_file.close
-
-          options['IdentityFile'] = key_file.path
-          options['IdentitiesOnly'] = 'yes'
-        end
-
-        if known_hosts
-          known_hosts_file = Tempfile.new('gitlab-shell-known-hosts')
-          known_hosts_file.chmod(0o400)
-          known_hosts_file.write(known_hosts)
-          known_hosts_file.close
-
-          options['StrictHostKeyChecking'] = 'yes'
-          options['UserKnownHostsFile'] = known_hosts_file.path
-        end
-
-        return yield({}) if options.empty?
-
-        script = Tempfile.new('gitlab-shell-ssh-wrapper')
-        script.chmod(0o755)
-        script.write(custom_ssh_script(options))
-        script.close
-
-        yield('GIT_SSH' => script.path)
-      ensure
-        key_file&.close!
-        known_hosts_file&.close!
-        script&.close!
-      end
-
-      private
-
-      def git_import_repository(source, timeout)
-        # Skip import if repo already exists
-        return false if File.exist?(repository_absolute_path)
-
-        masked_source = mask_password_in_url(source)
-
-        logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>."
-        cmd = %W(#{Gitlab.config.git.bin_path} clone --bare -- #{source} #{repository_absolute_path})
-
-        success = run_with_timeout(cmd, timeout, nil)
-
-        unless success
-          logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.")
-          FileUtils.rm_rf(repository_absolute_path)
-          return false
-        end
-
-        Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path)
-
-        # The project was imported successfully.
-        # Remove the origin URL since it may contain password.
-        remove_origin_in_repo
-
-        true
-      end
-
-      def gitaly_import_repository(source)
-        raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil)
-
-        Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source)
-        true
-      rescue GRPC::BadStatus => e
-        @output << e.message
-        false
-      end
-
-      def git_fork_repository(new_shard_name, new_repository_relative_path)
-        from_path = repository_absolute_path
-        new_shard_path = Gitlab.config.repositories.storages.fetch(new_shard_name).legacy_disk_path
-        to_path = File.join(new_shard_path, new_repository_relative_path)
-
-        # The repository cannot already exist
-        if File.exist?(to_path)
-          logger.error "fork-repository failed: destination repository <#{to_path}> already exists."
-          return false
-        end
-
-        # Ensure the namepsace / hashed storage directory exists
-        FileUtils.mkdir_p(File.dirname(to_path), mode: 0770)
-
-        logger.info "Forking repository from <#{from_path}> to <#{to_path}>."
-        cmd = %W(#{Gitlab.config.git.bin_path} clone --bare --no-local -- #{from_path} #{to_path})
-
-        run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path)
-      end
-
-      def gitaly_fork_repository(new_shard_name, new_repository_relative_path)
-        target_repository = Gitlab::Git::Repository.new(new_shard_name, new_repository_relative_path, nil)
-        raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil)
-
-        Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository)
-      rescue GRPC::BadStatus => e
-        logger.error "fork-repository failed: #{e.message}"
-        false
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/git/gitmodules_parser.rb b/lib/gitlab/git/gitmodules_parser.rb
index 4b505312f60..575e12390cd 100644
--- a/lib/gitlab/git/gitmodules_parser.rb
+++ b/lib/gitlab/git/gitmodules_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Gitaly note: JV: no RPC's here.
 
 module Gitlab
diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb
deleted file mode 100644
index 7c201c6169b..00000000000
--- a/lib/gitlab/git/hook.rb
+++ /dev/null
@@ -1,108 +0,0 @@
-# Gitaly note: JV: looks like this is only used by Gitlab::Git::HooksService in
-# app/services. We shouldn't bother migrating this until we know how
-# Gitlab::Git::HooksService will be migrated.
-
-module Gitlab
-  module Git
-    class Hook
-      GL_PROTOCOL = 'web'.freeze
-      attr_reader :name, :path, :repository
-
-      def initialize(name, repository)
-        @name = name
-        @repository = repository
-        @path = File.join(repo_path.strip, 'hooks', name)
-      end
-
-      def repo_path
-        repository.path
-      end
-
-      def exists?
-        File.exist?(path)
-      end
-
-      def trigger(gl_id, gl_username, oldrev, newrev, ref)
-        return [true, nil] unless exists?
-
-        Bundler.with_clean_env do
-          case name
-          when "pre-receive", "post-receive"
-            call_receive_hook(gl_id, gl_username, oldrev, newrev, ref)
-          when "update"
-            call_update_hook(gl_id, gl_username, oldrev, newrev, ref)
-          end
-        end
-      end
-
-      private
-
-      def call_receive_hook(gl_id, gl_username, oldrev, newrev, ref)
-        changes = [oldrev, newrev, ref].join(" ")
-
-        exit_status = false
-        exit_message = nil
-
-        vars = {
-          'GL_ID' => gl_id,
-          'GL_USERNAME' => gl_username,
-          'PWD' => repo_path,
-          'GL_PROTOCOL' => GL_PROTOCOL,
-          'GL_REPOSITORY' => repository.gl_repository
-        }
-
-        options = {
-          chdir: repo_path
-        }
-
-        Open3.popen3(vars, path, options) do |stdin, stdout, stderr, wait_thr|
-          exit_status = true
-          stdin.sync = true
-
-          # in git, pre- and post- receive hooks may just exit without
-          # reading stdin. We catch the exception to avoid a broken pipe
-          # warning
-          begin
-            # inject all the changes as stdin to the hook
-            changes.lines do |line|
-              stdin.puts line
-            end
-          rescue Errno::EPIPE
-          end
-
-          stdin.close
-
-          unless wait_thr.value == 0
-            exit_status = false
-            exit_message = retrieve_error_message(stderr, stdout)
-          end
-        end
-
-        [exit_status, exit_message]
-      end
-
-      def call_update_hook(gl_id, gl_username, oldrev, newrev, ref)
-        env = {
-          'GL_ID' => gl_id,
-          'GL_USERNAME' => gl_username,
-          'PWD' => repo_path
-        }
-
-        options = {
-          chdir: repo_path
-        }
-
-        args = [ref, oldrev, newrev]
-
-        stdout, stderr, status = Open3.capture3(env, path, *args, options)
-        [status.success?, Gitlab::Utils.nlbr(stderr.presence || stdout)]
-      end
-
-      def retrieve_error_message(stderr, stdout)
-        err_message = stderr.read
-        err_message = err_message.blank? ? stdout.read : err_message
-        Gitlab::Utils.nlbr(err_message)
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/git/hook_env.rb b/lib/gitlab/git/hook_env.rb
index 455e8451c10..892a069a3b7 100644
--- a/lib/gitlab/git/hook_env.rb
+++ b/lib/gitlab/git/hook_env.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Gitaly note: JV: no RPC's here.
 
 module Gitlab
@@ -17,18 +19,18 @@ module Gitlab
       ].freeze
 
       def self.set(gl_repository, env)
-        return unless RequestStore.active?
+        return unless Gitlab::SafeRequestStore.active?
 
         raise "missing gl_repository" if gl_repository.blank?
 
-        RequestStore.store[:gitlab_git_env] ||= {}
-        RequestStore.store[:gitlab_git_env][gl_repository] = whitelist_git_env(env)
+        Gitlab::SafeRequestStore[:gitlab_git_env] ||= {}
+        Gitlab::SafeRequestStore[:gitlab_git_env][gl_repository] = whitelist_git_env(env)
       end
 
       def self.all(gl_repository)
-        return {} unless RequestStore.active?
+        return {} unless Gitlab::SafeRequestStore.active?
 
-        h = RequestStore.fetch(:gitlab_git_env) { {} }
+        h = Gitlab::SafeRequestStore.fetch(:gitlab_git_env) { {} }
         h.fetch(gl_repository, {})
       end
 
diff --git a/lib/gitlab/git/hooks_service.rb b/lib/gitlab/git/hooks_service.rb
deleted file mode 100644
index f302b852b35..00000000000
--- a/lib/gitlab/git/hooks_service.rb
+++ /dev/null
@@ -1,37 +0,0 @@
-module Gitlab
-  module Git
-    class HooksService
-      PreReceiveError = Class.new(StandardError)
-
-      attr_accessor :oldrev, :newrev, :ref
-
-      def execute(pusher, repository, oldrev, newrev, ref)
-        @repository  = repository
-        @gl_id       = pusher.gl_id
-        @gl_username = pusher.username
-        @oldrev      = oldrev
-        @newrev      = newrev
-        @ref         = ref
-
-        %w(pre-receive update).each do |hook_name|
-          status, message = run_hook(hook_name)
-
-          unless status
-            raise PreReceiveError, message
-          end
-        end
-
-        yield(self).tap do
-          run_hook('post-receive')
-        end
-      end
-
-      private
-
-      def run_hook(name)
-        hook = Gitlab::Git::Hook.new(name, @repository)
-        hook.trigger(@gl_id, @gl_username, oldrev, newrev, ref)
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb
index d94082a3e30..3b9b516308f 100644
--- a/lib/gitlab/git/index.rb
+++ b/lib/gitlab/git/index.rb
@@ -1,157 +1,9 @@
-# Gitaly note: JV: When the time comes I think we will want to copy this
-# class into Gitaly. None of its methods look like they should be RPC's.
-# The RPC's will be at a higher level.
+# frozen_string_literal: true
 
 module Gitlab
   module Git
     class Index
       IndexError = Class.new(StandardError)
-
-      DEFAULT_MODE = 0o100644
-
-      ACTIONS = %w(create create_dir update move delete).freeze
-      ACTION_OPTIONS = %i(file_path previous_path content encoding).freeze
-
-      attr_reader :repository, :raw_index
-
-      def initialize(repository)
-        @repository = repository
-        @raw_index = repository.rugged.index
-      end
-
-      delegate :read_tree, :get, to: :raw_index
-
-      def apply(action, options)
-        validate_action!(action)
-        public_send(action, options.slice(*ACTION_OPTIONS)) # rubocop:disable GitlabSecurity/PublicSend
-      end
-
-      def write_tree
-        raw_index.write_tree(repository.rugged)
-      end
-
-      def dir_exists?(path)
-        raw_index.find { |entry| entry[:path].start_with?("#{path}/") }
-      end
-
-      def create(options)
-        options = normalize_options(options)
-
-        if get(options[:file_path])
-          raise IndexError, "A file with this name already exists"
-        end
-
-        add_blob(options)
-      end
-
-      def create_dir(options)
-        options = normalize_options(options)
-
-        if get(options[:file_path])
-          raise IndexError, "A file with this name already exists"
-        end
-
-        if dir_exists?(options[:file_path])
-          raise IndexError, "A directory with this name already exists"
-        end
-
-        options = options.dup
-        options[:file_path] += '/.gitkeep'
-        options[:content] = ''
-
-        add_blob(options)
-      end
-
-      def update(options)
-        options = normalize_options(options)
-
-        file_entry = get(options[:file_path])
-        unless file_entry
-          raise IndexError, "A file with this name doesn't exist"
-        end
-
-        add_blob(options, mode: file_entry[:mode])
-      end
-
-      def move(options)
-        options = normalize_options(options)
-
-        file_entry = get(options[:previous_path])
-        unless file_entry
-          raise IndexError, "A file with this name doesn't exist"
-        end
-
-        if get(options[:file_path])
-          raise IndexError, "A file with this name already exists"
-        end
-
-        raw_index.remove(options[:previous_path])
-
-        add_blob(options, mode: file_entry[:mode])
-      end
-
-      def delete(options)
-        options = normalize_options(options)
-
-        unless get(options[:file_path])
-          raise IndexError, "A file with this name doesn't exist"
-        end
-
-        raw_index.remove(options[:file_path])
-      end
-
-      private
-
-      def normalize_options(options)
-        options = options.dup
-        options[:file_path] = normalize_path(options[:file_path]) if options[:file_path]
-        options[:previous_path] = normalize_path(options[:previous_path]) if options[:previous_path]
-        options
-      end
-
-      def normalize_path(path)
-        unless path
-          raise IndexError, "You must provide a file path"
-        end
-
-        pathname = Gitlab::Git::PathHelper.normalize_path(path.dup)
-
-        pathname.each_filename do |segment|
-          if segment == '..'
-            raise IndexError, 'Path cannot include directory traversal'
-          end
-        end
-
-        pathname.to_s
-      end
-
-      def add_blob(options, mode: nil)
-        content = options[:content]
-        unless content
-          raise IndexError, "You must provide content"
-        end
-
-        content = Base64.decode64(content) if options[:encoding] == 'base64'
-
-        detect = CharlockHolmes::EncodingDetector.new.detect(content)
-        unless detect && detect[:type] == :binary
-          # When writing to the repo directly as we are doing here,
-          # the `core.autocrlf` config isn't taken into account.
-          content.gsub!("\r\n", "\n") if repository.autocrlf
-        end
-
-        oid = repository.rugged.write(content, :blob)
-
-        raw_index.add(path: options[:file_path], oid: oid, mode: mode || DEFAULT_MODE)
-      rescue Rugged::IndexError => e
-        raise IndexError, e.message
-      end
-
-      def validate_action!(action)
-        unless ACTIONS.include?(action.to_s)
-          raise ArgumentError, "Unknown action '#{action}'"
-        end
-      end
     end
   end
 end
diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb
index b9e5cf258f4..8e2a925dfea 100644
--- a/lib/gitlab/git/lfs_changes.rb
+++ b/lib/gitlab/git/lfs_changes.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     class LfsChanges
@@ -6,46 +8,12 @@ module Gitlab
         @newrev = newrev
       end
 
-      def new_pointers(object_limit: nil, not_in: nil)
-        @repository.gitaly_migrate(:blob_get_new_lfs_pointers) do |is_enabled|
-          if is_enabled
-            @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in)
-          else
-            git_new_pointers(object_limit, not_in)
-          end
-        end
+      def new_pointers(object_limit: nil, not_in: nil, dynamic_timeout: nil)
+        @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in, dynamic_timeout)
       end
 
       def all_pointers
-        @repository.gitaly_migrate(:blob_get_all_lfs_pointers) do |is_enabled|
-          if is_enabled
-            @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev)
-          else
-            git_all_pointers
-          end
-        end
-      end
-
-      private
-
-      def git_new_pointers(object_limit, not_in)
-        @new_pointers ||= begin
-          rev_list.new_objects(not_in: not_in, require_path: true) do |object_ids|
-            object_ids = object_ids.take(object_limit) if object_limit
-
-            Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids)
-          end
-        end
-      end
-
-      def git_all_pointers
-        rev_list.all_objects(require_path: true) do |object_ids|
-          Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids)
-        end
-      end
-
-      def rev_list
-        Gitlab::Git::RevList.new(@repository, newrev: @newrev)
+        @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev)
       end
     end
   end
diff --git a/lib/gitlab/git/lfs_pointer_file.rb b/lib/gitlab/git/lfs_pointer_file.rb
index 2ae0a889590..b7019a221ac 100644
--- a/lib/gitlab/git/lfs_pointer_file.rb
+++ b/lib/gitlab/git/lfs_pointer_file.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     class LfsPointerFile
diff --git a/lib/gitlab/git/merge_base.rb b/lib/gitlab/git/merge_base.rb
new file mode 100644
index 00000000000..b27f7038c26
--- /dev/null
+++ b/lib/gitlab/git/merge_base.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Git
+    class MergeBase
+      include Gitlab::Utils::StrongMemoize
+
+      def initialize(repository, refs)
+        @repository, @refs = repository, refs
+      end
+
+      # Returns the SHA of the first common ancestor
+      def sha
+        if unknown_refs.any?
+          raise UnknownRef, "Can't find merge base for unknown refs: #{unknown_refs.inspect}"
+        end
+
+        strong_memoize(:sha) do
+          @repository.merge_base(*commits_for_refs)
+        end
+      end
+
+      # Returns the merge base as a Gitlab::Git::Commit
+      def commit
+        return unless sha
+
+        @commit ||= @repository.commit_by(oid: sha)
+      end
+
+      # Returns the refs passed on initialization that aren't found in
+      # the repository, and thus cannot be used to find a merge base.
+      def unknown_refs
+        @unknown_refs ||= Hash[@refs.zip(commits_for_refs)]
+                            .select { |ref, commit| commit.nil? }.keys
+      end
+
+      private
+
+      def commits_for_refs
+        @commits_for_refs ||= @repository.commits_by(oids: @refs)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/object_pool.rb b/lib/gitlab/git/object_pool.rb
new file mode 100644
index 00000000000..1c6242b444a
--- /dev/null
+++ b/lib/gitlab/git/object_pool.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Git
+    class ObjectPool
+      # GL_REPOSITORY has to be passed for Gitlab::Git::Repositories, but not
+      # used for ObjectPools.
+      GL_REPOSITORY = ""
+
+      delegate :exists?, :size, to: :repository
+      delegate :unlink_repository, :delete, to: :object_pool_service
+
+      attr_reader :storage, :relative_path, :source_repository
+
+      def initialize(storage, relative_path, source_repository)
+        @storage = storage
+        @relative_path = relative_path
+        @source_repository = source_repository
+      end
+
+      def create
+        object_pool_service.create(source_repository)
+      end
+
+      def link(to_link_repo)
+        object_pool_service.link_repository(to_link_repo)
+      end
+
+      def gitaly_object_pool
+        Gitaly::ObjectPool.new(repository: to_gitaly_repository)
+      end
+
+      def to_gitaly_repository
+        Gitlab::GitalyClient::Util.repository(storage, relative_path, GL_REPOSITORY)
+      end
+
+      # Allows for reusing other RPCs by 'tricking' Gitaly to think its a repository
+      def repository
+        @repository ||= Gitlab::Git::Repository.new(storage, relative_path, GL_REPOSITORY)
+      end
+
+      private
+
+      def object_pool_service
+        @object_pool_service ||= Gitlab::GitalyClient::ObjectPoolService.new(self)
+      end
+
+      def relative_path_to(pool_member_path)
+        pool_path = Pathname.new("#{relative_path}#{File::SEPARATOR}")
+
+        Pathname.new(pool_member_path).relative_path_from(pool_path).to_s
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb
index 280def182d5..8797d3dce24 100644
--- a/lib/gitlab/git/operation_service.rb
+++ b/lib/gitlab/git/operation_service.rb
@@ -1,13 +1,15 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     class OperationService
-      include Gitlab::Git::Popen
-
       BranchUpdate = Struct.new(:newrev, :repo_created, :branch_created) do
         alias_method :repo_created?, :repo_created
         alias_method :branch_created?, :branch_created
 
         def self.from_gitaly(branch_update)
+          return if branch_update.nil?
+
           new(
             branch_update.commit_id,
             branch_update.repo_created,
@@ -15,177 +17,6 @@ module Gitlab
           )
         end
       end
-
-      attr_reader :user, :repository
-
-      def initialize(user, new_repository)
-        if user
-          user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id)
-          @user = user
-        end
-
-        # Refactoring aid
-        Gitlab::Git.check_namespace!(new_repository)
-
-        @repository = new_repository
-      end
-
-      def add_branch(branch_name, newrev)
-        ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name
-        oldrev = Gitlab::Git::BLANK_SHA
-
-        update_ref_in_hooks(ref, newrev, oldrev)
-      end
-
-      def rm_branch(branch)
-        ref = Gitlab::Git::BRANCH_REF_PREFIX + branch.name
-        oldrev = branch.target
-        newrev = Gitlab::Git::BLANK_SHA
-
-        update_ref_in_hooks(ref, newrev, oldrev)
-      end
-
-      def add_tag(tag_name, newrev, options = {})
-        ref = Gitlab::Git::TAG_REF_PREFIX + tag_name
-        oldrev = Gitlab::Git::BLANK_SHA
-
-        with_hooks(ref, newrev, oldrev) do |service|
-          # We want to pass the OID of the tag object to the hooks. For an
-          # annotated tag we don't know that OID until after the tag object
-          # (raw_tag) is created in the repository. That is why we have to
-          # update the value after creating the tag object. Only the
-          # "post-receive" hook will receive the correct value in this case.
-          raw_tag = repository.rugged.tags.create(tag_name, newrev, options)
-          service.newrev = raw_tag.target_id
-        end
-      end
-
-      def rm_tag(tag)
-        ref = Gitlab::Git::TAG_REF_PREFIX + tag.name
-        oldrev = tag.target
-        newrev = Gitlab::Git::BLANK_SHA
-
-        update_ref_in_hooks(ref, newrev, oldrev) do
-          repository.rugged.tags.delete(tag_name)
-        end
-      end
-
-      # Whenever `start_branch_name` is passed, if `branch_name` doesn't exist,
-      # it would be created from `start_branch_name`.
-      # If `start_repository` is passed, and the branch doesn't exist,
-      # it would try to find the commits from it instead of current repository.
-      def with_branch(
-        branch_name,
-        start_branch_name: nil,
-        start_repository: repository,
-        &block)
-
-        Gitlab::Git.check_namespace!(start_repository)
-        start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository)
-
-        start_branch_name = nil if start_repository.empty?
-
-        if start_branch_name && !start_repository.branch_exists?(start_branch_name)
-          raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.relative_path}"
-        end
-
-        update_branch_with_hooks(branch_name) do
-          repository.with_repo_branch_commit(
-            start_repository,
-            start_branch_name || branch_name,
-            &block)
-        end
-      end
-
-      def update_branch(branch_name, newrev, oldrev)
-        ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name
-        update_ref_in_hooks(ref, newrev, oldrev)
-      end
-
-      private
-
-      # Returns [newrev, should_run_after_create, should_run_after_create_branch]
-      def update_branch_with_hooks(branch_name)
-        update_autocrlf_option
-
-        was_empty = repository.empty?
-
-        # Make commit
-        newrev = yield
-
-        unless newrev
-          raise Gitlab::Git::CommitError.new('Failed to create commit')
-        end
-
-        branch = repository.find_branch(branch_name)
-        oldrev = find_oldrev_from_branch(newrev, branch)
-
-        ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name
-        update_ref_in_hooks(ref, newrev, oldrev)
-
-        BranchUpdate.new(newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev))
-      end
-
-      def find_oldrev_from_branch(newrev, branch)
-        return Gitlab::Git::BLANK_SHA unless branch
-
-        oldrev = branch.target
-
-        merge_base = repository.merge_base(newrev, branch.target)
-        raise Gitlab::Git::Repository::InvalidRef unless merge_base
-
-        if oldrev == merge_base
-          oldrev
-        else
-          raise Gitlab::Git::CommitError.new('Branch diverged')
-        end
-      end
-
-      def update_ref_in_hooks(ref, newrev, oldrev)
-        with_hooks(ref, newrev, oldrev) do
-          update_ref(ref, newrev, oldrev)
-        end
-      end
-
-      def with_hooks(ref, newrev, oldrev)
-        Gitlab::Git::HooksService.new.execute(
-          user,
-          repository,
-          oldrev,
-          newrev,
-          ref) do |service|
-
-          yield(service)
-        end
-      end
-
-      # Gitaly note: JV: wait with migrating #update_ref until we know how to migrate its call sites.
-      def update_ref(ref, newrev, oldrev)
-        # We use 'git update-ref' because libgit2/rugged currently does not
-        # offer 'compare and swap' ref updates. Without compare-and-swap we can
-        # (and have!) accidentally reset the ref to an earlier state, clobbering
-        # commits. See also https://github.com/libgit2/libgit2/issues/1534.
-        command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z]
-
-        output, status = popen(
-          command,
-          repository.path) do |stdin|
-          stdin.write("update #{ref}\x00#{newrev}\x00#{oldrev}\x00")
-        end
-
-        unless status.zero?
-          Gitlab::GitLogger.error("'git update-ref' in #{repository.path}: #{output}")
-          raise Gitlab::Git::CommitError.new(
-            "Could not update branch #{Gitlab::Git.branch_name(ref)}." \
-            " Please refresh and try again.")
-        end
-      end
-
-      def update_autocrlf_option
-        if repository.autocrlf != :input
-          repository.autocrlf = :input
-        end
-      end
     end
   end
 end
diff --git a/lib/gitlab/git/patches/collection.rb b/lib/gitlab/git/patches/collection.rb
new file mode 100644
index 00000000000..ad6b5d32abc
--- /dev/null
+++ b/lib/gitlab/git/patches/collection.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Git
+    module Patches
+      class Collection
+        MAX_PATCH_SIZE = 2.megabytes
+
+        def initialize(one_or_more_patches)
+          @patches = Array(one_or_more_patches).map do |patch_content|
+            Gitlab::Git::Patches::Patch.new(patch_content)
+          end
+        end
+
+        def content
+          @patches.map(&:content).join("\n")
+        end
+
+        def valid_size?
+          size < MAX_PATCH_SIZE
+        end
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        # `@patches` is not an `ActiveRecord` relation, but an `Enumerable`
+        # We're using sum from `ActiveSupport`
+        def size
+          @size ||= @patches.sum(&:size)
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/patches/commit_patches.rb b/lib/gitlab/git/patches/commit_patches.rb
new file mode 100644
index 00000000000..c62994432d3
--- /dev/null
+++ b/lib/gitlab/git/patches/commit_patches.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Git
+    module Patches
+      class CommitPatches
+        include Gitlab::Git::WrapsGitalyErrors
+
+        def initialize(user, repository, branch, patch_collection)
+          @user, @repository, @branch, @patches = user, repository, branch, patch_collection
+        end
+
+        def commit
+          repository.with_cache_hooks do
+            wrapped_gitaly_errors do
+              operation_service.user_commit_patches(user, branch, patches.content)
+            end
+          end
+        end
+
+        private
+
+        attr_reader :user, :repository, :branch, :patches
+
+        def operation_service
+          repository.raw.gitaly_operation_client
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/patches/patch.rb b/lib/gitlab/git/patches/patch.rb
new file mode 100644
index 00000000000..fe6ae1b5b00
--- /dev/null
+++ b/lib/gitlab/git/patches/patch.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Git
+    module Patches
+      class Patch
+        attr_reader :content
+
+        def initialize(content)
+          @content = content
+        end
+
+        def size
+          content.bytesize
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb
index 57b82a37d6c..e3a2031eeca 100644
--- a/lib/gitlab/git/path_helper.rb
+++ b/lib/gitlab/git/path_helper.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Gitaly note: JV: no RPC's here.
 
 module Gitlab
diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb
deleted file mode 100644
index f9f24ecc48d..00000000000
--- a/lib/gitlab/git/popen.rb
+++ /dev/null
@@ -1,104 +0,0 @@
-# Gitaly note: JV: no RPC's here.
-
-require 'open3'
-
-module Gitlab
-  module Git
-    module Popen
-      FAST_GIT_PROCESS_TIMEOUT = 15.seconds
-
-      def popen(cmd, path, vars = {}, lazy_block: nil)
-        unless cmd.is_a?(Array)
-          raise "System commands must be given as an array of strings"
-        end
-
-        path ||= Dir.pwd
-        vars['PWD'] = path
-        options = { chdir: path }
-
-        cmd_output = ""
-        cmd_status = 0
-        Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr|
-          stdout.set_encoding(Encoding::ASCII_8BIT)
-
-          yield(stdin) if block_given?
-          stdin.close
-
-          if lazy_block
-            cmd_output = lazy_block.call(stdout.lazy)
-            cmd_status = 0
-            break
-          else
-            cmd_output << stdout.read
-          end
-
-          cmd_output << stderr.read
-          cmd_status = wait_thr.value.exitstatus
-        end
-
-        [cmd_output, cmd_status]
-      end
-
-      def popen_with_timeout(cmd, timeout, path, vars = {})
-        unless cmd.is_a?(Array)
-          raise "System commands must be given as an array of strings"
-        end
-
-        path ||= Dir.pwd
-        vars['PWD'] = path
-
-        unless File.directory?(path)
-          FileUtils.mkdir_p(path)
-        end
-
-        rout, wout = IO.pipe
-        rerr, werr = IO.pipe
-
-        pid = Process.spawn(vars, *cmd, out: wout, err: werr, chdir: path, pgroup: true)
-
-        begin
-          status = process_wait_with_timeout(pid, timeout)
-
-          # close write ends so we could read them
-          wout.close
-          werr.close
-
-          cmd_output = rout.readlines.join
-          cmd_output << rerr.readlines.join # Copying the behaviour of `popen` which merges stderr into output
-
-          [cmd_output, status.exitstatus]
-        rescue Timeout::Error => e
-          kill_process_group_for_pid(pid)
-
-          raise e
-        ensure
-          wout.close unless wout.closed?
-          werr.close unless werr.closed?
-
-          rout.close
-          rerr.close
-        end
-      end
-
-      def process_wait_with_timeout(pid, timeout)
-        deadline = timeout.seconds.from_now
-        wait_time = 0.01
-
-        while deadline > Time.now
-          sleep(wait_time)
-          _, status = Process.wait2(pid, Process::WNOHANG)
-
-          return status unless status.nil?
-        end
-
-        raise Timeout::Error, "Timeout waiting for process ##{pid}"
-      end
-
-      def kill_process_group_for_pid(pid)
-        Process.kill("KILL", -pid)
-        Process.wait(pid)
-      rescue Errno::ESRCH
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/git/pre_receive_error.rb b/lib/gitlab/git/pre_receive_error.rb
new file mode 100644
index 00000000000..03caace6fce
--- /dev/null
+++ b/lib/gitlab/git/pre_receive_error.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Git
+    #
+    # PreReceiveError is special because its message gets displayed to users
+    # in the web UI. To prevent XSS we sanitize the message on
+    # initialization.
+    class PreReceiveError < StandardError
+      def initialize(msg = '')
+        super(nlbr(msg))
+      end
+
+      private
+
+      # In gitaly-ruby we override this method to do nothing, so that
+      # sanitization happens in gitlab-rails only.
+      def nlbr(str)
+        Gitlab::Utils.nlbr(str)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/push.rb b/lib/gitlab/git/push.rb
new file mode 100644
index 00000000000..b6577ba17f1
--- /dev/null
+++ b/lib/gitlab/git/push.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Git
+    class Push
+      include Gitlab::Utils::StrongMemoize
+
+      attr_reader :ref, :oldrev, :newrev
+
+      def initialize(project, oldrev, newrev, ref)
+        @project = project
+        @oldrev = oldrev.presence || Gitlab::Git::BLANK_SHA
+        @newrev = newrev.presence || Gitlab::Git::BLANK_SHA
+        @ref = ref
+      end
+
+      def branch_name
+        strong_memoize(:branch_name) do
+          Gitlab::Git.branch_name(@ref)
+        end
+      end
+
+      def branch_added?
+        Gitlab::Git.blank_ref?(@oldrev)
+      end
+
+      def branch_removed?
+        Gitlab::Git.blank_ref?(@newrev)
+      end
+
+      def branch_updated?
+        branch_push? && !branch_added? && !branch_removed?
+      end
+
+      def force_push?
+        Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev)
+      end
+
+      def branch_push?
+        strong_memoize(:branch_push) do
+          Gitlab::Git.branch_ref?(@ref)
+        end
+      end
+
+      def modified_paths
+        unless branch_updated?
+          raise ArgumentError, 'Unable to calculate modified paths!'
+        end
+
+        strong_memoize(:modified_paths) do
+          @project.repository.diff_stats(@oldrev, @newrev).paths
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/raw_diff_change.rb b/lib/gitlab/git/raw_diff_change.rb
index 98de9328071..e1002af40f6 100644
--- a/lib/gitlab/git/raw_diff_change.rb
+++ b/lib/gitlab/git/raw_diff_change.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     # This class behaves like a struct with fields :blob_id, :blob_size, :operation, :old_path, :new_path
diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb
index fa71a4e7ea7..eec91194949 100644
--- a/lib/gitlab/git/ref.rb
+++ b/lib/gitlab/git/ref.rb
@@ -1,4 +1,4 @@
-# Gitaly note: JV: probably no RPC's here (just one interaction with Rugged).
+# frozen_string_literal: true
 
 module Gitlab
   module Git
@@ -26,13 +26,6 @@ module Gitlab
         str.gsub(%r{\Arefs/heads/}, '')
       end
 
-      # Gitaly: this method will probably be migrated indirectly via its call sites.
-      def self.dereference_object(object)
-        object = object.target while object.is_a?(Rugged::Tag::Annotation)
-
-        object
-      end
-
       def initialize(repository, name, target, dereferenced_target)
         @name = Gitlab::Git.ref_name(name)
         @dereferenced_target = dereferenced_target
diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb
index ebe46722890..df3cd422527 100644
--- a/lib/gitlab/git/remote_mirror.rb
+++ b/lib/gitlab/git/remote_mirror.rb
@@ -1,87 +1,28 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     class RemoteMirror
-      def initialize(repository, ref_name)
-        @repository = repository
-        @ref_name = ref_name
-      end
-
-      def update(only_branches_matching: [])
-        @repository.gitaly_migrate(:remote_update_remote_mirror) do |is_enabled|
-          if is_enabled
-            gitaly_update(only_branches_matching)
-          else
-            rugged_update(only_branches_matching)
-          end
-        end
-      end
-
-      private
-
-      def gitaly_update(only_branches_matching)
-        @repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching)
-      end
+      include Gitlab::Git::WrapsGitalyErrors
 
-      def rugged_update(only_branches_matching)
-        local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching)
-        remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching)
+      attr_reader :repository, :ref_name, :only_branches_matching, :ssh_key, :known_hosts
 
-        updated_branches = changed_refs(local_branches, remote_branches)
-        push_branches(updated_branches.keys) if updated_branches.present?
-
-        delete_refs(local_branches, remote_branches)
-
-        local_tags = refs_obj(@repository.tags)
-        remote_tags = refs_obj(@repository.remote_tags(@ref_name))
-
-        updated_tags = changed_refs(local_tags, remote_tags)
-        @repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present?
-
-        delete_refs(local_tags, remote_tags)
-      end
-
-      def refs_obj(refs, only_refs_matching: [])
-        refs.each_with_object({}) do |ref, refs|
-          next if only_refs_matching.present? && !only_refs_matching.include?(ref.name)
-
-          refs[ref.name] = ref
-        end
-      end
-
-      def changed_refs(local_refs, remote_refs)
-        local_refs.select do |ref_name, ref|
-          remote_ref = remote_refs[ref_name]
-
-          remote_ref.nil? || ref.dereferenced_target != remote_ref.dereferenced_target
-        end
-      end
-
-      def push_branches(branches)
-        default_branch, branches = branches.partition do |branch|
-          @repository.root_ref == branch
-        end
-
-        # Push the default branch first so it works fine when remote mirror is empty.
-        branches.unshift(*default_branch)
-
-        @repository.push_remote_branches(@ref_name, branches)
-      end
-
-      def delete_refs(local_refs, remote_refs)
-        refs = refs_to_delete(local_refs, remote_refs)
-
-        @repository.delete_remote_branches(@ref_name, refs.keys) if refs.present?
-      end
-
-      def refs_to_delete(local_refs, remote_refs)
-        default_branch_id = @repository.commit.id
-
-        remote_refs.select do |remote_ref_name, remote_ref|
-          next false if local_refs[remote_ref_name] # skip if branch or tag exist in local repo
-
-          remote_ref_id = remote_ref.dereferenced_target.try(:id)
-
-          remote_ref_id && @repository.rugged_is_ancestor?(remote_ref_id, default_branch_id)
+      def initialize(repository, ref_name, only_branches_matching: [], ssh_key: nil, known_hosts: nil)
+        @repository = repository
+        @ref_name = ref_name
+        @only_branches_matching = only_branches_matching
+        @ssh_key = ssh_key
+        @known_hosts = known_hosts
+      end
+
+      def update
+        wrapped_gitaly_errors do
+          repository.gitaly_remote_client.update_remote_mirror(
+            ref_name,
+            only_branches_matching,
+            ssh_key: ssh_key,
+            known_hosts: known_hosts
+          )
         end
       end
     end
diff --git a/lib/gitlab/git/remote_repository.rb b/lib/gitlab/git/remote_repository.rb
index f40e59a8dd0..234541d8145 100644
--- a/lib/gitlab/git/remote_repository.rb
+++ b/lib/gitlab/git/remote_repository.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     #
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 4cbf20bfe76..786c90f9272 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -1,4 +1,5 @@
-# Gitlab::Git::Repository is a wrapper around native Rugged::Repository object
+# frozen_string_literal: true
+
 require 'tempfile'
 require 'forwardable'
 require "rubygems/package"
@@ -7,19 +8,12 @@ module Gitlab
   module Git
     class Repository
       include Gitlab::Git::RepositoryMirroring
-      include Gitlab::Git::Popen
+      include Gitlab::Git::WrapsGitalyErrors
       include Gitlab::EncodingHelper
       include Gitlab::Utils::StrongMemoize
 
-      ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[
-        GIT_OBJECT_DIRECTORY
-        GIT_ALTERNATE_OBJECT_DIRECTORIES
-      ].freeze
-      ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES = %w[
-        GIT_OBJECT_DIRECTORY_RELATIVE
-        GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE
-      ].freeze
       SEARCH_CONTEXT_LINES = 3
+      REV_LIST_COMMIT_LIMIT = 2_000
       # In https://gitlab.com/gitlab-org/gitaly/merge_requests/698
       # We copied these two prefixes into gitaly-go, so don't change these
       # or things will break! (REBASE_WORKTREE_PREFIX and SQUASH_WORKTREE_PREFIX)
@@ -40,19 +34,6 @@ module Gitlab
       ChecksumError = Class.new(StandardError)
 
       class << self
-        # Unlike `new`, `create` takes the repository path
-        def create(repo_path, bare: true, symlink_hooks_to: nil)
-          FileUtils.mkdir_p(repo_path, mode: 0770)
-
-          # Equivalent to `git --git-path=#{repo_path} init [--bare]`
-          repo = Rugged::Repository.init_at(repo_path, bare)
-          repo.close
-
-          create_hooks(repo_path, symlink_hooks_to) if symlink_hooks_to.present?
-
-          true
-        end
-
         def create_hooks(repo_path, global_hooks_path)
           local_hooks_path = File.join(repo_path, 'hooks')
           real_local_hooks_path = :not_found
@@ -86,10 +67,14 @@ module Gitlab
       # Relative path of repo
       attr_reader :relative_path
 
-      # Rugged repo object
-      attr_reader :rugged
+      attr_reader :storage, :gl_repository, :relative_path
 
-      attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path
+      # This remote name has to be stable for all types of repositories that
+      # can join an object pool. If it's structure ever changes, a migration
+      # has to be performed on the object pools to update the remote names.
+      # Else the pool can't be updated anymore and is left in an inconsistent
+      # state.
+      alias_method :object_pool_remote_name, :gl_repository
 
       # This initializer method is only used on the client side (gitlab-ce).
       # Gitaly-ruby uses a different initializer.
@@ -98,51 +83,33 @@ module Gitlab
         @relative_path = relative_path
         @gl_repository = gl_repository
 
-        @gitlab_projects = Gitlab::Git::GitlabProjects.new(
-          storage,
-          relative_path,
-          global_hooks_path: Gitlab.config.gitlab_shell.hooks_path,
-          logger: Rails.logger
-        )
-
         @name = @relative_path.split("/").last
       end
 
       def ==(other)
-        path == other.path
+        other.is_a?(self.class) && [storage, relative_path] == [other.storage, other.relative_path]
       end
 
+      alias_method :eql?, :==
+
+      def hash
+        [self.class, storage, relative_path].hash
+      end
+
+      # This method will be removed when Gitaly reaches v1.1.
       def path
-        @path ||= File.join(
+        File.join(
           Gitlab.config.repositories.storages[@storage].legacy_disk_path, @relative_path
         )
       end
 
       # Default branch in the repository
       def root_ref
-        @root_ref ||= gitaly_migrate(:root_ref) do |is_enabled|
-          if is_enabled
-            gitaly_ref_client.default_branch_name
-          else
-            discover_default_branch
-          end
-        end
-      end
-
-      def rugged
-        @rugged ||= circuit_breaker.perform do
-          Rugged::Repository.new(path, alternates: alternate_object_directories)
-        end
-      rescue Rugged::RepositoryError, Rugged::OSError
-        raise NoRepository.new('no repository for such path')
-      end
-
-      def cleanup
-        @rugged&.close
-      end
-
-      def circuit_breaker
-        @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage)
+        gitaly_ref_client.default_branch_name
+      rescue GRPC::NotFound => e
+        raise NoRepository.new(e.message)
+      rescue GRPC::Unknown => e
+        raise Gitlab::Git::CommandError.new(e.message)
       end
 
       def exists?
@@ -152,79 +119,36 @@ module Gitlab
       # Returns an Array of branch names
       # sorted by name ASC
       def branch_names
-        gitaly_migrate(:branch_names) do |is_enabled|
-          if is_enabled
-            gitaly_ref_client.branch_names
-          else
-            branches.map(&:name)
-          end
+        wrapped_gitaly_errors do
+          gitaly_ref_client.branch_names
         end
       end
 
       # Returns an Array of Branches
       def branches
-        gitaly_migrate(:branches) do |is_enabled|
-          if is_enabled
-            gitaly_ref_client.branches
-          else
-            branches_filter
-          end
+        wrapped_gitaly_errors do
+          gitaly_ref_client.branches
         end
       end
 
-      def reload_rugged
-        @rugged = nil
-      end
-
       # Directly find a branch with a simple name (e.g. master)
       #
-      # force_reload causes a new Rugged repository to be instantiated
-      #
-      # This is to work around a bug in libgit2 that causes in-memory refs to
-      # be stale/invalid when packed-refs is changed.
-      # See https://gitlab.com/gitlab-org/gitlab-ce/issues/15392#note_14538333
-      def find_branch(name, force_reload = false)
-        gitaly_migrate(:find_branch) do |is_enabled|
-          if is_enabled
-            gitaly_ref_client.find_branch(name)
-          else
-            reload_rugged if force_reload
-
-            rugged_ref = rugged.branches[name]
-            if rugged_ref
-              target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
-              Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
-            end
-          end
+      def find_branch(name)
+        wrapped_gitaly_errors do
+          gitaly_ref_client.find_branch(name)
         end
       end
 
       def local_branches(sort_by: nil)
-        gitaly_migrate(:local_branches) do |is_enabled|
-          if is_enabled
-            gitaly_ref_client.local_branches(sort_by: sort_by)
-          else
-            branches_filter(filter: :local, sort_by: sort_by)
-          end
+        wrapped_gitaly_errors do
+          gitaly_ref_client.local_branches(sort_by: sort_by)
         end
       end
 
       # Returns the number of valid branches
       def branch_count
-        gitaly_migrate(:branch_names) do |is_enabled|
-          if is_enabled
-            gitaly_ref_client.count_branch_names
-          else
-            rugged.branches.each(:local).count do |ref|
-              begin
-                ref.name && ref.target # ensures the branch is valid
-
-                true
-              rescue Rugged::ReferenceError
-                false
-              end
-            end
-          end
+        wrapped_gitaly_errors do
+          gitaly_ref_client.count_branch_names
         end
       end
 
@@ -245,50 +169,25 @@ module Gitlab
       # This refs by default not visible in project page and not cloned to client side.
       alias_method :has_visible_content?, :has_local_branches?
 
-      def has_local_branches_rugged?
-        rugged.branches.each(:local).any? do |ref|
-          begin
-            ref.name && ref.target # ensures the branch is valid
-
-            true
-          rescue Rugged::ReferenceError
-            false
-          end
-        end
-      end
-
       # Returns the number of valid tags
       def tag_count
-        gitaly_migrate(:tag_names) do |is_enabled|
-          if is_enabled
-            gitaly_ref_client.count_tag_names
-          else
-            rugged.tags.count
-          end
+        wrapped_gitaly_errors do
+          gitaly_ref_client.count_tag_names
         end
       end
 
       # Returns an Array of tag names
       def tag_names
-        gitaly_migrate(:tag_names) do |is_enabled|
-          if is_enabled
-            gitaly_ref_client.tag_names
-          else
-            rugged.tags.map { |t| t.name }
-          end
+        wrapped_gitaly_errors do
+          gitaly_ref_client.tag_names
         end
       end
 
       # Returns an Array of Tags
       #
-      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/390
       def tags
-        gitaly_migrate(:tags) do |is_enabled|
-          if is_enabled
-            tags_from_gitaly
-          else
-            tags_from_rugged
-          end
+        wrapped_gitaly_errors do
+          gitaly_ref_client.tags
         end
       end
 
@@ -296,13 +195,8 @@ module Gitlab
       #
       # Ref names must start with `refs/`.
       def ref_exists?(ref_name)
-        gitaly_migrate(:ref_exists,
-                      status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_ref_exists?(ref_name)
-          else
-            rugged_ref_exists?(ref_name)
-          end
+        wrapped_gitaly_errors do
+          gitaly_ref_exists?(ref_name)
         end
       end
 
@@ -310,12 +204,8 @@ module Gitlab
       #
       # name - The name of the tag as a String.
       def tag_exists?(name)
-        gitaly_migrate(:ref_exists_tags) do |is_enabled|
-          if is_enabled
-            gitaly_ref_exists?("refs/tags/#{name}")
-          else
-            rugged_tag_exists?(name)
-          end
+        wrapped_gitaly_errors do
+          gitaly_ref_exists?("refs/tags/#{name}")
         end
       end
 
@@ -323,20 +213,8 @@ module Gitlab
       #
       # name - The name of the branch as a String.
       def branch_exists?(name)
-        gitaly_migrate(:ref_exists_branches) do |is_enabled|
-          if is_enabled
-            gitaly_ref_exists?("refs/heads/#{name}")
-          else
-            rugged_branch_exists?(name)
-          end
-        end
-      end
-
-      def batch_existence(object_ids, existing: true)
-        filter_method = existing ? :select : :reject
-
-        object_ids.public_send(filter_method) do |oid| # rubocop:disable GitlabSecurity/PublicSend
-          rugged.exists?(oid)
+        wrapped_gitaly_errors do
+          gitaly_ref_exists?("refs/heads/#{name}")
         end
       end
 
@@ -346,61 +224,17 @@ module Gitlab
       end
 
       def delete_all_refs_except(prefixes)
-        gitaly_migrate(:ref_delete_refs) do |is_enabled|
-          if is_enabled
-            gitaly_ref_client.delete_refs(except_with_prefixes: prefixes)
-          else
-            delete_refs(*all_ref_names_except(prefixes))
-          end
+        wrapped_gitaly_errors do
+          gitaly_ref_client.delete_refs(except_with_prefixes: prefixes)
         end
       end
 
-      # Returns an Array of all ref names, except when it's matching pattern
-      #
-      # regexp - The pattern for ref names we don't want
-      def all_ref_names_except(prefixes)
-        rugged.references.reject do |ref|
-          prefixes.any? { |p| ref.name.start_with?(p) }
-        end.map(&:name)
-      end
-
-      # Discovers the default branch based on the repository's available branches
-      #
-      # - If no branches are present, returns nil
-      # - If one branch is present, returns its name
-      # - If two or more branches are present, returns current HEAD or master or first branch
-      def discover_default_branch
-        names = branch_names
-
-        return if names.empty?
-
-        return names[0] if names.length == 1
-
-        if rugged_head
-          extracted_name = Ref.extract_branch_name(rugged_head.name)
-
-          return extracted_name if names.include?(extracted_name)
-        end
-
-        if names.include?('master')
-          'master'
-        else
-          names[0]
-        end
-      end
-
-      def rugged_head
-        rugged.head
-      rescue Rugged::ReferenceError
-        nil
-      end
-
-      def archive_metadata(ref, storage_path, format = "tar.gz", append_sha:)
+      def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:)
         ref ||= root_ref
         commit = Gitlab::Git::Commit.find(self, ref)
         return {} if commit.nil?
 
-        prefix = archive_prefix(ref, commit.id, append_sha: append_sha)
+        prefix = archive_prefix(ref, commit.id, project_path, append_sha: append_sha)
 
         {
           'ArchivePrefix' => prefix,
@@ -412,16 +246,12 @@ module Gitlab
 
       # This is both the filename of the archive (missing the extension) and the
       # name of the top-level member of the archive under which all files go
-      #
-      # FIXME: The generated prefix is incorrect for projects with hashed
-      # storage enabled
-      def archive_prefix(ref, sha, append_sha:)
+      def archive_prefix(ref, sha, project_path, append_sha:)
         append_sha = (ref != sha) if append_sha.nil?
 
-        project_name = self.name.chomp('.git')
         formatted_ref = ref.tr('/', '-')
 
-        prefix_segments = [project_name, formatted_ref]
+        prefix_segments = [project_path, formatted_ref]
         prefix_segments << sha if append_sha
 
         prefix_segments.join('-')
@@ -466,18 +296,12 @@ module Gitlab
 
       # Return repo size in megabytes
       def size
-        size = gitaly_migrate(:repository_size) do |is_enabled|
-          if is_enabled
-            size_by_gitaly
-          else
-            size_by_shelling_out
-          end
-        end
+        size = gitaly_repository_client.repository_size
 
         (size.to_f / 1024).round(2)
       end
 
-      # Use the Rugged Walker API to build an array of commits.
+      # Build an array of commits.
       #
       # Usage.
       #   repo.log(
@@ -487,8 +311,6 @@ module Gitlab
       #     offset: 5,
       #     after: Time.new(2016, 4, 21, 14, 32, 10)
       #   )
-      #
-      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/446
       def log(options)
         default_options = {
           limit: 10,
@@ -509,69 +331,45 @@ module Gitlab
           raise ArgumentError.new("invalid Repository#log limit: #{limit.inspect}")
         end
 
-        gitaly_migrate(:find_commits) do |is_enabled|
-          if is_enabled
-            gitaly_commit_client.find_commits(options)
-          else
-            raw_log(options).map { |c| Commit.decorate(self, c) }
-          end
+        wrapped_gitaly_errors do
+          gitaly_commit_client.find_commits(options)
         end
       end
 
-      # Used in gitaly-ruby
-      def raw_log(options)
-        sha =
-          unless options[:all]
-            actual_ref = options[:ref] || root_ref
-            begin
-              sha_from_ref(actual_ref)
-            rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
-              # Return an empty array if the ref wasn't found
-              return []
-            end
-          end
-
-        log_by_shell(sha, options)
+      def new_commits(newrev)
+        wrapped_gitaly_errors do
+          gitaly_ref_client.list_new_commits(newrev)
+        end
       end
 
-      def count_commits(options)
-        count_commits_options = process_count_commits_options(options)
+      def new_blobs(newrev)
+        return [] if newrev.blank? || newrev == ::Gitlab::Git::BLANK_SHA
 
-        gitaly_migrate(:count_commits) do |is_enabled|
-          if is_enabled
-            count_commits_by_gitaly(count_commits_options)
-          else
-            count_commits_by_shelling_out(count_commits_options)
+        strong_memoize("new_blobs_#{newrev}") do
+          wrapped_gitaly_errors do
+            gitaly_ref_client.list_new_blobs(newrev, REV_LIST_COMMIT_LIMIT)
           end
         end
       end
 
-      # Return the object that +revspec+ points to.  If +revspec+ is an
-      # annotated tag, then return the tag's target instead.
-      def rev_parse_target(revspec)
-        obj = rugged.rev_parse(revspec)
-        Ref.dereference_object(obj)
-      end
-
-      # Return a collection of Rugged::Commits between the two revspec arguments.
-      # See http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for
-      # a detailed list of valid arguments.
-      #
-      # Gitaly note: JV: to be deprecated in favor of Commit.between
-      def rugged_commits_between(from, to)
-        walker = Rugged::Walker.new(rugged)
-        walker.sorting(Rugged::SORT_NONE | Rugged::SORT_REVERSE)
-
-        sha_from = sha_from_ref(from)
-        sha_to = sha_from_ref(to)
+      def count_commits(options)
+        options = process_count_commits_options(options.dup)
 
-        walker.push(sha_to)
-        walker.hide(sha_from)
+        wrapped_gitaly_errors do
+          if options[:left_right]
+            from = options[:from]
+            to = options[:to]
 
-        commits = walker.to_a
-        walker.reset
+            right_count = gitaly_commit_client
+              .commit_count("#{from}..#{to}", options)
+            left_count = gitaly_commit_client
+              .commit_count("#{to}..#{from}", options)
 
-        commits
+            [left_count, right_count]
+          else
+            gitaly_commit_client.commit_count(options[:ref], options)
+          end
+        end
       end
 
       # Counts the amount of commits between `from` and `to`.
@@ -584,65 +382,31 @@ module Gitlab
       def raw_changes_between(old_rev, new_rev)
         @raw_changes_between ||= {}
 
-        @raw_changes_between[[old_rev, new_rev]] ||= begin
-          return [] if new_rev.blank? || new_rev == Gitlab::Git::BLANK_SHA
+        @raw_changes_between[[old_rev, new_rev]] ||=
+          begin
+            return [] if new_rev.blank? || new_rev == Gitlab::Git::BLANK_SHA
 
-          gitaly_migrate(:raw_changes_between) do |is_enabled|
-            if is_enabled
+            wrapped_gitaly_errors do
               gitaly_repository_client.raw_changes_between(old_rev, new_rev)
                 .each_with_object([]) do |msg, arr|
                 msg.raw_changes.each { |change| arr << ::Gitlab::Git::RawDiffChange.new(change) }
               end
-            else
-              result = []
-
-              circuit_breaker.perform do
-                Open3.pipeline_r(git_diff_cmd(old_rev, new_rev), format_git_cat_file_script, git_cat_file_cmd) do |last_stdout, wait_threads|
-                  last_stdout.each_line { |line| result << ::Gitlab::Git::RawDiffChange.new(line.chomp!) }
-
-                  if wait_threads.any? { |waiter| !waiter.value&.success? }
-                    raise ::Gitlab::Git::Repository::GitError, "Unabled to obtain changes between #{old_rev} and #{new_rev}"
-                  end
-                end
-              end
-
-              result
             end
           end
-        end
       rescue ArgumentError => e
         raise Gitlab::Git::Repository::GitError.new(e)
       end
 
       # Returns the SHA of the most recent common ancestor of +from+ and +to+
-      def merge_base(from, to)
-        gitaly_migrate(:merge_base) do |is_enabled|
-          if is_enabled
-            gitaly_repository_client.find_merge_base(from, to)
-          else
-            rugged_merge_base(from, to)
-          end
+      def merge_base(*commits)
+        wrapped_gitaly_errors do
+          gitaly_repository_client.find_merge_base(*commits)
         end
       end
 
-      # Gitaly note: JV: check gitlab-ee before removing this method.
-      def rugged_is_ancestor?(ancestor_id, descendant_id)
-        return false if ancestor_id.nil? || descendant_id.nil?
-
-        rugged_merge_base(ancestor_id, descendant_id) == ancestor_id
-      rescue Rugged::OdbError
-        false
-      end
-
       # Returns true is +from+ is direct ancestor to +to+, otherwise false
       def ancestor?(from, to)
-        Gitlab::GitalyClient.migrate(:is_ancestor) do |is_enabled|
-          if is_enabled
-            gitaly_commit_client.ancestor?(from, to)
-          else
-            rugged_is_ancestor?(from, to)
-          end
-        end
+        gitaly_commit_client.ancestor?(from, to)
       end
 
       def merged_branch_names(branch_names = [])
@@ -652,12 +416,8 @@ module Gitlab
 
         return [] unless root_sha
 
-        branches = gitaly_migrate(:merged_branch_names) do |is_enabled|
-          if is_enabled
-            gitaly_merged_branch_names(branch_names, root_sha)
-          else
-            git_merged_branch_names(branch_names, root_sha)
-          end
+        branches = wrapped_gitaly_errors do
+          gitaly_merged_branch_names(branch_names, root_sha)
         end
 
         Set.new(branches)
@@ -668,35 +428,33 @@ module Gitlab
       # diff options.  The +options+ hash can also include :break_rewrites to
       # split larger rewrites into delete/add pairs.
       def diff(from, to, options = {}, *paths)
-        iterator = gitaly_migrate(:diff_between) do |is_enabled|
-          if is_enabled
-            gitaly_commit_client.diff(from, to, options.merge(paths: paths))
-          else
-            diff_patches(from, to, options, *paths)
-          end
-        end
+        iterator = gitaly_commit_client.diff(from, to, options.merge(paths: paths))
 
         Gitlab::Git::DiffCollection.new(iterator, options)
       end
 
+      def diff_stats(left_id, right_id)
+        if [left_id, right_id].any? { |ref| ref.blank? || Gitlab::Git.blank_ref?(ref) }
+          return empty_diff_stats
+        end
+
+        stats = wrapped_gitaly_errors do
+          gitaly_commit_client.diff_stats(left_id, right_id)
+        end
+
+        Gitlab::Git::DiffStatsCollection.new(stats)
+      rescue CommandError, TypeError
+        empty_diff_stats
+      end
+
       # Returns a RefName for a given SHA
       def ref_name_for_sha(ref_path, sha)
         raise ArgumentError, "sha can't be empty" unless sha.present?
 
-        gitaly_migrate(:find_ref_name) do |is_enabled|
-          if is_enabled
-            gitaly_ref_client.find_ref_name(sha, ref_path)
-          else
-            args = %W(for-each-ref --count=1 #{ref_path} --contains #{sha})
-
-            # Not found -> ["", 0]
-            # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0]
-            run_git(args).first.split.last
-          end
-        end
+        gitaly_ref_client.find_ref_name(sha, ref_path)
       end
 
-      # Get refs hash which key is is the commit id
+      # Get refs hash which key is the commit id
       # and value is a Gitlab::Git::Tag or Gitlab::Git::Branch
       # Note that both inherit from Gitlab::Git::Ref
       def refs_hash
@@ -713,41 +471,22 @@ module Gitlab
         @refs_hash
       end
 
-      # Lookup for rugged object by oid or ref name
-      def lookup(oid_or_ref_name)
-        rugged.rev_parse(oid_or_ref_name)
-      end
-
       # Returns url for submodule
       #
       # Ex.
       #   @repository.submodule_url_for('master', 'rack')
       #   # => git@localhost:rack.git
       #
-      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/329
       def submodule_url_for(ref, path)
-        Gitlab::GitalyClient.migrate(:submodule_url_for) do |is_enabled|
-          if is_enabled
-            gitaly_submodule_url_for(ref, path)
-          else
-            if submodules(ref).any?
-              submodule = submodules(ref)[path]
-              submodule['url'] if submodule
-            end
-          end
+        wrapped_gitaly_errors do
+          gitaly_submodule_url_for(ref, path)
         end
       end
 
       # Return total commits count accessible from passed ref
-      #
-      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/330
       def commit_count(ref)
-        gitaly_migrate(:commit_count) do |is_enabled|
-          if is_enabled
-            gitaly_commit_client.commit_count(ref)
-          else
-            rugged_commit_count(ref)
-          end
+        wrapped_gitaly_errors do
+          gitaly_commit_client.commit_count(ref)
         end
       end
 
@@ -776,38 +515,32 @@ module Gitlab
       end
 
       def add_branch(branch_name, user:, target:)
-        gitaly_operation_client.user_create_branch(branch_name, user, target)
-      rescue GRPC::FailedPrecondition => ex
-        raise InvalidRef, ex
+        wrapped_gitaly_errors do
+          gitaly_operation_client.user_create_branch(branch_name, user, target)
+        end
       end
 
       def add_tag(tag_name, user:, target:, message: nil)
-        gitaly_migrate(:operation_user_add_tag, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_add_tag(tag_name, user: user, target: target, message: message)
-          else
-            rugged_add_tag(tag_name, user: user, target: target, message: message)
-          end
+        wrapped_gitaly_errors do
+          gitaly_operation_client.add_tag(tag_name, user, target, message)
+        end
+      end
+
+      def update_branch(branch_name, user:, newrev:, oldrev:)
+        wrapped_gitaly_errors do
+          gitaly_operation_client.user_update_branch(branch_name, user, newrev, oldrev)
         end
       end
 
       def rm_branch(branch_name, user:)
-        gitaly_migrate(:operation_user_delete_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_operations_client.user_delete_branch(branch_name, user)
-          else
-            OperationService.new(user, self).rm_branch(find_branch(branch_name))
-          end
+        wrapped_gitaly_errors do
+          gitaly_operation_client.user_delete_branch(branch_name, user)
         end
       end
 
       def rm_tag(tag_name, user:)
-        gitaly_migrate(:operation_user_delete_tag) do |is_enabled|
-          if is_enabled
-            gitaly_operations_client.rm_tag(tag_name, user)
-          else
-            Gitlab::Git::OperationService.new(user, self).rm_tag(find_tag(tag_name))
-          end
+        wrapped_gitaly_errors do
+          gitaly_operation_client.rm_tag(tag_name, user)
         end
       end
 
@@ -816,142 +549,73 @@ module Gitlab
       end
 
       def merge(user, source_sha, target_branch, message, &block)
-        gitaly_migrate(:operation_user_merge_branch) do |is_enabled|
-          if is_enabled
-            gitaly_operation_client.user_merge_branch(user, source_sha, target_branch, message, &block)
-          else
-            rugged_merge(user, source_sha, target_branch, message, &block)
-          end
-        end
-      end
-
-      def rugged_merge(user, source_sha, target_branch, message)
-        committer = Gitlab::Git.committer_hash(email: user.email, name: user.name)
-
-        OperationService.new(user, self).with_branch(target_branch) do |start_commit|
-          our_commit = start_commit.sha
-          their_commit = source_sha
-
-          raise 'Invalid merge target' unless our_commit
-          raise 'Invalid merge source' unless their_commit
-
-          merge_index = rugged.merge_commits(our_commit, their_commit)
-          break if merge_index.conflicts?
-
-          options = {
-            parents: [our_commit, their_commit],
-            tree: merge_index.write_tree(rugged),
-            message: message,
-            author: committer,
-            committer: committer
-          }
-
-          commit_id = create_commit(options)
-
-          yield commit_id
-
-          commit_id
+        wrapped_gitaly_errors do
+          gitaly_operation_client.user_merge_branch(user, source_sha, target_branch, message, &block)
         end
-      rescue Gitlab::Git::CommitError # when merge_index.conflicts?
-        nil
       end
 
       def ff_merge(user, source_sha, target_branch)
-        gitaly_migrate(:operation_user_ff_branch) do |is_enabled|
-          if is_enabled
-            gitaly_ff_merge(user, source_sha, target_branch)
-          else
-            rugged_ff_merge(user, source_sha, target_branch)
-          end
+        wrapped_gitaly_errors do
+          gitaly_operation_client.user_ff_branch(user, source_sha, target_branch)
         end
       end
 
       def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
-        gitaly_migrate(:revert, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          args = {
-            user: user,
-            commit: commit,
-            branch_name: branch_name,
-            message: message,
-            start_branch_name: start_branch_name,
-            start_repository: start_repository
-          }
-
-          if is_enabled
-            gitaly_operations_client.user_revert(args)
-          else
-            rugged_revert(args)
-          end
-        end
-      end
-
-      def check_revert_content(target_commit, source_sha)
-        args = [target_commit.sha, source_sha]
-        args << { mainline: 1 } if target_commit.merge_commit?
-
-        revert_index = rugged.revert_commit(*args)
-        return false if revert_index.conflicts?
-
-        tree_id = revert_index.write_tree(rugged)
-        return false unless diff_exists?(source_sha, tree_id)
-
-        tree_id
-      end
+        args = {
+          user: user,
+          commit: commit,
+          branch_name: branch_name,
+          message: message,
+          start_branch_name: start_branch_name,
+          start_repository: start_repository
+        }
 
-      def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
-        gitaly_migrate(:cherry_pick) do |is_enabled|
-          args = {
-            user: user,
-            commit: commit,
-            branch_name: branch_name,
-            message: message,
-            start_branch_name: start_branch_name,
-            start_repository: start_repository
-          }
-
-          if is_enabled
-            gitaly_operations_client.user_cherry_pick(args)
-          else
-            rugged_cherry_pick(args)
-          end
+        wrapped_gitaly_errors do
+          gitaly_operation_client.user_revert(args)
         end
       end
 
-      def diff_exists?(sha1, sha2)
-        rugged.diff(sha1, sha2).size > 0
-      end
+      def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
+        args = {
+          user: user,
+          commit: commit,
+          branch_name: branch_name,
+          message: message,
+          start_branch_name: start_branch_name,
+          start_repository: start_repository
+        }
 
-      def user_to_committer(user)
-        Gitlab::Git.committer_hash(email: user.email, name: user.name)
+        wrapped_gitaly_errors do
+          gitaly_operation_client.user_cherry_pick(args)
+        end
       end
 
-      def create_commit(params = {})
-        params[:message].delete!("\r")
+      def update_submodule(user:, submodule:, commit_sha:, message:, branch:)
+        args = {
+          user: user,
+          submodule: submodule,
+          commit_sha: commit_sha,
+          branch: branch,
+          message: message
+        }
 
-        Rugged::Commit.create(rugged, params)
+        wrapped_gitaly_errors do
+          gitaly_operation_client.user_update_submodule(args)
+        end
       end
 
       # Delete the specified branch from the repository
       def delete_branch(branch_name)
-        gitaly_migrate(:delete_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_ref_client.delete_branch(branch_name)
-          else
-            rugged.branches.delete(branch_name)
-          end
+        wrapped_gitaly_errors do
+          gitaly_ref_client.delete_branch(branch_name)
         end
-      rescue Rugged::ReferenceError, CommandError => e
+      rescue CommandError => e
         raise DeleteBranchError, e
       end
 
       def delete_refs(*ref_names)
-        gitaly_migrate(:delete_refs,
-                      status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_delete_refs(*ref_names)
-          else
-            git_delete_refs(*ref_names)
-          end
+        wrapped_gitaly_errors do
+          gitaly_delete_refs(*ref_names)
         end
       end
 
@@ -961,58 +625,30 @@ module Gitlab
       #   create_branch("feature")
       #   create_branch("other-feature", "master")
       def create_branch(ref, start_point = "HEAD")
-        gitaly_migrate(:create_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_ref_client.create_branch(ref, start_point)
-          else
-            rugged_create_branch(ref, start_point)
-          end
+        wrapped_gitaly_errors do
+          gitaly_ref_client.create_branch(ref, start_point)
         end
       end
 
       # If `mirror_refmap` is present the remote is set as mirror with that mapping
       def add_remote(remote_name, url, mirror_refmap: nil)
-        gitaly_migrate(:remote_add_remote) do |is_enabled|
-          if is_enabled
-            gitaly_remote_client.add_remote(remote_name, url, mirror_refmap)
-          else
-            rugged_add_remote(remote_name, url, mirror_refmap)
-          end
+        wrapped_gitaly_errors do
+          gitaly_remote_client.add_remote(remote_name, url, mirror_refmap)
         end
       end
 
       def remove_remote(remote_name)
-        gitaly_migrate(:remote_remove_remote) do |is_enabled|
-          if is_enabled
-            gitaly_remote_client.remove_remote(remote_name)
-          else
-            rugged_remove_remote(remote_name)
-          end
+        wrapped_gitaly_errors do
+          gitaly_remote_client.remove_remote(remote_name)
         end
       end
 
-      # Update the specified remote using the values in the +options+ hash
-      #
-      # Example
-      # repo.update_remote("origin", url: "path/to/repo")
-      def remote_update(remote_name, url:)
-        # TODO: Implement other remote options
-        rugged.remotes.set_url(remote_name, url)
-        nil
-      end
+      def find_remote_root_ref(remote_name)
+        return unless remote_name.present?
 
-      AUTOCRLF_VALUES = {
-        "true" => true,
-        "false" => false,
-        "input" => :input
-      }.freeze
-
-      def autocrlf
-        AUTOCRLF_VALUES[rugged.config['core.autocrlf']]
-      end
-
-      def autocrlf=(value)
-        rugged.config['core.autocrlf'] = AUTOCRLF_VALUES.invert[value]
+        wrapped_gitaly_errors do
+          gitaly_remote_client.find_remote_root_ref(remote_name)
+        end
       end
 
       # Returns result like "git ls-files" , recursive and full file path
@@ -1020,48 +656,20 @@ module Gitlab
       # Ex.
       #   repo.ls_files('master')
       #
-      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/327
       def ls_files(ref)
-        gitaly_migrate(:ls_files) do |is_enabled|
-          if is_enabled
-            gitaly_ls_files(ref)
-          else
-            git_ls_files(ref)
-          end
-        end
+        gitaly_commit_client.ls_files(ref)
       end
 
-      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328
       def copy_gitattributes(ref)
-        Gitlab::GitalyClient.migrate(:apply_gitattributes) do |is_enabled|
-          if is_enabled
-            gitaly_copy_gitattributes(ref)
-          else
-            rugged_copy_gitattributes(ref)
-          end
+        wrapped_gitaly_errors do
+          gitaly_repository_client.apply_gitattributes(ref)
         end
-      rescue GRPC::InvalidArgument
-        raise InvalidRef
       end
 
       def info_attributes
         return @info_attributes if @info_attributes
 
-        content =
-          gitaly_migrate(:get_info_attributes, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-            if is_enabled
-              gitaly_repository_client.info_attributes
-            else
-              attributes_path = File.join(File.expand_path(path), 'info', 'attributes')
-
-              if File.exist?(attributes_path)
-                File.read(attributes_path)
-              else
-                ""
-              end
-            end
-          end
-
+        content = gitaly_repository_client.info_attributes
         @info_attributes = AttributesParser.new(content)
       end
 
@@ -1090,147 +698,46 @@ module Gitlab
       end
 
       def languages(ref = nil)
-        gitaly_migrate(:commit_languages, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_commit_client.languages(ref)
-          else
-            ref ||= rugged.head.target_id
-            languages = Linguist::Repository.new(rugged, ref).languages
-            total = languages.map(&:last).sum
-
-            languages = languages.map do |language|
-              name, share = language
-              color = Linguist::Language[name].color || "##{Digest::SHA256.hexdigest(name)[0...6]}"
-              {
-                value: (share.to_f * 100 / total).round(2),
-                label: name,
-                color: color,
-                highlight: color
-              }
-            end
-
-            languages.sort do |x, y|
-              y[:value] <=> x[:value]
-            end
-          end
+        wrapped_gitaly_errors do
+          gitaly_commit_client.languages(ref)
         end
       end
 
       def license_short_name
-        gitaly_migrate(:license_short_name,
-                       status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_repository_client.license_short_name
-          else
-            begin
-              # The licensee gem creates a Rugged object from the path:
-              # https://github.com/benbalter/licensee/blob/v8.7.0/lib/licensee/projects/git_project.rb
-              Licensee.license(path).try(:key)
-            rescue Rugged::Error
-            end
-          end
+        wrapped_gitaly_errors do
+          gitaly_repository_client.license_short_name
         end
       end
 
-      def with_repo_branch_commit(start_repository, start_branch_name)
-        Gitlab::Git.check_namespace!(start_repository)
-        start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository)
-
-        return yield nil if start_repository.empty?
-
-        if start_repository.same_repository?(self)
-          yield commit(start_branch_name)
-        else
-          start_commit_id = start_repository.commit_id(start_branch_name)
-
-          return yield nil unless start_commit_id
-
-          if branch_commit = commit(start_commit_id)
-            yield branch_commit
-          else
-            with_repo_tmp_commit(
-              start_repository, start_branch_name, start_commit_id) do |tmp_commit|
-              yield tmp_commit
-            end
-          end
+      def fetch_source_branch!(source_repository, source_branch, local_ref)
+        wrapped_gitaly_errors do
+          gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref)
         end
       end
 
-      def with_repo_tmp_commit(start_repository, start_branch_name, sha)
-        source_ref = start_branch_name
+      def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:)
+        tmp_ref = "refs/tmp/#{SecureRandom.hex}"
 
-        unless Gitlab::Git.branch_ref?(source_ref)
-          source_ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{source_ref}"
-        end
+        return unless fetch_source_branch!(source_repository, source_branch_name, tmp_ref)
 
-        tmp_ref = fetch_ref(
-          start_repository,
-          source_ref: source_ref,
-          target_ref: "refs/tmp/#{SecureRandom.hex}"
+        Gitlab::Git::Compare.new(
+          self,
+          target_branch_name,
+          tmp_ref,
+          straight: straight
         )
-
-        yield commit(sha)
       ensure
-        delete_refs(tmp_ref) if tmp_ref
-      end
-
-      def fetch_source_branch!(source_repository, source_branch, local_ref)
-        Gitlab::GitalyClient.migrate(:fetch_source_branch) do |is_enabled|
-          if is_enabled
-            gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref)
-          else
-            rugged_fetch_source_branch(source_repository, source_branch, local_ref)
-          end
-        end
+        delete_refs(tmp_ref)
       end
 
-      def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:)
-        Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-          with_repo_branch_commit(source_repository, source_branch_name) do |commit|
-            break unless commit
-
-            Gitlab::Git::Compare.new(
-              self,
-              target_branch_name,
-              commit.sha,
-              straight: straight
-            )
-          end
-        end
-      end
-
-      def write_ref(ref_path, ref, old_ref: nil, shell: true)
+      def write_ref(ref_path, ref, old_ref: nil)
         ref_path = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{ref_path}" unless ref_path.start_with?("refs/") || ref_path == "HEAD"
 
-        gitaly_migrate(:write_ref) do |is_enabled|
-          if is_enabled
-            gitaly_repository_client.write_ref(ref_path, ref, old_ref, shell)
-          else
-            local_write_ref(ref_path, ref, old_ref: old_ref, shell: shell)
-          end
+        wrapped_gitaly_errors do
+          gitaly_repository_client.write_ref(ref_path, ref, old_ref)
         end
       end
 
-      def fetch_ref(source_repository, source_ref:, target_ref:)
-        Gitlab::Git.check_namespace!(source_repository)
-        source_repository = RemoteRepository.new(source_repository) unless source_repository.is_a?(RemoteRepository)
-
-        message, status = GitalyClient.migrate(:fetch_ref) do |is_enabled|
-          if is_enabled
-            gitaly_fetch_ref(source_repository, source_ref: source_ref, target_ref: target_ref)
-          else
-            # When removing this code, also remove source_repository#path
-            # to remove deprecated method calls
-            local_fetch_ref(source_repository.path, source_ref: source_ref, target_ref: target_ref)
-          end
-        end
-
-        # Make sure ref was created, and raise Rugged::ReferenceError when not
-        raise Rugged::ReferenceError, message if status != 0
-
-        target_ref
-      end
-
       # Refactoring aid; allows us to copy code from app/models/repository.rb
       def commit(ref = 'HEAD')
         Gitlab::Git::Commit.find(self, ref)
@@ -1241,12 +748,28 @@ module Gitlab
       end
 
       def fetch_repository_as_mirror(repository)
-        gitaly_migrate(:remote_fetch_internal_remote) do |is_enabled|
-          if is_enabled
-            gitaly_remote_client.fetch_internal_remote(repository)
-          else
-            rugged_fetch_repository_as_mirror(repository)
-          end
+        wrapped_gitaly_errors do
+          gitaly_remote_client.fetch_internal_remote(repository)
+        end
+      end
+
+      # Fetch remote for repository
+      #
+      # remote - remote name
+      # ssh_auth - SSH known_hosts data and a private key to use for public-key authentication
+      # forced - should we use --force flag?
+      # no_tags - should we use --no-tags flag?
+      # prune - should we use --prune flag?
+      def fetch_remote(remote, ssh_auth: nil, forced: false, no_tags: false, prune: true)
+        wrapped_gitaly_errors do
+          gitaly_repository_client.fetch_remote(
+            remote,
+            ssh_auth: ssh_auth,
+            forced: forced,
+            no_tags: no_tags,
+            prune: prune,
+            timeout: GITLAB_PROJECTS_TIMEOUT
+          )
         end
       end
 
@@ -1259,20 +782,6 @@ module Gitlab
         Gitlab::Git::Blob.batch(self, items, blob_size_limit: blob_size_limit)
       end
 
-      def commit_index(user, branch_name, index, options)
-        committer = user_to_committer(user)
-
-        OperationService.new(user, self).with_branch(branch_name) do
-          commit_params = options.merge(
-            tree: index.write_tree(rugged),
-            author: committer,
-            committer: committer
-          )
-
-          create_commit(commit_params)
-        end
-      end
-
       def fsck
         msg, status = gitaly_repository_client.fsck
 
@@ -1280,16 +789,12 @@ module Gitlab
       end
 
       def create_from_bundle(bundle_path)
-        gitaly_migrate(:create_repo_from_bundle) do |is_enabled|
-          if is_enabled
-            gitaly_repository_client.create_from_bundle(bundle_path)
-          else
-            run_git!(%W(clone --bare -- #{bundle_path} #{path}), chdir: nil)
-            self.class.create_hooks(path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path))
-          end
-        end
+        # It's important to check that the linked-to file is actually a valid
+        # .bundle file as it is passed to `git clone`, which may otherwise
+        # interpret it as a pointer to another repository
+        ::Gitlab::Git::BundleFile.check!(bundle_path)
 
-        true
+        gitaly_repository_client.create_from_bundle(bundle_path)
       end
 
       def create_from_snapshot(url, auth)
@@ -1297,121 +802,77 @@ module Gitlab
       end
 
       def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:)
-        gitaly_migrate(:rebase) do |is_enabled|
-          if is_enabled
-            gitaly_rebase(user, rebase_id,
-                          branch: branch,
-                          branch_sha: branch_sha,
-                          remote_repository: remote_repository,
-                          remote_branch: remote_branch)
-          else
-            git_rebase(user, rebase_id,
-                       branch: branch,
-                       branch_sha: branch_sha,
-                       remote_repository: remote_repository,
-                       remote_branch: remote_branch)
-          end
+        wrapped_gitaly_errors do
+          gitaly_operation_client.user_rebase(user, rebase_id,
+                                            branch: branch,
+                                            branch_sha: branch_sha,
+                                            remote_repository: remote_repository,
+                                            remote_branch: remote_branch)
         end
       end
 
       def rebase_in_progress?(rebase_id)
-        gitaly_migrate(:rebase_in_progress) do |is_enabled|
-          if is_enabled
-            gitaly_repository_client.rebase_in_progress?(rebase_id)
-          else
-            fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id))
-          end
+        wrapped_gitaly_errors do
+          gitaly_repository_client.rebase_in_progress?(rebase_id)
         end
       end
 
       def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:)
-        gitaly_migrate(:squash) do |is_enabled|
-          if is_enabled
-            gitaly_operation_client.user_squash(user, squash_id, branch,
+        wrapped_gitaly_errors do
+          gitaly_operation_client.user_squash(user, squash_id, branch,
               start_sha, end_sha, author, message)
-          else
-            git_squash(user, squash_id, branch, start_sha, end_sha, author, message)
-          end
         end
       end
 
       def squash_in_progress?(squash_id)
-        gitaly_migrate(:squash_in_progress, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_repository_client.squash_in_progress?(squash_id)
-          else
-            fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id))
-          end
+        wrapped_gitaly_errors do
+          gitaly_repository_client.squash_in_progress?(squash_id)
         end
       end
 
-      def push_remote_branches(remote_name, branch_names, forced: true)
-        success = @gitlab_projects.push_branches(remote_name, GITLAB_PROJECTS_TIMEOUT, forced, branch_names)
-
-        success || gitlab_projects_error
-      end
-
-      def delete_remote_branches(remote_name, branch_names)
-        success = @gitlab_projects.delete_remote_branches(remote_name, branch_names)
-
-        success || gitlab_projects_error
-      end
-
-      def delete_remote_branches(remote_name, branch_names)
-        success = @gitlab_projects.delete_remote_branches(remote_name, branch_names)
-
-        success || gitlab_projects_error
-      end
-
       def bundle_to_disk(save_path)
-        gitaly_migrate(:bundle_to_disk) do |is_enabled|
-          if is_enabled
-            gitaly_repository_client.create_bundle(save_path)
-          else
-            run_git!(%W(bundle create #{save_path} --all))
-          end
+        wrapped_gitaly_errors do
+          gitaly_repository_client.create_bundle(save_path)
         end
 
         true
       end
 
-      # rubocop:disable Metrics/ParameterLists
       def multi_action(
         user, branch_name:, message:, actions:,
         author_email: nil, author_name: nil,
         start_branch_name: nil, start_repository: self)
 
-        gitaly_migrate(:operation_user_commit_files) do |is_enabled|
-          if is_enabled
-            gitaly_operation_client.user_commit_files(user, branch_name,
+        wrapped_gitaly_errors do
+          gitaly_operation_client.user_commit_files(user, branch_name,
               message, actions, author_email, author_name,
               start_branch_name, start_repository)
-          else
-            rugged_multi_action(user, branch_name, message, actions,
-              author_email, author_name, start_branch_name, start_repository)
-          end
         end
       end
-      # rubocop:enable Metrics/ParameterLists
 
       def write_config(full_path:)
         return unless full_path.present?
 
-        gitaly_migrate(:write_config) do |is_enabled|
-          if is_enabled
-            gitaly_repository_client.write_config(full_path: full_path)
-          else
-            rugged_write_config(full_path: full_path)
-          end
+        # This guard avoids Gitaly log/error spam
+        raise NoRepository, 'repository does not exist' unless exists?
+
+        set_config('gitlab.fullpath' => full_path)
+      end
+
+      def set_config(entries)
+        wrapped_gitaly_errors do
+          gitaly_repository_client.set_config(entries)
         end
       end
 
-      def gitaly_repository
-        Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository)
+      def delete_config(*keys)
+        wrapped_gitaly_errors do
+          gitaly_repository_client.delete_config(keys)
+        end
       end
 
-      def gitaly_operations_client
-        @gitaly_operations_client ||= Gitlab::GitalyClient::OperationService.new(self)
+      def gitaly_repository
+        Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository)
       end
 
       def gitaly_ref_client
@@ -1442,19 +903,9 @@ module Gitlab
         Gitlab::GitalyClient::ConflictsService.new(self, our_commit_oid, their_commit_oid)
       end
 
-      def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
-        Gitlab::GitalyClient.migrate(method, status: status, &block)
-      rescue GRPC::NotFound => e
-        raise NoRepository.new(e)
-      rescue GRPC::InvalidArgument => e
-        raise ArgumentError.new(e)
-      rescue GRPC::BadStatus => e
-        raise CommandError.new(e)
-      end
-
       def clean_stale_repository_files
-        gitaly_migrate(:repository_cleanup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          gitaly_repository_client.cleanup if is_enabled && exists?
+        wrapped_gitaly_errors do
+          gitaly_repository_client.cleanup if exists?
         end
       rescue Gitlab::Git::CommandError => e # Don't fail if we can't cleanup
         Rails.logger.error("Unable to clean repository on storage #{storage} with relative path #{relative_path}: #{e.message}")
@@ -1478,25 +929,14 @@ module Gitlab
         safe_query = Regexp.escape(query)
         ref ||= root_ref
 
-        gitaly_migrate(:search_files_by_content) do |is_enabled|
-          if is_enabled
-            gitaly_repository_client.search_files_by_content(ref, safe_query)
-          else
-            offset = 2
-            args = %W(grep -i -I -n -z --before-context #{offset} --after-context #{offset} -E -e #{safe_query} #{ref})
-
-            run_git(args).first.scrub.split(/^--\n/)
-          end
-        end
+        gitaly_repository_client.search_files_by_content(ref, safe_query)
       end
 
       def can_be_merged?(source_sha, target_branch)
-        gitaly_migrate(:can_be_merged) do |is_enabled|
-          if is_enabled
-            gitaly_can_be_merged?(source_sha, find_branch(target_branch, true).target)
-          else
-            rugged_can_be_merged?(source_sha, target_branch)
-          end
+        if target_sha = find_branch(target_branch)&.target
+          !gitaly_conflicts_client(source_sha, target_sha).conflicts?
+        else
+          false
         end
       end
 
@@ -1506,87 +946,27 @@ module Gitlab
 
         return [] if empty? || safe_query.blank?
 
-        gitaly_migrate(:search_files_by_name) do |is_enabled|
-          if is_enabled
-            gitaly_repository_client.search_files_by_name(ref, safe_query)
-          else
-            args = %W(ls-tree -r --name-status --full-tree #{ref} -- #{safe_query})
-
-            run_git(args).first.lines.map(&:strip)
-          end
-        end
+        gitaly_repository_client.search_files_by_name(ref, safe_query)
       end
 
       def find_commits_by_message(query, ref, path, limit, offset)
-        gitaly_migrate(:commits_by_message) do |is_enabled|
-          if is_enabled
-            find_commits_by_message_by_gitaly(query, ref, path, limit, offset)
-          else
-            find_commits_by_message_by_shelling_out(query, ref, path, limit, offset)
-          end
+        wrapped_gitaly_errors do
+          gitaly_commit_client
+            .commits_by_message(query, revision: ref, path: path, limit: limit, offset: offset)
+            .map { |c| commit(c) }
         end
       end
 
-      def shell_blame(sha, path)
-        output, _status = run_git(%W(blame -p #{sha} -- #{path}))
-        output
-      end
-
-      def last_commit_for_path(sha, path)
-        gitaly_migrate(:last_commit_for_path) do |is_enabled|
-          if is_enabled
-            last_commit_for_path_by_gitaly(sha, path)
-          else
-            last_commit_for_path_by_rugged(sha, path)
-          end
-        end
-      end
-
-      def rev_list(including: [], excluding: [], objects: false, &block)
-        args = ['rev-list']
-
-        args.push(*rev_list_param(including))
-
-        exclude_param = *rev_list_param(excluding)
-        if exclude_param.any?
-          args.push('--not')
-          args.push(*exclude_param)
+      def list_last_commits_for_tree(sha, path, offset: 0, limit: 25)
+        wrapped_gitaly_errors do
+          gitaly_commit_client.list_last_commits_for_tree(sha, path, offset: offset, limit: limit)
         end
-
-        args.push('--objects') if objects
-
-        run_git!(args, lazy_block: block)
-      end
-
-      def missed_ref(oldrev, newrev)
-        run_git!(['rev-list', '--max-count=1', oldrev, "^#{newrev}"])
       end
 
-      def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:)
-        base_args = %w(worktree add --detach)
-
-        # Note that we _don't_ want to test for `.present?` here: If the caller
-        # passes an non nil empty value it means it still wants sparse checkout
-        # but just isn't interested in any file, perhaps because it wants to
-        # checkout files in by a changeset but that changeset only adds files.
-        if sparse_checkout_files
-          # Create worktree without checking out
-          run_git!(base_args + ['--no-checkout', worktree_path], env: env)
-          worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path).chomp
-
-          configure_sparse_checkout(worktree_git_path, sparse_checkout_files)
-
-          # After sparse checkout configuration, checkout `branch` in worktree
-          run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env)
-        else
-          # Create worktree and checkout `branch` in it
-          run_git!(base_args + [worktree_path, branch], env: env)
+      def last_commit_for_path(sha, path)
+        wrapped_gitaly_errors do
+          gitaly_commit_client.last_commit_for_path(sha, path)
         end
-
-        yield
-      ensure
-        FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path)
-        FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path)
       end
 
       def checksum
@@ -1600,157 +980,13 @@ module Gitlab
 
       private
 
-      def uncached_has_local_branches?
-        gitaly_migrate(:has_local_branches, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_repository_client.has_local_branches?
-          else
-            has_local_branches_rugged?
-          end
-        end
-      end
-
-      def local_write_ref(ref_path, ref, old_ref: nil, shell: true)
-        if shell
-          shell_write_ref(ref_path, ref, old_ref)
-        else
-          rugged_write_ref(ref_path, ref)
-        end
-      end
-
-      def rugged_write_config(full_path:)
-        rugged.config['gitlab.fullpath'] = full_path
-      end
-
-      def shell_write_ref(ref_path, ref, old_ref)
-        raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ')
-        raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00")
-        raise ArgumentError, "invalid old_ref #{old_ref.inspect}" if !old_ref.nil? && old_ref.include?("\x00")
-
-        input = "update #{ref_path}\x00#{ref}\x00#{old_ref}\x00"
-        run_git!(%w[update-ref --stdin -z]) { |stdin| stdin.write(input) }
-      end
-
-      def rugged_write_ref(ref_path, ref)
-        rugged.references.create(ref_path, ref, force: true)
-      rescue Rugged::ReferenceError => ex
-        Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}"
-      rescue Rugged::OSError => ex
-        raise unless ex.message =~ /Failed to create locked file/ && ex.message =~ /File exists/
-
-        Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}"
-      end
-
-      def run_git(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block)
-        cmd = [Gitlab.config.git.bin_path, *args]
-        cmd.unshift("nice") if nice
-
-        object_directories = alternate_object_directories
-        if object_directories.any?
-          env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = object_directories.join(File::PATH_SEPARATOR)
-        end
-
-        circuit_breaker.perform do
-          popen(cmd, chdir, env, lazy_block: lazy_block, &block)
-        end
-      end
-
-      def run_git!(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block)
-        output, status = run_git(args, chdir: chdir, env: env, nice: nice, lazy_block: lazy_block, &block)
-
-        raise GitError, output unless status.zero?
-
-        output
-      end
-
-      def run_git_with_timeout(args, timeout, env: {})
-        circuit_breaker.perform do
-          popen_with_timeout([Gitlab.config.git.bin_path, *args], timeout, path, env)
-        end
-      end
-
-      # This function is duplicated in Gitaly-Go, don't change it!
-      # https://gitlab.com/gitlab-org/gitaly/merge_requests/698
-      def fresh_worktree?(path)
-        File.exist?(path) && !clean_stuck_worktree(path)
+      def empty_diff_stats
+        Gitlab::Git::DiffStatsCollection.new([])
       end
 
-      # This function is duplicated in Gitaly-Go, don't change it!
-      # https://gitlab.com/gitlab-org/gitaly/merge_requests/698
-      def clean_stuck_worktree(path)
-        return false unless File.mtime(path) < 15.minutes.ago
-
-        FileUtils.rm_rf(path)
-        true
-      end
-
-      # Adding a worktree means checking out the repository. For large repos,
-      # this can be very expensive, so set up sparse checkout for the worktree
-      # to only check out the files we're interested in.
-      def configure_sparse_checkout(worktree_git_path, files)
-        run_git!(%w(config core.sparseCheckout true))
-
-        return if files.empty?
-
-        worktree_info_path = File.join(worktree_git_path, 'info')
-        FileUtils.mkdir_p(worktree_info_path)
-        File.write(File.join(worktree_info_path, 'sparse-checkout'), files)
-      end
-
-      def rugged_fetch_source_branch(source_repository, source_branch, local_ref)
-        with_repo_branch_commit(source_repository, source_branch) do |commit|
-          if commit
-            write_ref(local_ref, commit.sha)
-            true
-          else
-            false
-          end
-        end
-      end
-
-      def worktree_path(prefix, id)
-        id = id.to_s
-        raise ArgumentError, "worktree id can't be empty" unless id.present?
-        raise ArgumentError, "worktree id can't contain slashes " if id.include?("/")
-
-        File.join(path, 'gitlab-worktree', "#{prefix}-#{id}")
-      end
-
-      def git_env_for_user(user)
-        {
-          'GIT_COMMITTER_NAME' => user.name,
-          'GIT_COMMITTER_EMAIL' => user.email,
-          'GL_ID' => Gitlab::GlId.gl_id(user),
-          'GL_PROTOCOL' => Gitlab::Git::Hook::GL_PROTOCOL,
-          'GL_REPOSITORY' => gl_repository
-        }
-      end
-
-      # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'.
-      def branches_filter(filter: nil, sort_by: nil)
-        branches = rugged.branches.each(filter).map do |rugged_ref|
-          begin
-            target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
-            Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
-          rescue Rugged::ReferenceError
-            # Omit invalid branch
-          end
-        end.compact
-
-        sort_branches(branches, sort_by)
-      end
-
-      def git_merged_branch_names(branch_names, root_sha)
-        git_arguments =
-          %W[branch --merged #{root_sha}
-             --format=%(refname:short)\ %(objectname)] + branch_names
-
-        lines = run_git(git_arguments).first.lines
-
-        lines.each_with_object([]) do |line, branches|
-          name, sha = line.strip.split(' ', 2)
-
-          branches << name if sha != root_sha
+      def uncached_has_local_branches?
+        wrapped_gitaly_errors do
+          gitaly_repository_client.has_local_branches?
         end
       end
 
@@ -1782,63 +1018,6 @@ module Gitlab
         end
       end
 
-      # Gitaly note: JV: although #log_by_shell shells out to Git I think the
-      # complexity is such that we should migrate it as Ruby before trying to
-      # do it in Go.
-      def log_by_shell(sha, options)
-        limit = options[:limit].to_i
-        offset = options[:offset].to_i
-        use_follow_flag = options[:follow] && options[:path].present?
-
-        # We will perform the offset in Ruby because --follow doesn't play well with --skip.
-        # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520
-        offset_in_ruby = use_follow_flag && options[:offset].present?
-        limit += offset if offset_in_ruby
-
-        cmd = %w[log]
-        cmd << "--max-count=#{limit}"
-        cmd << '--format=%H'
-        cmd << "--skip=#{offset}" unless offset_in_ruby
-        cmd << '--follow' if use_follow_flag
-        cmd << '--no-merges' if options[:skip_merges]
-        cmd << "--after=#{options[:after].iso8601}" if options[:after]
-        cmd << "--before=#{options[:before].iso8601}" if options[:before]
-
-        if options[:all]
-          cmd += %w[--all --reverse]
-        else
-          cmd << sha
-        end
-
-        # :path can be a string or an array of strings
-        if options[:path].present?
-          cmd << '--'
-          cmd += Array(options[:path])
-        end
-
-        raw_output, _status = run_git(cmd)
-        lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines
-
-        lines.map! { |c| Rugged::Commit.new(rugged, c.strip) }
-      end
-
-      # We are trying to deprecate this method because it does a lot of work
-      # but it seems to be used only to look up submodule URL's.
-      # https://gitlab.com/gitlab-org/gitaly/issues/329
-      def submodules(ref)
-        commit = rev_parse_target(ref)
-        return {} unless commit
-
-        begin
-          content = blob_content(commit, ".gitmodules")
-        rescue InvalidBlobName
-          return {}
-        end
-
-        parser = GitmodulesParser.new(content)
-        fill_submodule_ids(commit, parser.parse)
-      end
-
       def gitaly_submodule_url_for(ref, path)
         # We don't care about the contents so 1 byte is enough. Can't request 0 bytes, 0 means unlimited.
         commit_object = gitaly_commit_client.tree_entry(ref, path, 1)
@@ -1853,246 +1032,6 @@ module Gitlab
         found_module && found_module['url']
       end
 
-      def alternate_object_directories
-        relative_object_directories.map { |d| File.join(path, d) }
-      end
-
-      def relative_object_directories
-        Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact
-      end
-
-      # Get the content of a blob for a given commit.  If the blob is a commit
-      # (for submodules) then return the blob's OID.
-      def blob_content(commit, blob_name)
-        blob_entry = tree_entry(commit, blob_name)
-
-        unless blob_entry
-          raise InvalidBlobName.new("Invalid blob name: #{blob_name}")
-        end
-
-        case blob_entry[:type]
-        when :commit
-          blob_entry[:oid]
-        when :tree
-          raise InvalidBlobName.new("#{blob_name} is a tree, not a blob")
-        when :blob
-          rugged.lookup(blob_entry[:oid]).content
-        end
-      end
-
-      # Fill in the 'id' field of a submodule hash from its values
-      # as-of +commit+. Return a Hash consisting only of entries
-      # from the submodule hash for which the 'id' field is filled.
-      def fill_submodule_ids(commit, submodule_data)
-        submodule_data.each do |path, data|
-          id = begin
-            blob_content(commit, path)
-          rescue InvalidBlobName
-            nil
-          end
-          data['id'] = id
-        end
-        submodule_data.select { |path, data| data['id'] }
-      end
-
-      # Find the entry for +path+ in the tree for +commit+
-      def tree_entry(commit, path)
-        pathname = Pathname.new(path)
-        first = true
-        tmp_entry = nil
-
-        pathname.each_filename do |dir|
-          if first
-            tmp_entry = commit.tree[dir]
-            first = false
-          elsif tmp_entry.nil?
-            return nil
-          else
-            begin
-              tmp_entry = rugged.lookup(tmp_entry[:oid])
-            rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
-              return nil
-            end
-
-            return nil unless tmp_entry.type == :tree
-
-            tmp_entry = tmp_entry[dir]
-          end
-        end
-
-        tmp_entry
-      end
-
-      # Return the Rugged patches for the diff between +from+ and +to+.
-      def diff_patches(from, to, options = {}, *paths)
-        options ||= {}
-        break_rewrites = options[:break_rewrites]
-        actual_options = Gitlab::Git::Diff.filter_diff_options(options.merge(paths: paths))
-
-        diff = rugged.diff(from, to, actual_options)
-        diff.find_similar!(break_rewrites: break_rewrites)
-        diff.each_patch
-      end
-
-      def sort_branches(branches, sort_by)
-        case sort_by
-        when 'name'
-          branches.sort_by(&:name)
-        when 'updated_desc'
-          branches.sort do |a, b|
-            b.dereferenced_target.committed_date <=> a.dereferenced_target.committed_date
-          end
-        when 'updated_asc'
-          branches.sort do |a, b|
-            a.dereferenced_target.committed_date <=> b.dereferenced_target.committed_date
-          end
-        else
-          branches
-        end
-      end
-
-      def tags_from_rugged
-        rugged.references.each("refs/tags/*").map do |ref|
-          message = nil
-
-          if ref.target.is_a?(Rugged::Tag::Annotation)
-            tag_message = ref.target.message
-
-            if tag_message.respond_to?(:chomp)
-              message = tag_message.chomp
-            end
-          end
-
-          target_commit = Gitlab::Git::Commit.find(self, ref.target)
-          Gitlab::Git::Tag.new(self, {
-            name: ref.name,
-            target: ref.target,
-            target_commit: target_commit,
-            message: message
-          })
-        end.sort_by(&:name)
-      end
-
-      def last_commit_for_path_by_rugged(sha, path)
-        sha = last_commit_id_for_path_by_shelling_out(sha, path)
-        commit(sha)
-      end
-
-      def tags_from_gitaly
-        gitaly_ref_client.tags
-      end
-
-      def size_by_shelling_out
-        popen(%w(du -sk), path).first.strip.to_i
-      end
-
-      def size_by_gitaly
-        gitaly_repository_client.repository_size
-      end
-
-      def count_commits_by_gitaly(options)
-        if options[:left_right]
-          from = options[:from]
-          to = options[:to]
-
-          right_count = gitaly_commit_client
-            .commit_count("#{from}..#{to}", options)
-          left_count = gitaly_commit_client
-            .commit_count("#{to}..#{from}", options)
-
-          [left_count, right_count]
-        else
-          gitaly_commit_client.commit_count(options[:ref], options)
-        end
-      end
-
-      def count_commits_by_shelling_out(options)
-        cmd = count_commits_shelling_command(options)
-
-        raw_output, _status = run_git(cmd)
-
-        process_count_commits_raw_output(raw_output, options)
-      end
-
-      def count_commits_shelling_command(options)
-        cmd = %w[rev-list]
-        cmd << "--after=#{options[:after].iso8601}" if options[:after]
-        cmd << "--before=#{options[:before].iso8601}" if options[:before]
-        cmd << "--max-count=#{options[:max_count]}" if options[:max_count]
-        cmd << "--left-right" if options[:left_right]
-        cmd << '--count'
-
-        cmd << if options[:all]
-                 '--all'
-               elsif options[:ref]
-                 options[:ref]
-               else
-                 raise ArgumentError, "Please specify a valid ref or set the 'all' attribute to true"
-               end
-
-        cmd += %W[-- #{options[:path]}] if options[:path].present?
-        cmd
-      end
-
-      def process_count_commits_raw_output(raw_output, options)
-        if options[:left_right]
-          result = raw_output.scan(/\d+/).map(&:to_i)
-
-          if result.sum != options[:max_count]
-            result
-          else # Reaching max count, right is not accurate
-            right_option =
-              process_count_commits_options(options
-                .except(:left_right, :from, :to)
-                .merge(ref: options[:to]))
-
-            right = count_commits_by_shelling_out(right_option)
-
-            [result.first, right] # left should be accurate in the first call
-          end
-        else
-          raw_output.to_i
-        end
-      end
-
-      def gitaly_ls_files(ref)
-        gitaly_commit_client.ls_files(ref)
-      end
-
-      def git_ls_files(ref)
-        actual_ref = ref || root_ref
-
-        begin
-          sha_from_ref(actual_ref)
-        rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
-          # Return an empty array if the ref wasn't found
-          return []
-        end
-
-        cmd = %W(ls-tree -r --full-tree --full-name -- #{actual_ref})
-        raw_output, _status = run_git(cmd)
-
-        lines = raw_output.split("\n").map do |f|
-          stuff, path = f.split("\t")
-          _mode, type, _sha = stuff.split(" ")
-          path if type == "blob"
-          # Contain only blob type
-        end
-
-        lines.compact
-      end
-
-      # Returns true if the given ref name exists
-      #
-      # Ref names must start with `refs/`.
-      def rugged_ref_exists?(ref_name)
-        raise ArgumentError, 'invalid refname' unless ref_name.start_with?('refs/')
-
-        rugged.references.exist?(ref_name)
-      rescue Rugged::ReferenceError
-        false
-      end
-
       # Returns true if the given ref name exists
       #
       # Ref names must start with `refs/`.
@@ -2100,433 +1039,13 @@ module Gitlab
         gitaly_ref_client.ref_exists?(ref_name)
       end
 
-      # Returns true if the given tag exists
-      #
-      # name - The name of the tag as a String.
-      def rugged_tag_exists?(name)
-        !!rugged.tags[name]
-      end
-
-      # Returns true if the given branch exists
-      #
-      # name - The name of the branch as a String.
-      def rugged_branch_exists?(name)
-        rugged.branches.exists?(name)
-
-      # If the branch name is invalid (e.g. ".foo") Rugged will raise an error.
-      # Whatever code calls this method shouldn't have to deal with that so
-      # instead we just return `false` (which is true since a branch doesn't
-      # exist when it has an invalid name).
-      rescue Rugged::ReferenceError
-        false
-      end
-
-      def gitaly_add_tag(tag_name, user:, target:, message: nil)
-        gitaly_operations_client.add_tag(tag_name, user, target, message)
-      end
-
-      def rugged_add_tag(tag_name, user:, target:, message: nil)
-        target_object = Ref.dereference_object(lookup(target))
-        raise InvalidRef.new("target not found: #{target}") unless target_object
-
-        user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id)
-
-        options = nil # Use nil, not the empty hash. Rugged cares about this.
-        if message
-          options = {
-            message: message,
-            tagger: Gitlab::Git.committer_hash(email: user.email, name: user.name)
-          }
-        end
-
-        Gitlab::Git::OperationService.new(user, self).add_tag(tag_name, target_object.oid, options)
-
-        find_tag(tag_name)
-      rescue Rugged::ReferenceError => ex
-        raise InvalidRef, ex
-      rescue Rugged::TagError
-        raise TagExistsError
-      end
-
-      def rugged_create_branch(ref, start_point)
-        rugged_ref = rugged.branches.create(ref, start_point)
-        target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
-        Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
-      rescue Rugged::ReferenceError => e
-        raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ %r{'refs/heads/#{ref}'}
-
-        raise InvalidRef.new("Invalid reference #{start_point}")
-      end
-
       def gitaly_copy_gitattributes(revision)
         gitaly_repository_client.apply_gitattributes(revision)
       end
 
-      def rugged_copy_gitattributes(ref)
-        begin
-          commit = lookup(ref)
-        rescue Rugged::ReferenceError
-          raise InvalidRef.new("Ref #{ref} is invalid")
-        end
-
-        # Create the paths
-        info_dir_path = File.join(path, 'info')
-        info_attributes_path = File.join(info_dir_path, 'attributes')
-
-        begin
-          # Retrieve the contents of the blob
-          gitattributes_content = blob_content(commit, '.gitattributes')
-        rescue InvalidBlobName
-          # No .gitattributes found. Should now remove any info/attributes and return
-          File.delete(info_attributes_path) if File.exist?(info_attributes_path)
-          return
-        end
-
-        # Create the info directory if needed
-        Dir.mkdir(info_dir_path) unless File.directory?(info_dir_path)
-
-        # Write the contents of the .gitattributes file to info/attributes
-        # Use binary mode to prevent Rails from converting ASCII-8BIT to UTF-8
-        File.open(info_attributes_path, "wb") do |file|
-          file.write(gitattributes_content)
-        end
-      end
-
-      def rugged_revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
-        OperationService.new(user, self).with_branch(
-          branch_name,
-          start_branch_name: start_branch_name,
-          start_repository: start_repository
-        ) do |start_commit|
-
-          Gitlab::Git.check_namespace!(commit, start_repository)
-
-          revert_tree_id = check_revert_content(commit, start_commit.sha)
-          raise CreateTreeError unless revert_tree_id
-
-          committer = user_to_committer(user)
-
-          create_commit(message: message,
-                        author: committer,
-                        committer: committer,
-                        tree: revert_tree_id,
-                        parents: [start_commit.sha])
-        end
-      end
-
-      def rugged_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
-        OperationService.new(user, self).with_branch(
-          branch_name,
-          start_branch_name: start_branch_name,
-          start_repository: start_repository
-        ) do |start_commit|
-
-          Gitlab::Git.check_namespace!(commit, start_repository)
-
-          cherry_pick_tree_id = check_cherry_pick_content(commit, start_commit.sha)
-          raise CreateTreeError unless cherry_pick_tree_id
-
-          committer = user_to_committer(user)
-
-          create_commit(message: message,
-                        author: {
-                            email: commit.author_email,
-                            name: commit.author_name,
-                            time: commit.authored_date
-                        },
-                        committer: committer,
-                        tree: cherry_pick_tree_id,
-                        parents: [start_commit.sha])
-        end
-      end
-
-      def check_cherry_pick_content(target_commit, source_sha)
-        args = [target_commit.sha, source_sha]
-        args << 1 if target_commit.merge_commit?
-
-        cherry_pick_index = rugged.cherrypick_commit(*args)
-        return false if cherry_pick_index.conflicts?
-
-        tree_id = cherry_pick_index.write_tree(rugged)
-        return false unless diff_exists?(source_sha, tree_id)
-
-        tree_id
-      end
-
-      def gitaly_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:)
-        gitaly_operation_client.user_rebase(user, rebase_id,
-                                            branch: branch,
-                                            branch_sha: branch_sha,
-                                            remote_repository: remote_repository,
-                                            remote_branch: remote_branch)
-      end
-
-      def git_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:)
-        rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)
-        env = git_env_for_user(user)
-
-        if remote_repository.is_a?(RemoteRepository)
-          env.merge!(remote_repository.fetch_env)
-          remote_repo_path = GITALY_INTERNAL_URL
-        else
-          remote_repo_path = remote_repository.path
-        end
-
-        with_worktree(rebase_path, branch, env: env) do
-          run_git!(
-            %W(pull --rebase #{remote_repo_path} #{remote_branch}),
-            chdir: rebase_path, env: env
-          )
-
-          rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip
-
-          Gitlab::Git::OperationService.new(user, self)
-            .update_branch(branch, rebase_sha, branch_sha)
-
-          rebase_sha
-        end
-      end
-
-      def git_squash(user, squash_id, branch, start_sha, end_sha, author, message)
-        squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)
-        env = git_env_for_user(user).merge(
-          'GIT_AUTHOR_NAME' => author.name,
-          'GIT_AUTHOR_EMAIL' => author.email
-        )
-        diff_range = "#{start_sha}...#{end_sha}"
-        diff_files = run_git!(
-          %W(diff --name-only --diff-filter=ar --binary #{diff_range})
-        ).chomp
-
-        with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do
-          # Apply diff of the `diff_range` to the worktree
-          diff = run_git!(%W(diff --binary #{diff_range}))
-          run_git!(%w(apply --index --whitespace=nowarn), chdir: squash_path, env: env) do |stdin|
-            stdin.binmode
-            stdin.write(diff)
-          end
-
-          # Commit the `diff_range` diff
-          run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env)
-
-          # Return the squash sha. May print a warning for ambiguous refs, but
-          # we can ignore that with `--quiet` and just take the SHA, if present.
-          # HEAD here always refers to the current HEAD commit, even if there is
-          # another ref called HEAD.
-          run_git!(
-            %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env
-          ).chomp
-        end
-      end
-
-      def local_fetch_ref(source_path, source_ref:, target_ref:)
-        args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref})
-        run_git(args)
-      end
-
-      def gitaly_fetch_ref(source_repository, source_ref:, target_ref:)
-        args = %W(fetch --no-tags -f #{GITALY_INTERNAL_URL} #{source_ref}:#{target_ref})
-
-        run_git(args, env: source_repository.fetch_env)
-      end
-
-      def gitaly_ff_merge(user, source_sha, target_branch)
-        gitaly_operations_client.user_ff_branch(user, source_sha, target_branch)
-      rescue GRPC::FailedPrecondition => e
-        raise CommitError, e
-      end
-
-      def rugged_ff_merge(user, source_sha, target_branch)
-        OperationService.new(user, self).with_branch(target_branch) do |our_commit|
-          raise ArgumentError, 'Invalid merge target' unless our_commit
-
-          source_sha
-        end
-      rescue Rugged::ReferenceError, InvalidRef
-        raise ArgumentError, 'Invalid merge source'
-      end
-
-      def rugged_add_remote(remote_name, url, mirror_refmap)
-        rugged.remotes.create(remote_name, url)
-
-        set_remote_as_mirror(remote_name, refmap: mirror_refmap) if mirror_refmap
-      rescue Rugged::ConfigError
-        remote_update(remote_name, url: url)
-      end
-
-      def git_delete_refs(*ref_names)
-        instructions = ref_names.map do |ref|
-          "delete #{ref}\x00\x00"
-        end
-
-        message, status = run_git(%w[update-ref --stdin -z]) do |stdin|
-          stdin.write(instructions.join)
-        end
-
-        unless status.zero?
-          raise GitError.new("Could not delete refs #{ref_names}: #{message}")
-        end
-      end
-
       def gitaly_delete_refs(*ref_names)
         gitaly_ref_client.delete_refs(refs: ref_names) if ref_names.any?
       end
-
-      def rugged_remove_remote(remote_name)
-        # When a remote is deleted all its remote refs are deleted too, but in
-        # the case of mirrors we map its refs (that would usualy go under
-        # [remote_name]/) to the top level namespace. We clean the mapping so
-        # those don't get deleted.
-        if rugged.config["remote.#{remote_name}.mirror"]
-          rugged.config.delete("remote.#{remote_name}.fetch")
-        end
-
-        rugged.remotes.delete(remote_name)
-        true
-      rescue Rugged::ConfigError
-        false
-      end
-
-      def rugged_fetch_repository_as_mirror(repository)
-        remote_name = "tmp-#{SecureRandom.hex}"
-        repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository)
-
-        add_remote(remote_name, GITALY_INTERNAL_URL, mirror_refmap: :all_refs)
-        fetch_remote(remote_name, env: repository.fetch_env)
-      ensure
-        remove_remote(remote_name)
-      end
-
-      def rugged_multi_action(
-        user, branch_name, message, actions, author_email, author_name,
-        start_branch_name, start_repository)
-
-        OperationService.new(user, self).with_branch(
-          branch_name,
-          start_branch_name: start_branch_name,
-          start_repository: start_repository
-        ) do |start_commit|
-          index = Gitlab::Git::Index.new(self)
-          parents = []
-
-          if start_commit
-            index.read_tree(start_commit.rugged_commit.tree)
-            parents = [start_commit.sha]
-          end
-
-          actions.each { |opts| index.apply(opts.delete(:action), opts) }
-
-          committer = user_to_committer(user)
-          author = Gitlab::Git.committer_hash(email: author_email, name: author_name) || committer
-          options = {
-            tree: index.write_tree,
-            message: message,
-            parents: parents,
-            author: author,
-            committer: committer
-          }
-
-          create_commit(options)
-        end
-      end
-
-      def fetch_remote(remote_name = 'origin', env: nil)
-        run_git(['fetch', remote_name], env: env).last.zero?
-      end
-
-      def gitaly_can_be_merged?(their_commit, our_commit)
-        !gitaly_conflicts_client(our_commit, their_commit).conflicts?
-      end
-
-      def rugged_can_be_merged?(their_commit, our_commit)
-        !rugged.merge_commits(our_commit, their_commit).conflicts?
-      end
-
-      def gitlab_projects_error
-        raise CommandError, @gitlab_projects.output
-      end
-
-      def find_commits_by_message_by_shelling_out(query, ref, path, limit, offset)
-        ref ||= root_ref
-
-        args = %W(
-          log #{ref} --pretty=%H --skip #{offset}
-          --max-count #{limit} --grep=#{query} --regexp-ignore-case
-        )
-        args = args.concat(%W(-- #{path})) if path.present?
-
-        git_log_results = run_git(args).first.lines
-
-        git_log_results.map { |c| commit(c.chomp) }.compact
-      end
-
-      def find_commits_by_message_by_gitaly(query, ref, path, limit, offset)
-        gitaly_commit_client
-          .commits_by_message(query, revision: ref, path: path, limit: limit, offset: offset)
-          .map { |c| commit(c) }
-      end
-
-      def last_commit_for_path_by_gitaly(sha, path)
-        gitaly_commit_client.last_commit_for_path(sha, path)
-      end
-
-      def last_commit_id_for_path_by_shelling_out(sha, path)
-        args = %W(rev-list --max-count=1 #{sha} -- #{path})
-        run_git_with_timeout(args, Gitlab::Git::Popen::FAST_GIT_PROCESS_TIMEOUT).first.strip
-      end
-
-      def rugged_merge_base(from, to)
-        rugged.merge_base(from, to)
-      rescue Rugged::ReferenceError
-        nil
-      end
-
-      def rugged_commit_count(ref)
-        walker = Rugged::Walker.new(rugged)
-        walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE)
-        oid = rugged.rev_parse_oid(ref)
-        walker.push(oid)
-        walker.count
-      rescue Rugged::ReferenceError
-        0
-      end
-
-      def rev_list_param(spec)
-        spec == :all ? ['--all'] : spec
-      end
-
-      def sha_from_ref(ref)
-        rev_parse_target(ref).oid
-      end
-
-      def build_git_cmd(*args)
-        object_directories = alternate_object_directories.join(File::PATH_SEPARATOR)
-
-        env = { 'PWD' => self.path }
-        env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = object_directories if object_directories.present?
-
-        [
-          env,
-          ::Gitlab.config.git.bin_path,
-          *args,
-          { chdir: self.path }
-        ]
-      end
-
-      def git_diff_cmd(old_rev, new_rev)
-        old_rev = old_rev == ::Gitlab::Git::BLANK_SHA ? ::Gitlab::Git::EMPTY_TREE_ID : old_rev
-
-        build_git_cmd('diff', old_rev, new_rev, '--raw')
-      end
-
-      def git_cat_file_cmd
-        format = '%(objectname) %(objectsize) %(rest)'
-        build_git_cmd('cat-file', "--batch-check=#{format}")
-      end
-
-      def format_git_cat_file_script
-        File.expand_path('../support/format-git-cat-file-input', __FILE__)
-      end
     end
   end
 end
diff --git a/lib/gitlab/git/repository_cleaner.rb b/lib/gitlab/git/repository_cleaner.rb
new file mode 100644
index 00000000000..2d1d8435cf3
--- /dev/null
+++ b/lib/gitlab/git/repository_cleaner.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Git
+    class RepositoryCleaner
+      include Gitlab::Git::WrapsGitalyErrors
+
+      attr_reader :repository
+
+      # 'repository' is a Gitlab::Git::Repository
+      def initialize(repository)
+        @repository = repository
+      end
+
+      def apply_bfg_object_map(io)
+        wrapped_gitaly_errors do
+          gitaly_cleanup_client.apply_bfg_object_map(io)
+        end
+      end
+
+      private
+
+      def gitaly_cleanup_client
+        @gitaly_cleanup_client ||= Gitlab::GitalyClient::CleanupService.new(repository)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb
index e35ea5762eb..7e63a6dc7cb 100644
--- a/lib/gitlab/git/repository_mirroring.rb
+++ b/lib/gitlab/git/repository_mirroring.rb
@@ -1,94 +1,10 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     module RepositoryMirroring
-      REFMAPS = {
-        # With `:all_refs`, the repository is equivalent to the result of `git clone --mirror`
-        all_refs: '+refs/*:refs/*',
-        heads: '+refs/heads/*:refs/heads/*',
-        tags: '+refs/tags/*:refs/tags/*'
-      }.freeze
-
-      RemoteError = Class.new(StandardError)
-
-      def set_remote_as_mirror(remote_name, refmap: :all_refs)
-        set_remote_refmap(remote_name, refmap)
-
-        rugged.config["remote.#{remote_name}.mirror"] = true
-        rugged.config["remote.#{remote_name}.prune"] = true
-      end
-
-      def remote_tags(remote)
-        # Each line has this format: "dc872e9fa6963f8f03da6c8f6f264d0845d6b092\trefs/tags/v1.10.0\n"
-        # We want to convert it to: [{ 'v1.10.0' => 'dc872e9fa6963f8f03da6c8f6f264d0845d6b092' }, ...]
-        list_remote_tags(remote).map do |line|
-          target, path = line.strip.split("\t")
-
-          # When the remote repo does not have tags.
-          if target.nil? || path.nil?
-            Rails.logger.info "Empty or invalid list of tags for remote: #{remote}. Output: #{output}"
-            break []
-          end
-
-          name = path.split('/', 3).last
-          # We're only interested in tag references
-          # See: http://stackoverflow.com/questions/15472107/when-listing-git-ls-remote-why-theres-after-the-tag-name
-          next if name =~ /\^\{\}\Z/
-
-          target_commit = Gitlab::Git::Commit.find(self, target)
-          Gitlab::Git::Tag.new(self, {
-            name: name,
-            target: target,
-            target_commit: target_commit
-          })
-        end.compact
-      end
-
       def remote_branches(remote_name)
-        branches = []
-
-        rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref|
-          name = ref.name.sub(%r{\Arefs/remotes/#{remote_name}/}, '')
-
-          begin
-            target_commit = Gitlab::Git::Commit.find(self, ref.target)
-            branches << Gitlab::Git::Branch.new(self, name, ref.target, target_commit)
-          rescue Rugged::ReferenceError
-            # Omit invalid branch
-          end
-        end
-
-        branches
-      end
-
-      private
-
-      def set_remote_refmap(remote_name, refmap)
-        Array(refmap).each_with_index do |refspec, i|
-          refspec = REFMAPS[refspec] || refspec
-
-          # We need multiple `fetch` entries, but Rugged only allows replacing a config, not adding to it.
-          # To make sure we start from scratch, we set the first using rugged, and use `git` for any others
-          if i == 0
-            rugged.config["remote.#{remote_name}.fetch"] = refspec
-          else
-            run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
-          end
-        end
-      end
-
-      def list_remote_tags(remote)
-        tag_list, exit_code, error = nil
-        cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-remote --tags #{remote})
-
-        Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr|
-          tag_list  = stdout.read
-          error     = stderr.read
-          exit_code = wait_thr.value.exitstatus
-        end
-
-        raise RemoteError, error unless exit_code.zero?
-
-        tag_list.split("\n")
+        gitaly_ref_client.remote_branches(remote_name)
       end
     end
   end
diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb
deleted file mode 100644
index 38c3a55f96f..00000000000
--- a/lib/gitlab/git/rev_list.rb
+++ /dev/null
@@ -1,78 +0,0 @@
-# Gitaly note: JV: will probably be migrated indirectly by migrating the call sites.
-
-module Gitlab
-  module Git
-    class RevList
-      include Gitlab::Git::Popen
-
-      attr_reader :oldrev, :newrev, :repository
-
-      def initialize(repository, newrev:, oldrev: nil)
-        @oldrev = oldrev
-        @newrev = newrev
-        @repository = repository
-      end
-
-      # This method returns an array of new commit references
-      def new_refs
-        repository.rev_list(including: newrev, excluding: :all).split("\n")
-      end
-
-      # Finds newly added objects
-      # Returns an array of shas
-      #
-      # Can skip objects which do not have a path using required_path: true
-      # This skips commit objects and root trees, which might not be needed when
-      # looking for blobs
-      #
-      # When given a block it will yield objects as a lazy enumerator so
-      # the caller can limit work done instead of processing megabytes of data
-      def new_objects(require_path: nil, not_in: nil, &lazy_block)
-        opts = {
-          including: newrev,
-          excluding: not_in.nil? ? :all : not_in,
-          require_path: require_path
-        }
-
-        get_objects(opts, &lazy_block)
-      end
-
-      def all_objects(require_path: nil, &lazy_block)
-        get_objects(including: :all, require_path: require_path, &lazy_block)
-      end
-
-      # This methods returns an array of missed references
-      #
-      # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348.
-      def missed_ref
-        repository.missed_ref(oldrev, newrev).split("\n")
-      end
-
-      private
-
-      def execute(args)
-        repository.rev_list(args).split("\n")
-      end
-
-      def get_objects(including: [], excluding: [], require_path: nil)
-        opts = { including: including, excluding: excluding, objects: true }
-
-        repository.rev_list(opts) do |lazy_output|
-          objects = objects_from_output(lazy_output, require_path: require_path)
-
-          yield(objects)
-        end
-      end
-
-      def objects_from_output(object_output, require_path: nil)
-        object_output.map do |output_line|
-          sha, path = output_line.split(' ', 2)
-
-          next if require_path && path.to_s.empty?
-
-          sha
-        end.reject(&:nil?)
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/git/storage.rb b/lib/gitlab/git/storage.rb
deleted file mode 100644
index 5933312b0b5..00000000000
--- a/lib/gitlab/git/storage.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-module Gitlab
-  module Git
-    module Storage
-      class Inaccessible < StandardError
-        attr_reader :retry_after
-
-        def initialize(message = nil, retry_after = nil)
-          super(message)
-          @retry_after = retry_after
-        end
-      end
-
-      CircuitOpen = Class.new(Inaccessible)
-      Misconfiguration = Class.new(Inaccessible)
-      Failing = Class.new(Inaccessible)
-
-      REDIS_KEY_PREFIX = 'storage_accessible:'.freeze
-      REDIS_KNOWN_KEYS = "#{REDIS_KEY_PREFIX}known_keys_set".freeze
-
-      def self.redis
-        Gitlab::Redis::SharedState
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/git/storage/checker.rb b/lib/gitlab/git/storage/checker.rb
deleted file mode 100644
index 391f0d70583..00000000000
--- a/lib/gitlab/git/storage/checker.rb
+++ /dev/null
@@ -1,120 +0,0 @@
-module Gitlab
-  module Git
-    module Storage
-      class Checker
-        include CircuitBreakerSettings
-
-        attr_reader :storage_path, :storage, :hostname, :logger
-        METRICS_MUTEX = Mutex.new
-        STORAGE_TIMING_BUCKETS = [0.1, 0.15, 0.25, 0.33, 0.5, 1, 1.5, 2.5, 5, 10, 15].freeze
-
-        def self.check_all(logger = Rails.logger)
-          threads = Gitlab.config.repositories.storages.keys.map do |storage_name|
-            Thread.new do
-              Thread.current[:result] = new(storage_name, logger).check_with_lease
-            end
-          end
-
-          threads.map do |thread|
-            thread.join
-            thread[:result]
-          end
-        end
-
-        def self.check_histogram
-          @check_histogram ||=
-            METRICS_MUTEX.synchronize do
-              @check_histogram || Gitlab::Metrics.histogram(:circuitbreaker_storage_check_duration_seconds,
-                                                            'Storage check time in seconds',
-                                                            {},
-                                                            STORAGE_TIMING_BUCKETS
-                                                           )
-            end
-        end
-
-        def initialize(storage, logger = Rails.logger)
-          @storage = storage
-          config = Gitlab.config.repositories.storages[@storage]
-          @storage_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access { config.legacy_disk_path }
-          @logger = logger
-
-          @hostname = Gitlab::Environment.hostname
-        end
-
-        def check_with_lease
-          lease_key = "storage_check:#{cache_key}"
-          lease = Gitlab::ExclusiveLease.new(lease_key, timeout: storage_timeout)
-          result = { storage: storage, success: nil }
-
-          if uuid = lease.try_obtain
-            result[:success] = check
-
-            Gitlab::ExclusiveLease.cancel(lease_key, uuid)
-          else
-            logger.warn("#{hostname}: #{storage}: Skipping check, previous check still running")
-          end
-
-          result
-        end
-
-        def check
-          if perform_access_check
-            track_storage_accessible
-            true
-          else
-            track_storage_inaccessible
-            logger.error("#{hostname}: #{storage}: Not accessible.")
-            false
-          end
-        end
-
-        private
-
-        def perform_access_check
-          start_time = Gitlab::Metrics::System.monotonic_time
-
-          Gitlab::Git::Storage::ForkedStorageCheck.storage_available?(storage_path, storage_timeout, access_retries)
-        ensure
-          execution_time = Gitlab::Metrics::System.monotonic_time - start_time
-          self.class.check_histogram.observe({ storage: storage }, execution_time)
-        end
-
-        def track_storage_inaccessible
-          first_failure = current_failure_info.first_failure || Time.now
-          last_failure = Time.now
-
-          Gitlab::Git::Storage.redis.with do |redis|
-            redis.pipelined do
-              redis.hset(cache_key, :first_failure, first_failure.to_i)
-              redis.hset(cache_key, :last_failure, last_failure.to_i)
-              redis.hincrby(cache_key, :failure_count, 1)
-              redis.expire(cache_key, failure_reset_time)
-              maintain_known_keys(redis)
-            end
-          end
-        end
-
-        def track_storage_accessible
-          Gitlab::Git::Storage.redis.with do |redis|
-            redis.pipelined do
-              redis.hset(cache_key, :first_failure, nil)
-              redis.hset(cache_key, :last_failure, nil)
-              redis.hset(cache_key, :failure_count, 0)
-              maintain_known_keys(redis)
-            end
-          end
-        end
-
-        def maintain_known_keys(redis)
-          expire_time = Time.now.to_i + failure_reset_time
-          redis.zadd(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, expire_time, cache_key)
-          redis.zremrangebyscore(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, '-inf', Time.now.to_i)
-        end
-
-        def current_failure_info
-          FailureInfo.load(cache_key)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb
deleted file mode 100644
index 62427ac9cc4..00000000000
--- a/lib/gitlab/git/storage/circuit_breaker.rb
+++ /dev/null
@@ -1,78 +0,0 @@
-module Gitlab
-  module Git
-    module Storage
-      class CircuitBreaker
-        include CircuitBreakerSettings
-
-        attr_reader :storage,
-                    :hostname
-
-        delegate :last_failure, :failure_count, :no_failures?,
-                 to: :failure_info
-
-        def self.for_storage(storage)
-          cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do
-            Hash.new do |hash, storage_name|
-              hash[storage_name] = build(storage_name)
-            end
-          end
-
-          cached_circuitbreakers[storage]
-        end
-
-        def self.build(storage, hostname = Gitlab::Environment.hostname)
-          config = Gitlab.config.repositories.storages[storage]
-          Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-            if !config.present?
-              NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Storage '#{storage}' is not configured"))
-            elsif !config.legacy_disk_path.present?
-              NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Path for storage '#{storage}' is not configured"))
-            else
-              new(storage, hostname)
-            end
-          end
-        end
-
-        def initialize(storage, hostname)
-          @storage = storage
-          @hostname = hostname
-        end
-
-        def perform
-          return yield unless enabled?
-
-          check_storage_accessible!
-
-          yield
-        end
-
-        def circuit_broken?
-          return false if no_failures?
-
-          failure_count > failure_count_threshold
-        end
-
-        private
-
-        # The circuitbreaker can be enabled for the entire fleet using a Feature
-        # flag.
-        #
-        # Enabling it for a single host can be done setting the
-        # `GIT_STORAGE_CIRCUIT_BREAKER` environment variable.
-        def enabled?
-          ENV['GIT_STORAGE_CIRCUIT_BREAKER'].present? || Feature.enabled?('git_storage_circuit_breaker')
-        end
-
-        def failure_info
-          @failure_info ||= FailureInfo.load(cache_key)
-        end
-
-        def check_storage_accessible!
-          if circuit_broken?
-            raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_reset_time)
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/git/storage/circuit_breaker_settings.rb b/lib/gitlab/git/storage/circuit_breaker_settings.rb
deleted file mode 100644
index c9e225f187d..00000000000
--- a/lib/gitlab/git/storage/circuit_breaker_settings.rb
+++ /dev/null
@@ -1,37 +0,0 @@
-module Gitlab
-  module Git
-    module Storage
-      module CircuitBreakerSettings
-        def failure_count_threshold
-          application_settings.circuitbreaker_failure_count_threshold
-        end
-
-        def failure_reset_time
-          application_settings.circuitbreaker_failure_reset_time
-        end
-
-        def storage_timeout
-          application_settings.circuitbreaker_storage_timeout
-        end
-
-        def access_retries
-          application_settings.circuitbreaker_access_retries
-        end
-
-        def check_interval
-          application_settings.circuitbreaker_check_interval
-        end
-
-        def cache_key
-          @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}"
-        end
-
-        private
-
-        def application_settings
-          Gitlab::CurrentSettings.current_application_settings
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/git/storage/failure_info.rb b/lib/gitlab/git/storage/failure_info.rb
deleted file mode 100644
index 387279c110d..00000000000
--- a/lib/gitlab/git/storage/failure_info.rb
+++ /dev/null
@@ -1,39 +0,0 @@
-module Gitlab
-  module Git
-    module Storage
-      class FailureInfo
-        attr_accessor :first_failure, :last_failure, :failure_count
-
-        def self.reset_all!
-          Gitlab::Git::Storage.redis.with do |redis|
-            all_storage_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1)
-            redis.del(*all_storage_keys) unless all_storage_keys.empty?
-          end
-
-          RequestStore.delete(:circuitbreaker_cache)
-        end
-
-        def self.load(cache_key)
-          first_failure, last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis|
-            redis.hmget(cache_key, :first_failure, :last_failure, :failure_count)
-          end
-
-          last_failure = Time.at(last_failure.to_i) if last_failure.present?
-          first_failure = Time.at(first_failure.to_i) if first_failure.present?
-
-          new(first_failure, last_failure, failure_count.to_i)
-        end
-
-        def initialize(first_failure, last_failure, failure_count)
-          @first_failure = first_failure
-          @last_failure = last_failure
-          @failure_count = failure_count
-        end
-
-        def no_failures?
-          first_failure.blank? && last_failure.blank? && failure_count == 0
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/git/storage/forked_storage_check.rb b/lib/gitlab/git/storage/forked_storage_check.rb
deleted file mode 100644
index 0a4e557b59b..00000000000
--- a/lib/gitlab/git/storage/forked_storage_check.rb
+++ /dev/null
@@ -1,65 +0,0 @@
-module Gitlab
-  module Git
-    module Storage
-      module ForkedStorageCheck
-        extend self
-
-        def storage_available?(path, timeout_seconds = 5, retries = 1)
-          partial_timeout = timeout_seconds / retries
-          status = timeout_check(path, partial_timeout)
-
-          # If the status check did not succeed the first time, we retry a few
-          # more times to avoid one-off failures
-          current_attempts = 1
-          while current_attempts < retries && !status.success?
-            status = timeout_check(path, partial_timeout)
-            current_attempts += 1
-          end
-
-          status.success?
-        end
-
-        def timeout_check(path, timeout_seconds)
-          filesystem_check_pid = check_filesystem_in_process(path)
-
-          deadline = timeout_seconds.seconds.from_now.utc
-          wait_time = 0.01
-          status = nil
-
-          while status.nil?
-
-            if deadline > Time.now.utc
-              sleep(wait_time)
-              _pid, status = Process.wait2(filesystem_check_pid, Process::WNOHANG)
-            else
-              Process.kill('KILL', filesystem_check_pid)
-              # Blocking wait, so we are sure the process is gone before continuing
-              _pid, status = Process.wait2(filesystem_check_pid)
-            end
-          end
-
-          status
-        end
-
-        # This will spawn a new 2 processes to do the check:
-        # The outer child (waiter) will spawn another child process (stater).
-        #
-        # The stater is the process is performing the actual filesystem check
-        # the check might hang if the filesystem is acting up.
-        # In this case we will send a `KILL` to the waiter, which will still
-        # be responsive while the stater is hanging.
-        def check_filesystem_in_process(path)
-          spawn('ruby', '-e', ruby_check, path, [:out, :err] => '/dev/null')
-        end
-
-        def ruby_check
-          <<~RUBY_FILESYSTEM_CHECK
-          inner_pid = fork { File.stat(ARGV.first) }
-          Process.waitpid(inner_pid)
-          exit $?.exitstatus
-          RUBY_FILESYSTEM_CHECK
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb
deleted file mode 100644
index 90bbe85fd37..00000000000
--- a/lib/gitlab/git/storage/health.rb
+++ /dev/null
@@ -1,90 +0,0 @@
-module Gitlab
-  module Git
-    module Storage
-      class Health
-        attr_reader :storage_name, :info
-
-        def self.prefix_for_storage(storage_name)
-          "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage_name}:"
-        end
-
-        def self.for_all_storages
-          storage_names = Gitlab.config.repositories.storages.keys
-          results_per_storage = nil
-
-          Gitlab::Git::Storage.redis.with do |redis|
-            keys_per_storage = all_keys_for_storages(storage_names, redis)
-            results_per_storage = load_for_keys(keys_per_storage, redis)
-          end
-
-          results_per_storage.map do |name, info|
-            info.each { |i| i[:failure_count] = i[:failure_count].value.to_i }
-            new(name, info)
-          end
-        end
-
-        private_class_method def self.all_keys_for_storages(storage_names, redis)
-          keys_per_storage = {}
-          all_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1)
-
-          storage_names.each do |storage_name|
-            prefix = prefix_for_storage(storage_name)
-
-            keys_per_storage[storage_name] = all_keys.select { |key| key.starts_with?(prefix) }
-          end
-
-          keys_per_storage
-        end
-
-        private_class_method def self.load_for_keys(keys_per_storage, redis)
-          info_for_keys = {}
-
-          redis.pipelined do
-            keys_per_storage.each do |storage_name, keys_future|
-              info_for_storage = keys_future.map do |key|
-                { name: key, failure_count: redis.hget(key, :failure_count) }
-              end
-
-              info_for_keys[storage_name] = info_for_storage
-            end
-          end
-
-          info_for_keys
-        end
-
-        def self.for_failing_storages
-          for_all_storages.select(&:failing?)
-        end
-
-        def initialize(storage_name, info)
-          @storage_name = storage_name
-          @info = info
-        end
-
-        def failing_info
-          @failing_info ||= info.select { |info_for_host| info_for_host[:failure_count] > 0 }
-        end
-
-        def failing?
-          failing_info.any?
-        end
-
-        def failing_on_hosts
-          @failing_on_hosts ||= failing_info.map do |info_for_host|
-            info_for_host[:name].split(':').last
-          end
-        end
-
-        def failing_circuit_breakers
-          @failing_circuit_breakers ||= failing_on_hosts.map do |hostname|
-            CircuitBreaker.build(storage_name, hostname)
-          end
-        end
-
-        def total_failures
-          @total_failures ||= failing_info.sum { |info_for_host| info_for_host[:failure_count] }
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/git/storage/null_circuit_breaker.rb b/lib/gitlab/git/storage/null_circuit_breaker.rb
deleted file mode 100644
index 261c936c689..00000000000
--- a/lib/gitlab/git/storage/null_circuit_breaker.rb
+++ /dev/null
@@ -1,50 +0,0 @@
-module Gitlab
-  module Git
-    module Storage
-      class NullCircuitBreaker
-        include CircuitBreakerSettings
-
-        # These will have actual values
-        attr_reader :storage,
-                    :hostname
-
-        # These will always have nil values
-        attr_reader :storage_path
-
-        delegate :last_failure, :failure_count, :no_failures?,
-                 to: :failure_info
-
-        def initialize(storage, hostname, error: nil)
-          @storage = storage
-          @hostname = hostname
-          @error = error
-        end
-
-        def perform
-          @error ? raise(@error) : yield
-        end
-
-        def circuit_broken?
-          !!@error
-        end
-
-        def backing_off?
-          false
-        end
-
-        def failure_info
-          @failure_info ||=
-            if circuit_broken?
-              Gitlab::Git::Storage::FailureInfo.new(Time.now,
-                                                    Time.now,
-                                                    failure_count_threshold)
-            else
-              Gitlab::Git::Storage::FailureInfo.new(nil,
-                                                    nil,
-                                                    0)
-            end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/git/support/format-git-cat-file-input b/lib/gitlab/git/support/format-git-cat-file-input
deleted file mode 100755
index 2e93c646d0f..00000000000
--- a/lib/gitlab/git/support/format-git-cat-file-input
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env ruby
-
-# This script formats the output of the `git diff <old_rev> <new_rev> --raw`
-# command so it can be processed by `git cat-file`
-
-# We need to convert this:
-# ":100644 100644 5f53439... 85bc2f9... R060\tfiles/js/commit.js.coffee\tfiles/js/commit.coffee"
-# To:
-# "85bc2f9 R\tfiles/js/commit.js.coffee\tfiles/js/commit.coffee"
-
-ARGF.each do |line|
-  _, _, old_blob_id, new_blob_id, rest = line.split(/\s/, 5)
-
-  old_blob_id.gsub!(/[^\h]/, '')
-  new_blob_id.gsub!(/[^\h]/, '')
-
-  # We can't pass '0000000...' to `git cat-file` given it will not return info about the deleted file
-  blob_id = new_blob_id =~ /\A0+\z/ ? old_blob_id : new_blob_id
-
-  $stdout.puts "#{blob_id} #{rest}"
-end
diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb
index e44284572fd..23d989ff258 100644
--- a/lib/gitlab/git/tag.rb
+++ b/lib/gitlab/git/tag.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     class Tag < Ref
@@ -12,34 +14,15 @@ module Gitlab
 
       class << self
         def get_message(repository, tag_id)
-          BatchLoader.for({ repository: repository, tag_id: tag_id }).batch do |items, loader|
-            items_by_repo = items.group_by { |i| i[:repository] }
-
-            items_by_repo.each do |repo, items|
-              tag_ids = items.map { |i| i[:tag_id] }
-
-              messages = get_messages(repository, tag_ids)
-
-              messages.each do |id, message|
-                loader.call({ repository: repository, tag_id: id }, message)
-              end
+          BatchLoader.for(tag_id).batch(key: repository) do |tag_ids, loader, args|
+            get_messages(args[:key], tag_ids).each do |tag_id, message|
+              loader.call(tag_id, message)
             end
           end
         end
 
         def get_messages(repository, tag_ids)
-          repository.gitaly_migrate(:tag_messages) do |is_enabled|
-            if is_enabled
-              repository.gitaly_ref_client.get_tag_messages(tag_ids)
-            else
-              tag_ids.map do |id|
-                tag = repository.rugged.lookup(id)
-                message = tag.is_a?(Rugged::Commit) ? "" : tag.message
-
-                [id, message]
-              end.to_h
-            end
-          end
+          repository.gitaly_ref_client.get_tag_messages(tag_ids)
         end
       end
 
diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb
index b6ceb542dd1..51542bcaaa2 100644
--- a/lib/gitlab/git/tree.rb
+++ b/lib/gitlab/git/tree.rb
@@ -1,9 +1,10 @@
-# Gitaly note: JV: needs 1 RPC, migration is in progress.
+# frozen_string_literal: true
 
 module Gitlab
   module Git
     class Tree
       include Gitlab::EncodingHelper
+      extend Gitlab::Git::WrapsGitalyErrors
 
       attr_accessor :id, :root_id, :name, :path, :flat_path, :type,
         :mode, :commit_id, :submodule_url
@@ -17,12 +18,8 @@ module Gitlab
         def where(repository, sha, path = nil, recursive = false)
           path = nil if path == '' || path == '/'
 
-          Gitlab::GitalyClient.migrate(:tree_entries) do |is_enabled|
-            if is_enabled
-              repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive)
-            else
-              tree_entries_from_rugged(repository, sha, path, recursive)
-            end
+          wrapped_gitaly_errors do
+            repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive)
           end
         end
 
@@ -56,51 +53,6 @@ module Gitlab
             entry[:oid]
           end
         end
-
-        def tree_entries_from_rugged(repository, sha, path, recursive)
-          current_path_entries = get_tree_entries_from_rugged(repository, sha, path)
-          ordered_entries = []
-
-          current_path_entries.each do |entry|
-            ordered_entries << entry
-
-            if recursive && entry.dir?
-              ordered_entries.concat(tree_entries_from_rugged(repository, sha, entry.path, true))
-            end
-          end
-
-          ordered_entries
-        end
-
-        def get_tree_entries_from_rugged(repository, sha, path)
-          commit = repository.lookup(sha)
-          root_tree = commit.tree
-
-          tree = if path
-                   id = find_id_by_path(repository, root_tree.oid, path)
-                   if id
-                     repository.lookup(id)
-                   else
-                     []
-                   end
-                 else
-                   root_tree
-                 end
-
-          tree.map do |entry|
-            new(
-              id: entry[:oid],
-              root_id: root_tree.oid,
-              name: entry[:name],
-              type: entry[:type],
-              mode: entry[:filemode].to_s(8),
-              path: path ? File.join(path, entry[:name]) : entry[:name],
-              commit_id: sha
-            )
-          end
-        rescue Rugged::ReferenceError
-          []
-        end
       end
 
       def initialize(options)
diff --git a/lib/gitlab/git/user.rb b/lib/gitlab/git/user.rb
index e573cd0e143..2c798844798 100644
--- a/lib/gitlab/git/user.rb
+++ b/lib/gitlab/git/user.rb
@@ -1,10 +1,12 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     class User
       attr_reader :username, :name, :email, :gl_id
 
       def self.from_gitlab(gitlab_user)
-        new(gitlab_user.username, gitlab_user.name, gitlab_user.email, Gitlab::GlId.gl_id(gitlab_user))
+        new(gitlab_user.username, gitlab_user.name, gitlab_user.commit_email, Gitlab::GlId.gl_id(gitlab_user))
       end
 
       def self.from_gitaly(gitaly_user)
diff --git a/lib/gitlab/git/util.rb b/lib/gitlab/git/util.rb
index 4708f22dcb3..03c2c1367b0 100644
--- a/lib/gitlab/git/util.rb
+++ b/lib/gitlab/git/util.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Gitaly note: JV: no RPC's here.
 
 module Gitlab
diff --git a/lib/gitlab/git/version.rb b/lib/gitlab/git/version.rb
new file mode 100644
index 00000000000..64c89656167
--- /dev/null
+++ b/lib/gitlab/git/version.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Git
+    module Version
+      def self.git_version
+        Gitlab::VersionInfo.parse(Gitaly::Server.all.first.git_binary_version)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb
index 1ab8c4e0229..c43331bed60 100644
--- a/lib/gitlab/git/wiki.rb
+++ b/lib/gitlab/git/wiki.rb
@@ -1,15 +1,57 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     class Wiki
+      include Gitlab::Git::WrapsGitalyErrors
+
       DuplicatePageError = Class.new(StandardError)
       OperationError = Class.new(StandardError)
 
+      DEFAULT_PAGINATION = Kaminari.config.default_per_page
+
       CommitDetails = Struct.new(:user_id, :username, :name, :email, :message) do
         def to_h
           { user_id: user_id, username: username, name: name, email: email, message: message }
         end
       end
-      PageBlob = Struct.new(:name)
+
+      # GollumSlug inlines just enough knowledge from Gollum::Page to generate a
+      # slug, which is used when previewing pages that haven't been persisted
+      class GollumSlug
+        class << self
+          def cname(name, char_white_sub = '-', char_other_sub = '-')
+            if name.respond_to?(:gsub)
+              name.gsub(/\s/, char_white_sub).gsub(/[<>+]/, char_other_sub)
+            else
+              ''
+            end
+          end
+
+          def format_to_ext(format)
+            format == :markdown ? "md" : format.to_s
+          end
+
+          def canonicalize_filename(filename)
+            ::File.basename(filename, ::File.extname(filename)).tr('-', ' ')
+          end
+
+          def generate(title, format)
+            ext = format_to_ext(format.to_sym)
+            name = cname(title) + '.' + ext
+            canonical_name = canonicalize_filename(name)
+
+            path =
+              if name.include?('/')
+                name.sub(%r{/[^/]+$}, '/')
+              else
+                ''
+              end
+
+            path + cname(canonical_name, '-', '-')
+          end
+        end
+      end
 
       attr_reader :repository
 
@@ -27,63 +69,38 @@ module Gitlab
       end
 
       def write_page(name, format, content, commit_details)
-        @repository.gitaly_migrate(:wiki_write_page) do |is_enabled|
-          if is_enabled
-            gitaly_write_page(name, format, content, commit_details)
-          else
-            gollum_write_page(name, format, content, commit_details)
-          end
+        wrapped_gitaly_errors do
+          gitaly_write_page(name, format, content, commit_details)
         end
       end
 
       def delete_page(page_path, commit_details)
-        @repository.gitaly_migrate(:wiki_delete_page) do |is_enabled|
-          if is_enabled
-            gitaly_delete_page(page_path, commit_details)
-          else
-            gollum_delete_page(page_path, commit_details)
-          end
+        wrapped_gitaly_errors do
+          gitaly_delete_page(page_path, commit_details)
         end
       end
 
       def update_page(page_path, title, format, content, commit_details)
-        @repository.gitaly_migrate(:wiki_update_page) do |is_enabled|
-          if is_enabled
-            gitaly_update_page(page_path, title, format, content, commit_details)
-          else
-            gollum_update_page(page_path, title, format, content, commit_details)
-          end
+        wrapped_gitaly_errors do
+          gitaly_update_page(page_path, title, format, content, commit_details)
         end
       end
 
-      def pages(limit: nil)
-        @repository.gitaly_migrate(:wiki_get_all_pages) do |is_enabled|
-          if is_enabled
-            gitaly_get_all_pages
-          else
-            gollum_get_all_pages(limit: limit)
-          end
+      def pages(limit: 0)
+        wrapped_gitaly_errors do
+          gitaly_get_all_pages(limit: limit)
         end
       end
 
       def page(title:, version: nil, dir: nil)
-        @repository.gitaly_migrate(:wiki_find_page,
-                      status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_find_page(title: title, version: version, dir: dir)
-          else
-            gollum_find_page(title: title, version: version, dir: dir)
-          end
+        wrapped_gitaly_errors do
+          gitaly_find_page(title: title, version: version, dir: dir)
         end
       end
 
       def file(name, version)
-        @repository.gitaly_migrate(:wiki_find_file) do |is_enabled|
-          if is_enabled
-            gitaly_find_file(name, version)
-          else
-            gollum_find_file(name, version)
-          end
+        wrapped_gitaly_errors do
+          gitaly_find_file(name, version)
         end
       end
 
@@ -92,24 +109,15 @@ module Gitlab
       #  :per_page - The number of items per page.
       #  :limit    - Total number of items to return.
       def page_versions(page_path, options = {})
-        @repository.gitaly_migrate(:wiki_page_versions) do |is_enabled|
-          if is_enabled
-            versions = gitaly_wiki_client.page_versions(page_path, options)
-
-            # Gitaly uses gollum-lib to get the versions. Gollum defaults to 20
-            # per page, but also fetches 20 if `limit` or `per_page` < 20.
-            # Slicing returns an array with the expected number of items.
-            slice_bound = options[:limit] || options[:per_page] || Gollum::Page.per_page
-            versions[0..slice_bound]
-          else
-            current_page = gollum_page_by_path(page_path)
-
-            commits_from_page(current_page, options).map do |gitlab_git_commit|
-              gollum_page = gollum_wiki.page(current_page.title, gitlab_git_commit.id)
-              Gitlab::Git::WikiPageVersion.new(gitlab_git_commit, gollum_page&.format)
-            end
-          end
+        versions = wrapped_gitaly_errors do
+          gitaly_wiki_client.page_versions(page_path, options)
         end
+
+        # Gitaly uses gollum-lib to get the versions. Gollum defaults to 20
+        # per page, but also fetches 20 if `limit` or `per_page` < 20.
+        # Slicing returns an array with the expected number of items.
+        slice_bound = options[:limit] || options[:per_page] || DEFAULT_PAGINATION
+        versions[0..slice_bound]
       end
 
       def count_page_versions(page_path)
@@ -117,147 +125,23 @@ module Gitlab
       end
 
       def preview_slug(title, format)
-        # Adapted from gollum gem (Gollum::Wiki#preview_page) to avoid
-        # using Rugged through a Gollum::Wiki instance
-        page_class = Gollum::Page
-        page = page_class.new(nil)
-        ext = page_class.format_to_ext(format.to_sym)
-        name = page_class.cname(title) + '.' + ext
-        blob = PageBlob.new(name)
-        page.populate(blob)
-        page.url_path
+        GollumSlug.generate(title, format)
       end
 
       def page_formatted_data(title:, dir: nil, version: nil)
         version = version&.id
 
-        @repository.gitaly_migrate(:wiki_page_formatted_data, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_wiki_client.get_formatted_data(title: title, dir: dir, version: version)
-          else
-            # We don't use #page because if wiki_find_page feature is enabled, we would
-            # get a page without formatted_data.
-            gollum_find_page(title: title, dir: dir, version: version)&.formatted_data
-          end
+        wrapped_gitaly_errors do
+          gitaly_wiki_client.get_formatted_data(title: title, dir: dir, version: version)
         end
       end
 
-      def gollum_wiki
-        @gollum_wiki ||= Gollum::Wiki.new(@repository.path)
-      end
-
       private
 
-      # options:
-      #  :page     - The Integer page number.
-      #  :per_page - The number of items per page.
-      #  :limit    - Total number of items to return.
-      def commits_from_page(gollum_page, options = {})
-        unless options[:limit]
-          options[:offset] = ([1, options.delete(:page).to_i].max - 1) * Gollum::Page.per_page
-          options[:limit] = (options.delete(:per_page) || Gollum::Page.per_page).to_i
-        end
-
-        @repository.log(ref: gollum_page.last_version.id,
-                        path: gollum_page.path,
-                        limit: options[:limit],
-                        offset: options[:offset])
-      end
-
-      def gollum_page_by_path(page_path)
-        page_name = Gollum::Page.canonicalize_filename(page_path)
-        page_dir = File.split(page_path).first
-
-        gollum_wiki.paged(page_name, page_dir)
-      end
-
-      def new_page(gollum_page)
-        Gitlab::Git::WikiPage.new(gollum_page, new_version(gollum_page, gollum_page.version.id))
-      end
-
-      def new_version(gollum_page, commit_id)
-        Gitlab::Git::WikiPageVersion.new(version(commit_id), gollum_page&.format)
-      end
-
-      def version(commit_id)
-        commit_find_proc = -> { Gitlab::Git::Commit.find(@repository, commit_id) }
-
-        if RequestStore.active?
-          RequestStore.fetch([:wiki_version_commit, commit_id]) { commit_find_proc.call }
-        else
-          commit_find_proc.call
-        end
-      end
-
-      def assert_type!(object, klass)
-        unless object.is_a?(klass)
-          raise ArgumentError, "expected a #{klass}, got #{object.inspect}"
-        end
-      end
-
       def gitaly_wiki_client
         @gitaly_wiki_client ||= Gitlab::GitalyClient::WikiService.new(@repository)
       end
 
-      def gollum_write_page(name, format, content, commit_details)
-        assert_type!(format, Symbol)
-        assert_type!(commit_details, CommitDetails)
-
-        with_committer_with_hooks(commit_details) do |committer|
-          filename = File.basename(name)
-          dir = (tmp_dir = File.dirname(name)) == '.' ? '' : tmp_dir
-
-          gollum_wiki.write_page(filename, format, content, { committer: committer }, dir)
-        end
-      rescue Gollum::DuplicatePageError => e
-        raise Gitlab::Git::Wiki::DuplicatePageError, e.message
-      end
-
-      def gollum_delete_page(page_path, commit_details)
-        assert_type!(commit_details, CommitDetails)
-
-        with_committer_with_hooks(commit_details) do |committer|
-          gollum_wiki.delete_page(gollum_page_by_path(page_path), committer: committer)
-        end
-      end
-
-      def gollum_update_page(page_path, title, format, content, commit_details)
-        assert_type!(format, Symbol)
-        assert_type!(commit_details, CommitDetails)
-
-        with_committer_with_hooks(commit_details) do |committer|
-          page = gollum_page_by_path(page_path)
-          # Instead of performing two renames if the title has changed,
-          # the update_page will only update the format and content and
-          # the rename_page will do anything related to moving/renaming
-          gollum_wiki.update_page(page, page.name, format, content, committer: committer)
-          gollum_wiki.rename_page(page, title, committer: committer)
-        end
-      end
-
-      def gollum_find_page(title:, version: nil, dir: nil)
-        if version
-          version = Gitlab::Git::Commit.find(@repository, version).id
-        end
-
-        gollum_page = gollum_wiki.page(title, version, dir)
-        return unless gollum_page
-
-        new_page(gollum_page)
-      end
-
-      def gollum_find_file(name, version)
-        version ||= self.class.default_ref
-        gollum_file = gollum_wiki.file(name, version)
-        return unless gollum_file
-
-        Gitlab::Git::WikiFile.new(gollum_file)
-      end
-
-      def gollum_get_all_pages(limit: nil)
-        gollum_wiki.pages(limit: limit).map { |gollum_page| new_page(gollum_page) }
-      end
-
       def gitaly_write_page(name, format, content, commit_details)
         gitaly_wiki_client.write_page(name, format, content, commit_details)
       end
@@ -284,25 +168,11 @@ module Gitlab
         Gitlab::Git::WikiFile.new(wiki_file)
       end
 
-      def gitaly_get_all_pages
-        gitaly_wiki_client.get_all_pages.map do |wiki_page, version|
+      def gitaly_get_all_pages(limit: 0)
+        gitaly_wiki_client.get_all_pages(limit: limit).map do |wiki_page, version|
           Gitlab::Git::WikiPage.new(wiki_page, version)
         end
       end
-
-      def committer_with_hooks(commit_details)
-        Gitlab::Git::CommitterWithHooks.new(self, commit_details.to_h)
-      end
-
-      def with_committer_with_hooks(commit_details, &block)
-        committer = committer_with_hooks(commit_details)
-
-        yield committer
-
-        committer.commit
-
-        nil
-      end
     end
   end
 end
diff --git a/lib/gitlab/git/wiki_file.rb b/lib/gitlab/git/wiki_file.rb
index 84335aca4bc..c05a5adc00c 100644
--- a/lib/gitlab/git/wiki_file.rb
+++ b/lib/gitlab/git/wiki_file.rb
@@ -1,19 +1,16 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     class WikiFile
       attr_reader :mime_type, :raw_data, :name, :path
 
-      # This class is meant to be serializable so that it can be constructed
-      # by Gitaly and sent over the network to GitLab.
-      #
-      # Because Gollum::File is not serializable we must get all the data from
-      # 'gollum_file' during initialization, and NOT store it in an instance
-      # variable.
-      def initialize(gollum_file)
-        @mime_type = gollum_file.mime_type
-        @raw_data = gollum_file.raw_data
-        @name = gollum_file.name
-        @path = gollum_file.path
+      # This class wraps Gitlab::GitalyClient::WikiFile
+      def initialize(gitaly_file)
+        @mime_type = gitaly_file.mime_type
+        @raw_data = gitaly_file.raw_data
+        @name = gitaly_file.name
+        @path = gitaly_file.path
       end
     end
   end
diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb
index 669ae11a423..f6cac398548 100644
--- a/lib/gitlab/git/wiki_page.rb
+++ b/lib/gitlab/git/wiki_page.rb
@@ -1,27 +1,19 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     class WikiPage
       attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical, :formatted_data
 
-      # This class is meant to be serializable so that it can be constructed
-      # by Gitaly and sent over the network to GitLab.
-      #
-      # Because Gollum::Page is not serializable we must get all the data from
-      # 'gollum_page' during initialization, and NOT store it in an instance
-      # variable.
-      #
-      # Note that 'version' is a WikiPageVersion instance which it itself
-      # serializable. That means it's OK to store 'version' in an instance
-      # variable.
-      def initialize(gollum_page, version)
-        @url_path = gollum_page.url_path
-        @title = gollum_page.title
-        @format = gollum_page.format
-        @path = gollum_page.path
-        @raw_data = gollum_page.raw_data
-        @name = gollum_page.name
-        @historical = gollum_page.historical?
-        @formatted_data = gollum_page.formatted_data if gollum_page.is_a?(Gollum::Page)
+      # This class abstracts away Gitlab::GitalyClient::WikiPage
+      def initialize(gitaly_page, version)
+        @url_path = gitaly_page.url_path
+        @title = gitaly_page.title
+        @format = gitaly_page.format
+        @path = gitaly_page.path
+        @raw_data = gitaly_page.raw_data
+        @name = gitaly_page.name
+        @historical = gitaly_page.historical?
 
         @version = version
       end
diff --git a/lib/gitlab/git/wiki_page_version.rb b/lib/gitlab/git/wiki_page_version.rb
index 55f1afedcab..475a9d4d1b9 100644
--- a/lib/gitlab/git/wiki_page_version.rb
+++ b/lib/gitlab/git/wiki_page_version.rb
@@ -1,13 +1,10 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Git
     class WikiPageVersion
       attr_reader :commit, :format
 
-      # This class is meant to be serializable so that it can be constructed
-      # by Gitaly and sent over the network to GitLab.
-      #
-      # Both 'commit' (a Gitlab::Git::Commit) and 'format' (a string) are
-      # serializable.
       def initialize(commit, format)
         @commit = commit
         @format = format
diff --git a/lib/gitlab/git/wraps_gitaly_errors.rb b/lib/gitlab/git/wraps_gitaly_errors.rb
new file mode 100644
index 00000000000..9963bcfbf1c
--- /dev/null
+++ b/lib/gitlab/git/wraps_gitaly_errors.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Git
+    module WrapsGitalyErrors
+      def wrapped_gitaly_errors(&block)
+        yield block
+      rescue GRPC::NotFound => e
+        raise Gitlab::Git::Repository::NoRepository.new(e)
+      rescue GRPC::InvalidArgument => e
+        raise ArgumentError.new(e)
+      rescue GRPC::BadStatus => e
+        raise Gitlab::Git::CommandError.new(e)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index db7c29be94b..010bd0e520c 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -1,12 +1,21 @@
+# frozen_string_literal: true
+
 # Check a user's access to perform a git action. All public methods in this
 # class return an instance of `GitlabAccessStatus`
 module Gitlab
   class GitAccess
+    include Gitlab::Utils::StrongMemoize
+
     UnauthorizedError = Class.new(StandardError)
     NotFoundError = Class.new(StandardError)
     ProjectCreationError = Class.new(StandardError)
+    TimeoutError = Class.new(StandardError)
     ProjectMovedError = Class.new(NotFoundError)
 
+    # Use the magic string '_any' to indicate we do not know what the
+    # changes are. This is also what gitlab-shell does.
+    ANY = '_any'
+
     ERROR_MESSAGES = {
       upload: 'You are not allowed to upload code for this project.',
       download: 'You are not allowed to download code from this project.',
@@ -19,14 +28,22 @@ module Gitlab
       upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.',
       receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.',
       read_only: 'The repository is temporarily read-only. Please try again later.',
-      cannot_push_to_read_only: "You can't push code to a read-only GitLab instance."
+      cannot_push_to_read_only: "You can't push code to a read-only GitLab instance.",
+      push_code: 'You are not allowed to push code to this project.'
     }.freeze
 
-    DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze
-    PUSH_COMMANDS = %w{ git-receive-pack }.freeze
+    INTERNAL_TIMEOUT = 50.seconds.freeze
+    LOG_HEADER = <<~MESSAGE
+      Push operation timed out
+
+      Timing information for debugging purposes:
+    MESSAGE
+
+    DOWNLOAD_COMMANDS = %w{git-upload-pack git-upload-archive}.freeze
+    PUSH_COMMANDS = %w{git-receive-pack}.freeze
     ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS
 
-    attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type
+    attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type, :changes, :logger
 
     def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil, auth_result_type: nil)
       @actor    = actor
@@ -40,12 +57,19 @@ module Gitlab
     end
 
     def check(cmd, changes)
+      @logger = Checks::TimedLogger.new(timeout: INTERNAL_TIMEOUT, header: LOG_HEADER)
+      @changes = changes
+
       check_protocol!
       check_valid_actor!
       check_active_user!
       check_authentication_abilities!(cmd)
       check_command_disabled!(cmd)
       check_command_existence!(cmd)
+
+      custom_action = check_custom_action(cmd)
+      return custom_action if custom_action
+
       check_db_accessibility!(cmd)
 
       ensure_project_on_push!(cmd, changes)
@@ -58,10 +82,10 @@ module Gitlab
       when *DOWNLOAD_COMMANDS
         check_download_access!
       when *PUSH_COMMANDS
-        check_push_access!(changes)
+        check_push_access!
       end
 
-      true
+      ::Gitlab::GitAccessResult::Success.new
     end
 
     def guest_can_download_code?
@@ -88,6 +112,10 @@ module Gitlab
 
     private
 
+    def check_custom_action(cmd)
+      nil
+    end
+
     def check_valid_actor!
       return unless actor.is_a?(Key)
 
@@ -176,7 +204,7 @@ module Gitlab
 
     def ensure_project_on_push!(cmd, changes)
       return if project || deploy_key?
-      return unless receive_pack?(cmd) && changes == '_any' && authentication_abilities.include?(:push_code)
+      return unless receive_pack?(cmd) && changes == ANY && authentication_abilities.include?(:push_code)
 
       namespace = Namespace.find_by_full_path(namespace_path)
 
@@ -218,7 +246,7 @@ module Gitlab
       end
     end
 
-    def check_push_access!(changes)
+    def check_push_access!
       if project.repository_read_only?
         raise UnauthorizedError, ERROR_MESSAGES[:read_only]
       end
@@ -233,38 +261,50 @@ module Gitlab
         raise UnauthorizedError, ERROR_MESSAGES[:upload]
       end
 
-      return if changes.blank? # Allow access this is needed for EE.
-
-      check_change_access!(changes)
+      check_change_access!
     end
 
-    def check_change_access!(changes)
-      # If there are worktrees with a HEAD pointing to a non-existent object,
-      # calls to `git rev-list --all` will fail in git 2.15+. This should also
-      # clear stale lock files.
-      project.repository.clean_stale_repository_files
-
-      changes_list = Gitlab::ChangesList.new(changes)
+    def check_change_access!
+      # Deploy keys with write access can push anything
+      return if deploy_key?
 
-      # Iterate over all changes to find if user allowed all of them to be applied
-      changes_list.each.with_index do |change, index|
-        first_change = index == 0
+      if changes == ANY
+        can_push = user_access.can_do_action?(:push_code) ||
+          project.any_branch_allows_collaboration?(user_access.user)
 
-        # If user does not have access to make at least one change, cancel all
-        # push by allowing the exception to bubble up
-        check_single_change_access(change, skip_lfs_integrity_check: !first_change)
+        unless can_push
+          raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code]
+        end
+      else
+        # If there are worktrees with a HEAD pointing to a non-existent object,
+        # calls to `git rev-list --all` will fail in git 2.15+. This should also
+        # clear stale lock files.
+        project.repository.clean_stale_repository_files
+
+        # Iterate over all changes to find if user allowed all of them to be applied
+        changes_list.each.with_index do |change, index|
+          first_change = index == 0
+
+          # If user does not have access to make at least one change, cancel all
+          # push by allowing the exception to bubble up
+          check_single_change_access(change, skip_lfs_integrity_check: !first_change)
+        end
       end
     end
 
     def check_single_change_access(change, skip_lfs_integrity_check: false)
-      Checks::ChangeAccess.new(
+      change_access = Checks::ChangeAccess.new(
         change,
         user_access: user_access,
         project: project,
-        skip_authorization: deploy_key?,
         skip_lfs_integrity_check: skip_lfs_integrity_check,
-        protocol: protocol
-      ).exec
+        protocol: protocol,
+        logger: logger
+      )
+
+      change_access.exec
+    rescue Checks::TimedLogger::TimeoutError
+      raise TimeoutError, logger.full_message
     end
 
     def deploy_key
@@ -321,6 +361,10 @@ module Gitlab
 
     protected
 
+    def changes_list
+      @changes_list ||= Gitlab::ChangesList.new(changes == ANY ? [] : changes)
+    end
+
     def user
       return @user if defined?(@user)
 
diff --git a/lib/gitlab/git_access_result/custom_action.rb b/lib/gitlab/git_access_result/custom_action.rb
new file mode 100644
index 00000000000..a05a4baed82
--- /dev/null
+++ b/lib/gitlab/git_access_result/custom_action.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GitAccessResult
+    class CustomAction
+      attr_reader :payload, :message
+
+      # Example of payload:
+      #
+      # {
+      #   'action' => 'geo_proxy_to_primary',
+      #   'data' => {
+      #     'api_endpoints' => %w{geo/proxy_git_push_ssh/info_refs geo/proxy_git_push_ssh/push},
+      #     'gl_username' => user.username,
+      #     'primary_repo' => geo_primary_http_url_to_repo(project_or_wiki)
+      #   }
+      # }
+      #
+      def initialize(payload, message)
+        @payload = payload
+        @message = message
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git_access_result/success.rb b/lib/gitlab/git_access_result/success.rb
new file mode 100644
index 00000000000..7bb9f24cb0e
--- /dev/null
+++ b/lib/gitlab/git_access_result/success.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GitAccessResult
+    class Success
+    end
+  end
+end
diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb
index a5b3902ebf4..0af91957fa8 100644
--- a/lib/gitlab/git_access_wiki.rb
+++ b/lib/gitlab/git_access_wiki.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class GitAccessWiki < GitAccess
     ERROR_MESSAGES = {
@@ -13,7 +15,7 @@ module Gitlab
       authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_wiki_code)
     end
 
-    def check_single_change_access(change, _options = {})
+    def check_change_access!
       unless user_access.can_do_action?(:create_wiki)
         raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki]
       end
diff --git a/lib/gitlab/git_logger.rb b/lib/gitlab/git_logger.rb
index 9e02ccc0f44..dac4ddd320f 100644
--- a/lib/gitlab/git_logger.rb
+++ b/lib/gitlab/git_logger.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class GitLogger < Gitlab::Logger
     def self.file_name_noext
diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb
index 742118b76a8..426436c2164 100644
--- a/lib/gitlab/git_post_receive.rb
+++ b/lib/gitlab/git_post_receive.rb
@@ -1,23 +1,27 @@
+# frozen_string_literal: true
+
 module Gitlab
   class GitPostReceive
     include Gitlab::Identifier
-    attr_reader :project, :identifier, :changes
+    attr_reader :project, :identifier, :changes, :push_options
 
-    def initialize(project, identifier, changes)
+    def initialize(project, identifier, changes, push_options)
       @project = project
       @identifier = identifier
       @changes = deserialize_changes(changes)
+      @push_options = push_options
     end
 
-    def identify(revision)
-      super(identifier, project, revision)
+    def identify
+      super(identifier)
     end
 
     def changes_refs
-      return enum_for(:changes_refs) unless block_given?
+      return changes unless block_given?
 
       changes.each do |change|
-        oldrev, newrev, ref = change.strip.split(' ')
+        change.strip!
+        oldrev, newrev, ref = change.split(' ')
 
         yield oldrev, newrev, ref
       end
@@ -26,13 +30,10 @@ module Gitlab
     private
 
     def deserialize_changes(changes)
-      changes = utf8_encode_changes(changes)
-      changes.lines
+      utf8_encode_changes(changes).each_line
     end
 
     def utf8_encode_changes(changes)
-      changes = changes.dup
-
       changes.force_encoding('UTF-8')
       return changes if changes.valid_encoding?
 
diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb
index 2e3e4fc3f1f..3f13ebeb9d0 100644
--- a/lib/gitlab/git_ref_validator.rb
+++ b/lib/gitlab/git_ref_validator.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Gitaly note: JV: does not need to be migrated, works without a repo.
 
 module Gitlab
@@ -7,11 +9,15 @@ module Gitlab
     #
     # Returns true for a valid reference name, false otherwise
     def validate(ref_name)
-      return false if ref_name.start_with?('refs/heads/')
-      return false if ref_name.start_with?('refs/remotes/')
+      not_allowed_prefixes = %w(refs/heads/ refs/remotes/ -)
+      return false if ref_name.start_with?(*not_allowed_prefixes)
+      return false if ref_name == 'HEAD'
 
-      Gitlab::Utils.system_silent(
-        %W(#{Gitlab.config.git.bin_path} check-ref-format --branch #{ref_name}))
+      begin
+        Rugged::Reference.valid_name?("refs/heads/#{ref_name}")
+      rescue ArgumentError
+        return false
+      end
     end
   end
 end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 550294916a4..0ab53f8f706 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'base64'
 
 require 'gitaly'
@@ -7,11 +9,6 @@ require 'grpc/health/v1/health_services_pb'
 module Gitlab
   module GitalyClient
     include Gitlab::Metrics::Methods
-    module MigrationStatus
-      DISABLED = 1
-      OPT_IN = 2
-      OPT_OUT = 3
-    end
 
     class TooManyInvocationsError < StandardError
       attr_reader :call_site, :invocation_count, :max_call_stack
@@ -23,21 +20,17 @@ module Gitlab
         stacks = most_invoked_stack.join('\n') if most_invoked_stack
 
         msg = "GitalyClient##{call_site} called #{invocation_count} times from single request. Potential n+1?"
-        msg << "\nThe following call site called into Gitaly #{max_call_stack} times:\n#{stacks}\n" if stacks
+        msg = "#{msg}\nThe following call site called into Gitaly #{max_call_stack} times:\n#{stacks}\n" if stacks
 
         super(msg)
       end
     end
 
-    SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze
+    PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m
+    SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'
     MAXIMUM_GITALY_CALLS = 35
     CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
 
-    # We have a mechanism to let GitLab automatically opt in to all Gitaly
-    # features. We want to be able to exclude some features from automatic
-    # opt-in. That is what EXPLICIT_OPT_IN_REQUIRED is for.
-    EXPLICIT_OPT_IN_REQUIRED = [Gitlab::GitalyClient::StorageSettings::DISK_ACCESS_DENIED_FLAG].freeze
-
     MUTEX = Mutex.new
 
     class << self
@@ -46,11 +39,6 @@ module Gitlab
 
     self.query_time = 0
 
-    define_histogram :gitaly_migrate_call_duration_seconds do
-      docstring "Gitaly migration call execution timings"
-      base_labels gitaly_enabled: nil, feature: nil
-    end
-
     define_histogram :gitaly_controller_action_duration_seconds do
       docstring "Gitaly endpoint histogram by controller and action combination"
       base_labels Gitlab::Metrics::Transaction::BASE_LABELS.merge(gitaly_service: nil, rpc: nil)
@@ -63,11 +51,49 @@ module Gitlab
         @stubs[storage][name] ||= begin
           klass = stub_class(name)
           addr = stub_address(storage)
-          klass.new(addr, :this_channel_is_insecure)
+          creds = stub_creds(storage)
+          klass.new(addr, creds, interceptors: interceptors)
         end
       end
     end
 
+    def self.interceptors
+      return [] unless Gitlab::Tracing.enabled?
+
+      [Gitlab::Tracing::GRPCInterceptor.instance]
+    end
+    private_class_method :interceptors
+
+    def self.stub_cert_paths
+      cert_paths = Dir["#{OpenSSL::X509::DEFAULT_CERT_DIR}/*"]
+      cert_paths << OpenSSL::X509::DEFAULT_CERT_FILE if File.exist? OpenSSL::X509::DEFAULT_CERT_FILE
+      cert_paths
+    end
+
+    def self.stub_certs
+      return @certs if @certs
+
+      @certs = stub_cert_paths.flat_map do |cert_file|
+        File.read(cert_file).scan(PEM_REGEX).map do |cert|
+          begin
+            OpenSSL::X509::Certificate.new(cert).to_pem
+          rescue OpenSSL::OpenSSLError => e
+            Rails.logger.error "Could not load certificate #{cert_file} #{e}"
+            Gitlab::Sentry.track_exception(e, extra: { cert_file: cert_file })
+            nil
+          end
+        end.compact
+      end.uniq.join("\n")
+    end
+
+    def self.stub_creds(storage)
+      if URI(address(storage)).scheme == 'tls'
+        GRPC::Core::ChannelCredentials.new stub_certs
+      else
+        :this_channel_is_insecure
+      end
+    end
+
     def self.stub_class(name)
       if name == :health_check
         Grpc::Health::V1::Health::Stub
@@ -77,9 +103,7 @@ module Gitlab
     end
 
     def self.stub_address(storage)
-      addr = address(storage)
-      addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp'
-      addr
+      address(storage).sub(%r{^tcp://|^tls://}, '')
     end
 
     def self.clear_stubs!
@@ -101,15 +125,19 @@ module Gitlab
         raise "storage #{storage.inspect} is missing a gitaly_address"
       end
 
-      unless URI(address).scheme.in?(%w(tcp unix))
-        raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix'"
+      unless URI(address).scheme.in?(%w(tcp unix tls))
+        raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'"
       end
 
       address
     end
 
     def self.address_metadata(storage)
-      Base64.strict_encode64(JSON.dump({ storage => { 'address' => address(storage), 'token' => token(storage) } }))
+      Base64.strict_encode64(JSON.dump(storage => connection_data(storage)))
+    end
+
+    def self.connection_data(storage)
+      { 'address' => address(storage), 'token' => token(storage) }
     end
 
     # All Gitaly RPC call sites should use GitalyClient.call. This method
@@ -129,7 +157,6 @@ module Gitlab
     def self.call(storage, service, rpc, request, remote_storage: nil, timeout: nil)
       start = Gitlab::Metrics::System.monotonic_time
       request_hash = request.is_a?(Google::Protobuf::MessageExts) ? request.to_h : {}
-      @current_call_id ||= SecureRandom.uuid
 
       enforce_gitaly_request_limits(:call)
 
@@ -142,15 +169,13 @@ module Gitlab
     ensure
       duration = Gitlab::Metrics::System.monotonic_time - start
 
-      # Keep track, seperately, for the performance bar
+      # Keep track, separately, for the performance bar
       self.query_time += duration
       gitaly_controller_action_duration_seconds.observe(
         current_transaction_labels.merge(gitaly_service: service.to_s, rpc: rpc.to_s),
         duration)
 
-      add_call_details(id: @current_call_id, feature: service, duration: duration, request: request_hash)
-
-      @current_call_id = nil
+      add_call_details(feature: "#{service}##{rpc}", duration: duration, request: request_hash, rpc: rpc)
     end
 
     def self.handle_grpc_unavailable!(ex)
@@ -179,10 +204,29 @@ module Gitlab
     end
     private_class_method :current_transaction_labels
 
+    # For some time related tasks we can't rely on `Time.now` since it will be
+    # affected by Timecop in some tests, and the clock of some gitaly-related
+    # components (grpc's c-core and gitaly server) use system time instead of
+    # timecop's time, so tests will fail.
+    # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will circumvent
+    # timecop.
+    def self.real_time
+      Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))
+    end
+    private_class_method :real_time
+
+    def self.authorization_token(storage)
+      token = token(storage).to_s
+      issued_at = real_time.to_i.to_s
+      hmac = OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, token, issued_at)
+
+      "v2.#{hmac}.#{issued_at}"
+    end
+    private_class_method :authorization_token
+
     def self.request_kwargs(storage, timeout, remote_storage: nil)
-      encoded_token = Base64.strict_encode64(token(storage).to_s)
       metadata = {
-        'authorization' => "Bearer #{encoded_token}",
+        'authorization' => "Bearer #{authorization_token(storage)}",
         'client_name' => CLIENT_NAME
       }
 
@@ -190,6 +234,9 @@ module Gitlab
       feature = feature_stack && feature_stack[0]
       metadata['call_site'] = feature.to_s if feature
       metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage
+      metadata['x-gitlab-correlation-id'] = Gitlab::CorrelationId.current_id if Gitlab::CorrelationId.current_id
+
+      metadata.merge!(server_feature_flags)
 
       result = { metadata: metadata }
 
@@ -198,17 +245,20 @@ module Gitlab
 
       return result unless timeout > 0
 
-      # Do not use `Time.now` for deadline calculation, since it
-      # will be affected by Timecop in some tests, but grpc's c-core
-      # uses system time instead of timecop's time, so tests will fail
-      # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will
-      # circumvent timecop
-      deadline = Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + timeout
+      deadline = real_time + timeout
       result[:deadline] = deadline
 
       result
     end
 
+    SERVER_FEATURE_FLAGS = %w[].freeze
+
+    def self.server_feature_flags
+      SERVER_FEATURE_FLAGS.map do |f|
+        ["gitaly-feature-#{f.tr('_', '-')}", feature_enabled?(f).to_s]
+      end.to_h
+    end
+
     def self.token(storage)
       params = Gitlab.config.repositories.storages[storage]
       raise "storage not found: #{storage.inspect}" if params.nil?
@@ -216,77 +266,14 @@ module Gitlab
       params['gitaly_token'].presence || Gitlab.config.gitaly['token']
     end
 
-    # Evaluates whether a feature toggle is on or off
-    def self.feature_enabled?(feature_name, status: MigrationStatus::OPT_IN)
-      # Disabled features are always off!
-      return false if status == MigrationStatus::DISABLED
-
-      feature = Feature.get("gitaly_#{feature_name}")
-
-      # If the feature has been set, always evaluate
-      if Feature.persisted?(feature)
-        if feature.percentage_of_time_value > 0
-          # Probabilistically enable this feature
-          return Random.rand() * 100 < feature.percentage_of_time_value
-        end
-
-        return feature.enabled?
-      end
-
-      # If the feature has not been set, the default depends
-      # on it's status
-      case status
-      when MigrationStatus::OPT_OUT
-        true
-      when MigrationStatus::OPT_IN
-        opt_into_all_features? && !EXPLICIT_OPT_IN_REQUIRED.include?(feature_name)
-      else
-        false
-      end
-    end
-
-    # opt_into_all_features? returns true when the current environment
-    # is one in which we opt into features automatically
-    def self.opt_into_all_features?
-      Rails.env.development? || ENV["GITALY_FEATURE_DEFAULT_ON"] == "1"
-    end
-    private_class_method :opt_into_all_features?
-
-    def self.migrate(feature, status: MigrationStatus::OPT_IN)
-      # Enforce limits at both the `migrate` and `call` sites to ensure that
-      # problems are not hidden by a feature being disabled
-      enforce_gitaly_request_limits(:migrate)
-
-      is_enabled  = feature_enabled?(feature, status: status)
-      metric_name = feature.to_s
-      metric_name += "_gitaly" if is_enabled
-
-      Gitlab::Metrics.measure(metric_name) do
-        # Some migrate calls wrap other migrate calls
-        allow_n_plus_1_calls do
-          feature_stack = Thread.current[:gitaly_feature_stack] ||= []
-          feature_stack.unshift(feature)
-          begin
-            start = Gitlab::Metrics::System.monotonic_time
-            @current_call_id = SecureRandom.uuid
-            call_details = { id: @current_call_id }
-            yield is_enabled
-          ensure
-            total_time = Gitlab::Metrics::System.monotonic_time - start
-            gitaly_migrate_call_duration_seconds.observe({ gitaly_enabled: is_enabled, feature: feature }, total_time)
-            feature_stack.shift
-            Thread.current[:gitaly_feature_stack] = nil if feature_stack.empty?
-
-            add_call_details(call_details.merge(feature: feature, duration: total_time))
-          end
-        end
-      end
+    def self.feature_enabled?(feature_name)
+      Feature.enabled?("gitaly_#{feature_name}")
     end
 
     # Ensures that Gitaly is not being abuse through n+1 misuse etc
     def self.enforce_gitaly_request_limits(call_site)
       # Only count limits in request-response environments (not sidekiq for example)
-      return unless RequestStore.active?
+      return unless Gitlab::SafeRequestStore.active?
 
       # This is this actual number of times this call was made. Used for information purposes only
       actual_call_count = increment_call_count("gitaly_#{call_site}_actual")
@@ -310,7 +297,7 @@ module Gitlab
     end
 
     def self.allow_n_plus_1_calls
-      return yield unless RequestStore.active?
+      return yield unless Gitlab::SafeRequestStore.active?
 
       begin
         increment_call_count(:gitaly_call_count_exception_block_depth)
@@ -321,63 +308,44 @@ module Gitlab
     end
 
     def self.get_call_count(key)
-      RequestStore.store[key] || 0
+      Gitlab::SafeRequestStore[key] || 0
     end
     private_class_method :get_call_count
 
     def self.increment_call_count(key)
-      RequestStore.store[key] ||= 0
-      RequestStore.store[key] += 1
+      Gitlab::SafeRequestStore[key] ||= 0
+      Gitlab::SafeRequestStore[key] += 1
     end
     private_class_method :increment_call_count
 
     def self.decrement_call_count(key)
-      RequestStore.store[key] -= 1
+      Gitlab::SafeRequestStore[key] -= 1
     end
     private_class_method :decrement_call_count
 
-    # Returns an estimate of the number of Gitaly calls made for this
-    # request
+    # Returns the of the number of Gitaly calls made for this request
     def self.get_request_count
-      return 0 unless RequestStore.active?
-
-      gitaly_migrate_count = get_call_count("gitaly_migrate_actual")
-      gitaly_call_count = get_call_count("gitaly_call_actual")
-
-      # Using the maximum of migrate and call_count will provide an
-      # indicator of how many Gitaly calls will be made, even
-      # before a feature is enabled. This provides us with a single
-      # metric, but not an exact number, but this tradeoff is acceptable
-      if gitaly_migrate_count > gitaly_call_count
-        gitaly_migrate_count
-      else
-        gitaly_call_count
-      end
+      get_call_count("gitaly_call_actual")
     end
 
     def self.reset_counts
-      return unless RequestStore.active?
+      return unless Gitlab::SafeRequestStore.active?
 
-      %w[migrate call].each do |call_site|
-        RequestStore.store["gitaly_#{call_site}_actual"] = 0
-        RequestStore.store["gitaly_#{call_site}_permitted"] = 0
-      end
+      Gitlab::SafeRequestStore["gitaly_call_actual"] = 0
+      Gitlab::SafeRequestStore["gitaly_call_permitted"] = 0
     end
 
     def self.add_call_details(details)
-      id = details.delete(:id)
+      return unless Gitlab::SafeRequestStore[:peek_enabled]
 
-      return unless id && RequestStore.active? && RequestStore.store[:peek_enabled]
-
-      RequestStore.store['gitaly_call_details'] ||= {}
-      RequestStore.store['gitaly_call_details'][id] ||= {}
-      RequestStore.store['gitaly_call_details'][id].merge!(details)
+      Gitlab::SafeRequestStore['gitaly_call_details'] ||= []
+      Gitlab::SafeRequestStore['gitaly_call_details'] << details
     end
 
     def self.list_call_details
-      return {} unless RequestStore.active? && RequestStore.store[:peek_enabled]
+      return [] unless Gitlab::SafeRequestStore[:peek_enabled]
 
-      RequestStore.store['gitaly_call_details'] || {}
+      Gitlab::SafeRequestStore['gitaly_call_details'] || []
     end
 
     def self.expected_server_version
@@ -385,13 +353,13 @@ module Gitlab
       path.read.chomp
     end
 
-    def self.timestamp(t)
-      Google::Protobuf::Timestamp.new(seconds: t.to_i)
+    def self.timestamp(time)
+      Google::Protobuf::Timestamp.new(seconds: time.to_i)
     end
 
     # The default timeout on all Gitaly calls
     def self.default_timeout
-      return 0 if Sidekiq.server?
+      return no_timeout if Sidekiq.server?
 
       timeout(:gitaly_timeout_default)
     end
@@ -404,6 +372,10 @@ module Gitlab
       timeout(:gitaly_timeout_medium)
     end
 
+    def self.no_timeout
+      0
+    end
+
     def self.timeout(timeout_name)
       Gitlab::CurrentSettings.current_application_settings[timeout_name]
     end
@@ -411,22 +383,22 @@ module Gitlab
 
     # Count a stack. Used for n+1 detection
     def self.count_stack
-      return unless RequestStore.active?
+      return unless Gitlab::SafeRequestStore.active?
 
-      stack_string = caller.drop(1).join("\n")
+      stack_string = Gitlab::Profiler.clean_backtrace(caller).drop(1).join("\n")
 
-      RequestStore.store[:stack_counter] ||= Hash.new
+      Gitlab::SafeRequestStore[:stack_counter] ||= Hash.new
 
-      count = RequestStore.store[:stack_counter][stack_string] || 0
-      RequestStore.store[:stack_counter][stack_string] = count + 1
+      count = Gitlab::SafeRequestStore[:stack_counter][stack_string] || 0
+      Gitlab::SafeRequestStore[:stack_counter][stack_string] = count + 1
     end
     private_class_method :count_stack
 
     # Returns a count for the stack which called Gitaly the most times. Used for n+1 detection
     def self.max_call_count
-      return 0 unless RequestStore.active?
+      return 0 unless Gitlab::SafeRequestStore.active?
 
-      stack_counter = RequestStore.store[:stack_counter]
+      stack_counter = Gitlab::SafeRequestStore[:stack_counter]
       return 0 unless stack_counter
 
       stack_counter.values.max
@@ -435,9 +407,9 @@ module Gitlab
 
     # Returns the stacks that calls Gitaly the most times. Used for n+1 detection
     def self.max_stacks
-      return nil unless RequestStore.active?
+      return nil unless Gitlab::SafeRequestStore.active?
 
-      stack_counter = RequestStore.store[:stack_counter]
+      stack_counter = Gitlab::SafeRequestStore[:stack_counter]
       return nil unless stack_counter
 
       max = max_call_count
diff --git a/lib/gitlab/gitaly_client/attributes_bag.rb b/lib/gitlab/gitaly_client/attributes_bag.rb
index 198a1de91c7..3f1a0ef4888 100644
--- a/lib/gitlab/gitaly_client/attributes_bag.rb
+++ b/lib/gitlab/gitaly_client/attributes_bag.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     # This module expects an `ATTRS` const to be defined on the subclass
diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb
index 28554208984..39547328210 100644
--- a/lib/gitlab/gitaly_client/blob_service.rb
+++ b/lib/gitlab/gitaly_client/blob_service.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     class BlobService
@@ -13,9 +15,9 @@ module Gitlab
           oid: oid,
           limit: limit
         )
-        response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_blob, request)
+        response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_blob, request, timeout: GitalyClient.fast_timeout)
 
-        data = ''
+        data = []
         blob = nil
         response.each do |msg|
           if blob.nil?
@@ -27,6 +29,8 @@ module Gitlab
 
         return nil if blob.oid.blank?
 
+        data = data.join
+
         Gitlab::Git::Blob.new(
           id: blob.oid,
           size: blob.size,
@@ -43,7 +47,7 @@ module Gitlab
           blob_ids: blob_ids
         )
 
-        response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request)
+        response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request, timeout: GitalyClient.medium_timeout)
 
         map_lfs_pointers(response)
       end
@@ -66,13 +70,13 @@ module Gitlab
           :blob_service,
           :get_blobs,
           request,
-          timeout: GitalyClient.default_timeout
+          timeout: GitalyClient.fast_timeout
         )
 
         GitalyClient::BlobsStitcher.new(response)
       end
 
-      def get_new_lfs_pointers(revision, limit, not_in)
+      def get_new_lfs_pointers(revision, limit, not_in, dynamic_timeout = nil)
         request = Gitaly::GetNewLFSPointersRequest.new(
           repository: @gitaly_repo,
           revision: encode_binary(revision),
@@ -85,7 +89,20 @@ module Gitlab
           request.not_in_refs += not_in
         end
 
-        response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_new_lfs_pointers, request)
+        timeout =
+          if dynamic_timeout
+            [dynamic_timeout, GitalyClient.medium_timeout].min
+          else
+            GitalyClient.medium_timeout
+          end
+
+        response = GitalyClient.call(
+          @gitaly_repo.storage_name,
+          :blob_service,
+          :get_new_lfs_pointers,
+          request,
+          timeout: timeout
+        )
 
         map_lfs_pointers(response)
       end
@@ -96,7 +113,7 @@ module Gitlab
           revision: encode_binary(revision)
         )
 
-        response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_all_lfs_pointers, request)
+        response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_all_lfs_pointers, request, timeout: GitalyClient.medium_timeout)
 
         map_lfs_pointers(response)
       end
diff --git a/lib/gitlab/gitaly_client/blobs_stitcher.rb b/lib/gitlab/gitaly_client/blobs_stitcher.rb
index 5ca592ff812..01bab854082 100644
--- a/lib/gitlab/gitaly_client/blobs_stitcher.rb
+++ b/lib/gitlab/gitaly_client/blobs_stitcher.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     class BlobsStitcher
diff --git a/lib/gitlab/gitaly_client/cleanup_service.rb b/lib/gitlab/gitaly_client/cleanup_service.rb
new file mode 100644
index 00000000000..3e8d6a773ca
--- /dev/null
+++ b/lib/gitlab/gitaly_client/cleanup_service.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GitalyClient
+    class CleanupService
+      attr_reader :repository, :gitaly_repo, :storage
+
+      # 'repository' is a Gitlab::Git::Repository
+      def initialize(repository)
+        @repository = repository
+        @gitaly_repo = repository.gitaly_repository
+        @storage = repository.storage
+      end
+
+      def apply_bfg_object_map(io)
+        first_request = Gitaly::ApplyBfgObjectMapRequest.new(repository: gitaly_repo)
+
+        enum = Enumerator.new do |y|
+          y.yield first_request
+
+          while data = io.read(RepositoryService::MAX_MSG_SIZE)
+            y.yield Gitaly::ApplyBfgObjectMapRequest.new(object_map: data)
+            break if io&.eof?
+          end
+        end
+
+        GitalyClient.call(
+          storage,
+          :cleanup_service,
+          :apply_bfg_object_map,
+          enum,
+          timeout: GitalyClient.no_timeout
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index a4cc64de80d..4e46cb9f05c 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     class CommitService
@@ -70,12 +72,19 @@ module Gitlab
 
       def commit_deltas(commit)
         request = Gitaly::CommitDeltaRequest.new(diff_from_parent_request_params(commit))
-        response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request)
+        response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request, timeout: GitalyClient.fast_timeout)
 
         response.flat_map { |msg| msg.deltas }
       end
 
       def tree_entry(ref, path, limit = nil)
+        if Pathname.new(path).cleanpath.to_s.start_with?('../')
+          # The TreeEntry RPC should return an empty reponse in this case but in
+          # Gitaly 0.107.0 and earlier we get an exception instead. This early return
+          # saves us a Gitaly roundtrip while also avoiding the exception.
+          return
+        end
+
         request = Gitaly::TreeEntryRequest.new(
           repository: @gitaly_repo,
           revision: encode_binary(ref),
@@ -86,7 +95,7 @@ module Gitlab
         response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request, timeout: GitalyClient.medium_timeout)
 
         entry = nil
-        data = ''
+        data = []
         response.each do |msg|
           if entry.nil?
             entry = msg
@@ -96,7 +105,7 @@ module Gitlab
 
           data << msg.data
         end
-        entry.data = data
+        entry.data = data.join
 
         entry unless entry.oid.blank?
       end
@@ -141,6 +150,24 @@ module Gitlab
         GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count
       end
 
+      def list_last_commits_for_tree(revision, path, offset: 0, limit: 25)
+        request = Gitaly::ListLastCommitsForTreeRequest.new(
+          repository: @gitaly_repo,
+          revision: encode_binary(revision),
+          path: encode_binary(path.to_s),
+          offset: offset,
+          limit: limit
+        )
+
+        response = GitalyClient.call(@repository.storage, :commit_service, :list_last_commits_for_tree, request, timeout: GitalyClient.medium_timeout)
+
+        response.each_with_object({}) do |gitaly_response, hsh|
+          gitaly_response.commits.each do |commit_for_tree|
+            hsh[commit_for_tree.path] = Gitlab::Git::Commit.new(@repository, commit_for_tree.commit)
+          end
+        end
+      end
+
       def last_commit_for_path(revision, path)
         request = Gitaly::LastCommitForPathRequest.new(
           repository: @gitaly_repo,
@@ -165,6 +192,17 @@ module Gitlab
         consume_commits_response(response)
       end
 
+      def diff_stats(left_commit_sha, right_commit_sha)
+        request = Gitaly::DiffStatsRequest.new(
+          repository: @gitaly_repo,
+          left_commit_id: left_commit_sha,
+          right_commit_id: right_commit_sha
+        )
+
+        response = GitalyClient.call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout)
+        response.flat_map(&:stats)
+      end
+
       def find_all_commits(opts = {})
         request = Gitaly::FindAllCommitsRequest.new(
           repository: @gitaly_repo,
@@ -179,6 +217,8 @@ module Gitlab
       end
 
       def list_commits_by_oid(oids)
+        return [] if oids.empty?
+
         request = Gitaly::ListCommitsByOidRequest.new(repository: @gitaly_repo, oid: oids)
 
         response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout)
@@ -216,31 +256,33 @@ module Gitlab
         )
 
         response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout)
-        response.reduce("") { |memo, msg| memo << msg.data }
+        response.reduce([]) { |memo, msg| memo << msg.data }.join
       end
 
       def find_commit(revision)
-        if RequestStore.active?
-          # We don't use RequeStstore.fetch(key) { ... } directly because `revision`
-          # can be a branch name, so we can't use it as a key as it could point
-          # to another commit later on (happens a lot in tests).
+        if Gitlab::SafeRequestStore.active?
+          # We don't use Gitlab::SafeRequestStore.fetch(key) { ... } directly
+          # because `revision` can be a branch name, so we can't use it as a key
+          # as it could point to another commit later on (happens a lot in
+          # tests).
           key = {
             storage: @gitaly_repo.storage_name,
             relative_path: @gitaly_repo.relative_path,
             commit_id: revision
           }
-          return RequestStore[key] if RequestStore.exist?(key)
+          return Gitlab::SafeRequestStore[key] if Gitlab::SafeRequestStore.exist?(key)
 
           commit = call_find_commit(revision)
           return unless commit
 
           key[:commit_id] = commit.id
-          RequestStore[key] = commit
+          Gitlab::SafeRequestStore[key] = commit
         else
           call_find_commit(revision)
         end
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def patch(revision)
         request = Gitaly::CommitPatchRequest.new(
           repository: @gitaly_repo,
@@ -250,6 +292,7 @@ module Gitlab
 
         response.sum(&:data)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def commit_stats(revision)
         request = Gitaly::CommitStatsRequest.new(
@@ -293,7 +336,7 @@ module Gitlab
           end
         end
 
-        response = GitalyClient.call(@repository.storage, :commit_service, :filter_shas_with_signatures, enum)
+        response = GitalyClient.call(@repository.storage, :commit_service, :filter_shas_with_signatures, enum, timeout: GitalyClient.fast_timeout)
 
         response.flat_map do |msg|
           msg.shas.map { |sha| EncodingHelper.encode!(sha) }
@@ -304,8 +347,8 @@ module Gitlab
         request = Gitaly::ExtractCommitSignatureRequest.new(repository: @gitaly_repo, commit_id: commit_id)
         response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request)
 
-        signature = ''.b
-        signed_text = ''.b
+        signature = +''.b
+        signed_text = +''.b
 
         response.each do |message|
           signature << message.signature
@@ -315,13 +358,15 @@ module Gitlab
         return if signature.blank? && signed_text.blank?
 
         [signature, signed_text]
+      rescue GRPC::InvalidArgument => ex
+        raise ArgumentError, ex
       end
 
       def get_commit_signatures(commit_ids)
         request = Gitaly::GetCommitSignaturesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids)
-        response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_signatures, request)
+        response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_signatures, request, timeout: GitalyClient.fast_timeout)
 
-        signatures = Hash.new { |h, k| h[k] = [''.b, ''.b] }
+        signatures = Hash.new { |h, k| h[k] = [+''.b, +''.b] }
         current_commit_id = nil
 
         response.each do |message|
@@ -332,13 +377,15 @@ module Gitlab
         end
 
         signatures
+      rescue GRPC::InvalidArgument => ex
+        raise ArgumentError, ex
       end
 
       def get_commit_messages(commit_ids)
         request = Gitaly::GetCommitMessagesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids)
-        response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_messages, request)
+        response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_messages, request, timeout: GitalyClient.fast_timeout)
 
-        messages = Hash.new { |h, k| h[k] = ''.b }
+        messages = Hash.new { |h, k| h[k] = +''.b }
         current_commit_id = nil
 
         response.each do |rpc_message|
@@ -355,8 +402,8 @@ module Gitlab
       def call_commit_diff(request_params, options = {})
         request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false)
         request_params[:enforce_limits] = options.fetch(:limits, true)
-        request_params[:collapse_diffs] = request_params[:enforce_limits] || !options.fetch(:expanded, true)
-        request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h)
+        request_params[:collapse_diffs] = !options.fetch(:expanded, true)
+        request_params.merge!(Gitlab::Git::DiffCollection.limits(options).to_h)
 
         request = Gitaly::CommitDiffRequest.new(request_params)
         response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request, timeout: GitalyClient.medium_timeout)
@@ -386,8 +433,8 @@ module Gitlab
         end
       end
 
-      def encode_repeated(a)
-        Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| encode_binary(s) } )
+      def encode_repeated(array)
+        Google::Protobuf::RepeatedField.new(:bytes, array.map { |s| encode_binary(s) } )
       end
 
       def call_find_commit(revision)
diff --git a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb
index c275a065bce..0e00f6e8c44 100644
--- a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb
+++ b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     class ConflictFilesStitcher
@@ -17,7 +19,7 @@ module Gitlab
 
               current_file = file_from_gitaly_header(gitaly_file.header)
             else
-              current_file.raw_content << gitaly_file.content
+              current_file.raw_content = "#{current_file.raw_content}#{gitaly_file.content}"
             end
           end
         end
diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb
index e14734495a8..6304f998563 100644
--- a/lib/gitlab/gitaly_client/conflicts_service.rb
+++ b/lib/gitlab/gitaly_client/conflicts_service.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     class ConflictsService
@@ -25,10 +27,12 @@ module Gitlab
 
       def conflicts?
         list_conflict_files.any?
-      rescue GRPC::FailedPrecondition
-        # The server raises this exception when it encounters ConflictSideMissing, which
-        # means a conflict exists but its `theirs` or `ours` data is nil due to a non-existent
-        # file in one of the trees.
+      rescue GRPC::FailedPrecondition, GRPC::Unknown
+        # The server raises FailedPrecondition when it encounters
+        # ConflictSideMissing, which means a conflict exists but its `theirs` or
+        # `ours` data is nil due to a non-existent file in one of the trees.
+        #
+        # GRPC::Unknown comes from Rugged::ReferenceError and Rugged::OdbError.
         true
       end
 
@@ -46,7 +50,7 @@ module Gitlab
           end
         end
 
-        response = GitalyClient.call(@repository.storage, :conflicts_service, :resolve_conflicts, req_enum, remote_storage: target_repository.storage)
+        response = GitalyClient.call(@repository.storage, :conflicts_service, :resolve_conflicts, req_enum, remote_storage: target_repository.storage, timeout: GitalyClient.medium_timeout)
 
         if response.resolution_error.present?
           raise Gitlab::Git::Conflict::Resolver::ResolutionError, response.resolution_error
diff --git a/lib/gitlab/gitaly_client/diff.rb b/lib/gitlab/gitaly_client/diff.rb
index d98a0ce988f..dd192ccde1a 100644
--- a/lib/gitlab/gitaly_client/diff.rb
+++ b/lib/gitlab/gitaly_client/diff.rb
@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     class Diff
-      ATTRS = %i(from_path to_path old_mode new_mode from_id to_id patch overflow_marker collapsed).freeze
+      ATTRS = %i(from_path to_path old_mode new_mode from_id to_id patch overflow_marker collapsed too_large).freeze
 
       include AttributesBag
     end
diff --git a/lib/gitlab/gitaly_client/diff_stitcher.rb b/lib/gitlab/gitaly_client/diff_stitcher.rb
index da243ee2d1a..98d327a7329 100644
--- a/lib/gitlab/gitaly_client/diff_stitcher.rb
+++ b/lib/gitlab/gitaly_client/diff_stitcher.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     class DiffStitcher
@@ -20,7 +22,7 @@ module Gitlab
 
             current_diff = GitalyClient::Diff.new(diff_params)
           else
-            current_diff.patch += diff_msg.raw_patch_data
+            current_diff.patch = "#{current_diff.patch}#{diff_msg.raw_patch_data}"
           end
 
           if diff_msg.end_of_patch
diff --git a/lib/gitlab/gitaly_client/health_check_service.rb b/lib/gitlab/gitaly_client/health_check_service.rb
index 6c1213f5e20..0c495f60633 100644
--- a/lib/gitlab/gitaly_client/health_check_service.rb
+++ b/lib/gitlab/gitaly_client/health_check_service.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     class HealthCheckService
diff --git a/lib/gitlab/gitaly_client/namespace_service.rb b/lib/gitlab/gitaly_client/namespace_service.rb
index bd7c345ac01..f0be3cbebd2 100644
--- a/lib/gitlab/gitaly_client/namespace_service.rb
+++ b/lib/gitlab/gitaly_client/namespace_service.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     class NamespaceService
@@ -8,31 +10,31 @@ module Gitlab
       def exists?(name)
         request = Gitaly::NamespaceExistsRequest.new(storage_name: @storage, name: name)
 
-        gitaly_client_call(:namespace_exists, request).exists
+        gitaly_client_call(:namespace_exists, request, timeout: GitalyClient.fast_timeout).exists
       end
 
       def add(name)
         request = Gitaly::AddNamespaceRequest.new(storage_name: @storage, name: name)
 
-        gitaly_client_call(:add_namespace, request)
+        gitaly_client_call(:add_namespace, request, timeout: GitalyClient.fast_timeout)
       end
 
       def remove(name)
         request = Gitaly::RemoveNamespaceRequest.new(storage_name: @storage, name: name)
 
-        gitaly_client_call(:remove_namespace, request)
+        gitaly_client_call(:remove_namespace, request, timeout: nil)
       end
 
       def rename(from, to)
         request = Gitaly::RenameNamespaceRequest.new(storage_name: @storage, from: from, to: to)
 
-        gitaly_client_call(:rename_namespace, request)
+        gitaly_client_call(:rename_namespace, request, timeout: GitalyClient.fast_timeout)
       end
 
       private
 
-      def gitaly_client_call(type, request)
-        GitalyClient.call(@storage, :namespace_service, type, request)
+      def gitaly_client_call(type, request, timeout: nil)
+        GitalyClient.call(@storage, :namespace_service, type, request, timeout: timeout)
       end
     end
   end
diff --git a/lib/gitlab/gitaly_client/notification_service.rb b/lib/gitlab/gitaly_client/notification_service.rb
index 326e6f7dafc..873c3e4086d 100644
--- a/lib/gitlab/gitaly_client/notification_service.rb
+++ b/lib/gitlab/gitaly_client/notification_service.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     class NotificationService
diff --git a/lib/gitlab/gitaly_client/object_pool_service.rb b/lib/gitlab/gitaly_client/object_pool_service.rb
new file mode 100644
index 00000000000..6e7ede5fd18
--- /dev/null
+++ b/lib/gitlab/gitaly_client/object_pool_service.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GitalyClient
+    class ObjectPoolService
+      attr_reader :object_pool, :storage
+
+      def initialize(object_pool)
+        @object_pool = object_pool.gitaly_object_pool
+        @storage = object_pool.storage
+      end
+
+      def create(repository)
+        request = Gitaly::CreateObjectPoolRequest.new(
+          object_pool: object_pool,
+          origin: repository.gitaly_repository)
+
+        GitalyClient.call(storage, :object_pool_service, :create_object_pool, request)
+      end
+
+      def delete
+        request = Gitaly::DeleteObjectPoolRequest.new(object_pool: object_pool)
+
+        GitalyClient.call(storage, :object_pool_service, :delete_object_pool, request)
+      end
+
+      def link_repository(repository)
+        request = Gitaly::LinkRepositoryToObjectPoolRequest.new(
+          object_pool: object_pool,
+          repository: repository.gitaly_repository
+        )
+
+        GitalyClient.call(storage, :object_pool_service, :link_repository_to_object_pool,
+                          request, timeout: GitalyClient.fast_timeout)
+      end
+
+      def unlink_repository(repository)
+        request = Gitaly::UnlinkRepositoryFromObjectPoolRequest.new(
+          object_pool: object_pool,
+          repository: repository.gitaly_repository
+        )
+
+        GitalyClient.call(storage, :object_pool_service, :unlink_repository_from_object_pool,
+                          request, timeout: GitalyClient.fast_timeout)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb
index 44b0e517bf0..22d2d149e65 100644
--- a/lib/gitlab/gitaly_client/operation_service.rb
+++ b/lib/gitlab/gitaly_client/operation_service.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     class OperationService
@@ -17,10 +19,10 @@ module Gitlab
           user: Gitlab::Git::User.from_gitlab(user).to_gitaly
         )
 
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request)
+        response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.medium_timeout)
 
         if pre_receive_error = response.pre_receive_error.presence
-          raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error
+          raise Gitlab::Git::PreReceiveError, pre_receive_error
         end
       end
 
@@ -33,9 +35,9 @@ module Gitlab
           message: encode_binary(message.to_s)
         )
 
-        response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request)
+        response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.medium_timeout)
         if pre_receive_error = response.pre_receive_error.presence
-          raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error
+          raise Gitlab::Git::PreReceiveError, pre_receive_error
         elsif response.exists
           raise Gitlab::Git::Repository::TagExistsError
         end
@@ -56,7 +58,7 @@ module Gitlab
           :user_create_branch, request)
 
         if response.pre_receive_error.present?
-          raise Gitlab::Git::HooksService::PreReceiveError.new(response.pre_receive_error)
+          raise Gitlab::Git::PreReceiveError.new(response.pre_receive_error)
         end
 
         branch = response.branch
@@ -64,6 +66,24 @@ module Gitlab
 
         target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit)
         Gitlab::Git::Branch.new(@repository, branch.name, target_commit.id, target_commit)
+      rescue GRPC::FailedPrecondition => ex
+        raise Gitlab::Git::Repository::InvalidRef, ex
+      end
+
+      def user_update_branch(branch_name, user, newrev, oldrev)
+        request = Gitaly::UserUpdateBranchRequest.new(
+          repository: @gitaly_repo,
+          branch_name: encode_binary(branch_name),
+          user: Gitlab::Git::User.from_gitlab(user).to_gitaly,
+          newrev: encode_binary(newrev),
+          oldrev: encode_binary(oldrev)
+        )
+
+        response = GitalyClient.call(@repository.storage, :operation_service, :user_update_branch, request)
+
+        if pre_receive_error = response.pre_receive_error.presence
+          raise Gitlab::Git::PreReceiveError, pre_receive_error
+        end
       end
 
       def user_delete_branch(branch_name, user)
@@ -76,7 +96,7 @@ module Gitlab
         response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_branch, request)
 
         if pre_receive_error = response.pre_receive_error.presence
-          raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error
+          raise Gitlab::Git::PreReceiveError, pre_receive_error
         end
       end
 
@@ -106,7 +126,7 @@ module Gitlab
         second_response = response_enum.next
 
         if second_response.pre_receive_error.present?
-          raise Gitlab::Git::HooksService::PreReceiveError, second_response.pre_receive_error
+          raise Gitlab::Git::PreReceiveError, second_response.pre_receive_error
         end
 
         branch_update = second_response.branch_update
@@ -126,13 +146,16 @@ module Gitlab
           branch: encode_binary(target_branch)
         )
 
-        branch_update = GitalyClient.call(
+        response = GitalyClient.call(
           @repository.storage,
           :operation_service,
           :user_ff_branch,
           request
-        ).branch_update
-        Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update)
+        )
+
+        Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
+      rescue GRPC::FailedPrecondition => e
+        raise Gitlab::Git::CommitError, e
       end
 
       def user_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
@@ -175,7 +198,7 @@ module Gitlab
         )
 
         if response.pre_receive_error.presence
-          raise Gitlab::Git::HooksService::PreReceiveError, response.pre_receive_error
+          raise Gitlab::Git::PreReceiveError, response.pre_receive_error
         elsif response.git_error.presence
           raise Gitlab::Git::Repository::GitError, response.git_error
         else
@@ -209,6 +232,32 @@ module Gitlab
         response.squash_sha
       end
 
+      def user_update_submodule(user:, submodule:, commit_sha:, branch:, message:)
+        request = Gitaly::UserUpdateSubmoduleRequest.new(
+          repository: @gitaly_repo,
+          user: Gitlab::Git::User.from_gitlab(user).to_gitaly,
+          commit_sha: commit_sha,
+          branch: encode_binary(branch),
+          submodule: encode_binary(submodule),
+          commit_message: encode_binary(message)
+        )
+
+        response = GitalyClient.call(
+          @repository.storage,
+          :operation_service,
+          :user_update_submodule,
+          request
+        )
+
+        if response.pre_receive_error.present?
+          raise Gitlab::Git::PreReceiveError, response.pre_receive_error
+        elsif response.commit_error.present?
+          raise Gitlab::Git::CommitError, response.commit_error
+        else
+          Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
+        end
+      end
+
       def user_commit_files(
         user, branch_name, commit_message, actions, author_email, author_name,
         start_branch_name, start_repository)
@@ -242,7 +291,7 @@ module Gitlab
           :user_commit_files, req_enum, remote_storage: start_repository.storage)
 
         if (pre_receive_error = response.pre_receive_error.presence)
-          raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error
+          raise Gitlab::Git::PreReceiveError, pre_receive_error
         end
 
         if (index_error = response.index_error.presence)
@@ -252,6 +301,29 @@ module Gitlab
         Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
       end
 
+      def user_commit_patches(user, branch_name, patches)
+        header = Gitaly::UserApplyPatchRequest::Header.new(
+          repository: @gitaly_repo,
+          user: Gitlab::Git::User.from_gitlab(user).to_gitaly,
+          target_branch: encode_binary(branch_name)
+        )
+        reader = binary_stringio(patches)
+
+        chunks = Enumerator.new do |chunk|
+          chunk.yield Gitaly::UserApplyPatchRequest.new(header: header)
+
+          until reader.eof?
+            patch_chunk = reader.read(MAX_MSG_SIZE)
+
+            chunk.yield(Gitaly::UserApplyPatchRequest.new(patches: patch_chunk))
+          end
+        end
+
+        response = GitalyClient.call(@repository.storage, :operation_service, :user_apply_patch, chunks)
+
+        Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
+      end
+
       private
 
       def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
@@ -272,7 +344,8 @@ module Gitlab
           :operation_service,
           :"user_#{rpc}",
           request,
-          remote_storage: start_repository.storage
+          remote_storage: start_repository.storage,
+          timeout: GitalyClient.medium_timeout
         )
 
         handle_cherry_pick_or_revert_response(response)
@@ -280,14 +353,14 @@ module Gitlab
 
       def handle_cherry_pick_or_revert_response(response)
         if response.pre_receive_error.presence
-          raise Gitlab::Git::HooksService::PreReceiveError, response.pre_receive_error
+          raise Gitlab::Git::PreReceiveError, response.pre_receive_error
         elsif response.commit_error.presence
           raise Gitlab::Git::CommitError, response.commit_error
         elsif response.create_tree_error.presence
           raise Gitlab::Git::Repository::CreateTreeError, response.create_tree_error
-        else
-          Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
         end
+
+        Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
       end
 
       def user_commit_files_request_header(
@@ -311,7 +384,9 @@ module Gitlab
           action: action[:action].upcase.to_sym,
           file_path: encode_binary(action[:file_path]),
           previous_path: encode_binary(action[:previous_path]),
-          base64_content: action[:encoding] == 'base64'
+          base64_content: action[:encoding] == 'base64',
+          execute_filemode: !!action[:execute_filemode],
+          infer_content: !!action[:infer_content]
         )
       rescue RangeError
         raise ArgumentError, "Unknown action '#{action[:action]}'"
diff --git a/lib/gitlab/gitaly_client/queue_enumerator.rb b/lib/gitlab/gitaly_client/queue_enumerator.rb
index b8018029552..3a412102abe 100644
--- a/lib/gitlab/gitaly_client/queue_enumerator.rb
+++ b/lib/gitlab/gitaly_client/queue_enumerator.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     class QueueEnumerator
diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb
index 3ac46be6208..d5633d167ac 100644
--- a/lib/gitlab/gitaly_client/ref_service.rb
+++ b/lib/gitlab/gitaly_client/ref_service.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     class RefService
@@ -12,37 +14,44 @@ module Gitlab
 
       def branches
         request = Gitaly::FindAllBranchesRequest.new(repository: @gitaly_repo)
-        response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request)
+        response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request, timeout: GitalyClient.fast_timeout)
 
         consume_find_all_branches_response(response)
       end
 
+      def remote_branches(remote_name)
+        request = Gitaly::FindAllRemoteBranchesRequest.new(repository: @gitaly_repo, remote_name: remote_name)
+        response = GitalyClient.call(@repository.storage, :ref_service, :find_all_remote_branches, request)
+
+        consume_find_all_remote_branches_response(remote_name, response)
+      end
+
       def merged_branches(branch_names = [])
         request = Gitaly::FindAllBranchesRequest.new(
           repository: @gitaly_repo,
           merged_only: true,
           merged_branches: branch_names.map { |s| encode_binary(s) }
         )
-        response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request)
+        response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request, timeout: GitalyClient.fast_timeout)
 
         consume_find_all_branches_response(response)
       end
 
       def default_branch_name
         request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo)
-        response = GitalyClient.call(@storage, :ref_service, :find_default_branch_name, request)
+        response = GitalyClient.call(@storage, :ref_service, :find_default_branch_name, request, timeout: GitalyClient.fast_timeout)
         Gitlab::Git.branch_name(response.name)
       end
 
       def branch_names
         request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo)
-        response = GitalyClient.call(@storage, :ref_service, :find_all_branch_names, request)
+        response = GitalyClient.call(@storage, :ref_service, :find_all_branch_names, request, timeout: GitalyClient.fast_timeout)
         consume_refs_response(response) { |name| Gitlab::Git.branch_name(name) }
       end
 
       def tag_names
         request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo)
-        response = GitalyClient.call(@storage, :ref_service, :find_all_tag_names, request)
+        response = GitalyClient.call(@storage, :ref_service, :find_all_tag_names, request, timeout: GitalyClient.fast_timeout)
         consume_refs_response(response) { |name| Gitlab::Git.tag_name(name) }
       end
 
@@ -56,6 +65,42 @@ module Gitlab
         encode!(response.name.dup)
       end
 
+      def list_new_commits(newrev)
+        request = Gitaly::ListNewCommitsRequest.new(
+          repository: @gitaly_repo,
+          commit_id: newrev
+        )
+
+        response = GitalyClient
+          .call(@storage, :ref_service, :list_new_commits, request, timeout: GitalyClient.medium_timeout)
+
+        commits = []
+        response.each do |msg|
+          msg.commits.each do |c|
+            commits << Gitlab::Git::Commit.new(@repository, c)
+          end
+        end
+
+        commits
+      end
+
+      def list_new_blobs(newrev, limit = 0)
+        request = Gitaly::ListNewBlobsRequest.new(
+          repository: @gitaly_repo,
+          commit_id: newrev,
+          limit: limit
+        )
+
+        response = GitalyClient
+          .call(@storage, :ref_service, :list_new_blobs, request, timeout: GitalyClient.medium_timeout)
+
+        response.flat_map do |msg|
+          # Returns an Array of Gitaly::NewBlobObject objects
+          # Available methods are: #size, #oid and #path
+          msg.new_blob_objects
+        end
+      end
+
       def count_tag_names
         tag_names.count
       end
@@ -67,19 +112,19 @@ module Gitlab
       def local_branches(sort_by: nil)
         request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo)
         request.sort_by = sort_by_param(sort_by) if sort_by
-        response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request)
+        response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request, timeout: GitalyClient.fast_timeout)
         consume_find_local_branches_response(response)
       end
 
       def tags
         request = Gitaly::FindAllTagsRequest.new(repository: @gitaly_repo)
-        response = GitalyClient.call(@storage, :ref_service, :find_all_tags, request)
+        response = GitalyClient.call(@storage, :ref_service, :find_all_tags, request, timeout: GitalyClient.medium_timeout)
         consume_tags_response(response)
       end
 
       def ref_exists?(ref_name)
         request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: encode_binary(ref_name))
-        response = GitalyClient.call(@storage, :ref_service, :ref_exists, request)
+        response = GitalyClient.call(@storage, :ref_service, :ref_exists, request, timeout: GitalyClient.fast_timeout)
         response.value
       rescue GRPC::InvalidArgument => e
         raise ArgumentError, e.message
@@ -91,7 +136,7 @@ module Gitlab
           name: encode_binary(branch_name)
         )
 
-        response = GitalyClient.call(@repository.storage, :ref_service, :find_branch, request)
+        response = GitalyClient.call(@repository.storage, :ref_service, :find_branch, request, timeout: GitalyClient.medium_timeout)
         branch = response.branch
         return unless branch
 
@@ -140,7 +185,7 @@ module Gitlab
           except_with_prefix: except_with_prefixes.map { |r| encode_binary(r) }
         )
 
-        response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request)
+        response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.default_timeout)
 
         raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present?
       end
@@ -153,7 +198,7 @@ module Gitlab
           limit: limit
         )
 
-        stream = GitalyClient.call(@repository.storage, :ref_service, :list_tag_names_containing_commit, request)
+        stream = GitalyClient.call(@repository.storage, :ref_service, :list_tag_names_containing_commit, request, timeout: GitalyClient.medium_timeout)
 
         consume_ref_contains_sha_response(stream, :tag_names)
       end
@@ -166,16 +211,16 @@ module Gitlab
           limit: limit
         )
 
-        stream = GitalyClient.call(@repository.storage, :ref_service, :list_branch_names_containing_commit, request)
+        stream = GitalyClient.call(@repository.storage, :ref_service, :list_branch_names_containing_commit, request, timeout: GitalyClient.medium_timeout)
 
         consume_ref_contains_sha_response(stream, :branch_names)
       end
 
       def get_tag_messages(tag_ids)
         request = Gitaly::GetTagMessagesRequest.new(repository: @gitaly_repo, tag_ids: tag_ids)
-        response = GitalyClient.call(@repository.storage, :ref_service, :get_tag_messages, request)
+        response = GitalyClient.call(@repository.storage, :ref_service, :get_tag_messages, request, timeout: GitalyClient.fast_timeout)
 
-        messages = Hash.new { |h, k| h[k] = ''.b }
+        messages = Hash.new { |h, k| h[k] = +''.b }
         current_tag_id = nil
 
         response.each do |rpc_message|
@@ -224,6 +269,18 @@ module Gitlab
         end
       end
 
+      def consume_find_all_remote_branches_response(remote_name, response)
+        remote_name += '/' unless remote_name.ends_with?('/')
+
+        response.flat_map do |message|
+          message.branches.map do |branch|
+            target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit)
+            branch_name = branch.name.sub(remote_name, '')
+            Gitlab::Git::Branch.new(@repository, branch_name, branch.target_commit.id, target_commit)
+          end
+        end
+      end
+
       def consume_tags_response(response)
         response.flat_map do |message|
           message.tags.map { |gitaly_tag| Gitlab::Git::Tag.new(@repository, gitaly_tag) }
diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb
index f2d699d9dfb..81fac37ee68 100644
--- a/lib/gitlab/gitaly_client/remote_service.rb
+++ b/lib/gitlab/gitaly_client/remote_service.rb
@@ -1,6 +1,10 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     class RemoteService
+      include Gitlab::EncodingHelper
+
       MAX_MSG_SIZE = 128.kilobytes.freeze
 
       def self.exists?(remote_url)
@@ -28,13 +32,13 @@ module Gitlab
           mirror_refmaps: Array.wrap(mirror_refmaps).map(&:to_s)
         )
 
-        GitalyClient.call(@storage, :remote_service, :add_remote, request)
+        GitalyClient.call(@storage, :remote_service, :add_remote, request, timeout: GitalyClient.fast_timeout)
       end
 
       def remove_remote(name)
         request = Gitaly::RemoveRemoteRequest.new(repository: @gitaly_repo, name: name)
 
-        response = GitalyClient.call(@storage, :remote_service, :remove_remote, request)
+        response = GitalyClient.call(@storage, :remote_service, :remove_remote, request, timeout: GitalyClient.fast_timeout)
 
         response.result
       end
@@ -52,13 +56,30 @@ module Gitlab
         response.result
       end
 
-      def update_remote_mirror(ref_name, only_branches_matching)
+      def find_remote_root_ref(remote_name)
+        request = Gitaly::FindRemoteRootRefRequest.new(
+          repository: @gitaly_repo,
+          remote: remote_name
+        )
+
+        response = GitalyClient.call(@storage, :remote_service,
+                                     :find_remote_root_ref, request)
+
+        encode_utf8(response.ref)
+      end
+
+      def update_remote_mirror(ref_name, only_branches_matching, ssh_key: nil, known_hosts: nil)
         req_enum = Enumerator.new do |y|
-          y.yield Gitaly::UpdateRemoteMirrorRequest.new(
+          first_request = Gitaly::UpdateRemoteMirrorRequest.new(
             repository: @gitaly_repo,
             ref_name: ref_name
           )
 
+          first_request.ssh_key = ssh_key if ssh_key.present?
+          first_request.known_hosts = known_hosts if known_hosts.present?
+
+          y.yield(first_request)
+
           current_size = 0
 
           slices = only_branches_matching.slice_before do |branch_name|
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index ee01f5a5bd9..8a1abfbf874 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     class RepositoryService
@@ -21,7 +23,7 @@ module Gitlab
 
       def cleanup
         request = Gitaly::CleanupRequest.new(repository: @gitaly_repo)
-        GitalyClient.call(@storage, :repository_service, :cleanup, request)
+        GitalyClient.call(@storage, :repository_service, :cleanup, request, timeout: GitalyClient.fast_timeout)
       end
 
       def garbage_collect(create_bitmap)
@@ -41,22 +43,24 @@ module Gitlab
 
       def repository_size
         request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo)
-        response = GitalyClient.call(@storage, :repository_service, :repository_size, request)
+        response = GitalyClient.call(@storage, :repository_service, :repository_size, request, timeout: GitalyClient.medium_timeout)
         response.size
       end
 
       def apply_gitattributes(revision)
         request = Gitaly::ApplyGitattributesRequest.new(repository: @gitaly_repo, revision: encode_binary(revision))
-        GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request)
+        GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request, timeout: GitalyClient.fast_timeout)
+      rescue GRPC::InvalidArgument => ex
+        raise Gitlab::Git::Repository::InvalidRef, ex
       end
 
       def info_attributes
         request = Gitaly::GetInfoAttributesRequest.new(repository: @gitaly_repo)
 
-        response = GitalyClient.call(@storage, :repository_service, :get_info_attributes, request)
-        response.each_with_object("") do |message, attributes|
+        response = GitalyClient.call(@storage, :repository_service, :get_info_attributes, request, timeout: GitalyClient.fast_timeout)
+        response.each_with_object([]) do |message, attributes|
           attributes << message.attributes
-        end
+        end.join
       end
 
       def fetch_remote(remote, ssh_auth:, forced:, no_tags:, timeout:, prune: true)
@@ -65,7 +69,7 @@ module Gitlab
           no_tags: no_tags, timeout: timeout, no_prune: !prune
         )
 
-        if ssh_auth&.ssh_import?
+        if ssh_auth&.ssh_mirror_url?
           if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
             request.ssh_key = ssh_auth.ssh_private_key
           end
@@ -80,7 +84,7 @@ module Gitlab
 
       def create_repository
         request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo)
-        GitalyClient.call(@storage, :repository_service, :create_repository, request)
+        GitalyClient.call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.medium_timeout)
       end
 
       def has_local_branches?
@@ -96,7 +100,7 @@ module Gitlab
           revisions: revisions.map { |r| encode_binary(r) }
         )
 
-        response = GitalyClient.call(@storage, :repository_service, :find_merge_base, request)
+        response = GitalyClient.call(@storage, :repository_service, :find_merge_base, request, timeout: GitalyClient.fast_timeout)
         response.base.presence
       end
 
@@ -196,42 +200,38 @@ module Gitlab
       end
 
       def create_bundle(save_path)
-        request = Gitaly::CreateBundleRequest.new(repository: @gitaly_repo)
-        response = GitalyClient.call(
-          @storage,
-          :repository_service,
+        gitaly_fetch_stream_to_file(
+          save_path,
           :create_bundle,
-          request,
-          timeout: GitalyClient.default_timeout
+          Gitaly::CreateBundleRequest,
+          GitalyClient.no_timeout
         )
+      end
 
-        File.open(save_path, 'wb') do |f|
-          response.each do |message|
-            f.write(message.data)
-          end
-        end
+      def backup_custom_hooks(save_path)
+        gitaly_fetch_stream_to_file(
+          save_path,
+          :backup_custom_hooks,
+          Gitaly::BackupCustomHooksRequest,
+          GitalyClient.default_timeout
+        )
       end
 
       def create_from_bundle(bundle_path)
-        request = Gitaly::CreateRepositoryFromBundleRequest.new(repository: @gitaly_repo)
-        enum = Enumerator.new do |y|
-          File.open(bundle_path, 'rb') do |f|
-            while data = f.read(MAX_MSG_SIZE)
-              request.data = data
-
-              y.yield request
-
-              request = Gitaly::CreateRepositoryFromBundleRequest.new
-            end
-          end
-        end
-
-        GitalyClient.call(
-          @storage,
-          :repository_service,
+        gitaly_repo_stream_request(
+          bundle_path,
           :create_repository_from_bundle,
-          enum,
-          timeout: GitalyClient.default_timeout
+          Gitaly::CreateRepositoryFromBundleRequest,
+          GitalyClient.no_timeout
+        )
+      end
+
+      def restore_custom_hooks(custom_hooks_path)
+        gitaly_repo_stream_request(
+          custom_hooks_path,
+          :restore_custom_hooks,
+          Gitaly::RestoreCustomHooksRequest,
+          GitalyClient.default_timeout
         )
       end
 
@@ -247,37 +247,54 @@ module Gitlab
           :repository_service,
           :create_repository_from_snapshot,
           request,
-          timeout: GitalyClient.default_timeout
+          timeout: GitalyClient.no_timeout
         )
       end
 
-      def write_ref(ref_path, ref, old_ref, shell)
+      def write_ref(ref_path, ref, old_ref)
         request = Gitaly::WriteRefRequest.new(
           repository: @gitaly_repo,
           ref: ref_path.b,
-          revision: ref.b,
-          shell: shell
+          revision: ref.b
         )
         request.old_revision = old_ref.b unless old_ref.nil?
 
-        response = GitalyClient.call(@storage, :repository_service, :write_ref, request)
+        GitalyClient.call(@storage, :repository_service, :write_ref, request, timeout: GitalyClient.fast_timeout)
+      end
 
-        raise Gitlab::Git::CommandError, encode!(response.error) if response.error.present?
+      def set_config(entries)
+        return if entries.empty?
 
-        true
+        request = Gitaly::SetConfigRequest.new(repository: @gitaly_repo)
+        entries.each do |key, value|
+          request.entries << build_set_config_entry(key, value)
+        end
+
+        GitalyClient.call(
+          @storage,
+          :repository_service,
+          :set_config,
+          request,
+          timeout: GitalyClient.fast_timeout
+        )
+
+        nil
       end
 
-      def write_config(full_path:)
-        request = Gitaly::WriteConfigRequest.new(repository: @gitaly_repo, full_path: full_path)
-        response = GitalyClient.call(
+      def delete_config(keys)
+        return if keys.empty?
+
+        request = Gitaly::DeleteConfigRequest.new(repository: @gitaly_repo, keys: keys)
+
+        GitalyClient.call(
           @storage,
           :repository_service,
-          :write_config,
+          :delete_config,
           request,
           timeout: GitalyClient.fast_timeout
         )
 
-        raise Gitlab::Git::OSError.new(response.error) unless response.error.empty?
+        nil
       end
 
       def license_short_name
@@ -290,7 +307,7 @@ module Gitlab
 
       def calculate_checksum
         request  = Gitaly::CalculateChecksumRequest.new(repository: @gitaly_repo)
-        response = GitalyClient.call(@storage, :repository_service, :calculate_checksum, request)
+        response = GitalyClient.call(@storage, :repository_service, :calculate_checksum, request, timeout: GitalyClient.fast_timeout)
         response.checksum.presence
       rescue GRPC::DataLoss => e
         raise Gitlab::Git::Repository::InvalidRepository.new(e)
@@ -299,18 +316,78 @@ module Gitlab
       def raw_changes_between(from, to)
         request = Gitaly::GetRawChangesRequest.new(repository: @gitaly_repo, from_revision: from, to_revision: to)
 
-        GitalyClient.call(@storage, :repository_service, :get_raw_changes, request)
+        GitalyClient.call(@storage, :repository_service, :get_raw_changes, request, timeout: GitalyClient.fast_timeout)
       end
 
       def search_files_by_name(ref, query)
         request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: query)
-        GitalyClient.call(@storage, :repository_service, :search_files_by_name, request).flat_map(&:files)
+        GitalyClient.call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files)
       end
 
       def search_files_by_content(ref, query)
         request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query)
         GitalyClient.call(@storage, :repository_service, :search_files_by_content, request).flat_map(&:matches)
       end
+
+      private
+
+      def gitaly_fetch_stream_to_file(save_path, rpc_name, request_class, timeout)
+        request = request_class.new(repository: @gitaly_repo)
+        response = GitalyClient.call(
+          @storage,
+          :repository_service,
+          rpc_name,
+          request,
+          timeout: timeout
+        )
+
+        File.open(save_path, 'wb') do |f|
+          response.each do |message|
+            f.write(message.data)
+          end
+        end
+        # If the file is empty means that we received an empty stream, we delete the file
+        FileUtils.rm(save_path) if File.zero?(save_path)
+      end
+
+      def gitaly_repo_stream_request(file_path, rpc_name, request_class, timeout)
+        request = request_class.new(repository: @gitaly_repo)
+        enum = Enumerator.new do |y|
+          File.open(file_path, 'rb') do |f|
+            while data = f.read(MAX_MSG_SIZE)
+              request.data = data
+
+              y.yield request
+              request = request_class.new
+            end
+          end
+        end
+
+        GitalyClient.call(
+          @storage,
+          :repository_service,
+          rpc_name,
+          enum,
+          timeout: timeout
+        )
+      end
+
+      def build_set_config_entry(key, value)
+        entry = Gitaly::SetConfigRequest::Entry.new(key: key)
+
+        case value
+        when String
+          entry.value_str = value
+        when Integer
+          entry.value_int32 = value
+        when TrueClass, FalseClass
+          entry.value_bool = value
+        else
+          raise InvalidArgument, "invalid git config value: #{value.inspect}"
+        end
+
+        entry
+      end
     end
   end
 end
diff --git a/lib/gitlab/gitaly_client/server_service.rb b/lib/gitlab/gitaly_client/server_service.rb
index 2e1076d1f66..0ade6942db9 100644
--- a/lib/gitlab/gitaly_client/server_service.rb
+++ b/lib/gitlab/gitaly_client/server_service.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     # Meant for extraction of server data, and later maybe to perform misc task
@@ -9,7 +11,7 @@ module Gitlab
       end
 
       def info
-        GitalyClient.call(@storage, :server_service, :server_info, Gitaly::ServerInfoRequest.new)
+        GitalyClient.call(@storage, :server_service, :server_info, Gitaly::ServerInfoRequest.new, timeout: GitalyClient.fast_timeout)
       end
     end
   end
diff --git a/lib/gitlab/gitaly_client/storage_service.rb b/lib/gitlab/gitaly_client/storage_service.rb
index eb0e910665b..4edcb0b8ba9 100644
--- a/lib/gitlab/gitaly_client/storage_service.rb
+++ b/lib/gitlab/gitaly_client/storage_service.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     class StorageService
@@ -5,6 +7,14 @@ module Gitlab
         @storage = storage
       end
 
+      # Returns all directories in the git storage directory, lexically ordered
+      def list_directories(depth: 1)
+        request = Gitaly::ListDirectoriesRequest.new(storage_name: @storage, depth: depth)
+
+        GitalyClient.call(@storage, :storage_service, :list_directories, request)
+          .flat_map(&:paths)
+      end
+
       # Delete all repositories in the storage. This is a slow and VERY DESTRUCTIVE operation.
       def delete_all_repositories
         request = Gitaly::DeleteAllRepositoriesRequest.new(storage_name: @storage)
diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb
index 02fcb413abd..754cccb6b3f 100644
--- a/lib/gitlab/gitaly_client/storage_settings.rb
+++ b/lib/gitlab/gitaly_client/storage_settings.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     # This is a chokepoint that is meant to help us stop remove all places
@@ -13,7 +15,7 @@ module Gitlab
         Storage is invalid because it has no `path` key.
 
         For source installations, update your config/gitlab.yml Refer to gitlab.yml.example for an updated example.
-        If you're using the Gitlab Development Kit, you can update your configuration running `gdk reconfigure`.
+        If you're using the GitLab Development Kit, you can update your configuration running `gdk reconfigure`.
       MSG
 
       # This class will give easily recognizable NoMethodErrors
@@ -60,8 +62,8 @@ module Gitlab
 
       private
 
-      def method_missing(m, *args, &block)
-        @hash.public_send(m, *args, &block) # rubocop:disable GitlabSecurity/PublicSend
+      def method_missing(msg, *args, &block)
+        @hash.public_send(msg, *args, &block) # rubocop:disable GitlabSecurity/PublicSend
       end
     end
   end
diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb
index 9c19c51d412..dce5d6a8ad0 100644
--- a/lib/gitlab/gitaly_client/util.rb
+++ b/lib/gitlab/gitaly_client/util.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     module Util
diff --git a/lib/gitlab/gitaly_client/wiki_file.rb b/lib/gitlab/gitaly_client/wiki_file.rb
index 47c60c92484..ef2b23732d1 100644
--- a/lib/gitlab/gitaly_client/wiki_file.rb
+++ b/lib/gitlab/gitaly_client/wiki_file.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     class WikiFile
diff --git a/lib/gitlab/gitaly_client/wiki_page.rb b/lib/gitlab/gitaly_client/wiki_page.rb
index a02d15db5dd..757a429fb8a 100644
--- a/lib/gitlab/gitaly_client/wiki_page.rb
+++ b/lib/gitlab/gitaly_client/wiki_page.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitalyClient
     class WikiPage
diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb
index 2dfe055a496..2b3d622af4d 100644
--- a/lib/gitlab/gitaly_client/wiki_service.rb
+++ b/lib/gitlab/gitaly_client/wiki_service.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'stringio'
 
 module Gitlab
@@ -69,7 +71,7 @@ module Gitlab
           commit_details: gitaly_commit_details(commit_details)
         )
 
-        GitalyClient.call(@repository.storage, :wiki_service, :wiki_delete_page, request)
+        GitalyClient.call(@repository.storage, :wiki_service, :wiki_delete_page, request, timeout: GitalyClient.medium_timeout)
       end
 
       def find_page(title:, version: nil, dir: nil)
@@ -80,14 +82,14 @@ module Gitlab
           directory: encode_binary(dir)
         )
 
-        response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_page, request)
+        response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_page, request, timeout: GitalyClient.fast_timeout)
 
         wiki_page_from_iterator(response)
       end
 
-      def get_all_pages
-        request = Gitaly::WikiGetAllPagesRequest.new(repository: @gitaly_repo)
-        response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_all_pages, request)
+      def get_all_pages(limit: 0)
+        request = Gitaly::WikiGetAllPagesRequest.new(repository: @gitaly_repo, limit: limit)
+        response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_all_pages, request, timeout: GitalyClient.medium_timeout)
         pages = []
 
         loop do
@@ -110,10 +112,10 @@ module Gitlab
           repository: @gitaly_repo,
           page_path: encode_binary(page_path),
           page: options[:page] || 1,
-          per_page: options[:per_page] || Gollum::Page.per_page
+          per_page: options[:per_page] || Gitlab::Git::Wiki::DEFAULT_PAGINATION
         )
 
-        stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_page_versions, request)
+        stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_page_versions, request, timeout: GitalyClient.medium_timeout)
 
         versions = []
         stream.each do |message|
@@ -132,14 +134,14 @@ module Gitlab
           revision: encode_binary(revision)
         )
 
-        response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_file, request)
+        response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_file, request, timeout: GitalyClient.fast_timeout)
         wiki_file = nil
 
         response.each do |message|
           next unless message.name.present? || wiki_file
 
           if wiki_file
-            wiki_file.raw_data << message.raw_data
+            wiki_file.raw_data = "#{wiki_file.raw_data}#{message.raw_data}"
           else
             wiki_file = GitalyClient::WikiFile.new(message.to_h)
             # All gRPC strings in a response are frozen, so we get
@@ -160,7 +162,7 @@ module Gitlab
         )
 
         response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_formatted_data, request)
-        response.reduce("") { |memo, msg| memo << msg.data }
+        response.reduce([]) { |memo, msg| memo << msg.data }.join
       end
 
       private
diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb
index 65b5e30c70f..14a6d6443ec 100644
--- a/lib/gitlab/github_import.rb
+++ b/lib/gitlab/github_import.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GithubImport
     def self.refmap
@@ -10,24 +12,6 @@ module Gitlab
       Client.new(token_to_use, parallel: parallel)
     end
 
-    # Inserts a raw row and returns the ID of the inserted row.
-    #
-    # attributes - The attributes/columns to set.
-    # relation - An ActiveRecord::Relation to use for finding the ID of the row
-    #            when using MySQL.
-    def self.insert_and_return_id(attributes, relation)
-      # We use bulk_insert here so we can bypass any queries executed by
-      # callbacks or validation rules, as doing this wouldn't scale when
-      # importing very large projects.
-      result = Gitlab::Database
-        .bulk_insert(relation.table_name, [attributes], return_ids: true)
-
-      # MySQL doesn't support returning the IDs of a bulk insert in a way that
-      # is not a pain, so in this case we'll issue an extra query instead.
-      result.first ||
-        relation.where(iid: attributes[:iid]).limit(1).pluck(:id).first
-    end
-
     # Returns the ID of the ghost user.
     def self.ghost_user_id
       key = 'github-import/ghost-user-id'
diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb
index 8274f37d358..d562958e955 100644
--- a/lib/gitlab/github_import/importer/diff_note_importer.rb
+++ b/lib/gitlab/github_import/importer/diff_note_importer.rb
@@ -13,7 +13,7 @@ module Gitlab
           @note = note
           @project = project
           @client = client
-          @user_finder = UserFinder.new(project, client)
+          @user_finder = GithubImport::UserFinder.new(project, client)
         end
 
         def execute
diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb
index 31fefebf787..656d46b6a7d 100644
--- a/lib/gitlab/github_import/importer/issue_importer.rb
+++ b/lib/gitlab/github_import/importer/issue_importer.rb
@@ -4,6 +4,8 @@ module Gitlab
   module GithubImport
     module Importer
       class IssueImporter
+        include Gitlab::Import::DatabaseHelpers
+
         attr_reader :project, :issue, :client, :user_finder, :milestone_finder,
                     :issuable_finder
 
@@ -19,7 +21,7 @@ module Gitlab
           @issue = issue
           @project = project
           @client = client
-          @user_finder = UserFinder.new(project, client)
+          @user_finder = GithubImport::UserFinder.new(project, client)
           @milestone_finder = MilestoneFinder.new(project)
           @issuable_finder = GithubImport::IssuableFinder.new(project, issue)
         end
@@ -55,7 +57,7 @@ module Gitlab
             updated_at: issue.updated_at
           }
 
-          GithubImport.insert_and_return_id(attributes, project.issues)
+          insert_and_return_id(attributes, project.issues)
         rescue ActiveRecord::InvalidForeignKey
           # It's possible the project has been deleted since scheduling this
           # job. In this case we'll just skip creating the issue.
diff --git a/lib/gitlab/github_import/importer/labels_importer.rb b/lib/gitlab/github_import/importer/labels_importer.rb
index a73033d35ba..80246fa1b77 100644
--- a/lib/gitlab/github_import/importer/labels_importer.rb
+++ b/lib/gitlab/github_import/importer/labels_importer.rb
@@ -10,11 +10,13 @@ module Gitlab
 
         # project - An instance of `Project`.
         # client - An instance of `Gitlab::GithubImport::Client`.
+        # rubocop: disable CodeReuse/ActiveRecord
         def initialize(project, client)
           @project = project
           @client = client
           @existing_labels = project.labels.pluck(:title).to_set
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         def execute
           bulk_insert(Label, build_labels)
diff --git a/lib/gitlab/github_import/importer/lfs_object_importer.rb b/lib/gitlab/github_import/importer/lfs_object_importer.rb
new file mode 100644
index 00000000000..195383fd3e9
--- /dev/null
+++ b/lib/gitlab/github_import/importer/lfs_object_importer.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GithubImport
+    module Importer
+      class LfsObjectImporter
+        attr_reader :lfs_object, :project
+
+        # lfs_object - An instance of `Gitlab::GithubImport::Representation::LfsObject`.
+        # project - An instance of `Project`.
+        def initialize(lfs_object, project, _)
+          @lfs_object = lfs_object
+          @project = project
+        end
+
+        def lfs_download_object
+          LfsDownloadObject.new(oid: lfs_object.oid, size: lfs_object.size, link: lfs_object.link)
+        end
+
+        def execute
+          Projects::LfsPointers::LfsDownloadService.new(project, lfs_download_object).execute
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb
new file mode 100644
index 00000000000..6046e30d4ef
--- /dev/null
+++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GithubImport
+    module Importer
+      class LfsObjectsImporter
+        include ParallelScheduling
+
+        def importer_class
+          LfsObjectImporter
+        end
+
+        def representation_class
+          Representation::LfsObject
+        end
+
+        def sidekiq_worker_class
+          ImportLfsObjectWorker
+        end
+
+        def collection_method
+          :lfs_objects
+        end
+
+        def each_object_to_import
+          lfs_objects = Projects::LfsPointers::LfsImportService.new(project).execute
+
+          lfs_objects.each do |object|
+            yield object
+          end
+        rescue StandardError => e
+          Rails.logger.error("The Lfs import process failed. #{e.message}")
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb
index c53480e828a..87cf2c8b598 100644
--- a/lib/gitlab/github_import/importer/milestones_importer.rb
+++ b/lib/gitlab/github_import/importer/milestones_importer.rb
@@ -10,11 +10,13 @@ module Gitlab
 
         # project - An instance of `Project`
         # client - An instance of `Gitlab::GithubImport::Client`
+        # rubocop: disable CodeReuse/ActiveRecord
         def initialize(project, client)
           @project = project
           @client = client
           @existing_milestones = project.milestones.pluck(:iid).to_set
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         def execute
           bulk_insert(Milestone, build_milestones)
diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb
index c890f2df360..2b06d1b3baf 100644
--- a/lib/gitlab/github_import/importer/note_importer.rb
+++ b/lib/gitlab/github_import/importer/note_importer.rb
@@ -13,7 +13,7 @@ module Gitlab
           @note = note
           @project = project
           @client = client
-          @user_finder = UserFinder.new(project, client)
+          @user_finder = GithubImport::UserFinder.new(project, client)
         end
 
         def execute
diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb
index 49d859f9624..ae7c4cf1b38 100644
--- a/lib/gitlab/github_import/importer/pull_request_importer.rb
+++ b/lib/gitlab/github_import/importer/pull_request_importer.rb
@@ -4,6 +4,8 @@ module Gitlab
   module GithubImport
     module Importer
       class PullRequestImporter
+        include Gitlab::Import::MergeRequestHelpers
+
         attr_reader :pull_request, :project, :client, :user_finder,
                     :milestone_finder, :issuable_finder
 
@@ -15,75 +17,56 @@ module Gitlab
           @pull_request = pull_request
           @project = project
           @client = client
-          @user_finder = UserFinder.new(project, client)
+          @user_finder = GithubImport::UserFinder.new(project, client)
           @milestone_finder = MilestoneFinder.new(project)
           @issuable_finder =
             GithubImport::IssuableFinder.new(project, pull_request)
         end
 
         def execute
-          if (mr_id = create_merge_request)
-            issuable_finder.cache_database_id(mr_id)
+          mr, already_exists = create_merge_request
+
+          if mr
+            insert_git_data(mr, already_exists)
+            issuable_finder.cache_database_id(mr.id)
           end
         end
 
         # Creates the merge request and returns its ID.
         #
         # This method will return `nil` if the merge request could not be
-        # created.
+        # created, otherwise it will return an Array containing the following
+        # values:
+        #
+        # 1. A MergeRequest instance.
+        # 2. A boolean indicating if the MR already exists.
         def create_merge_request
           author_id, author_found = user_finder.author_id_for(pull_request)
 
           description = MarkdownText
             .format(pull_request.description, pull_request.author, author_found)
 
-          # This work must be wrapped in a transaction as otherwise we can leave
-          # behind incomplete data in the event of an error. This can then lead
-          # to duplicate key errors when jobs are retried.
-          MergeRequest.transaction do
-            attributes = {
-              iid: pull_request.iid,
-              title: pull_request.truncated_title,
-              description: description,
-              source_project_id: project.id,
-              target_project_id: project.id,
-              source_branch: pull_request.formatted_source_branch,
-              target_branch: pull_request.target_branch,
-              state: pull_request.state,
-              milestone_id: milestone_finder.id_for(pull_request),
-              author_id: author_id,
-              assignee_id: user_finder.assignee_id_for(pull_request),
-              created_at: pull_request.created_at,
-              updated_at: pull_request.updated_at
-            }
-
-            # When creating merge requests there are a lot of hooks that may
-            # run, for many different reasons. Many of these hooks (e.g. the
-            # ones used for rendering Markdown) are completely unnecessary and
-            # may even lead to transaction timeouts.
-            #
-            # To ensure importing pull requests has a minimal impact and can
-            # complete in a reasonable time we bypass all the hooks by inserting
-            # the row and then retrieving it. We then only perform the
-            # additional work that is strictly necessary.
-            merge_request_id = GithubImport
-              .insert_and_return_id(attributes, project.merge_requests)
+          attributes = {
+            iid: pull_request.iid,
+            title: pull_request.truncated_title,
+            description: description,
+            source_project_id: project.id,
+            target_project_id: project.id,
+            source_branch: pull_request.formatted_source_branch,
+            target_branch: pull_request.target_branch,
+            state: pull_request.state,
+            milestone_id: milestone_finder.id_for(pull_request),
+            author_id: author_id,
+            assignee_id: user_finder.assignee_id_for(pull_request),
+            created_at: pull_request.created_at,
+            updated_at: pull_request.updated_at
+          }
 
-            merge_request = project.merge_requests.find(merge_request_id)
-
-            # These fields are set so we can create the correct merge request
-            # diffs.
-            merge_request.source_branch_sha = pull_request.source_branch_sha
-            merge_request.target_branch_sha = pull_request.target_branch_sha
-
-            merge_request.keep_around_commit
-            merge_request.merge_request_diffs.create
+          create_merge_request_without_hooks(project, attributes, pull_request.iid)
+        end
 
-            merge_request.id
-          end
-        rescue ActiveRecord::InvalidForeignKey
-          # It's possible the project has been deleted since scheduling this
-          # job. In this case we'll just skip creating the merge request.
+        def insert_git_data(merge_request, already_exists)
+          insert_or_replace_git_data(merge_request, pull_request.source_branch_sha, pull_request.target_branch_sha, already_exists)
         end
       end
     end
diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb
index e70361c163b..a52866c4b08 100644
--- a/lib/gitlab/github_import/importer/pull_requests_importer.rb
+++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb
@@ -43,7 +43,7 @@ module Gitlab
           Rails.logger
             .info("GitHub importer finished updating repository for #{pname}")
 
-          repository_updates_counter.increment(project: pname)
+          repository_updates_counter.increment
         end
 
         def update_repository?(pr)
diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb
index 100f459fdcc..0e7c9ee0d00 100644
--- a/lib/gitlab/github_import/importer/releases_importer.rb
+++ b/lib/gitlab/github_import/importer/releases_importer.rb
@@ -10,11 +10,13 @@ module Gitlab
 
         # project - An instance of `Project`
         # client - An instance of `Gitlab::GithubImport::Client`
+        # rubocop: disable CodeReuse/ActiveRecord
         def initialize(project, client)
           @project = project
           @client = client
           @existing_tags = project.releases.pluck(:tag).to_set
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         def execute
           bulk_insert(Release, build_releases)
diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb
index 01168abde6c..bc3ea9e9226 100644
--- a/lib/gitlab/github_import/importer/repository_importer.rb
+++ b/lib/gitlab/github_import/importer/repository_importer.rb
@@ -14,11 +14,13 @@ module Gitlab
         end
 
         # Returns true if we should import the wiki for the project.
+        # rubocop: disable CodeReuse/ActiveRecord
         def import_wiki?
           client.repository(project.import_source)&.has_wiki &&
             !project.wiki_repository_exists? &&
             Gitlab::GitalyClient::RemoteService.exists?(wiki_url)
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         # Imports the repository data.
         #
@@ -78,7 +80,7 @@ module Gitlab
         end
 
         def fail_import(message)
-          project.mark_import_as_failed(message)
+          project.import_state.mark_as_failed(message)
           false
         end
       end
diff --git a/lib/gitlab/github_import/label_finder.rb b/lib/gitlab/github_import/label_finder.rb
index 9be071141db..d2479a8f565 100644
--- a/lib/gitlab/github_import/label_finder.rb
+++ b/lib/gitlab/github_import/label_finder.rb
@@ -18,6 +18,7 @@ module Gitlab
         Caching.read_integer(cache_key_for(name))
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def build_cache
         mapping = @project
           .labels
@@ -28,6 +29,7 @@ module Gitlab
 
         Caching.write_multiple(mapping)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def cache_key_for(name)
         CACHE_KEY % { project: project.id, name: name }
diff --git a/lib/gitlab/github_import/milestone_finder.rb b/lib/gitlab/github_import/milestone_finder.rb
index 208d15dc144..5625730e796 100644
--- a/lib/gitlab/github_import/milestone_finder.rb
+++ b/lib/gitlab/github_import/milestone_finder.rb
@@ -21,6 +21,7 @@ module Gitlab
         Caching.read_integer(cache_key_for(issuable.milestone_number))
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def build_cache
         mapping = @project
           .milestones
@@ -31,6 +32,7 @@ module Gitlab
 
         Caching.write_multiple(mapping)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def cache_key_for(iid)
         CACHE_KEY % { project: project.id, iid: iid }
diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb
index b02b123c98e..9d81441d96e 100644
--- a/lib/gitlab/github_import/parallel_importer.rb
+++ b/lib/gitlab/github_import/parallel_importer.rb
@@ -15,6 +15,15 @@ module Gitlab
         true
       end
 
+      # This is a workaround for a Ruby 2.3.7 bug. rspec-mocks cannot restore
+      # the visibility of prepended modules. See
+      # https://github.com/rspec/rspec-mocks/issues/1231 for more details.
+      if Rails.env.test?
+        def self.requires_ci_cd_setup?
+          raise NotImplementedError
+        end
+      end
+
       def initialize(project)
         @project = project
       end
@@ -32,8 +41,7 @@ module Gitlab
         Gitlab::SidekiqStatus
           .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION)
 
-        project.ensure_import_state
-        project.import_state&.update_column(:jid, jid)
+        project.import_state.update_column(:jid, jid)
 
         Stage::ImportRepositoryWorker
           .perform_async(project.id)
diff --git a/lib/gitlab/github_import/representation/expose_attribute.rb b/lib/gitlab/github_import/representation/expose_attribute.rb
index c3405759631..d2438ee8094 100644
--- a/lib/gitlab/github_import/representation/expose_attribute.rb
+++ b/lib/gitlab/github_import/representation/expose_attribute.rb
@@ -6,7 +6,7 @@ module Gitlab
       module ExposeAttribute
         extend ActiveSupport::Concern
 
-        module ClassMethods
+        class_methods do
           # Defines getter methods for the given attribute names.
           #
           # Example:
diff --git a/lib/gitlab/github_import/representation/lfs_object.rb b/lib/gitlab/github_import/representation/lfs_object.rb
new file mode 100644
index 00000000000..a4606173f49
--- /dev/null
+++ b/lib/gitlab/github_import/representation/lfs_object.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GithubImport
+    module Representation
+      class LfsObject
+        include ToHash
+        include ExposeAttribute
+
+        attr_reader :attributes
+
+        expose_attribute :oid, :link, :size
+
+        # Builds a lfs_object
+        def self.from_api_response(lfs_object)
+          new({ oid: lfs_object.oid, link: lfs_object.link, size: lfs_object.size })
+        end
+
+        # Builds a new lfs_object using a Hash that was built from a JSON payload.
+        def self.from_json_hash(raw_hash)
+          new(Representation.symbolize_hash(raw_hash))
+        end
+
+        # attributes - A Hash containing the raw lfs_object details. The keys of this
+        #              Hash must be Symbols.
+        def initialize(attributes)
+          @attributes = attributes
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/sequential_importer.rb b/lib/gitlab/github_import/sequential_importer.rb
index 4f7324536a0..6a181caf65d 100644
--- a/lib/gitlab/github_import/sequential_importer.rb
+++ b/lib/gitlab/github_import/sequential_importer.rb
@@ -19,7 +19,8 @@ module Gitlab
         Importer::PullRequestsImporter,
         Importer::IssuesImporter,
         Importer::DiffNotesImporter,
-        Importer::NotesImporter
+        Importer::NotesImporter,
+        Importer::LfsObjectsImporter
       ].freeze
 
       # project - The project to import the data into.
@@ -41,8 +42,6 @@ module Gitlab
           klass.new(project, client, parallel: false).execute
         end
 
-        project.repository.after_import
-
         true
       end
     end
diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb
index be1259662a7..30283f147ef 100644
--- a/lib/gitlab/github_import/user_finder.rb
+++ b/lib/gitlab/github_import/user_finder.rb
@@ -136,13 +136,17 @@ module Gitlab
         Caching.write(ID_FOR_EMAIL_CACHE_KEY % email, gitlab_id)
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def query_id_for_github_id(id)
         User.for_github_id(id).pluck(:id).first
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def query_id_for_github_email(email)
         User.by_any_email(email).pluck(:id).first
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       # Reads an ID from the cache.
       #
diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb
index 22719e9a003..86474159f8b 100644
--- a/lib/gitlab/gitlab_import/client.rb
+++ b/lib/gitlab/gitlab_import/client.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitlabImport
     class Client
@@ -32,15 +34,15 @@ module Gitlab
         api.get("/api/v4/user").parsed
       end
 
-      def issues(project_identifier)
-        lazy_page_iterator(PER_PAGE) do |page|
-          api.get("/api/v4/projects/#{project_identifier}/issues?per_page=#{PER_PAGE}&page=#{page}").parsed
+      def issues(project_identifier, **kwargs)
+        lazy_page_iterator(**kwargs) do |page, per_page|
+          api.get("/api/v4/projects/#{project_identifier}/issues?per_page=#{per_page}&page=#{page}").parsed
         end
       end
 
-      def issue_comments(project_identifier, issue_id)
-        lazy_page_iterator(PER_PAGE) do |page|
-          api.get("/api/v4/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{PER_PAGE}&page=#{page}").parsed
+      def issue_comments(project_identifier, issue_id, **kwargs)
+        lazy_page_iterator(**kwargs) do |page, per_page|
+          api.get("/api/v4/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{per_page}&page=#{page}").parsed
         end
       end
 
@@ -48,23 +50,27 @@ module Gitlab
         api.get("/api/v4/projects/#{id}").parsed
       end
 
-      def projects
-        lazy_page_iterator(PER_PAGE) do |page|
-          api.get("/api/v4/projects?per_page=#{PER_PAGE}&page=#{page}").parsed
+      def projects(**kwargs)
+        lazy_page_iterator(**kwargs) do |page, per_page|
+          api.get("/api/v4/projects?per_page=#{per_page}&page=#{page}&simple=true&membership=true").parsed
         end
       end
 
       private
 
-      def lazy_page_iterator(per_page)
+      def lazy_page_iterator(starting_page: 1, page_limit: nil, per_page: PER_PAGE)
         Enumerator.new do |y|
-          page = 1
+          page = starting_page
+          page_limit = (starting_page - 1) + page_limit if page_limit
+
           loop do
-            items = yield(page)
+            items = yield(page, per_page)
+
             items.each do |item|
               y << item
             end
-            break if items.empty? || items.size < per_page
+
+            break if items.empty? || items.size < per_page || (page_limit && page >= page_limit)
 
             page += 1
           end
diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb
index 195672f5a12..e84863deba8 100644
--- a/lib/gitlab/gitlab_import/importer.rb
+++ b/lib/gitlab/gitlab_import/importer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitlabImport
     class Importer
@@ -22,22 +24,22 @@ module Gitlab
           issues = client.issues(project_identifier)
 
           issues.each do |issue|
-            body = @formatter.author_line(issue["author"]["name"])
-            body += issue["description"]
+            body = [@formatter.author_line(issue["author"]["name"])]
+            body << issue["description"]
 
             comments = client.issue_comments(project_identifier, issue["iid"])
 
             if comments.any?
-              body += @formatter.comments_header
+              body << @formatter.comments_header
             end
 
             comments.each do |comment|
-              body += @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"])
+              body << @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"])
             end
 
             project.issues.create!(
               iid: issue["iid"],
-              description: body,
+              description: body.join,
               title: issue["title"],
               state: issue["state"],
               updated_at: issue["updated_at"],
@@ -52,10 +54,12 @@ module Gitlab
 
       private
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def gitlab_user_id(project, gitlab_id)
         user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'gitlab'", gitlab_id.to_s)
         (user && user.id) || project.creator_id
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb
index 430b8c10058..35feea17351 100644
--- a/lib/gitlab/gitlab_import/project_creator.rb
+++ b/lib/gitlab/gitlab_import/project_creator.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GitlabImport
     class ProjectCreator
diff --git a/lib/gitlab/gl_id.rb b/lib/gitlab/gl_id.rb
index a53d156b41f..1ed842c2264 100644
--- a/lib/gitlab/gl_id.rb
+++ b/lib/gitlab/gl_id.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GlId
     def self.gl_id(user)
diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb
index 07c0abcce23..435b74806e7 100644
--- a/lib/gitlab/gl_repository.rb
+++ b/lib/gitlab/gl_repository.rb
@@ -1,9 +1,12 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GlRepository
     def self.gl_repository(project, is_wiki)
       "#{is_wiki ? 'wiki' : 'project'}-#{project.id}"
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def self.parse(gl_repository)
       match_data = /\A(project|wiki)-([1-9][0-9]*)\z/.match(gl_repository)
       unless match_data
@@ -16,5 +19,6 @@ module Gitlab
 
       [project, wiki]
     end
+    # rubocop: enable CodeReuse/ActiveRecord
   end
 end
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index deaa14c8434..9b1794eec91 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # rubocop:disable Metrics/AbcSize
 
 module Gitlab
@@ -6,7 +8,7 @@ module Gitlab
 
     def add_gon_variables
       gon.api_version            = 'v4'
-      gon.default_avatar_url     = URI.join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s
+      gon.default_avatar_url     = default_avatar_url
       gon.max_file_size          = Gitlab::CurrentSettings.max_attachment_size
       gon.asset_host             = ActionController::Base.asset_host
       gon.webpack_public_path    = webpack_public_path
@@ -30,5 +32,30 @@ module Gitlab
         gon.current_user_avatar_url = current_user.avatar_url
       end
     end
+
+    # Exposes the state of a feature flag to the frontend code.
+    #
+    # name - The name of the feature flag, e.g. `my_feature`.
+    # args - Any additional arguments to pass to `Feature.enabled?`. This allows
+    #        you to check if a flag is enabled for a particular user.
+    def push_frontend_feature_flag(name, *args)
+      var_name = name.to_s.camelize(:lower)
+      enabled = Feature.enabled?(name, *args)
+
+      # Here the `true` argument signals gon that the value should be merged
+      # into any existing ones, instead of overwriting them. This allows you to
+      # use this method to push multiple feature flags.
+      gon.push({ features: { var_name => enabled } }, true)
+    end
+
+    def default_avatar_url
+      # We can't use ActionController::Base.helpers.image_url because it
+      # doesn't return an actual URL because request is nil for some reason.
+      #
+      # We also can't use Gitlab::Utils.append_path because the image path
+      # may be an absolute URL.
+      URI.join(Gitlab.config.gitlab.url,
+               ActionController::Base.helpers.image_path('no_avatar.png')).to_s
+    end
   end
 end
diff --git a/lib/gitlab/google_code_import/client.rb b/lib/gitlab/google_code_import/client.rb
index b1dbf554e41..52d714880b5 100644
--- a/lib/gitlab/google_code_import/client.rb
+++ b/lib/gitlab/google_code_import/client.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GoogleCodeImport
     class Client
diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb
index 46b49128140..1e7203cb82a 100644
--- a/lib/gitlab/google_code_import/importer.rb
+++ b/lib/gitlab/google_code_import/importer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GoogleCodeImport
     class Importer
@@ -78,6 +80,7 @@ module Gitlab
         end
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def import_issues
         return unless repo.issues
 
@@ -101,7 +104,7 @@ module Gitlab
             if username.start_with?("@")
               username = username[1..-1]
 
-              if user = User.find_by(username: username)
+              if user = UserFinder.new(username).find_by_username
                 assignee_id = user.id
               end
             end
@@ -123,6 +126,7 @@ module Gitlab
           import_issue_comments(issue, comments)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def import_issue_labels(raw_issue)
         labels = []
@@ -200,27 +204,27 @@ module Gitlab
         "Status: #{name}"
       end
 
-      def linkify_issues(s)
-        s = s.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2')
-        s = s.gsub(/([Cc]omment) #([0-9]+)/, '\1 \2')
-        s
+      def linkify_issues(str)
+        str = str.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2')
+        str = str.gsub(/([Cc]omment) #([0-9]+)/, '\1 \2')
+        str
       end
 
-      def escape_for_markdown(s)
+      def escape_for_markdown(str)
         # No headings and lists
-        s = s.gsub(/^#/, "\\#")
-        s = s.gsub(/^-/, "\\-")
+        str = str.gsub(/^#/, "\\#")
+        str = str.gsub(/^-/, "\\-")
 
         # No inline code
-        s = s.gsub("`", "\\`")
+        str = str.gsub("`", "\\`")
 
         # Carriage returns make me sad
-        s = s.delete("\r")
+        str = str.delete("\r")
 
         # Markdown ignores single newlines, but we need them as <br />.
-        s = s.gsub("\n", "  \n")
+        str = str.gsub("\n", "  \n")
 
-        s
+        str
       end
 
       def create_label(name)
diff --git a/lib/gitlab/google_code_import/project_creator.rb b/lib/gitlab/google_code_import/project_creator.rb
index 326cfcaa8af..eaef85acb98 100644
--- a/lib/gitlab/google_code_import/project_creator.rb
+++ b/lib/gitlab/google_code_import/project_creator.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GoogleCodeImport
     class ProjectCreator
diff --git a/lib/gitlab/google_code_import/repository.rb b/lib/gitlab/google_code_import/repository.rb
index ad33fc2cad2..19627c8cd35 100644
--- a/lib/gitlab/google_code_import/repository.rb
+++ b/lib/gitlab/google_code_import/repository.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GoogleCodeImport
     class Repository
diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb
index 413872d7e08..32f61b1d65c 100644
--- a/lib/gitlab/gpg.rb
+++ b/lib/gitlab/gpg.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Gpg
     extend self
@@ -54,7 +56,11 @@ module Gitlab
         fingerprints = CurrentKeyChain.fingerprints_from_key(key)
 
         GPGME::Key.find(:public, fingerprints).flat_map do |raw_key|
-          raw_key.uids.map { |uid| { name: uid.name, email: uid.email.downcase } }
+          raw_key.uids.each_with_object([]) do |uid, arr|
+            name = uid.name.force_encoding('UTF-8')
+            email = uid.email.force_encoding('UTF-8')
+            arr << { name: name, email: email.downcase } if name.valid_encoding? && email.valid_encoding?
+          end
         end
       end
     end
@@ -67,8 +73,10 @@ module Gitlab
       if MUTEX.locked? && MUTEX.owned?
         optimistic_using_tmp_keychain(&block)
       else
-        MUTEX.synchronize do
-          optimistic_using_tmp_keychain(&block)
+        ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+          MUTEX.synchronize do
+            optimistic_using_tmp_keychain(&block)
+          end
         end
       end
     end
diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb
index 6d2278d0876..4fbb87385c3 100644
--- a/lib/gitlab/gpg/commit.rb
+++ b/lib/gitlab/gpg/commit.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Gpg
     class Commit
@@ -26,6 +28,7 @@ module Gitlab
         !!(signature_text && signed_text)
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def signature
         return unless has_signature?
 
@@ -36,13 +39,13 @@ module Gitlab
 
         @signature = create_cached_signature!
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def update_signature!(cached_signature)
         using_keychain do |gpg_key|
-          cached_signature.update_attributes!(attributes(gpg_key))
+          cached_signature.update!(attributes(gpg_key))
+          @signature = cached_signature
         end
-
-        @signature = cached_signature
       end
 
       private
@@ -55,11 +58,15 @@ module Gitlab
           # the proper signature.
           # NOTE: the invoked method is #fingerprint but it's only returning
           # 16 characters (the format used by keyid) instead of 40.
-          gpg_key = find_gpg_key(verified_signature.fingerprint)
+          fingerprint = verified_signature&.fingerprint
+
+          break unless fingerprint
+
+          gpg_key = find_gpg_key(fingerprint)
 
           if gpg_key
             Gitlab::Gpg::CurrentKeyChain.add(gpg_key.key)
-            @verified_signature = nil
+            clear_memoization(:verified_signature)
           end
 
           yield gpg_key
@@ -67,9 +74,16 @@ module Gitlab
       end
 
       def verified_signature
-        @verified_signature ||= GPGME::Crypto.new.verify(signature_text, signed_text: signed_text) do |verified_signature|
+        strong_memoize(:verified_signature) { gpgme_signature }
+      end
+
+      def gpgme_signature
+        GPGME::Crypto.new.verify(signature_text, signed_text: signed_text) do |verified_signature|
+          # Return the first signature for now: https://gitlab.com/gitlab-org/gitlab-ce/issues/54932
           break verified_signature
         end
+      rescue GPGME::Error
+        nil
       end
 
       def create_cached_signature!
@@ -88,7 +102,7 @@ module Gitlab
           commit_sha: @commit.sha,
           project: @commit.project,
           gpg_key: gpg_key,
-          gpg_key_primary_keyid: gpg_key&.keyid || verified_signature.fingerprint,
+          gpg_key_primary_keyid: gpg_key&.keyid || verified_signature&.fingerprint,
           gpg_key_user_name: user_infos[:name],
           gpg_key_user_email: user_infos[:email],
           verification_status: verification_status
@@ -98,7 +112,7 @@ module Gitlab
       def verification_status(gpg_key)
         return :unknown_key unless gpg_key
         return :unverified_key unless gpg_key.verified?
-        return :unverified unless verified_signature.valid?
+        return :unverified unless verified_signature&.valid?
 
         if gpg_key.verified_and_belongs_to_email?(@commit.committer_email)
           :verified
@@ -113,9 +127,11 @@ module Gitlab
         gpg_key&.verified_user_infos&.first || gpg_key&.user_infos&.first || {}
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def find_gpg_key(keyid)
         GpgKey.find_by(primary_keyid: keyid) || GpgKeySubkey.find_by(keyid: keyid)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb
index 1991911ef6a..d892d27a917 100644
--- a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb
+++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Gpg
     class InvalidGpgSignatureUpdater
@@ -5,6 +7,7 @@ module Gitlab
         @gpg_key = gpg_key
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def run
         GpgSignature
           .select(:id, :commit_sha, :project_id)
@@ -12,6 +15,7 @@ module Gitlab
           .where(gpg_key_primary_keyid: @gpg_key.keyids)
           .find_each { |sig| sig.gpg_commit&.update_signature!(sig) }
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb
index 1e1fdabca93..9bb1e8fc7a2 100644
--- a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb
+++ b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb
@@ -1,9 +1,15 @@
+# frozen_string_literal: true
+
 module Gitlab
   module GrapeLogging
     module Formatters
       class LogrageWithTimestamp
+        include Gitlab::EncodingHelper
+
         def call(severity, datetime, _, data)
           time = data.delete :time
+          data[:params] = process_params(data)
+
           attributes = {
             time: datetime.utc.iso8601(3),
             severity: severity,
@@ -13,6 +19,27 @@ module Gitlab
           }.merge(data)
           ::Lograge.formatter.call(attributes) + "\n"
         end
+
+        private
+
+        def process_params(data)
+          return [] unless data.has_key?(:params)
+
+          data[:params]
+            .each_pair
+            .map { |k, v| { key: k, value: utf8_encode_values(v) } }
+        end
+
+        def utf8_encode_values(data)
+          case data
+          when Hash
+            data.merge(data) { |k, v| utf8_encode_values(v) }
+          when Array
+            data.map { |v| utf8_encode_values(v) }
+          when String
+            encode_utf8(data)
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb b/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb
new file mode 100644
index 00000000000..fa4c5d86d44
--- /dev/null
+++ b/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# This module adds additional correlation id the grape logger
+module Gitlab
+  module GrapeLogging
+    module Loggers
+      class CorrelationIdLogger < ::GrapeLogging::Loggers::Base
+        def parameters(_, _)
+          { Gitlab::CorrelationId::LOG_KEY => Gitlab::CorrelationId.current_id }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/grape_logging/loggers/perf_logger.rb b/lib/gitlab/grape_logging/loggers/perf_logger.rb
new file mode 100644
index 00000000000..e3b9c59bd6e
--- /dev/null
+++ b/lib/gitlab/grape_logging/loggers/perf_logger.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# This module adds additional performance metrics to the grape logger
+module Gitlab
+  module GrapeLogging
+    module Loggers
+      class PerfLogger < ::GrapeLogging::Loggers::Base
+        def parameters(_, _)
+          { gitaly_calls: Gitlab::GitalyClient.get_request_count }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb b/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb
index 0adac79f25a..705e23adff2 100644
--- a/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb
+++ b/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # This grape_logging module (https://github.com/aserafin/grape_logging) makes it
 # possible to log how much time an API request was queued by Workhorse.
 module Gitlab
diff --git a/lib/gitlab/grape_logging/loggers/route_logger.rb b/lib/gitlab/grape_logging/loggers/route_logger.rb
new file mode 100644
index 00000000000..f3146b4dfd9
--- /dev/null
+++ b/lib/gitlab/grape_logging/loggers/route_logger.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# This grape_logging module (https://github.com/aserafin/grape_logging) makes it
+# possible to log the details of the action
+module Gitlab
+  module GrapeLogging
+    module Loggers
+      class RouteLogger < ::GrapeLogging::Loggers::Base
+        def parameters(request, _)
+          endpoint = request.env[Grape::Env::API_ENDPOINT]
+          route = endpoint&.route&.pattern&.origin
+
+          return {} unless route
+
+          { route: route }
+        rescue
+          # endpoint.route calls env[Grape::Env::GRAPE_ROUTING_ARGS][:route_info]
+          # but env[Grape::Env::GRAPE_ROUTING_ARGS] is nil in the case of a 405 response
+          # so we're rescuing exceptions and bailing out
+          {}
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/grape_logging/loggers/user_logger.rb b/lib/gitlab/grape_logging/loggers/user_logger.rb
index fa172861967..6caa6c715e7 100644
--- a/lib/gitlab/grape_logging/loggers/user_logger.rb
+++ b/lib/gitlab/grape_logging/loggers/user_logger.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # This grape_logging module (https://github.com/aserafin/grape_logging) makes it
 # possible to log the user who performed the Grape API action by retrieving
 # the user context from the request environment.
diff --git a/lib/gitlab/graphql.rb b/lib/gitlab/graphql.rb
new file mode 100644
index 00000000000..8a59e83974f
--- /dev/null
+++ b/lib/gitlab/graphql.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    StandardGraphqlError = Class.new(StandardError)
+
+    def self.enabled?
+      Feature.enabled?(:graphql, default_enabled: true)
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/authorize.rb b/lib/gitlab/graphql/authorize.rb
new file mode 100644
index 00000000000..5e48bf9043d
--- /dev/null
+++ b/lib/gitlab/graphql/authorize.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    # Allow fields to declare permissions their objects must have. The field
+    # will be set to nil unless all required permissions are present.
+    module Authorize
+      extend ActiveSupport::Concern
+
+      def self.use(schema_definition)
+        schema_definition.instrument(:field, Instrumentation.new)
+      end
+
+      def required_permissions
+        # If the `#authorize` call is used on multiple classes, we add the
+        # permissions specified on a subclass, to the ones that were specified
+        # on it's superclass.
+        @required_permissions ||= if self.respond_to?(:superclass) && superclass.respond_to?(:required_permissions)
+                                    superclass.required_permissions.dup
+                                  else
+                                    []
+                                  end
+      end
+
+      def authorize(*permissions)
+        required_permissions.concat(permissions)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb
new file mode 100644
index 00000000000..a56c4f6368d
--- /dev/null
+++ b/lib/gitlab/graphql/authorize/authorize_resource.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Authorize
+      module AuthorizeResource
+        extend ActiveSupport::Concern
+
+        included do
+          extend Gitlab::Graphql::Authorize
+        end
+
+        def find_object(*args)
+          raise NotImplementedError, "Implement #find_object in #{self.class.name}"
+        end
+
+        def authorized_find(*args)
+          object = find_object(*args)
+
+          object if authorized?(object)
+        end
+
+        def authorized_find!(*args)
+          object = find_object(*args)
+          authorize!(object)
+
+          object
+        end
+
+        def authorize!(object)
+          unless authorized?(object)
+            raise Gitlab::Graphql::Errors::ResourceNotAvailable,
+                  "The resource that you are attempting to access does not exist or you don't have permission to perform this action"
+          end
+        end
+
+        def authorized?(object)
+          self.class.required_permissions.all? do |ability|
+            # The actions could be performed across multiple objects. In which
+            # case the current user is common, and we could benefit from the
+            # caching in `DeclarativePolicy`.
+            Ability.allowed?(current_user, ability, object, scope: :user)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/authorize/instrumentation.rb b/lib/gitlab/graphql/authorize/instrumentation.rb
new file mode 100644
index 00000000000..d638d2b43ee
--- /dev/null
+++ b/lib/gitlab/graphql/authorize/instrumentation.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Authorize
+      class Instrumentation
+        # Replace the resolver for the field with one that will only return the
+        # resolved object if the permissions check is successful.
+        #
+        # Collections are not supported. Apply permissions checks for those at the
+        # database level instead, to avoid loading superfluous data from the DB
+        def instrument(_type, field)
+          field_definition = field.metadata[:type_class]
+          return field unless field_definition.respond_to?(:required_permissions)
+          return field if field_definition.required_permissions.empty?
+
+          old_resolver = field.resolve_proc
+
+          new_resolver = -> (obj, args, ctx) do
+            resolved_obj = old_resolver.call(obj, args, ctx)
+            checker = build_checker(ctx[:current_user], field_definition.required_permissions)
+
+            if resolved_obj.respond_to?(:then)
+              resolved_obj.then(&checker)
+            else
+              checker.call(resolved_obj)
+            end
+          end
+
+          field.redefine do
+            resolve(new_resolver)
+          end
+        end
+
+        private
+
+        def build_checker(current_user, abilities)
+          proc do |obj|
+            # Load the elements if they weren't loaded by BatchLoader yet
+            obj = obj.sync if obj.respond_to?(:sync)
+            obj if abilities.all? { |ability| Ability.allowed?(current_user, ability, obj) }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/connections.rb b/lib/gitlab/graphql/connections.rb
new file mode 100644
index 00000000000..fbccdfa7b08
--- /dev/null
+++ b/lib/gitlab/graphql/connections.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Connections
+      def self.use(_schema)
+        GraphQL::Relay::BaseConnection.register_connection_implementation(
+          ActiveRecord::Relation,
+          Gitlab::Graphql::Connections::KeysetConnection
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/connections/keyset_connection.rb b/lib/gitlab/graphql/connections/keyset_connection.rb
new file mode 100644
index 00000000000..851054c0393
--- /dev/null
+++ b/lib/gitlab/graphql/connections/keyset_connection.rb
@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Connections
+      class KeysetConnection < GraphQL::Relay::BaseConnection
+        def cursor_from_node(node)
+          encode(node[order_field].to_s)
+        end
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def sliced_nodes
+          @sliced_nodes ||=
+            begin
+              sliced = nodes
+
+              sliced = sliced.where(before_slice) if before.present?
+              sliced = sliced.where(after_slice) if after.present?
+
+              sliced
+            end
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def paged_nodes
+          if first && last
+            raise Gitlab::Graphql::Errors::ArgumentError.new("Can only provide either `first` or `last`, not both")
+          end
+
+          if last
+            sliced_nodes.last(limit_value)
+          else
+            sliced_nodes.limit(limit_value)
+          end
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        private
+
+        def before_slice
+          if sort_direction == :asc
+            table[order_field].lt(decode(before))
+          else
+            table[order_field].gt(decode(before))
+          end
+        end
+
+        def after_slice
+          if sort_direction == :asc
+            table[order_field].gt(decode(after))
+          else
+            table[order_field].lt(decode(after))
+          end
+        end
+
+        def limit_value
+          @limit_value ||= [first, last, max_page_size].compact.min
+        end
+
+        def table
+          nodes.arel_table
+        end
+
+        def order_info
+          @order_info ||= nodes.order_values.first
+        end
+
+        def order_field
+          @order_field ||= order_info&.expr&.name || nodes.primary_key
+        end
+
+        def sort_direction
+          @order_direction ||= order_info&.direction || :desc
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/errors.rb b/lib/gitlab/graphql/errors.rb
new file mode 100644
index 00000000000..fe74549e322
--- /dev/null
+++ b/lib/gitlab/graphql/errors.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Errors
+      BaseError = Class.new(GraphQL::ExecutionError)
+      ArgumentError = Class.new(BaseError)
+      ResourceNotAvailable = Class.new(BaseError)
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/expose_permissions.rb b/lib/gitlab/graphql/expose_permissions.rb
new file mode 100644
index 00000000000..365b7cca24f
--- /dev/null
+++ b/lib/gitlab/graphql/expose_permissions.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module ExposePermissions
+      extend ActiveSupport::Concern
+      prepended do
+        def self.expose_permissions(permission_type, description: 'Permissions for the current user on the resource')
+          field :user_permissions, permission_type,
+                description: description,
+                null: false,
+                resolve: -> (obj, _, _) { obj }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/loaders/batch_model_loader.rb b/lib/gitlab/graphql/loaders/batch_model_loader.rb
new file mode 100644
index 00000000000..5a0099dc6b1
--- /dev/null
+++ b/lib/gitlab/graphql/loaders/batch_model_loader.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Loaders
+      class BatchModelLoader
+        attr_reader :model_class, :model_id
+
+        def initialize(model_class, model_id)
+          @model_class, @model_id = model_class, model_id
+        end
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def find
+          BatchLoader.for({ model: model_class, id: model_id }).batch do |loader_info, loader|
+            per_model = loader_info.group_by { |info| info[:model] }
+            per_model.each do |model, info|
+              ids = info.map { |i| i[:id] }
+              results = model.where(id: ids)
+
+              results.each { |record| loader.call({ model: model, id: record.id }, record) }
+            end
+          end
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/mount_mutation.rb b/lib/gitlab/graphql/mount_mutation.rb
new file mode 100644
index 00000000000..9048967d4e1
--- /dev/null
+++ b/lib/gitlab/graphql/mount_mutation.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module MountMutation
+      extend ActiveSupport::Concern
+
+      class_methods do
+        def mount_mutation(mutation_class)
+          # Using an underscored field name symbol will make `graphql-ruby`
+          # standardize the field name
+          field mutation_class.graphql_name.underscore.to_sym,
+                mutation: mutation_class
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/present.rb b/lib/gitlab/graphql/present.rb
new file mode 100644
index 00000000000..7f69bf601d6
--- /dev/null
+++ b/lib/gitlab/graphql/present.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Present
+      extend ActiveSupport::Concern
+      prepended do
+        def self.present_using(kls)
+          @presenter_class = kls
+        end
+
+        def self.presenter_class
+          @presenter_class
+        end
+      end
+
+      def self.use(schema_definition)
+        schema_definition.instrument(:field, Instrumentation.new)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/present/instrumentation.rb b/lib/gitlab/graphql/present/instrumentation.rb
new file mode 100644
index 00000000000..ab03c40c22d
--- /dev/null
+++ b/lib/gitlab/graphql/present/instrumentation.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Present
+      class Instrumentation
+        def instrument(type, field)
+          return field unless field.metadata[:type_class]
+
+          presented_in = field.metadata[:type_class].owner
+          return field unless presented_in.respond_to?(:presenter_class)
+          return field unless presented_in.presenter_class
+
+          old_resolver = field.resolve_proc
+
+          resolve_with_presenter = -> (presented_type, args, context) do
+            # We need to wrap the original presentation type into a type that
+            # uses the presenter as an object.
+            object = presented_type.object
+
+            if object.is_a?(presented_in.presenter_class)
+              next old_resolver.call(presented_type, args, context)
+            end
+
+            presenter = presented_in.presenter_class.new(object, **context.to_h)
+            wrapped = presented_type.class.new(presenter, context)
+
+            old_resolver.call(wrapped, args, context)
+          end
+
+          field.redefine do
+            resolve(resolve_with_presenter)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/variables.rb b/lib/gitlab/graphql/variables.rb
new file mode 100644
index 00000000000..b13ea37c21f
--- /dev/null
+++ b/lib/gitlab/graphql/variables.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    class Variables
+      Invalid = Class.new(Gitlab::Graphql::StandardGraphqlError)
+
+      def initialize(param)
+        @param = param
+      end
+
+      def to_h
+        ensure_hash(@param)
+      end
+
+      private
+
+      # Handle form data, JSON body, or a blank value
+      def ensure_hash(ambiguous_param)
+        case ambiguous_param
+        when String
+          if ambiguous_param.present?
+            ensure_hash(JSON.parse(ambiguous_param))
+          else
+            {}
+          end
+        when Hash, ActionController::Parameters
+          ambiguous_param
+        when nil
+          {}
+        else
+          raise Invalid, "Unexpected parameter: #{ambiguous_param}"
+        end
+      rescue JSON::ParserError => e
+        raise Invalid.new(e)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphs/commits.rb b/lib/gitlab/graphs/commits.rb
index 3caf9036459..66e1b2e78b4 100644
--- a/lib/gitlab/graphs/commits.rb
+++ b/lib/gitlab/graphs/commits.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Graphs
     class Commits
@@ -18,7 +20,7 @@ module Gitlab
       end
 
       def commit_per_day
-        @commit_per_day ||= @commits.size / (@duration + 1)
+        @commit_per_day ||= (@commits.size.to_f / (@duration + 1)).round(1)
       end
 
       def collect_data
diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb
new file mode 100644
index 00000000000..bf463077dcc
--- /dev/null
+++ b/lib/gitlab/hashed_storage/migrator.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module HashedStorage
+    # Hashed Storage Migrator
+    #
+    # This is responsible for scheduling and flagging projects
+    # to be migrated from Legacy to Hashed storage, either one by one or in bulk.
+    class Migrator
+      BATCH_SIZE = 100
+
+      # Schedule a range of projects to be bulk migrated with #bulk_migrate asynchronously
+      #
+      # @param [Integer] start first project id for the range
+      # @param [Integer] finish last project id for the range
+      def bulk_schedule(start:, finish:)
+        ::HashedStorage::MigratorWorker.perform_async(start, finish)
+      end
+
+      # Start migration of projects from specified range
+      #
+      # Flagging a project to be migrated is a synchronous action
+      # but the migration runs through async jobs
+      #
+      # @param [Integer] start first project id for the range
+      # @param [Integer] finish last project id for the range
+      # rubocop: disable CodeReuse/ActiveRecord
+      def bulk_migrate(start:, finish:)
+        projects = build_relation(start, finish)
+
+        projects.with_route.find_each(batch_size: BATCH_SIZE) do |project|
+          migrate(project)
+        end
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      # Flag a project to be migrated to Hashed Storage
+      #
+      # @param [Project] project that will be migrated
+      def migrate(project)
+        Rails.logger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..."
+
+        project.migrate_to_hashed_storage!
+      rescue => err
+        Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}")
+      end
+
+      def rollback(project)
+        # TODO: implement rollback strategy
+      end
+
+      private
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def build_relation(start, finish)
+        relation = Project
+        table = Project.arel_table
+
+        relation = relation.where(table[:id].gteq(start)) if start
+        relation = relation.where(table[:id].lteq(finish)) if finish
+
+        relation
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+    end
+  end
+end
diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb
index 8aba42ccfce..38f552fab03 100644
--- a/lib/gitlab/hashed_storage/rake_helper.rb
+++ b/lib/gitlab/hashed_storage/rake_helper.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module HashedStorage
     module RakeHelper
@@ -9,27 +11,45 @@ module Gitlab
         ENV.fetch('LIMIT', 500).to_i
       end
 
+      def self.range_from
+        ENV['ID_FROM']
+      end
+
+      def self.range_to
+        ENV['ID_TO']
+      end
+
+      def self.range_single_item?
+        !range_from.nil? && range_from == range_to
+      end
+
+      # rubocop: disable CodeReuse/ActiveRecord
       def self.project_id_batches(&block)
-        Project.with_unmigrated_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
+        Project.with_unmigrated_storage.in_batches(of: batch_size, start: range_from, finish: range_to) do |relation| # rubocop: disable Cop/InBatches
           ids = relation.pluck(:id)
 
           yield ids.min, ids.max
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def self.legacy_attachments_relation
         Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments])
           JOIN projects
             ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
         SQL
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def self.hashed_attachments_relation
         Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments])
           JOIN projects
           ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
         SQL
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def self.relation_summary(relation_name, relation)
         relation_count = relation.count
@@ -50,6 +70,7 @@ module Gitlab
         end
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def self.listing(relation_name, relation)
         relation_count = relation_summary(relation_name, relation)
         return unless relation_count > 0
@@ -66,6 +87,7 @@ module Gitlab
           break if index + 1 >= limit
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/gitlab/health_checks/base_abstract_check.rb b/lib/gitlab/health_checks/base_abstract_check.rb
index 8b365dab185..1d31f59999c 100644
--- a/lib/gitlab/health_checks/base_abstract_check.rb
+++ b/lib/gitlab/health_checks/base_abstract_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module HealthChecks
     module BaseAbstractCheck
diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb
index e27e16ddaf6..2bcd25cd3cc 100644
--- a/lib/gitlab/health_checks/db_check.rb
+++ b/lib/gitlab/health_checks/db_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module HealthChecks
     class DbCheck
@@ -17,7 +19,7 @@ module Gitlab
         def check
           catch_timeout 10.seconds do
             if Gitlab::Database.postgresql?
-              ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')
+              ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')&.to_s
             else
               ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.first&.to_s
             end
diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb
deleted file mode 100644
index fcbf266b80b..00000000000
--- a/lib/gitlab/health_checks/fs_shards_check.rb
+++ /dev/null
@@ -1,168 +0,0 @@
-module Gitlab
-  module HealthChecks
-    class FsShardsCheck
-      extend BaseAbstractCheck
-      RANDOM_STRING = SecureRandom.hex(1000).freeze
-      COMMAND_TIMEOUT = '1'.freeze
-      TIMEOUT_EXECUTABLE = 'timeout'.freeze
-
-      class << self
-        def readiness
-          repository_storages.map do |storage_name|
-            begin
-              if !storage_circuitbreaker_test(storage_name)
-                HealthChecks::Result.new(false, 'circuitbreaker tripped', shard: storage_name)
-              elsif !storage_stat_test(storage_name)
-                HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name)
-              else
-                with_temp_file(storage_name) do |tmp_file_path|
-                  if !storage_write_test(tmp_file_path)
-                    HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name)
-                  elsif !storage_read_test(tmp_file_path)
-                    HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name)
-                  else
-                    HealthChecks::Result.new(true, nil, shard: storage_name)
-                  end
-                end
-              end
-            rescue RuntimeError => ex
-              message = "unexpected error #{ex} when checking storage #{storage_name}"
-              Rails.logger.error(message)
-              HealthChecks::Result.new(false, message, shard: storage_name)
-            end
-          end
-        end
-
-        def metrics
-          repository_storages.flat_map do |storage_name|
-            [
-              storage_stat_metrics(storage_name),
-              storage_write_metrics(storage_name),
-              storage_read_metrics(storage_name),
-              storage_circuitbreaker_metrics(storage_name)
-            ].flatten
-          end
-        end
-
-        private
-
-        def operation_metrics(ok_metric, latency_metric, **labels)
-          result, elapsed = yield
-          [
-            metric(latency_metric, elapsed, **labels),
-            metric(ok_metric, result ? 1 : 0, **labels)
-          ]
-        rescue RuntimeError => ex
-          Rails.logger.error("unexpected error #{ex} when checking #{ok_metric}")
-          [metric(ok_metric, 0, **labels)]
-        end
-
-        def repository_storages
-          storages_paths.keys
-        end
-
-        def storages_paths
-          Gitlab.config.repositories.storages
-        end
-
-        def exec_with_timeout(cmd_args, *args, &block)
-          Gitlab::Popen.popen([TIMEOUT_EXECUTABLE, COMMAND_TIMEOUT].concat(cmd_args), *args, &block)
-        end
-
-        def with_temp_file(storage_name)
-          temp_file_path = Dir::Tmpname.create(%w(fs_shards_check +deleted), storage_path(storage_name)) { |path| path }
-          yield temp_file_path
-        ensure
-          delete_test_file(temp_file_path)
-        end
-
-        def storage_path(storage_name)
-          Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-            storages_paths[storage_name]&.legacy_disk_path
-          end
-        end
-
-        # All below test methods use shell commands to perform actions on storage volumes.
-        # In case a storage volume have connectivity problems causing pure Ruby IO operation to wait indefinitely,
-        # we can rely on shell commands to be terminated once `timeout` kills them.
-        #
-        # However we also fallback to pure Ruby file operations in case a specific shell command is missing
-        # so we are still able to perform healthchecks and gather metrics from such system.
-
-        def delete_test_file(tmp_path)
-          _, status = exec_with_timeout(%W{ rm -f #{tmp_path} })
-          status.zero?
-        rescue Errno::ENOENT
-          File.delete(tmp_path) rescue Errno::ENOENT
-        end
-
-        def storage_stat_test(storage_name)
-          stat_path = File.join(storage_path(storage_name), '.')
-          begin
-            _, status = exec_with_timeout(%W{ stat #{stat_path} })
-            status.zero?
-          rescue Errno::ENOENT
-            File.exist?(stat_path) && File::Stat.new(stat_path).readable?
-          end
-        end
-
-        def storage_write_test(tmp_path)
-          _, status = exec_with_timeout(%W{ tee #{tmp_path} }) do |stdin|
-            stdin.write(RANDOM_STRING)
-          end
-          status.zero?
-        rescue Errno::ENOENT
-          written_bytes = File.write(tmp_path, RANDOM_STRING) rescue Errno::ENOENT
-          written_bytes == RANDOM_STRING.length
-        end
-
-        def storage_read_test(tmp_path)
-          _, status = exec_with_timeout(%W{ diff #{tmp_path} - }) do |stdin|
-            stdin.write(RANDOM_STRING)
-          end
-          status.zero?
-        rescue Errno::ENOENT
-          file_contents = File.read(tmp_path) rescue Errno::ENOENT
-          file_contents == RANDOM_STRING
-        end
-
-        def storage_circuitbreaker_test(storage_name)
-          Gitlab::Git::Storage::CircuitBreaker.build(storage_name).perform { "OK" }
-        rescue Gitlab::Git::Storage::Inaccessible
-          nil
-        end
-
-        def storage_stat_metrics(storage_name)
-          operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, shard: storage_name) do
-            with_timing { storage_stat_test(storage_name) }
-          end
-        end
-
-        def storage_write_metrics(storage_name)
-          operation_metrics(:filesystem_writable, :filesystem_write_latency_seconds, shard: storage_name) do
-            with_temp_file(storage_name) do |tmp_file_path|
-              with_timing { storage_write_test(tmp_file_path) }
-            end
-          end
-        end
-
-        def storage_read_metrics(storage_name)
-          operation_metrics(:filesystem_readable, :filesystem_read_latency_seconds, shard: storage_name) do
-            with_temp_file(storage_name) do |tmp_file_path|
-              storage_write_test(tmp_file_path) # writes data used by read test
-              with_timing { storage_read_test(tmp_file_path) }
-            end
-          end
-        end
-
-        def storage_circuitbreaker_metrics(storage_name)
-          operation_metrics(:filesystem_circuitbreaker,
-                            :filesystem_circuitbreaker_latency_seconds,
-                            shard: storage_name) do
-            with_timing { storage_circuitbreaker_test(storage_name) }
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/health_checks/gitaly_check.rb b/lib/gitlab/health_checks/gitaly_check.rb
index 11416c002e3..898733fea5d 100644
--- a/lib/gitlab/health_checks/gitaly_check.rb
+++ b/lib/gitlab/health_checks/gitaly_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module HealthChecks
     class GitalyCheck
@@ -13,14 +15,14 @@ module Gitlab
         end
 
         def metrics
-          repository_storages.flat_map do |storage_name|
-            result, elapsed = with_timing { check(storage_name) }
-            labels = { shard: storage_name }
+          Gitaly::Server.all.flat_map do |server|
+            result, elapsed = with_timing { server.read_writeable? }
+            labels = { shard: server.storage }
 
             [
-              metric("#{metric_prefix}_success", successful?(result) ? 1 : 0, **labels),
+              metric("#{metric_prefix}_success", result ? 1 : 0, **labels),
               metric("#{metric_prefix}_latency_seconds", elapsed, **labels)
-            ].flatten
+            ]
           end
         end
 
@@ -36,10 +38,6 @@ module Gitlab
           METRIC_PREFIX
         end
 
-        def successful?(result)
-          result[:success]
-        end
-
         def repository_storages
           storages.keys
         end
diff --git a/lib/gitlab/health_checks/metric.rb b/lib/gitlab/health_checks/metric.rb
index d62d9136886..62a5216d159 100644
--- a/lib/gitlab/health_checks/metric.rb
+++ b/lib/gitlab/health_checks/metric.rb
@@ -1,3 +1,6 @@
-module Gitlab::HealthChecks # rubocop:disable Naming/FileName
+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+module Gitlab::HealthChecks
   Metric = Struct.new(:name, :value, :labels)
 end
diff --git a/lib/gitlab/health_checks/prometheus_text_format.rb b/lib/gitlab/health_checks/prometheus_text_format.rb
index b3c759b4730..2a8f9d31cd5 100644
--- a/lib/gitlab/health_checks/prometheus_text_format.rb
+++ b/lib/gitlab/health_checks/prometheus_text_format.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module HealthChecks
     class PrometheusTextFormat
diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb
index 0eb9b77634a..0c8fe83893b 100644
--- a/lib/gitlab/health_checks/redis/cache_check.rb
+++ b/lib/gitlab/health_checks/redis/cache_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module HealthChecks
     module Redis
@@ -19,11 +21,13 @@ module Gitlab
             result == 'PONG'
           end
 
+          # rubocop: disable CodeReuse/ActiveRecord
           def check
             catch_timeout 10.seconds do
               Gitlab::Redis::Cache.with(&:ping)
             end
           end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb
index f322fe831b8..b1e33b9f459 100644
--- a/lib/gitlab/health_checks/redis/queues_check.rb
+++ b/lib/gitlab/health_checks/redis/queues_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module HealthChecks
     module Redis
@@ -19,11 +21,13 @@ module Gitlab
             result == 'PONG'
           end
 
+          # rubocop: disable CodeReuse/ActiveRecord
           def check
             catch_timeout 10.seconds do
               Gitlab::Redis::Queues.with(&:ping)
             end
           end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb
index 8ceb0a0aa46..f7e46fce134 100644
--- a/lib/gitlab/health_checks/redis/redis_check.rb
+++ b/lib/gitlab/health_checks/redis/redis_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module HealthChecks
     module Redis
diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb
index 07e6f707998..285ac271929 100644
--- a/lib/gitlab/health_checks/redis/shared_state_check.rb
+++ b/lib/gitlab/health_checks/redis/shared_state_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module HealthChecks
     module Redis
@@ -19,11 +21,13 @@ module Gitlab
             result == 'PONG'
           end
 
+          # rubocop: disable CodeReuse/ActiveRecord
           def check
             catch_timeout 10.seconds do
               Gitlab::Redis::SharedState.with(&:ping)
             end
           end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/health_checks/result.rb b/lib/gitlab/health_checks/result.rb
index e323e2c9723..d32a6980eb8 100644
--- a/lib/gitlab/health_checks/result.rb
+++ b/lib/gitlab/health_checks/result.rb
@@ -1,3 +1,6 @@
-module Gitlab::HealthChecks # rubocop:disable Naming/FileName
+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+module Gitlab::HealthChecks
   Result = Struct.new(:success, :message, :labels)
 end
diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb
index 96945ce5b20..3588260d6eb 100644
--- a/lib/gitlab/health_checks/simple_abstract_check.rb
+++ b/lib/gitlab/health_checks/simple_abstract_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module HealthChecks
     module SimpleAbstractCheck
diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb
index 5408a1a6838..a4e60bbd828 100644
--- a/lib/gitlab/highlight.rb
+++ b/lib/gitlab/highlight.rb
@@ -1,20 +1,28 @@
+# frozen_string_literal: true
+
 module Gitlab
   class Highlight
-    def self.highlight(blob_name, blob_content, repository: nil, plain: false)
-      new(blob_name, blob_content, repository: repository)
+    TIMEOUT_BACKGROUND = 30.seconds
+    TIMEOUT_FOREGROUND = 3.seconds
+    MAXIMUM_TEXT_HIGHLIGHT_SIZE = 1.megabyte
+
+    def self.highlight(blob_name, blob_content, language: nil, plain: false)
+      new(blob_name, blob_content, language: language)
         .highlight(blob_content, continue: false, plain: plain)
     end
 
     attr_reader :blob_name
 
-    def initialize(blob_name, blob_content, repository: nil)
+    def initialize(blob_name, blob_content, language: nil)
       @formatter = Rouge::Formatters::HTMLGitlab
-      @repository = repository
+      @language = language
       @blob_name = blob_name
       @blob_content = blob_content
     end
 
     def highlight(text, continue: true, plain: false)
+      plain ||= text.length > MAXIMUM_TEXT_HIGHLIGHT_SIZE
+
       highlighted_text = highlight_text(text, continue: continue, plain: plain)
       highlighted_text = link_dependencies(text, highlighted_text) if blob_name
       highlighted_text
@@ -31,11 +39,9 @@ module Gitlab
     private
 
     def custom_language
-      language_name = @repository && @repository.gitattribute(@blob_name, 'gitlab-language')
-
-      return nil unless language_name
+      return nil unless @language
 
-      Rouge::Lexer.find_fancy(language_name)
+      Rouge::Lexer.find_fancy(@language)
     end
 
     def highlight_text(text, continue: true, plain: false)
@@ -51,11 +57,20 @@ module Gitlab
     end
 
     def highlight_rich(text, continue: true)
-      @formatter.format(lexer.lex(text, continue: continue), tag: lexer.tag).html_safe
+      tag = lexer.tag
+      tokens = lexer.lex(text, continue: continue)
+      Timeout.timeout(timeout_time) { @formatter.format(tokens, tag: tag).html_safe }
+    rescue Timeout::Error => e
+      Gitlab::Sentry.track_exception(e)
+      highlight_plain(text)
     rescue
       highlight_plain(text)
     end
 
+    def timeout_time
+      Sidekiq.server? ? TIMEOUT_BACKGROUND : TIMEOUT_FOREGROUND
+    end
+
     def link_dependencies(text, highlighted_text)
       Gitlab::DependencyLinker.link(blob_name, text, highlighted_text)
     end
diff --git a/lib/gitlab/hook_data/base_builder.rb b/lib/gitlab/hook_data/base_builder.rb
new file mode 100644
index 00000000000..d54175bce81
--- /dev/null
+++ b/lib/gitlab/hook_data/base_builder.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module HookData
+    class BaseBuilder
+      attr_accessor :object
+
+      MARKDOWN_SIMPLE_IMAGE = %r{
+          #{::Gitlab::Regex.markdown_code_or_html_blocks}
+        |
+          (?<image>
+            !
+            \[(?<title>[^\n]*?)\]
+            \((?<url>(?!(https?://|//))[^\n]+?)\)
+          )
+      }mx.freeze
+
+      def initialize(object)
+        @object = object
+      end
+
+      private
+
+      def absolute_image_urls(markdown_text)
+        return markdown_text unless markdown_text.present?
+
+        markdown_text.gsub(MARKDOWN_SIMPLE_IMAGE) do
+          if $~[:image]
+            url = $~[:url]
+            url = "#{uploads_prefix}#{url}" if url.start_with?('/uploads')
+            url = "/#{url}" unless url.start_with?('/')
+
+            "![#{$~[:title]}](#{Gitlab.config.gitlab.url}#{url})"
+          else
+            $~[0]
+          end
+        end
+      end
+
+      def uploads_prefix
+        project&.full_path || ''
+      end
+
+      def project
+        return unless object.respond_to?(:project)
+
+        object.project
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/hook_data/issuable_builder.rb b/lib/gitlab/hook_data/issuable_builder.rb
index 6ab36676127..0803df65632 100644
--- a/lib/gitlab/hook_data/issuable_builder.rb
+++ b/lib/gitlab/hook_data/issuable_builder.rb
@@ -1,13 +1,11 @@
+# frozen_string_literal: true
+
 module Gitlab
   module HookData
-    class IssuableBuilder
+    class IssuableBuilder < BaseBuilder
       CHANGES_KEYS = %i[previous current].freeze
 
-      attr_accessor :issuable
-
-      def initialize(issuable)
-        @issuable = issuable
-      end
+      alias_method :issuable, :object
 
       def build(user: nil, changes: {})
         hook_data = {
@@ -32,7 +30,7 @@ module Gitlab
       end
 
       def safe_keys
-        issuable_builder::SAFE_HOOK_ATTRIBUTES + issuable_builder::SAFE_HOOK_RELATIONS
+        issuable_builder.safe_hook_attributes + issuable_builder::SAFE_HOOK_RELATIONS
       end
 
       private
diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb
index f9b1a3caf5e..c99353b9d49 100644
--- a/lib/gitlab/hook_data/issue_builder.rb
+++ b/lib/gitlab/hook_data/issue_builder.rb
@@ -1,53 +1,54 @@
+# frozen_string_literal: true
+
 module Gitlab
   module HookData
-    class IssueBuilder
-      SAFE_HOOK_ATTRIBUTES = %i[
-        assignee_id
-        author_id
-        closed_at
-        confidential
-        created_at
-        description
-        due_date
-        id
-        iid
-        last_edited_at
-        last_edited_by_id
-        milestone_id
-        moved_to_id
-        project_id
-        relative_position
-        state
-        time_estimate
-        title
-        updated_at
-        updated_by_id
-      ].freeze
-
+    class IssueBuilder < BaseBuilder
       SAFE_HOOK_RELATIONS = %i[
         assignees
         labels
         total_time_spent
       ].freeze
 
-      attr_accessor :issue
-
-      def initialize(issue)
-        @issue = issue
+      def self.safe_hook_attributes
+        %i[
+          assignee_id
+          author_id
+          closed_at
+          confidential
+          created_at
+          description
+          due_date
+          id
+          iid
+          last_edited_at
+          last_edited_by_id
+          milestone_id
+          moved_to_id
+          project_id
+          relative_position
+          state
+          time_estimate
+          title
+          updated_at
+          updated_by_id
+        ].freeze
       end
 
+      alias_method :issue, :object
+
       def build
         attrs = {
-          url: Gitlab::UrlBuilder.build(issue),
-          total_time_spent: issue.total_time_spent,
-          human_total_time_spent: issue.human_total_time_spent,
-          human_time_estimate: issue.human_time_estimate,
-          assignee_ids: issue.assignee_ids,
-          assignee_id: issue.assignee_ids.first # This key is deprecated
+            description: absolute_image_urls(issue.description),
+            url: Gitlab::UrlBuilder.build(issue),
+            total_time_spent: issue.total_time_spent,
+            human_total_time_spent: issue.human_total_time_spent,
+            human_time_estimate: issue.human_time_estimate,
+            assignee_ids: issue.assignee_ids,
+            assignee_id: issue.assignee_ids.first # This key is deprecated
         }
 
-        issue.attributes.with_indifferent_access.slice(*SAFE_HOOK_ATTRIBUTES)
-          .merge!(attrs)
+        issue.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes)
+            .merge!(attrs)
       end
     end
   end
diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb
index aff786864f2..ad38e26e40a 100644
--- a/lib/gitlab/hook_data/merge_request_builder.rb
+++ b/lib/gitlab/hook_data/merge_request_builder.rb
@@ -1,33 +1,37 @@
+# frozen_string_literal: true
+
 module Gitlab
   module HookData
-    class MergeRequestBuilder
-      SAFE_HOOK_ATTRIBUTES = %i[
-        assignee_id
-        author_id
-        created_at
-        description
-        head_pipeline_id
-        id
-        iid
-        last_edited_at
-        last_edited_by_id
-        merge_commit_sha
-        merge_error
-        merge_params
-        merge_status
-        merge_user_id
-        merge_when_pipeline_succeeds
-        milestone_id
-        source_branch
-        source_project_id
-        state
-        target_branch
-        target_project_id
-        time_estimate
-        title
-        updated_at
-        updated_by_id
-      ].freeze
+    class MergeRequestBuilder < BaseBuilder
+      def self.safe_hook_attributes
+        %i[
+          assignee_id
+          author_id
+          created_at
+          description
+          head_pipeline_id
+          id
+          iid
+          last_edited_at
+          last_edited_by_id
+          merge_commit_sha
+          merge_error
+          merge_params
+          merge_status
+          merge_user_id
+          merge_when_pipeline_succeeds
+          milestone_id
+          source_branch
+          source_project_id
+          state
+          target_branch
+          target_project_id
+          time_estimate
+          title
+          updated_at
+          updated_by_id
+        ].freeze
+      end
 
       SAFE_HOOK_RELATIONS = %i[
         assignee
@@ -35,14 +39,11 @@ module Gitlab
         total_time_spent
       ].freeze
 
-      attr_accessor :merge_request
-
-      def initialize(merge_request)
-        @merge_request = merge_request
-      end
+      alias_method :merge_request, :object
 
       def build
         attrs = {
+          description: absolute_image_urls(merge_request.description),
           url: Gitlab::UrlBuilder.build(merge_request),
           source: merge_request.source_project.try(:hook_attrs),
           target: merge_request.target_project.hook_attrs,
@@ -53,8 +54,8 @@ module Gitlab
           human_time_estimate: merge_request.human_time_estimate
         }
 
-        merge_request.attributes.with_indifferent_access.slice(*SAFE_HOOK_ATTRIBUTES)
-          .merge!(attrs)
+        merge_request.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes)
+            .merge!(attrs)
       end
     end
   end
diff --git a/lib/gitlab/hook_data/note_builder.rb b/lib/gitlab/hook_data/note_builder.rb
new file mode 100644
index 00000000000..ae30ef6364b
--- /dev/null
+++ b/lib/gitlab/hook_data/note_builder.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module HookData
+    class NoteBuilder < BaseBuilder
+      SAFE_HOOK_ATTRIBUTES = %i[
+        attachment
+        author_id
+        change_position
+        commit_id
+        created_at
+        discussion_id
+        id
+        line_code
+        note
+        noteable_id
+        noteable_type
+        original_position
+        position
+        project_id
+        resolved_at
+        resolved_by_id
+        resolved_by_push
+        st_diff
+        system
+        type
+        updated_at
+        updated_by_id
+      ].freeze
+
+      alias_method :note, :object
+
+      def build
+        note
+          .attributes
+          .with_indifferent_access
+          .slice(*SAFE_HOOK_ATTRIBUTES)
+          .merge(
+            description: absolute_image_urls(note.note),
+            url: Gitlab::UrlBuilder.build(note)
+          )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/hook_data/wiki_page_builder.rb b/lib/gitlab/hook_data/wiki_page_builder.rb
new file mode 100644
index 00000000000..67f06b1ca46
--- /dev/null
+++ b/lib/gitlab/hook_data/wiki_page_builder.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module HookData
+    class WikiPageBuilder < BaseBuilder
+      alias_method :wiki_page, :object
+
+      def build
+        wiki_page
+          .attributes
+          .merge(
+            'content' => absolute_image_urls(wiki_page.content)
+          )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb
index 9aca3b0fb26..bcd9e2be35f 100644
--- a/lib/gitlab/http.rb
+++ b/lib/gitlab/http.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # This class is used as a proxy for all outbounding http connection
 # coming from callbacks, services and hooks. The direct use of the HTTParty
 # is discouraged because it can lead to several security problems, like SSRF
@@ -5,9 +7,16 @@
 module Gitlab
   class HTTP
     BlockedUrlError = Class.new(StandardError)
+    RedirectionTooDeep = Class.new(StandardError)
 
     include HTTParty # rubocop:disable Gitlab/HTTParty
 
     connection_adapter ProxyHTTPConnectionAdapter
+
+    def self.perform_request(http_method, path, options, &block)
+      super
+    rescue HTTParty::RedirectionTooDeep
+      raise RedirectionTooDeep
+    end
   end
 end
diff --git a/lib/gitlab/http_io.rb b/lib/gitlab/http_io.rb
new file mode 100644
index 00000000000..6a9fb85b054
--- /dev/null
+++ b/lib/gitlab/http_io.rb
@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+##
+# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html)
+# source: https://gitlab.com/snippets/1685610
+module Gitlab
+  class HttpIO
+    BUFFER_SIZE = 128.kilobytes
+
+    InvalidURLError = Class.new(StandardError)
+    FailedToGetChunkError = Class.new(StandardError)
+
+    attr_reader :uri, :size
+    attr_reader :tell
+    attr_reader :chunk, :chunk_range
+
+    alias_method :pos, :tell
+
+    def initialize(url, size)
+      raise InvalidURLError unless ::Gitlab::UrlSanitizer.valid?(url)
+
+      @uri = URI(url)
+      @size = size
+      @tell = 0
+    end
+
+    def close
+      # no-op
+    end
+
+    def binmode
+      # no-op
+    end
+
+    def binmode?
+      true
+    end
+
+    def path
+      nil
+    end
+
+    def url
+      @uri.to_s
+    end
+
+    def seek(pos, where = IO::SEEK_SET)
+      new_pos =
+        case where
+        when IO::SEEK_END
+          size + pos
+        when IO::SEEK_SET
+          pos
+        when IO::SEEK_CUR
+          tell + pos
+        else
+          -1
+        end
+
+      raise 'new position is outside of file' if new_pos < 0 || new_pos > size
+
+      @tell = new_pos
+    end
+
+    def eof?
+      tell == size
+    end
+
+    def each_line
+      until eof?
+        line = readline
+        break if line.nil?
+
+        yield(line)
+      end
+    end
+
+    def read(length = nil, outbuf = nil)
+      out = []
+
+      length ||= size - tell
+
+      until length <= 0 || eof?
+        data = get_chunk
+        break if data.empty?
+
+        chunk_bytes = [BUFFER_SIZE - chunk_offset, length].min
+        data_slice = data.byteslice(0, chunk_bytes)
+
+        out << data_slice
+        @tell += data_slice.bytesize
+        length -= data_slice.bytesize
+      end
+
+      out = out.join
+
+      # If outbuf is passed, we put the output into the buffer. This supports IO.copy_stream functionality
+      if outbuf
+        outbuf.replace(out)
+      end
+
+      out
+    end
+
+    def readline
+      out = []
+
+      until eof?
+        data = get_chunk
+        new_line = data.index("\n")
+
+        if !new_line.nil?
+          out << data[0..new_line]
+          @tell += new_line + 1
+          break
+        else
+          out << data
+          @tell += data.bytesize
+        end
+      end
+
+      out.join
+    end
+
+    def write(data)
+      raise NotImplementedError
+    end
+
+    def truncate(offset)
+      raise NotImplementedError
+    end
+
+    def flush
+      raise NotImplementedError
+    end
+
+    def present?
+      true
+    end
+
+    private
+
+    ##
+    # The below methods are not implemented in IO class
+    #
+    def in_range?
+      @chunk_range&.include?(tell)
+    end
+
+    def get_chunk
+      unless in_range?
+        response = Net::HTTP.start(uri.hostname, uri.port, proxy_from_env: true, use_ssl: uri.scheme == 'https') do |http|
+          http.request(request)
+        end
+
+        raise FailedToGetChunkError unless response.code == '200' || response.code == '206'
+
+        @chunk = response.body.force_encoding(Encoding::BINARY)
+        @chunk_range = response.content_range
+
+        ##
+        # Note: If provider does not return content_range, then we set it as we requested
+        # Provider: minio
+        # - When the file size is larger than requested Content-range, the Content-range is included in responses with Net::HTTPPartialContent 206
+        # - When the file size is smaller than requested Content-range, the Content-range is included in responses with Net::HTTPPartialContent 206
+        # Provider: AWS
+        # - When the file size is larger than requested Content-range, the Content-range is included in responses with Net::HTTPPartialContent 206
+        # - When the file size is smaller than requested Content-range, the Content-range is included in responses with Net::HTTPPartialContent 206
+        # Provider: GCS
+        # - When the file size is larger than requested Content-range, the Content-range is included in responses with Net::HTTPPartialContent 206
+        # - When the file size is smaller than requested Content-range, the Content-range is included in responses with Net::HTTPOK 200
+        @chunk_range ||= (chunk_start...(chunk_start + @chunk.bytesize))
+      end
+
+      @chunk[chunk_offset..BUFFER_SIZE]
+    end
+
+    def request
+      Net::HTTP::Get.new(uri).tap do |request|
+        request.set_range(chunk_start, BUFFER_SIZE)
+      end
+    end
+
+    def chunk_offset
+      tell % BUFFER_SIZE
+    end
+
+    def chunk_start
+      (tell / BUFFER_SIZE) * BUFFER_SIZE
+    end
+
+    def chunk_end
+      [chunk_start + BUFFER_SIZE, size].min
+    end
+  end
+end
diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb
index 3772ef11c7f..7e0398f09af 100644
--- a/lib/gitlab/i18n.rb
+++ b/lib/gitlab/i18n.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module I18n
     extend self
@@ -5,6 +7,7 @@ module Gitlab
     AVAILABLE_LANGUAGES = {
       'en' => 'English',
       'es' => 'Español',
+      'gl_ES' => 'Galego',
       'de' => 'Deutsch',
       'fr' => 'Français',
       'pt_BR' => 'Português (Brasil)',
@@ -21,7 +24,9 @@ module Gitlab
       'nl_NL' => 'Nederlands',
       'tr_TR' => 'Türkçe',
       'id_ID' => 'Bahasa Indonesia',
-      'fil_PH' => 'Filipino'
+      'fil_PH' => 'Filipino',
+      'pl_PL' => 'Polski',
+      'cs_CZ' => 'Čeština'
     }.freeze
 
     def available_locales
diff --git a/lib/gitlab/i18n/metadata_entry.rb b/lib/gitlab/i18n/metadata_entry.rb
index 35d57459a3d..3764e379681 100644
--- a/lib/gitlab/i18n/metadata_entry.rb
+++ b/lib/gitlab/i18n/metadata_entry.rb
@@ -1,18 +1,29 @@
+# frozen_string_literal: true
+
 module Gitlab
   module I18n
     class MetadataEntry
       attr_reader :entry_data
 
+      # Avoid testing too many plurals if `nplurals` was incorrectly set.
+      # Based on info on https://www.gnu.org/software/gettext/manual/html_node/Plural-forms.html
+      # which mentions special cases for numbers ending in 2 digits
+      MAX_FORMS_TO_TEST = 101
+
       def initialize(entry_data)
         @entry_data = entry_data
       end
 
-      def expected_plurals
+      def expected_forms
         return nil unless plural_information
 
         plural_information['nplurals'].to_i
       end
 
+      def forms_to_test
+        @forms_to_test ||= [expected_forms, MAX_FORMS_TO_TEST].compact.min
+      end
+
       private
 
       def plural_information
diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb
index 7d3ff8c7f58..3e9a035010f 100644
--- a/lib/gitlab/i18n/po_linter.rb
+++ b/lib/gitlab/i18n/po_linter.rb
@@ -1,6 +1,10 @@
+# frozen_string_literal: true
+
 module Gitlab
   module I18n
     class PoLinter
+      include Gitlab::Utils::StrongMemoize
+
       attr_reader :po_path, :translation_entries, :metadata_entry, :locale
 
       VARIABLE_REGEX = /%{\w*}|%[a-z]/.freeze
@@ -34,7 +38,7 @@ module Gitlab
         end
 
         @translation_entries = entries.map do |entry_data|
-          Gitlab::I18n::TranslationEntry.new(entry_data, metadata_entry.expected_plurals)
+          Gitlab::I18n::TranslationEntry.new(entry_data, metadata_entry.expected_forms)
         end
 
         nil
@@ -48,7 +52,7 @@ module Gitlab
 
         translation_entries.each do |entry|
           errors_for_entry = validate_entry(entry)
-          errors[join_message(entry.msgid)] = errors_for_entry if errors_for_entry.any?
+          errors[entry.msgid] = errors_for_entry if errors_for_entry.any?
         end
 
         errors
@@ -62,6 +66,7 @@ module Gitlab
         validate_newlines(errors, entry)
         validate_number_of_plurals(errors, entry)
         validate_unescaped_chars(errors, entry)
+        validate_translation(errors, entry)
 
         errors
       end
@@ -81,35 +86,39 @@ module Gitlab
       end
 
       def validate_number_of_plurals(errors, entry)
-        return unless metadata_entry&.expected_plurals
+        return unless metadata_entry&.expected_forms
         return unless entry.translated?
 
-        if entry.has_plural? && entry.all_translations.size != metadata_entry.expected_plurals
-          errors << "should have #{metadata_entry.expected_plurals} "\
-                    "#{'translations'.pluralize(metadata_entry.expected_plurals)}"
+        if entry.has_plural? && entry.all_translations.size != metadata_entry.expected_forms
+          errors << "should have #{metadata_entry.expected_forms} "\
+                    "#{'translations'.pluralize(metadata_entry.expected_forms)}"
         end
       end
 
       def validate_newlines(errors, entry)
-        if entry.msgid_contains_newlines?
+        if entry.msgid_has_multiple_lines?
           errors << 'is defined over multiple lines, this breaks some tooling.'
         end
 
-        if entry.plural_id_contains_newlines?
+        if entry.plural_id_has_multiple_lines?
           errors << 'plural is defined over multiple lines, this breaks some tooling.'
         end
 
-        if entry.translations_contain_newlines?
+        if entry.translations_have_multiple_lines?
           errors << 'has translations defined over multiple lines, this breaks some tooling.'
         end
       end
 
       def validate_variables(errors, entry)
         if entry.has_singular_translation?
+          validate_variables_in_message(errors, entry.msgid, entry.msgid)
+
           validate_variables_in_message(errors, entry.msgid, entry.singular_translation)
         end
 
         if entry.has_plural?
+          validate_variables_in_message(errors, entry.plural_id, entry.plural_id)
+
           entry.plural_translations.each do |translation|
             validate_variables_in_message(errors, entry.plural_id, translation)
           end
@@ -117,41 +126,98 @@ module Gitlab
       end
 
       def validate_variables_in_message(errors, message_id, message_translation)
-        message_id = join_message(message_id)
         required_variables = message_id.scan(VARIABLE_REGEX)
 
         validate_unnamed_variables(errors, required_variables)
-        validate_translation(errors, message_id, required_variables)
         validate_variable_usage(errors, message_translation, required_variables)
       end
 
-      def validate_translation(errors, message_id, used_variables)
+      def validate_translation(errors, entry)
+        Gitlab::I18n.with_locale(locale) do
+          if entry.has_plural?
+            translate_plural(entry)
+          else
+            translate_singular(entry)
+          end
+        end
+
+      # `sprintf` could raise an `ArgumentError` when invalid passing something
+      # other than a Hash when using named variables
+      #
+      # `sprintf` could raise `TypeError` when passing a wrong type when using
+      # unnamed variables
+      #
+      # FastGettext::Translation could raise `RuntimeError` (raised as a string),
+      # or as subclassess `NoTextDomainConfigured` & `InvalidFormat`
+      #
+      # `FastGettext::Translation` could raise `ArgumentError` as subclassess
+      # `InvalidEncoding`, `IllegalSequence` & `InvalidCharacter`
+      rescue ArgumentError, TypeError, RuntimeError => e
+        errors << "Failure translating to #{locale}: #{e.message}"
+      end
+
+      def translate_singular(entry)
+        used_variables = entry.msgid.scan(VARIABLE_REGEX)
         variables = fill_in_variables(used_variables)
 
-        begin
-          Gitlab::I18n.with_locale(locale) do
-            translated = if message_id.include?('|')
-                           FastGettext::Translation.s_(message_id)
-                         else
-                           FastGettext::Translation._(message_id)
-                         end
+        translation = if entry.msgid.include?('|')
+                        FastGettext::Translation.s_(entry.msgid)
+                      else
+                        FastGettext::Translation._(entry.msgid)
+                      end
+
+        translation % variables if used_variables.any?
+      end
+
+      def translate_plural(entry)
+        used_variables = entry.plural_id.scan(VARIABLE_REGEX)
+        variables = fill_in_variables(used_variables)
+
+        numbers_covering_all_plurals.map do |number|
+          translation = FastGettext::Translation.n_(entry.msgid, entry.plural_id, number)
+
+          translation % variables if used_variables.any?
+        end
+      end
+
+      def numbers_covering_all_plurals
+        @numbers_covering_all_plurals ||= calculate_numbers_covering_all_plurals
+      end
+
+      def calculate_numbers_covering_all_plurals
+        required_numbers = []
+        discovered_indexes = []
+        counter = 0
 
-            translated % variables
+        while discovered_indexes.size < metadata_entry.forms_to_test && counter < Gitlab::I18n::MetadataEntry::MAX_FORMS_TO_TEST
+          index_for_count = index_for_pluralization(counter)
+
+          unless discovered_indexes.include?(index_for_count)
+            discovered_indexes << index_for_count
+            required_numbers << counter
           end
 
-        # `sprintf` could raise an `ArgumentError` when invalid passing something
-        # other than a Hash when using named variables
-        #
-        # `sprintf` could raise `TypeError` when passing a wrong type when using
-        # unnamed variables
-        #
-        # FastGettext::Translation could raise `RuntimeError` (raised as a string),
-        # or as subclassess `NoTextDomainConfigured` & `InvalidFormat`
-        #
-        # `FastGettext::Translation` could raise `ArgumentError` as subclassess
-        # `InvalidEncoding`, `IllegalSequence` & `InvalidCharacter`
-        rescue ArgumentError, TypeError, RuntimeError => e
-          errors << "Failure translating to #{locale} with #{variables}: #{e.message}"
+          counter += 1
+        end
+
+        required_numbers
+      end
+
+      def index_for_pluralization(counter)
+        # This calls the C function that defines the pluralization rule, it can
+        # return a boolean (`false` represents 0, `true` represents 1) or an integer
+        # that specifies the plural form to be used for the given number
+        pluralization_result = Gitlab::I18n.with_locale(locale) do
+          FastGettext.pluralisation_rule.call(counter)
+        end
+
+        case pluralization_result
+        when false
+          0
+        when true
+          1
+        else
+          pluralization_result
         end
       end
 
@@ -172,16 +238,20 @@ module Gitlab
       end
 
       def validate_unnamed_variables(errors, variables)
-        if  variables.size > 1 && variables.any? { |variable_name| unnamed_variable?(variable_name) }
+        unnamed_variables, named_variables = variables.partition { |name| unnamed_variable?(name) }
+
+        if unnamed_variables.any? && named_variables.any?
+          errors << 'is combining named variables with unnamed variables'
+        end
+
+        if unnamed_variables.size > 1
           errors << 'is combining multiple unnamed variables'
         end
       end
 
       def validate_variable_usage(errors, translation, required_variables)
-        translation = join_message(translation)
-
         # We don't need to validate when the message is empty.
-        # In this case we fall back to the default, which has all the the
+        # In this case we fall back to the default, which has all the
         # required variables.
         return if translation.empty?
 
@@ -205,10 +275,6 @@ module Gitlab
       def validate_flags(errors, entry)
         errors << "is marked #{entry.flag}" if entry.flag
       end
-
-      def join_message(message)
-        Array(message).join
-      end
     end
   end
 end
diff --git a/lib/gitlab/i18n/translation_entry.rb b/lib/gitlab/i18n/translation_entry.rb
index e6c95afca7e..19c10b2e402 100644
--- a/lib/gitlab/i18n/translation_entry.rb
+++ b/lib/gitlab/i18n/translation_entry.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module I18n
     class TranslationEntry
@@ -11,11 +13,11 @@ module Gitlab
       end
 
       def msgid
-        entry_data[:msgid]
+        @msgid ||= Array(entry_data[:msgid]).join
       end
 
       def plural_id
-        entry_data[:msgid_plural]
+        @plural_id ||= Array(entry_data[:msgid_plural]).join
       end
 
       def has_plural?
@@ -23,12 +25,11 @@ module Gitlab
       end
 
       def singular_translation
-        all_translations.first if has_singular_translation?
+        all_translations.first.to_s if has_singular_translation?
       end
 
       def all_translations
-        @all_translations ||= entry_data.fetch_values(*translation_keys)
-                                .reject(&:empty?)
+        @all_translations ||= translation_entries.map { |translation| Array(translation).join }
       end
 
       def translated?
@@ -54,16 +55,16 @@ module Gitlab
         nplurals > 1 || !has_plural?
       end
 
-      def msgid_contains_newlines?
-        msgid.is_a?(Array)
+      def msgid_has_multiple_lines?
+        entry_data[:msgid].is_a?(Array)
       end
 
-      def plural_id_contains_newlines?
-        plural_id.is_a?(Array)
+      def plural_id_has_multiple_lines?
+        entry_data[:msgid_plural].is_a?(Array)
       end
 
-      def translations_contain_newlines?
-        all_translations.any? { |translation| translation.is_a?(Array) }
+      def translations_have_multiple_lines?
+        translation_entries.any? { |translation| translation.is_a?(Array) }
       end
 
       def msgid_contains_unescaped_chars?
@@ -84,6 +85,11 @@ module Gitlab
 
       private
 
+      def translation_entries
+        @translation_entries ||= entry_data.fetch_values(*translation_keys)
+                                   .reject(&:empty?)
+      end
+
       def translation_keys
         @translation_keys ||= entry_data.keys.select { |key| key.to_s =~ /\Amsgstr(\[\d+\])?\z/ }
       end
diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb
index 3f3f10596c5..d5f94ad04f1 100644
--- a/lib/gitlab/identifier.rb
+++ b/lib/gitlab/identifier.rb
@@ -1,11 +1,11 @@
+# frozen_string_literal: true
+
 # Detect user based on identifier like
-# key-13 or user-36 or last commit
+# key-13 or user-36
 module Gitlab
   module Identifier
-    def identify(identifier, project = nil, newrev = nil)
-      if identifier.blank?
-        identify_using_commit(project, newrev)
-      elsif identifier =~ /\Auser-\d+\Z/
+    def identify(identifier)
+      if identifier =~ /\Auser-\d+\Z/
         # git push over http
         identify_using_user(identifier)
       elsif identifier =~ /\Akey-\d+\Z/
@@ -14,20 +14,8 @@ module Gitlab
       end
     end
 
-    # Tries to identify a user based on a commit SHA.
-    def identify_using_commit(project, ref)
-      return if project.nil? && ref.nil?
-
-      commit = project.commit(ref)
-
-      return if !commit || !commit.author_email
-
-      identify_with_cache(:email, commit.author_email) do
-        commit.author
-      end
-    end
-
     # Tries to identify a user based on a user identifier (e.g. "user-123").
+    # rubocop: disable CodeReuse/ActiveRecord
     def identify_using_user(identifier)
       user_id = identifier.gsub("user-", "")
 
@@ -35,6 +23,7 @@ module Gitlab
         User.find_by(id: user_id)
       end
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     # Tries to identify a user based on an SSH key identifier (e.g. "key-123").
     def identify_using_ssh_key(identifier)
diff --git a/lib/gitlab/import/database_helpers.rb b/lib/gitlab/import/database_helpers.rb
new file mode 100644
index 00000000000..5b3f30d894a
--- /dev/null
+++ b/lib/gitlab/import/database_helpers.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Import
+    module DatabaseHelpers
+      # Inserts a raw row and returns the ID of the inserted row.
+      #
+      # attributes - The attributes/columns to set.
+      # relation - An ActiveRecord::Relation to use for finding the ID of the row
+      #            when using MySQL.
+      # rubocop: disable CodeReuse/ActiveRecord
+      def insert_and_return_id(attributes, relation)
+        # We use bulk_insert here so we can bypass any queries executed by
+        # callbacks or validation rules, as doing this wouldn't scale when
+        # importing very large projects.
+        result = Gitlab::Database
+                 .bulk_insert(relation.table_name, [attributes], return_ids: true)
+
+        # MySQL doesn't support returning the IDs of a bulk insert in a way that
+        # is not a pain, so in this case we'll issue an extra query instead.
+        result.first ||
+          relation.where(iid: attributes[:iid]).limit(1).pluck(:id).first
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+    end
+  end
+end
diff --git a/lib/gitlab/import/logger.rb b/lib/gitlab/import/logger.rb
new file mode 100644
index 00000000000..ab3e822a4e9
--- /dev/null
+++ b/lib/gitlab/import/logger.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Import
+    class Logger < ::Gitlab::JsonLogger
+      def self.file_name_noext
+        'importer'
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import/merge_request_creator.rb b/lib/gitlab/import/merge_request_creator.rb
new file mode 100644
index 00000000000..8291372bba9
--- /dev/null
+++ b/lib/gitlab/import/merge_request_creator.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This module is designed for importers that need to create many merge
+# requests quickly.  When creating merge requests there are a lot of hooks
+# that may run, for many different reasons. Many of these hooks (e.g. the ones
+# used for rendering Markdown) are completely unnecessary and may even lead to
+# transaction timeouts.
+#
+# To ensure importing merge requests has a minimal impact and can complete in
+# a reasonable time we bypass all the hooks by inserting the row and then
+# retrieving it. We then only perform the additional work that is strictly
+# necessary.
+module Gitlab
+  module Import
+    class MergeRequestCreator
+      include ::Gitlab::Import::DatabaseHelpers
+      include ::Gitlab::Import::MergeRequestHelpers
+
+      attr_accessor :project
+
+      def initialize(project)
+        @project = project
+      end
+
+      def execute(attributes)
+        source_branch_sha = attributes.delete(:source_branch_sha)
+        target_branch_sha = attributes.delete(:target_branch_sha)
+        iid = attributes[:iid]
+
+        merge_request, already_exists = create_merge_request_without_hooks(project, attributes, iid)
+
+        if merge_request
+          insert_or_replace_git_data(merge_request, source_branch_sha, target_branch_sha, already_exists)
+        end
+
+        merge_request
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import/merge_request_helpers.rb b/lib/gitlab/import/merge_request_helpers.rb
new file mode 100644
index 00000000000..fa3ff6c3f12
--- /dev/null
+++ b/lib/gitlab/import/merge_request_helpers.rb
@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Import
+    module MergeRequestHelpers
+      include DatabaseHelpers
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def create_merge_request_without_hooks(project, attributes, iid)
+        # This work must be wrapped in a transaction as otherwise we can leave
+        # behind incomplete data in the event of an error. This can then lead
+        # to duplicate key errors when jobs are retried.
+        MergeRequest.transaction do
+          # When creating merge requests there are a lot of hooks that may
+          # run, for many different reasons. Many of these hooks (e.g. the
+          # ones used for rendering Markdown) are completely unnecessary and
+          # may even lead to transaction timeouts.
+          #
+          # To ensure importing pull requests has a minimal impact and can
+          # complete in a reasonable time we bypass all the hooks by inserting
+          # the row and then retrieving it. We then only perform the
+          # additional work that is strictly necessary.
+          merge_request_id = insert_and_return_id(attributes, project.merge_requests)
+
+          merge_request = project.merge_requests.reload.find(merge_request_id)
+
+          [merge_request, false]
+        end
+      rescue ActiveRecord::InvalidForeignKey
+        # It's possible the project has been deleted since scheduling this
+        # job. In this case we'll just skip creating the merge request.
+        []
+      rescue ActiveRecord::RecordNotUnique
+        # It's possible we previously created the MR, but failed when updating
+        # the Git data. In this case we'll just continue working on the
+        # existing row.
+        [project.merge_requests.find_by(iid: iid), true]
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def insert_or_replace_git_data(merge_request, source_branch_sha, target_branch_sha, already_exists = false)
+        # These fields are set so we can create the correct merge request
+        # diffs.
+        merge_request.source_branch_sha = source_branch_sha
+        merge_request.target_branch_sha = target_branch_sha
+
+        merge_request.keep_around_commit
+
+        # MR diffs normally use an "after_save" hook to pull data from Git.
+        # All of this happens in the transaction started by calling
+        # create/save/etc. This in turn can lead to these transactions being
+        # held open for much longer than necessary. To work around this we
+        # first save the diff, then populate it.
+        diff =
+          if already_exists
+            merge_request.merge_request_diffs.take ||
+              merge_request.merge_request_diffs.build
+          else
+            merge_request.merge_request_diffs.build
+          end
+
+        diff.importing = true
+        diff.save
+        diff.save_git_content
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+    end
+  end
+end
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index b713fa7e1cd..f63a5ece71e 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     extend self
 
     # For every version update, the version history in import_export.md has to be kept up to date.
-    VERSION = '0.2.3'.freeze
+    VERSION = '0.2.4'.freeze
     FILENAME_LIMIT = 50
 
     def export_path(relative_path:)
diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
index aef371d81eb..d39b6fe5955 100644
--- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     module AfterExportStrategies
       class BaseAfterExportStrategy
+        extend Gitlab::ImportExport::CommandLineUtil
         include ActiveModel::Validations
         extend Forwardable
 
@@ -24,9 +27,10 @@ module Gitlab
         end
 
         def execute(current_user, project)
-          return unless project&.export_project_path
-
           @project = project
+
+          return unless @project.export_status == :finished
+
           @current_user = current_user
 
           if invalid?
@@ -51,9 +55,12 @@ module Gitlab
         end
 
         def self.lock_file_path(project)
-          return unless project&.export_path
+          return unless project.export_path || export_file_exists?
 
-          File.join(project.export_path, AFTER_EXPORT_LOCK_FILE_NAME)
+          lock_path = project.import_export_shared.archive_path
+
+          mkdir_p(lock_path)
+          File.join(lock_path, AFTER_EXPORT_LOCK_FILE_NAME)
         end
 
         protected
@@ -77,6 +84,10 @@ module Gitlab
         def log_validation_errors
           errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) }
         end
+
+        def export_file_exists?
+          project.export_file_exists?
+        end
       end
     end
   end
diff --git a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb
index 4371a7eff56..1b391314a74 100644
--- a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     module AfterExportStrategies
diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
index 938664a95a1..b30900f7c61 100644
--- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     module AfterExportStrategies
@@ -23,7 +25,7 @@ module Gitlab
         def strategy_execute
           handle_response_error(send_file)
 
-          project.remove_exported_project_file
+          project.remove_exports
         end
 
         def handle_response_error(response)
@@ -38,14 +40,16 @@ module Gitlab
         private
 
         def send_file
-          export_file = File.open(project.export_project_path)
-
-          Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options(export_file)) # rubocop:disable GitlabSecurity/PublicSend
+          Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend
         ensure
           export_file.close if export_file
         end
 
-        def send_file_options(export_file)
+        def export_file
+          project.export_file.open
+        end
+
+        def send_file_options
           {
             body_stream: export_file,
             headers: headers
@@ -53,7 +57,11 @@ module Gitlab
         end
 
         def headers
-          { 'Content-Length' => File.size(project.export_project_path).to_s }
+          { 'Content-Length' => export_size.to_s }
+        end
+
+        def export_size
+          project.export_file.file.size
         end
       end
     end
diff --git a/lib/gitlab/import_export/after_export_strategy_builder.rb b/lib/gitlab/import_export/after_export_strategy_builder.rb
index 7eabcae2380..37394f46a99 100644
--- a/lib/gitlab/import_export/after_export_strategy_builder.rb
+++ b/lib/gitlab/import_export/after_export_strategy_builder.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class AfterExportStrategyBuilder
diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb
index 7c9fc5c15bb..93b37b7bc5f 100644
--- a/lib/gitlab/import_export/attribute_cleaner.rb
+++ b/lib/gitlab/import_export/attribute_cleaner.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class AttributeCleaner
diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb
index 0c8fda07294..409243e68a5 100644
--- a/lib/gitlab/import_export/attributes_finder.rb
+++ b/lib/gitlab/import_export/attributes_finder.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class AttributesFinder
diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb
index cfa595629f4..be1b97bd7a7 100644
--- a/lib/gitlab/import_export/avatar_restorer.rb
+++ b/lib/gitlab/import_export/avatar_restorer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class AvatarRestorer
@@ -19,7 +21,7 @@ module Gitlab
       private
 
       def avatar_export_file
-        @avatar_export_file ||= Dir["#{avatar_export_path}/*"].first
+        @avatar_export_file ||= Dir["#{avatar_export_path}/**/*"].find { |f| File.file?(f) }
       end
 
       def avatar_export_path
diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb
index 998c21e2586..47ca898c690 100644
--- a/lib/gitlab/import_export/avatar_saver.rb
+++ b/lib/gitlab/import_export/avatar_saver.rb
@@ -1,8 +1,8 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class AvatarSaver
-      include Gitlab::ImportExport::CommandLineUtil
-
       def initialize(project:, shared:)
         @project = project
         @shared = shared
@@ -11,21 +11,15 @@ module Gitlab
       def save
         return true unless @project.avatar.exists?
 
-        copy_files(avatar_path, avatar_export_path)
+        Gitlab::ImportExport::UploadsManager.new(
+          project: @project,
+          shared: @shared,
+          relative_export_path: 'avatar'
+        ).save
       rescue => e
         @shared.error(e)
         false
       end
-
-      private
-
-      def avatar_export_path
-        File.join(@shared.export_path, 'avatar', @project.avatar_identifier)
-      end
-
-      def avatar_path
-        @project.avatar.path
-      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb
index 2f163db936b..bdecff0931c 100644
--- a/lib/gitlab/import_export/command_line_util.rb
+++ b/lib/gitlab/import_export/command_line_util.rb
@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     module CommandLineUtil
-      DEFAULT_MODE = 0700
+      UNTAR_MASK = 'u+rwX,go+rX,go-w'
+      DEFAULT_DIR_MODE = 0700
 
       def tar_czf(archive:, dir:)
         tar_with_options(archive: archive, dir: dir, options: 'czf')
@@ -12,18 +15,34 @@ module Gitlab
       end
 
       def mkdir_p(path)
-        FileUtils.mkdir_p(path, mode: DEFAULT_MODE)
-        FileUtils.chmod(DEFAULT_MODE, path)
+        FileUtils.mkdir_p(path, mode: DEFAULT_DIR_MODE)
+        FileUtils.chmod(DEFAULT_DIR_MODE, path)
       end
 
       private
 
+      def download_or_copy_upload(uploader, upload_path)
+        if uploader.upload.local?
+          copy_files(uploader.path, upload_path)
+        else
+          download(uploader.url, upload_path)
+        end
+      end
+
+      def download(url, upload_path)
+        File.open(upload_path, 'w') do |file|
+          # Download (stream) file from the uploader's location
+          IO.copy_stream(URI.parse(url).open, file)
+        end
+      end
+
       def tar_with_options(archive:, dir:, options:)
         execute(%W(tar -#{options} #{archive} -C #{dir} .))
       end
 
       def untar_with_options(archive:, dir:, options:)
         execute(%W(tar -#{options} #{archive} -C #{dir}))
+        execute(%W(chmod -R #{UNTAR_MASK} #{dir}))
       end
 
       def execute(cmd)
diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb
index 788eedf2686..454dc778b6b 100644
--- a/lib/gitlab/import_export/error.rb
+++ b/lib/gitlab/import_export/error.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     Error = Class.new(StandardError)
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 0f4c3498036..05432f433e7 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -1,23 +1,29 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class FileImporter
       include Gitlab::ImportExport::CommandLineUtil
 
       MAX_RETRIES = 8
+      IGNORED_FILENAMES = %w(. ..).freeze
 
       def self.import(*args)
         new(*args).import
       end
 
-      def initialize(archive_file:, shared:)
+      def initialize(project:, archive_file:, shared:)
+        @project = project
         @archive_file = archive_file
         @shared = shared
       end
 
       def import
         mkdir_p(@shared.export_path)
+        mkdir_p(@shared.archive_path)
 
-        remove_symlinks!
+        remove_symlinks
+        copy_archive
 
         wait_for_archived_file do
           decompress_archive
@@ -26,7 +32,8 @@ module Gitlab
         @shared.error(e)
         false
       ensure
-        remove_symlinks!
+        remove_import_file
+        remove_symlinks
       end
 
       private
@@ -50,7 +57,15 @@ module Gitlab
         result
       end
 
-      def remove_symlinks!
+      def copy_archive
+        return if @archive_file
+
+        @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
+
+        download_or_copy_upload(@project.import_export_upload.import_file, @archive_file)
+      end
+
+      def remove_symlinks
         extracted_files.each do |path|
           FileUtils.rm(path) if File.lstat(path).symlink?
         end
@@ -58,8 +73,12 @@ module Gitlab
         true
       end
 
+      def remove_import_file
+        FileUtils.rm_rf(@archive_file)
+      end
+
       def extracted_files
-        Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| f =~ %r{.*/\.{1,2}$} }
+        Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| IGNORED_FILENAMES.include?(File.basename(f)) }
       end
     end
   end
diff --git a/lib/gitlab/import_export/group_project_object_builder.rb b/lib/gitlab/import_export/group_project_object_builder.rb
new file mode 100644
index 00000000000..1c62591ed5a
--- /dev/null
+++ b/lib/gitlab/import_export/group_project_object_builder.rb
@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    # Given a class, it finds or creates a new object
+    # (initializes in the case of Label) at group or project level.
+    # If it does not exist in the group, it creates it at project level.
+    #
+    # Example:
+    #   `GroupProjectObjectBuilder.build(Label, label_attributes)`
+    #    finds or initializes a label with the given attributes.
+    #
+    # It also adds some logic around Group Labels/Milestones for edge cases.
+    class GroupProjectObjectBuilder
+      def self.build(*args)
+        Project.transaction do
+          new(*args).find
+        end
+      end
+
+      def initialize(klass, attributes)
+        @klass = klass < Label ? Label : klass
+        @attributes = attributes
+        @group = @attributes['group']
+        @project = @attributes['project']
+      end
+
+      def find
+        find_object || @klass.create(project_attributes)
+      end
+
+      private
+
+      def find_object
+        @klass.where(where_clause).first
+      end
+
+      def where_clause
+        @attributes.slice('title').map do |key, value|
+          scope_clause = table[:project_id].eq(@project.id)
+          scope_clause = scope_clause.or(table[:group_id].eq(@group.id)) if @group
+
+          table[key].eq(value).and(scope_clause)
+        end.reduce(:or)
+      end
+
+      def table
+        @table ||= @klass.arel_table
+      end
+
+      def project_attributes
+        @attributes.except('group').tap do |atts|
+          if label?
+            atts['type'] = 'ProjectLabel' # Always create project labels
+          elsif milestone?
+            if atts['group_id'] # Transform new group milestones into project ones
+              atts['iid'] = nil
+              atts.delete('group_id')
+            else
+              claim_iid
+            end
+          end
+        end
+      end
+
+      def label?
+        @klass == Label
+      end
+
+      def milestone?
+        @klass == Milestone
+      end
+
+      # If an existing group milestone used the IID
+      # claim the IID back and set the group milestone to use one available
+      # This is necessary to fix situations like the following:
+      #  - Importing into a user namespace project with exported group milestones
+      #    where the IID of the Group milestone could conflict with a project one.
+      def claim_iid
+        # The milestone has to be a group milestone, as it's the only case where
+        # we set the IID as the maximum. The rest of them are fixed.
+        milestone = @project.milestones.find_by(iid: @attributes['iid'])
+
+        return unless milestone
+
+        milestone.iid = nil
+        milestone.ensure_project_iid!
+        milestone.save!
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/hash_util.rb b/lib/gitlab/import_export/hash_util.rb
index d4adeeb3797..b6ce89a973b 100644
--- a/lib/gitlab/import_export/hash_util.rb
+++ b/lib/gitlab/import_export/hash_util.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class HashUtil
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index da3667faf7a..add7ee58da6 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -19,12 +19,17 @@ project_tree:
     - milestone:
       - events:
         - :push_event_payload
+    - resource_label_events:
+      - label:
+          :priorities
     - :issue_assignees
   - snippets:
     - :award_emoji
     - notes:
         :author
-  - :releases
+  - releases:
+    - :author
+    - :links
   - project_members:
     - :user
   - merge_requests:
@@ -45,7 +50,10 @@ project_tree:
     - milestone:
       - events:
         - :push_event_payload
-  - pipelines:
+    - resource_label_events:
+      - label:
+          :priorities
+  - ci_pipelines:
     - notes:
       - :author
       - events:
@@ -56,7 +64,6 @@ project_tree:
   - :triggers
   - :pipeline_schedules
   - :services
-  - :hooks
   - protected_branches:
     - :merge_access_levels
     - :push_access_levels
@@ -64,8 +71,10 @@ project_tree:
     - :create_access_levels
   - :project_feature
   - :custom_attributes
+  - :prometheus_metrics
   - :project_badges
   - :ci_cd_settings
+  - :error_tracking_setting
 
 # Only include the following attributes for the models specified.
 included_attributes:
@@ -85,18 +94,18 @@ excluded_attributes:
     - :path
     - :namespace_id
     - :creator_id
+    - :pool_repository_id
     - :import_url
     - :import_status
     - :avatar
     - :import_type
     - :import_source
-    - :import_error
     - :mirror
     - :runners_token
+    - :runners_token_encrypted
     - :repository_storage
     - :repository_read_only
     - :lfs_enabled
-    - :import_jid
     - :created_at
     - :updated_at
     - :id
@@ -107,6 +116,17 @@ excluded_attributes:
     - :storage_version
     - :remote_mirror_available_overridden
     - :description_html
+    - :repository_languages
+    - :bfg_object_map
+  namespaces:
+    - :runners_token
+    - :runners_token_encrypted
+  project_import_state:
+    - :last_error
+    - :jid
+  prometheus_metrics:
+    - :common
+    - :identifier
   snippets:
     - :expired_at
   merge_request_diff:
@@ -125,13 +145,28 @@ excluded_attributes:
   statuses:
     - :trace
     - :token
+    - :token_encrypted
     - :when
     - :artifacts_file
     - :artifacts_metadata
+    - :commands
   push_event_payload:
     - :event_id
   project_badges:
     - :group_id
+  resource_label_events:
+    - :reference
+    - :reference_html
+    - :epic_id
+  runners:
+    - :token
+    - :token_encrypted
+  services:
+    - :template
+  error_tracking_setting:
+    - :encrypted_token
+    - :encrypted_token_iv
+    - :enabled
 
 methods:
   labels:
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index 63cab07324a..767f1b5de0e 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class Importer
@@ -35,7 +37,8 @@ module Gitlab
       end
 
       def import_file
-        Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file,
+        Gitlab::ImportExport::FileImporter.import(project: @project,
+                                                  archive_file: @archive_file,
                                                   shared: @shared)
       end
 
@@ -91,7 +94,12 @@ module Gitlab
       end
 
       def remove_import_file
-        FileUtils.rm_rf(@archive_file)
+        upload = @project.import_export_upload
+
+        return unless upload&.import_file&.file
+
+        upload.remove_import_file!
+        upload.save!
       end
 
       def overwrite_project
diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb
index b48f63bcd7e..477499e1688 100644
--- a/lib/gitlab/import_export/json_hash_builder.rb
+++ b/lib/gitlab/import_export/json_hash_builder.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     # Generates a hash that conforms with http://apidock.com/rails/Hash/to_json
diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb
index b28c3c161b7..345c7880e30 100644
--- a/lib/gitlab/import_export/lfs_restorer.rb
+++ b/lib/gitlab/import_export/lfs_restorer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class LfsRestorer
diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb
index 29410e2331c..954f6f00078 100644
--- a/lib/gitlab/import_export/lfs_saver.rb
+++ b/lib/gitlab/import_export/lfs_saver.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class LfsSaver
diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb
index 8b8e48aac76..6be95a16513 100644
--- a/lib/gitlab/import_export/members_mapper.rb
+++ b/lib/gitlab/import_export/members_mapper.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class MembersMapper
@@ -45,9 +47,9 @@ module Gitlab
       end
 
       def ensure_default_member!
-        @project.project_members.destroy_all
+        @project.project_members.destroy_all # rubocop: disable DestroyAll
 
-        ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true)
+        ProjectMember.create!(user: @user, access_level: ProjectMember::MAINTAINER, source_id: @project.id, importing: true)
       end
 
       def add_team_member(member, existing_user = nil)
diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb
index f3d7407383c..040a70d6775 100644
--- a/lib/gitlab/import_export/merge_request_parser.rb
+++ b/lib/gitlab/import_export/merge_request_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class MergeRequestParser
@@ -25,8 +27,13 @@ module Gitlab
         @project.repository.create_branch(@merge_request.target_branch, @merge_request.target_branch_sha)
       end
 
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1295
       def fetch_ref
-        @project.repository.fetch_ref(@project.repository, source_ref: @diff_head_sha, target_ref: @merge_request.source_branch)
+        target_ref = Gitlab::Git::BRANCH_REF_PREFIX + @merge_request.source_branch
+
+        unless @project.repository.fetch_source_branch!(@project.repository, @diff_head_sha, target_ref)
+          Rails.logger.warn("Import/Export warning: Failed to create #{target_ref} for MR: #{@merge_request.iid}")
+        end
       end
 
       def branch_exists?(branch_name)
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 4eb67fbe11e..51001750a6c 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class ProjectTreeRestorer
-      # Relations which cannot have both group_id and project_id at the same time
-      RESTRICT_PROJECT_AND_GROUP = %i(milestone milestones).freeze
+      # Relations which cannot be saved at project level (and have a group assigned)
+      GROUP_MODELS = [GroupLabel, Milestone].freeze
 
       def initialize(user:, shared:, project:)
         @path = File.join(shared.export_path, 'project.json')
@@ -24,6 +26,8 @@ module Gitlab
 
         @project_members = @tree_hash.delete('project_members')
 
+        RelationRenameService.rename(@tree_hash)
+
         ActiveRecord::Base.uncached do
           ActiveRecord::Base.no_touching do
             create_relations
@@ -70,12 +74,23 @@ module Gitlab
       def save_relation_hash(relation_hash_batch, relation_key)
         relation_hash = create_relation(relation_key, relation_hash_batch)
 
+        remove_group_models(relation_hash) if relation_hash.is_a?(Array)
+
         @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash)
 
         # Restore the project again, extra query that skips holding the AR objects in memory
         @restored_project = Project.find(@project_id)
       end
 
+      # Remove project models that became group models as we found them at group level.
+      # This no longer required saving them at the root project level.
+      # For example, in the case of an existing group label that matched the title.
+      def remove_group_models(relation_hash)
+        relation_hash.reject! do |value|
+          GROUP_MODELS.include?(value.class) && value.group_id
+        end
+      end
+
       def default_relation_list
         reader.tree.reject do |model|
           model.is_a?(Hash) && model[:project_members]
@@ -83,13 +98,16 @@ module Gitlab
       end
 
       def restore_project
-        @project.update_columns(project_params)
+        Gitlab::Timeless.timeless(@project) do
+          @project.update(project_params)
+        end
+
         @project
       end
 
       def project_params
         @project_params ||= begin
-          attrs = json_params.merge(override_params)
+          attrs = json_params.merge(override_params).merge(visibility_level)
 
           # Cleaning all imported and overridden params
           Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: attrs,
@@ -109,6 +127,13 @@ module Gitlab
         end
       end
 
+      def visibility_level
+        level = override_params['visibility_level'] || json_params['visibility_level'] || @project.visibility_level
+        level = @project.group.visibility_level if @project.group && level > @project.group.visibility_level
+
+        { 'visibility_level' => level }
+      end
+
       # Given a relation hash containing one or more models and its relationships,
       # loops through each model and each object from a model type and
       # and assigns its correspondent attributes hash from +tree_hash+
@@ -122,16 +147,25 @@ module Gitlab
         return if tree_hash[relation_key].blank?
 
         tree_array = [tree_hash[relation_key]].flatten
+        null_iid_pipelines = []
 
         # Avoid keeping a possible heavy object in memory once we are done with it
-        while relation_item = tree_array.shift
+        while relation_item = (tree_array.shift || null_iid_pipelines.shift)
+          if nil_iid_pipeline?(relation_key, relation_item) && tree_array.any?
+            # Move pipelines with NULL IIDs to the end
+            # so they don't clash with existing IIDs.
+            null_iid_pipelines << relation_item
+
+            next
+          end
+
           # The transaction at this level is less speedy than one single transaction
           # But we can't have it in the upper level or GC won't get rid of the AR objects
           # after we save the batch.
           Project.transaction do
             process_sub_relation(relation, relation_item)
 
-            # For every subrelation that hangs from Project, save the associated records alltogether
+            # For every subrelation that hangs from Project, save the associated records altogether
             # This effectively batches all records per subrelation item, only keeping those in memory
             # We have to keep in mind that more batch granularity << Memory, but >> Slowness
             if save
@@ -170,7 +204,7 @@ module Gitlab
       def create_relation(relation, relation_hash_list)
         relation_array = [relation_hash_list].flatten.map do |relation_hash|
           Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym,
-                                                       relation_hash: parsed_relation_hash(relation_hash, relation.to_sym),
+                                                       relation_hash: relation_hash,
                                                        members_mapper: members_mapper,
                                                        user: @user,
                                                        project: @restored_project,
@@ -180,24 +214,16 @@ module Gitlab
         relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
       end
 
-      def parsed_relation_hash(relation_hash, relation_type)
-        if RESTRICT_PROJECT_AND_GROUP.include?(relation_type)
-          params = {}
-          params['group_id'] = restored_project.group.try(:id) if relation_hash['group_id']
-          params['project_id'] = restored_project.id if relation_hash['project_id']
-        else
-          params = { 'group_id' => restored_project.group.try(:id), 'project_id' => restored_project.id }
-        end
-
-        relation_hash.merge(params)
-      end
-
       def reader
         @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
       end
 
       def excluded_keys_for_relation(relation)
-        @reader.attributes_finder.find_excluded_keys(relation)
+        reader.attributes_finder.find_excluded_keys(relation)
+      end
+
+      def nil_iid_pipeline?(relation_key, relation_item)
+        relation_key == 'ci_pipelines' && relation_item['iid'].nil?
       end
     end
   end
diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb
index 5510c0b8b2f..2255635acdf 100644
--- a/lib/gitlab/import_export/project_tree_saver.rb
+++ b/lib/gitlab/import_export/project_tree_saver.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class ProjectTreeSaver
@@ -32,6 +34,8 @@ module Gitlab
 
         project_json['project_members'] += group_members_json
 
+        RelationRenameService.add_new_associations(project_json)
+
         project_json.to_json
       end
 
diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb
index e621c40fc7a..bc0d18e03fa 100644
--- a/lib/gitlab/import_export/reader.rb
+++ b/lib/gitlab/import_export/reader.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class Reader
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index c5cf290f191..099b488f68e 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -1,13 +1,17 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class RelationFactory
       OVERRIDES = { snippets: :project_snippets,
+                    ci_pipelines: 'Ci::Pipeline',
                     pipelines: 'Ci::Pipeline',
                     stages: 'Ci::Stage',
                     statuses: 'commit_status',
                     triggers: 'Ci::Trigger',
                     pipeline_schedules: 'Ci::PipelineSchedule',
                     builds: 'Ci::Build',
+                    runners: 'Ci::Runner',
                     hooks: 'ProjectHook',
                     merge_access_levels: 'ProtectedBranch::MergeAccessLevel',
                     push_access_levels: 'ProtectedBranch::PushAccessLevel',
@@ -19,7 +23,9 @@ module Gitlab
                     custom_attributes: 'ProjectCustomAttribute',
                     project_badges: 'Badge',
                     metrics: 'MergeRequest::Metrics',
-                    ci_cd_settings: 'ProjectCiCdSetting' }.freeze
+                    ci_cd_settings: 'ProjectCiCdSetting',
+                    error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting',
+                    links: 'Releases::Link' }.freeze
 
       USER_REFERENCES = %w[author_id assignee_id updated_by_id merged_by_id latest_closed_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id].freeze
 
@@ -31,7 +37,7 @@ module Gitlab
 
       EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature].freeze
 
-      TOKEN_RESET_MODELS = %w[Ci::Trigger Ci::Build ProjectHook].freeze
+      TOKEN_RESET_MODELS = %w[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze
 
       def self.create(*args)
         new(*args).create
@@ -47,13 +53,15 @@ module Gitlab
       end
 
       def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:, excluded_keys: [])
-        @relation_name = OVERRIDES[relation_sym] || relation_sym
+        @relation_name = self.class.overrides[relation_sym] || relation_sym
         @relation_hash = relation_hash.except('noteable_id')
         @members_mapper = members_mapper
         @user = user
         @project = project
         @imported_object_retries = 0
 
+        @relation_hash['project_id'] = @project.id
+
         # Remove excluded keys from relation_hash
         # We don't do this in the parsed_relation_hash because of the 'transformed attributes'
         # For example, MergeRequestDiffFiles exports its diff attribute as utf8_diff. Then,
@@ -74,23 +82,25 @@ module Gitlab
         generate_imported_object
       end
 
+      def self.overrides
+        OVERRIDES
+      end
+
       private
 
       def setup_models
         case @relation_name
         when :merge_request_diff_files       then setup_diff
         when :notes                          then setup_note
-        when :project_label, :project_labels then setup_label
-        when :milestone, :milestones         then setup_milestone
-        when 'Ci::Pipeline'                  then setup_pipeline
-        else
-          @relation_hash['project_id'] = @project.id
         end
 
         update_user_references
         update_project_references
+        update_group_references
         remove_duplicate_assignees
 
+        setup_pipeline if @relation_name == 'Ci::Pipeline'
+
         reset_tokens!
         remove_encrypted_attributes!
       end
@@ -141,6 +151,7 @@ module Gitlab
         if BUILD_MODELS.include?(@relation_name)
           @relation_hash.delete('trace') # old export files have trace
           @relation_hash.delete('token')
+          @relation_hash.delete('commands')
 
           imported_object
         elsif @relation_name == :merge_requests
@@ -151,39 +162,23 @@ module Gitlab
       end
 
       def update_project_references
-        project_id = @relation_hash.delete('project_id')
-
         # If source and target are the same, populate them with the new project ID.
         if @relation_hash['source_project_id']
-          @relation_hash['source_project_id'] = same_source_and_target? ? project_id : MergeRequestParser::FORKED_PROJECT_ID
+          @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID
         end
 
-        # project_id may not be part of the export, but we always need to populate it if required.
-        @relation_hash['project_id'] = project_id
-        @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id']
+        @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id']
       end
 
       def same_source_and_target?
         @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id']
       end
 
-      def setup_label
-        # If there's no group, move the label to a project label
-        if @relation_hash['type'] == 'GroupLabel' && @relation_hash['group_id']
-          @relation_hash['project_id'] = nil
-          @relation_name = :group_label
-        else
-          @relation_hash['group_id'] = nil
-          @relation_hash['type'] = 'ProjectLabel'
-        end
-      end
+      def update_group_references
+        return unless EXISTING_OBJECT_CHECK.include?(@relation_name)
+        return unless @relation_hash['group_id']
 
-      def setup_milestone
-        if @relation_hash['group_id']
-          @relation_hash['group_id'] = @project.group.id
-        else
-          @relation_hash['project_id'] = @project.id
-        end
+        @relation_hash['group_id'] = @project.group&.id
       end
 
       def reset_tokens!
@@ -223,7 +218,7 @@ module Gitlab
 
       def update_note_for_missing_author(author_name)
         @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank?
-        @relation_hash['note'] += missing_author_note(@relation_hash['updated_at'], author_name)
+        @relation_hash['note'] = "#{@relation_hash['note']}#{missing_author_note(@relation_hash['updated_at'], author_name)}"
       end
 
       def admin_user?
@@ -271,15 +266,7 @@ module Gitlab
       end
 
       def existing_object
-        @existing_object ||=
-          begin
-            existing_object = find_or_create_object!
-
-            # Done in two steps, as MySQL behaves differently than PostgreSQL using
-            # the +find_or_create_by+ method and does not return the ID the second time.
-            existing_object.update!(parsed_relation_hash)
-            existing_object
-          end
+        @existing_object ||= find_or_create_object!
       end
 
       def unknown_service?
@@ -288,29 +275,16 @@ module Gitlab
       end
 
       def find_or_create_object!
-        finder_attributes = if @relation_name == :group_label
-                              %w[title group_id]
-                            elsif parsed_relation_hash['project_id']
-                              %w[title project_id]
-                            else
-                              %w[title group_id]
-                            end
-
-        finder_hash = parsed_relation_hash.slice(*finder_attributes)
-
-        if label?
-          label = relation_class.find_or_initialize_by(finder_hash)
-          parsed_relation_hash.delete('priorities') if label.persisted?
-
-          label.save!
-          label
-        else
-          relation_class.find_or_create_by(finder_hash)
+        return relation_class.find_or_create_by(project_id: @project.id) if @relation_name == :project_feature
+
+        # Can't use IDs as validation exists calling `group` or `project` attributes
+        finder_hash = parsed_relation_hash.tap do |hash|
+          hash['group'] = @project.group if relation_class.attribute_method?('group_id')
+          hash['project'] = @project
+          hash.delete('project_id')
         end
-      end
 
-      def label?
-        @relation_name.to_s.include?('label')
+        GroupProjectObjectBuilder.build(relation_class, finder_hash)
       end
     end
   end
diff --git a/lib/gitlab/import_export/relation_rename_service.rb b/lib/gitlab/import_export/relation_rename_service.rb
new file mode 100644
index 00000000000..179bde5e21e
--- /dev/null
+++ b/lib/gitlab/import_export/relation_rename_service.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# This class is intended to help with relation renames within Gitlab versions
+# and allow compatibility between versions.
+# If you have to change one relationship name that is imported/exported,
+# you should add it to the RENAMES constant indicating the old name and the
+# new one.
+# The behavior of these renamed relationships should be transient and it should
+# only last one release until you completely remove the renaming from the list.
+#
+# When importing, this class will check the project hash and:
+# - if only the old relationship name is found, it will rename it with the new one
+# - if only the new relationship name is found, it will do nothing
+# - if it finds both, it will use the new relationship data
+#
+# When exporting, this class will duplicate the keys in the resulting file.
+# This way, if we open the file in an old version of the exporter it will work
+# and also it will with the newer versions.
+module Gitlab
+  module ImportExport
+    class RelationRenameService
+      RENAMES = {
+        'pipelines' => 'ci_pipelines' # Added in 11.6, remove in 11.7
+      }.freeze
+
+      def self.rename(tree_hash)
+        return unless tree_hash&.present?
+
+        RENAMES.each do |old_name, new_name|
+          old_entry = tree_hash.delete(old_name)
+
+          next if tree_hash[new_name]
+          next unless old_entry
+
+          tree_hash[new_name] = old_entry
+        end
+      end
+
+      def self.add_new_associations(tree_hash)
+        RENAMES.each do |old_name, new_name|
+          next if tree_hash.key?(old_name)
+
+          tree_hash[old_name] = tree_hash[new_name]
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb
index 5a9bbceac67..91167a9c4fb 100644
--- a/lib/gitlab/import_export/repo_restorer.rb
+++ b/lib/gitlab/import_export/repo_restorer.rb
@@ -1,8 +1,9 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class RepoRestorer
       include Gitlab::ImportExport::CommandLineUtil
-      include Gitlab::ShellAdapter
 
       def initialize(project:, shared:, path_to_bundle:)
         @project = project
diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb
index 695462c7dd2..a60618dfcec 100644
--- a/lib/gitlab/import_export/repo_saver.rb
+++ b/lib/gitlab/import_export/repo_saver.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class RepoSaver
@@ -26,10 +28,6 @@ module Gitlab
         @shared.error(e)
         false
       end
-
-      def path_to_repo
-        @project.repository.path_to_repo
-      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb
index 2daeba90a51..72f575db095 100644
--- a/lib/gitlab/import_export/saver.rb
+++ b/lib/gitlab/import_export/saver.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class Saver
@@ -15,15 +17,20 @@ module Gitlab
       def save
         if compress_and_save
           remove_export_path
+
           Rails.logger.info("Saved project export #{archive_file}")
-          archive_file
+
+          save_upload
         else
-          @shared.error(Gitlab::ImportExport::Error.new("Unable to save #{archive_file} into #{@shared.export_path}"))
+          @shared.error(Gitlab::ImportExport::Error.new(error_message))
           false
         end
       rescue => e
         @shared.error(e)
         false
+      ensure
+        remove_archive
+        remove_export_path
       end
 
       private
@@ -36,9 +43,25 @@ module Gitlab
         FileUtils.rm_rf(@shared.export_path)
       end
 
+      def remove_archive
+        FileUtils.rm_rf(@shared.archive_path)
+      end
+
       def archive_file
         @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
       end
+
+      def save_upload
+        upload = ImportExportUpload.find_or_initialize_by(project: @project)
+
+        File.open(archive_file) { |file| upload.export_file = file }
+
+        upload.save!
+      end
+
+      def error_message
+        "Unable to save #{archive_file} into #{@shared.export_path}."
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb
index 6d7c36ce38b..947caaaefee 100644
--- a/lib/gitlab/import_export/shared.rb
+++ b/lib/gitlab/import_export/shared.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class Shared
@@ -6,6 +8,7 @@ module Gitlab
       def initialize(project)
         @project = project
         @errors = []
+        @logger = Gitlab::Import::Logger.build
       end
 
       def active_export_count
@@ -21,19 +24,16 @@ module Gitlab
       end
 
       def error(error)
-        error_out(error.message, caller[0].dup)
-        add_error_message(error.message)
+        log_error(message: error.message, caller: caller[0].dup)
+        log_debug(backtrace: error.backtrace&.join("\n"))
+
+        Gitlab::Sentry.track_acceptable_exception(error, extra: log_base_data)
 
-        # Debug:
-        if error.backtrace
-          Rails.logger.error("Import/Export backtrace: #{error.backtrace.join("\n")}")
-        else
-          Rails.logger.error("No backtrace found")
-        end
+        add_error_message(error.message)
       end
 
-      def add_error_message(error_message)
-        @errors << error_message
+      def add_error_message(message)
+        @errors << filtered_error_message(message)
       end
 
       def after_export_in_progress?
@@ -50,8 +50,25 @@ module Gitlab
         @project.disk_path
       end
 
-      def error_out(message, caller)
-        Rails.logger.error("Import/Export error raised on #{caller}: #{message}")
+      def log_error(details)
+        @logger.error(log_base_data.merge(details))
+      end
+
+      def log_debug(details)
+        @logger.debug(log_base_data.merge(details))
+      end
+
+      def log_base_data
+        {
+          importer: 'Import/Export',
+          import_jid: @project&.import_state&.import_jid,
+          project_id: @project&.id,
+          project_path: @project&.full_path
+        }
+      end
+
+      def filtered_error_message(message)
+        Projects::ImportErrorFilter.filter_message(message)
       end
 
       def after_export_lock_file
diff --git a/lib/gitlab/import_export/statistics_restorer.rb b/lib/gitlab/import_export/statistics_restorer.rb
index bcdd9c12c85..3fafb01c37c 100644
--- a/lib/gitlab/import_export/statistics_restorer.rb
+++ b/lib/gitlab/import_export/statistics_restorer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class StatisticsRestorer
diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb
new file mode 100644
index 00000000000..e232198150a
--- /dev/null
+++ b/lib/gitlab/import_export/uploads_manager.rb
@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    class UploadsManager
+      include Gitlab::ImportExport::CommandLineUtil
+
+      UPLOADS_BATCH_SIZE = 100
+
+      def initialize(project:, shared:, relative_export_path: 'uploads')
+        @project = project
+        @shared = shared
+        @relative_export_path = relative_export_path
+      end
+
+      def save
+        copy_project_uploads
+
+        true
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      def restore
+        Dir["#{uploads_export_path}/**/*"].each do |upload|
+          next if File.directory?(upload)
+
+          add_upload(upload)
+        end
+
+        true
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def add_upload(upload)
+        uploader_context = FileUploader.extract_dynamic_path(upload).named_captures.symbolize_keys
+
+        UploadService.new(@project, File.open(upload, 'r'), FileUploader, uploader_context).execute.to_h
+      end
+
+      def copy_project_uploads
+        each_uploader do |uploader|
+          next unless uploader.file
+
+          if uploader.upload.local?
+            next unless uploader.upload.exist?
+
+            copy_files(uploader.absolute_path, File.join(uploads_export_path, uploader.upload.path))
+          else
+            download_and_copy(uploader)
+          end
+        end
+      end
+
+      def uploads_export_path
+        @uploads_export_path ||= File.join(@shared.export_path, @relative_export_path)
+      end
+
+      def each_uploader
+        avatar_path = @project.avatar&.upload&.path
+
+        if @relative_export_path == 'avatar'
+          yield(@project.avatar)
+        else
+          project_uploads_except_avatar(avatar_path).find_each(batch_size: UPLOADS_BATCH_SIZE) do |upload|
+            yield(upload.build_uploader)
+          end
+        end
+      end
+
+      def project_uploads_except_avatar(avatar_path)
+        return @project.uploads unless avatar_path
+
+        @project.uploads.where("path != ?", avatar_path)
+      end
+
+      def download_and_copy(upload)
+        secret = upload.try(:secret) || ''
+        upload_path = File.join(uploads_export_path, secret, upload.filename)
+
+        mkdir_p(File.join(uploads_export_path, secret))
+
+        download_or_copy_upload(upload, upload_path)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb
index df19354b76e..5f422dcbefa 100644
--- a/lib/gitlab/import_export/uploads_restorer.rb
+++ b/lib/gitlab/import_export/uploads_restorer.rb
@@ -1,10 +1,13 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class UploadsRestorer < UploadsSaver
       def restore
-        return true unless File.directory?(uploads_export_path)
-
-        copy_files(uploads_export_path, uploads_path)
+        Gitlab::ImportExport::UploadsManager.new(
+          project: @project,
+          shared: @shared
+        ).restore
       rescue => e
         @shared.error(e)
         false
diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb
index 2f08dda55fd..be1066c30b2 100644
--- a/lib/gitlab/import_export/uploads_saver.rb
+++ b/lib/gitlab/import_export/uploads_saver.rb
@@ -1,29 +1,22 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class UploadsSaver
-      include Gitlab::ImportExport::CommandLineUtil
-
       def initialize(project:, shared:)
         @project = project
         @shared = shared
       end
 
       def save
-        return true unless File.directory?(uploads_path)
-
-        copy_files(uploads_path, uploads_export_path)
+        Gitlab::ImportExport::UploadsManager.new(
+          project: @project,
+          shared: @shared
+        ).save
       rescue => e
         @shared.error(e)
         false
       end
-
-      def uploads_path
-        FileUploader.absolute_base_dir(@project)
-      end
-
-      def uploads_export_path
-        File.join(@shared.export_path, 'uploads')
-      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb
index bd3c3ee3b2f..6d978d00ea5 100644
--- a/lib/gitlab/import_export/version_checker.rb
+++ b/lib/gitlab/import_export/version_checker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class VersionChecker
diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb
index 7cf88298642..8230c0f1e77 100644
--- a/lib/gitlab/import_export/version_saver.rb
+++ b/lib/gitlab/import_export/version_saver.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class VersionSaver
diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb
index 5fa2e101e29..7303bcf61a4 100644
--- a/lib/gitlab/import_export/wiki_repo_saver.rb
+++ b/lib/gitlab/import_export/wiki_repo_saver.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class WikiRepoSaver < RepoSaver
@@ -22,12 +24,8 @@ module Gitlab
         "project.wiki.bundle"
       end
 
-      def path_to_repo
-        @wiki.repository.path_to_repo
-      end
-
       def wiki_repository_exists?
-        File.exist?(@wiki.repository.path_to_repo) && !@wiki.repository.empty?
+        @wiki.repository.exists? && !@wiki.repository.empty?
       end
     end
   end
diff --git a/lib/gitlab/import_export/wiki_restorer.rb b/lib/gitlab/import_export/wiki_restorer.rb
index f33bfb332ab..28b5e7449cd 100644
--- a/lib/gitlab/import_export/wiki_restorer.rb
+++ b/lib/gitlab/import_export/wiki_restorer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ImportExport
     class WikiRestorer < RepoRestorer
diff --git a/lib/gitlab/import_formatter.rb b/lib/gitlab/import_formatter.rb
index 4e611e7f16c..d4ba4d1181d 100644
--- a/lib/gitlab/import_formatter.rb
+++ b/lib/gitlab/import_formatter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class ImportFormatter
     def comment(author, date, body)
diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb
index 60d5fa4d29a..67952ca0f2d 100644
--- a/lib/gitlab/import_sources.rb
+++ b/lib/gitlab/import_sources.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Gitlab::ImportSources module
 #
 # Define import sources that can be used
@@ -8,37 +10,43 @@ module Gitlab
     ImportSource = Struct.new(:name, :title, :importer)
 
     # We exclude `bare_repository` here as it has no import class associated
-    ImportTable = [
-      ImportSource.new('github',         'GitHub',        Gitlab::GithubImport::ParallelImporter),
-      ImportSource.new('bitbucket',      'Bitbucket',     Gitlab::BitbucketImport::Importer),
-      ImportSource.new('gitlab',         'GitLab.com',    Gitlab::GitlabImport::Importer),
-      ImportSource.new('google_code',    'Google Code',   Gitlab::GoogleCodeImport::Importer),
-      ImportSource.new('fogbugz',        'FogBugz',       Gitlab::FogbugzImport::Importer),
-      ImportSource.new('git',            'Repo by URL',   nil),
-      ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer),
-      ImportSource.new('gitea',          'Gitea',         Gitlab::LegacyGithubImport::Importer)
+    IMPORT_TABLE = [
+      ImportSource.new('github',           'GitHub',           Gitlab::GithubImport::ParallelImporter),
+      ImportSource.new('bitbucket',        'Bitbucket Cloud',  Gitlab::BitbucketImport::Importer),
+      ImportSource.new('bitbucket_server', 'Bitbucket Server', Gitlab::BitbucketServerImport::Importer),
+      ImportSource.new('gitlab',           'GitLab.com',       Gitlab::GitlabImport::Importer),
+      ImportSource.new('google_code',      'Google Code',      Gitlab::GoogleCodeImport::Importer),
+      ImportSource.new('fogbugz',          'FogBugz',          Gitlab::FogbugzImport::Importer),
+      ImportSource.new('git',              'Repo by URL',      nil),
+      ImportSource.new('gitlab_project',   'GitLab export',    Gitlab::ImportExport::Importer),
+      ImportSource.new('gitea',            'Gitea',            Gitlab::LegacyGithubImport::Importer),
+      ImportSource.new('manifest',         'Manifest file',    nil)
     ].freeze
 
     class << self
       def options
-        @options ||= Hash[ImportTable.map { |importer| [importer.title, importer.name] }]
+        Hash[import_table.map { |importer| [importer.title, importer.name] }]
       end
 
       def values
-        @values ||= ImportTable.map(&:name)
+        import_table.map(&:name)
       end
 
       def importer_names
-        @importer_names ||= ImportTable.select(&:importer).map(&:name)
+        import_table.select(&:importer).map(&:name)
       end
 
       def importer(name)
-        ImportTable.find { |import_source| import_source.name == name }.importer
+        import_table.find { |import_source| import_source.name == name }.importer
       end
 
       def title(name)
         options.key(name)
       end
+
+      def import_table
+        IMPORT_TABLE
+      end
     end
   end
 end
diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb
index d323cb9dadf..cc0c633b943 100644
--- a/lib/gitlab/incoming_email.rb
+++ b/lib/gitlab/incoming_email.rb
@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+
 module Gitlab
   module IncomingEmail
-    UNSUBSCRIBE_SUFFIX = '+unsubscribe'.freeze
-    WILDCARD_PLACEHOLDER = '%{key}'.freeze
+    UNSUBSCRIBE_SUFFIX        = '-unsubscribe'.freeze
+    UNSUBSCRIBE_SUFFIX_LEGACY = '+unsubscribe'.freeze
+    WILDCARD_PLACEHOLDER      = '%{key}'.freeze
 
     class << self
       def enabled?
@@ -20,6 +23,7 @@ module Gitlab
         config.address.sub(WILDCARD_PLACEHOLDER, key)
       end
 
+      # example: incoming+1234567890abcdef1234567890abcdef-unsubscribe@incoming.gitlab.com
       def unsubscribe_address(key)
         config.address.sub(WILDCARD_PLACEHOLDER, "#{key}#{UNSUBSCRIBE_SUFFIX}")
       end
diff --git a/lib/gitlab/insecure_key_fingerprint.rb b/lib/gitlab/insecure_key_fingerprint.rb
index f85b6e9197f..e4f0e9d2c73 100644
--- a/lib/gitlab/insecure_key_fingerprint.rb
+++ b/lib/gitlab/insecure_key_fingerprint.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   #
   # Calculates the fingerprint of a given key without using
diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb
index 0c9de72329c..351d15605e0 100644
--- a/lib/gitlab/issuable_metadata.rb
+++ b/lib/gitlab/issuable_metadata.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module IssuableMetadata
     def issuable_meta_data(issuable_collection, collection_type)
diff --git a/lib/gitlab/issuable_sorter.rb b/lib/gitlab/issuable_sorter.rb
index d392214867a..42bbfb32d0b 100644
--- a/lib/gitlab/issuable_sorter.rb
+++ b/lib/gitlab/issuable_sorter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module IssuableSorter
     class << self
diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb
index 505810964bc..659fb1472d2 100644
--- a/lib/gitlab/issuables_count_for_state.rb
+++ b/lib/gitlab/issuables_count_for_state.rb
@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 module Gitlab
   # Class for counting and caching the number of issuables per state.
   class IssuablesCountForState
-    # The name of the RequestStore cache key.
+    # The name of the Gitlab::SafeRequestStore cache key.
     CACHE_KEY = :issuables_count_for_state
 
     # The state values that can be safely casted to a Symbol.
@@ -10,12 +12,7 @@ module Gitlab
     # finder - The finder class to use for retrieving the issuables.
     def initialize(finder)
       @finder = finder
-      @cache =
-        if RequestStore.active?
-          RequestStore[CACHE_KEY] ||= initialize_cache
-        else
-          initialize_cache
-        end
+      @cache = Gitlab::SafeRequestStore[CACHE_KEY] ||= initialize_cache
     end
 
     def for_state_or_opened(state = nil)
diff --git a/lib/gitlab/issues_labels.rb b/lib/gitlab/issues_labels.rb
index b8ca7f2f55f..17c9cb969df 100644
--- a/lib/gitlab/issues_labels.rb
+++ b/lib/gitlab/issues_labels.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class IssuesLabels
     class << self
diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb
index f7a8eae0be4..e97e961771c 100644
--- a/lib/gitlab/job_waiter.rb
+++ b/lib/gitlab/job_waiter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   # JobWaiter can be used to wait for a number of Sidekiq jobs to complete.
   #
diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb
new file mode 100644
index 00000000000..1adf83739ad
--- /dev/null
+++ b/lib/gitlab/json_cache.rb
@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class JsonCache
+    attr_reader :backend, :cache_key_with_version, :namespace
+
+    def initialize(options = {})
+      @backend = options.fetch(:backend, Rails.cache)
+      @namespace = options.fetch(:namespace, nil)
+      @cache_key_with_version = options.fetch(:cache_key_with_version, true)
+    end
+
+    def active?
+      if backend.respond_to?(:active?)
+        backend.active?
+      else
+        true
+      end
+    end
+
+    def cache_key(key)
+      expanded_cache_key = [namespace, key].compact
+
+      if cache_key_with_version
+        expanded_cache_key << Rails.version
+      end
+
+      expanded_cache_key.join(':')
+    end
+
+    def expire(key)
+      backend.delete(cache_key(key))
+    end
+
+    def read(key, klass = nil)
+      value = backend.read(cache_key(key))
+      value = parse_value(value, klass) if value
+      value
+    end
+
+    def write(key, value, options = nil)
+      backend.write(cache_key(key), value.to_json, options)
+    end
+
+    def fetch(key, options = {}, &block)
+      klass = options.delete(:as)
+      value = read(key, klass)
+
+      return value unless value.nil?
+
+      value = yield
+
+      write(key, value, options)
+
+      value
+    end
+
+    private
+
+    def parse_value(raw, klass)
+      value = ActiveSupport::JSON.decode(raw)
+
+      case value
+      when Hash then parse_entry(value, klass)
+      when Array then parse_entries(value, klass)
+      else
+        value
+      end
+    rescue ActiveSupport::JSON.parse_error
+      nil
+    end
+
+    def parse_entry(raw, klass)
+      klass.new(raw) if valid_entry?(raw, klass)
+    end
+
+    def valid_entry?(raw, klass)
+      return false unless klass && raw.is_a?(Hash)
+
+      (raw.keys - klass.attribute_names).empty?
+    end
+
+    def parse_entries(values, klass)
+      values.map { |value| parse_entry(value, klass) }.compact
+    end
+  end
+end
diff --git a/lib/gitlab/json_logger.rb b/lib/gitlab/json_logger.rb
new file mode 100644
index 00000000000..a5a5759cc89
--- /dev/null
+++ b/lib/gitlab/json_logger.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class JsonLogger < ::Gitlab::Logger
+    def self.file_name_noext
+      raise NotImplementedError
+    end
+
+    def format_message(severity, timestamp, progname, message)
+      data = {}
+      data[:severity] = severity
+      data[:time] = timestamp.utc.iso8601(3)
+      data[Gitlab::CorrelationId::LOG_KEY] = Gitlab::CorrelationId.current_id
+
+      case message
+      when String
+        data[:message] = message
+      when Hash
+        data.merge!(message)
+      end
+
+      data.to_json + "\n"
+    end
+  end
+end
diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb
index da43bd0af4b..a9957a85d48 100644
--- a/lib/gitlab/kubernetes.rb
+++ b/lib/gitlab/kubernetes.rb
@@ -1,6 +1,12 @@
+# frozen_string_literal: true
+
 module Gitlab
   # Helper methods to do with Kubernetes network services & resources
   module Kubernetes
+    def self.build_header_hash
+      Hash.new { |h, k| h[k] = [] }
+    end
+
     # This is the comand that is run to start a terminal session. Kubernetes
     # expects `command=foo&command=bar, not `command[]=foo&command[]=bar`
     EXEC_COMMAND = URI.encode_www_form(
@@ -37,13 +43,14 @@ module Gitlab
           selectors:    { pod: pod_name, container: container["name"] },
           url:          container_exec_url(api_url, namespace, pod_name, container["name"]),
           subprotocols: ['channel.k8s.io'],
-          headers:      Hash.new { |h, k| h[k] = [] },
+          headers:      ::Gitlab::Kubernetes.build_header_hash,
           created_at:   created_at
         }
       end
     end
 
     def add_terminal_auth(terminal, token:, max_session_time:, ca_pem: nil)
+      terminal[:headers] ||= ::Gitlab::Kubernetes.build_header_hash
       terminal[:headers]['Authorization'] << "Bearer #{token}"
       terminal[:max_session_time] = max_session_time
       terminal[:ca_pem] = ca_pem if ca_pem.present?
@@ -78,6 +85,8 @@ module Gitlab
     end
 
     def to_kubeconfig(url:, namespace:, token:, ca_pem: nil)
+      return unless token.present?
+
       config = {
         apiVersion: 'v1',
         clusters: [
@@ -106,7 +115,7 @@ module Gitlab
 
       kubeconfig_embed_ca_pem(config, ca_pem) if ca_pem
 
-      config.deep_stringify_keys
+      YAML.dump(config.deep_stringify_keys)
     end
 
     private
diff --git a/lib/gitlab/kubernetes/cluster_role_binding.rb b/lib/gitlab/kubernetes/cluster_role_binding.rb
new file mode 100644
index 00000000000..ebea8aff5be
--- /dev/null
+++ b/lib/gitlab/kubernetes/cluster_role_binding.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    class ClusterRoleBinding
+      attr_reader :name, :cluster_role_name, :subjects
+
+      def initialize(name, cluster_role_name, subjects)
+        @name = name
+        @cluster_role_name = cluster_role_name
+        @subjects = subjects
+      end
+
+      def generate
+        ::Kubeclient::Resource.new.tap do |resource|
+          resource.metadata = metadata
+          resource.roleRef = role_ref
+          resource.subjects = subjects
+        end
+      end
+
+      private
+
+      def metadata
+        { name: name }
+      end
+
+      def role_ref
+        {
+          apiGroup: 'rbac.authorization.k8s.io',
+          kind: 'ClusterRole',
+          name: cluster_role_name
+        }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/kubernetes/config_map.rb b/lib/gitlab/kubernetes/config_map.rb
index 95e1054919d..0bcaaa03974 100644
--- a/lib/gitlab/kubernetes/config_map.rb
+++ b/lib/gitlab/kubernetes/config_map.rb
@@ -1,21 +1,27 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Kubernetes
     class ConfigMap
-      def initialize(name, values)
+      def initialize(name, files)
         @name = name
-        @values = values
+        @files = files
       end
 
       def generate
         resource = ::Kubeclient::Resource.new
         resource.metadata = metadata
-        resource.data = { values: values }
+        resource.data = files
         resource
       end
 
+      def config_map_name
+        "values-content-configuration-#{name}"
+      end
+
       private
 
-      attr_reader :name, :values
+      attr_reader :name, :files
 
       def metadata
         {
@@ -25,10 +31,6 @@ module Gitlab
         }
       end
 
-      def config_map_name
-        "values-content-configuration-#{name}"
-      end
-
       def namespace
         Gitlab::Kubernetes::Helm::NAMESPACE
       end
diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb
index 0f0588b8b23..bbac15c7710 100644
--- a/lib/gitlab/kubernetes/helm.rb
+++ b/lib/gitlab/kubernetes/helm.rb
@@ -1,8 +1,14 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Kubernetes
     module Helm
-      HELM_VERSION = '2.7.0'.freeze
+      HELM_VERSION = '2.12.2'.freeze
+      KUBECTL_VERSION = '1.11.0'.freeze
       NAMESPACE = 'gitlab-managed-apps'.freeze
+      SERVICE_ACCOUNT = 'tiller'.freeze
+      CLUSTER_ROLE_BINDING = 'tiller-admin'.freeze
+      CLUSTER_ROLE = 'cluster-admin'.freeze
     end
   end
 end
diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb
index 2edd34109ba..b9903e37f40 100644
--- a/lib/gitlab/kubernetes/helm/api.rb
+++ b/lib/gitlab/kubernetes/helm/api.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Kubernetes
     module Helm
@@ -8,9 +10,23 @@ module Gitlab
         end
 
         def install(command)
-          @namespace.ensure_exists!
-          create_config_map(command) if command.config_map?
-          @kubeclient.create_pod(command.pod_resource)
+          namespace.ensure_exists!
+
+          create_service_account(command)
+          create_cluster_role_binding(command)
+          create_config_map(command)
+
+          delete_pod!(command.pod_name)
+          kubeclient.create_pod(command.pod_resource)
+        end
+
+        def update(command)
+          namespace.ensure_exists!
+
+          update_config_map(command)
+
+          delete_pod!(command.pod_name)
+          kubeclient.create_pod(command.pod_resource)
         end
 
         ##
@@ -20,25 +36,87 @@ module Gitlab
         #
         # values: "Pending", "Running", "Succeeded", "Failed", "Unknown"
         #
-        def installation_status(pod_name)
-          @kubeclient.get_pod(pod_name, @namespace.name).status.phase
+        def status(pod_name)
+          kubeclient.get_pod(pod_name, namespace.name).status.phase
         end
 
-        def installation_log(pod_name)
-          @kubeclient.get_pod_log(pod_name, @namespace.name).body
+        def log(pod_name)
+          kubeclient.get_pod_log(pod_name, namespace.name).body
         end
 
-        def delete_installation_pod!(pod_name)
-          @kubeclient.delete_pod(pod_name, @namespace.name)
+        def delete_pod!(pod_name)
+          kubeclient.delete_pod(pod_name, namespace.name)
+        rescue ::Kubeclient::ResourceNotFoundError
+          # no-op
+        end
+
+        def get_config_map(config_map_name)
+          namespace.ensure_exists!
+
+          kubeclient.get_config_map(config_map_name, namespace.name)
         end
 
         private
 
+        attr_reader :kubeclient, :namespace
+
         def create_config_map(command)
           command.config_map_resource.tap do |config_map_resource|
-            @kubeclient.create_config_map(config_map_resource)
+            if config_map_exists?(config_map_resource)
+              kubeclient.update_config_map(config_map_resource)
+            else
+              kubeclient.create_config_map(config_map_resource)
+            end
+          end
+        end
+
+        def update_config_map(command)
+          command.config_map_resource.tap do |config_map_resource|
+            kubeclient.update_config_map(config_map_resource)
           end
         end
+
+        def create_service_account(command)
+          command.service_account_resource.tap do |service_account_resource|
+            break unless service_account_resource
+
+            if service_account_exists?(service_account_resource)
+              kubeclient.update_service_account(service_account_resource)
+            else
+              kubeclient.create_service_account(service_account_resource)
+            end
+          end
+        end
+
+        def create_cluster_role_binding(command)
+          command.cluster_role_binding_resource.tap do |cluster_role_binding_resource|
+            break unless cluster_role_binding_resource
+
+            if cluster_role_binding_exists?(cluster_role_binding_resource)
+              kubeclient.update_cluster_role_binding(cluster_role_binding_resource)
+            else
+              kubeclient.create_cluster_role_binding(cluster_role_binding_resource)
+            end
+          end
+        end
+
+        def config_map_exists?(resource)
+          kubeclient.get_config_map(resource.metadata.name, resource.metadata.namespace)
+        rescue ::Kubeclient::ResourceNotFoundError
+          false
+        end
+
+        def service_account_exists?(resource)
+          kubeclient.get_service_account(resource.metadata.name, resource.metadata.namespace)
+        rescue ::Kubeclient::ResourceNotFoundError
+          false
+        end
+
+        def cluster_role_binding_exists?(resource)
+          kubeclient.get_cluster_role_binding(resource.metadata.name)
+        rescue ::Kubeclient::ResourceNotFoundError
+          false
+        end
       end
     end
   end
diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb
index 3d778da90c7..2bcb428b25d 100644
--- a/lib/gitlab/kubernetes/helm/base_command.rb
+++ b/lib/gitlab/kubernetes/helm/base_command.rb
@@ -1,42 +1,66 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Kubernetes
     module Helm
-      class BaseCommand
-        attr_reader :name
-
-        def initialize(name)
-          @name = name
-        end
-
+      module BaseCommand
         def pod_resource
-          Gitlab::Kubernetes::Helm::Pod.new(self, namespace).generate
+          pod_service_account_name = rbac? ? service_account_name : nil
+
+          Gitlab::Kubernetes::Helm::Pod.new(self, namespace, service_account_name: pod_service_account_name).generate
         end
 
         def generate_script
           <<~HEREDOC
-            set -eo pipefail
-            ALPINE_VERSION=$(cat /etc/alpine-release | cut -d '.' -f 1,2)
-            echo http://mirror.clarkson.edu/alpine/v$ALPINE_VERSION/main >> /etc/apk/repositories
-            echo http://mirror1.hs-esslingen.de/pub/Mirrors/alpine/v$ALPINE_VERSION/main >> /etc/apk/repositories
-            apk add -U ca-certificates openssl >/dev/null
-            wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v#{Gitlab::Kubernetes::Helm::HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null
-            mv /tmp/linux-amd64/helm /usr/bin/
+            set -xeo pipefail
           HEREDOC
         end
 
-        def config_map?
-          false
-        end
-
         def pod_name
           "install-#{name}"
         end
 
+        def config_map_resource
+          Gitlab::Kubernetes::ConfigMap.new(name, files).generate
+        end
+
+        def service_account_resource
+          nil
+        end
+
+        def cluster_role_binding_resource
+          nil
+        end
+
+        def file_names
+          files.keys
+        end
+
+        def name
+          raise "Not implemented"
+        end
+
+        def rbac?
+          raise "Not implemented"
+        end
+
+        def files
+          raise "Not implemented"
+        end
+
         private
 
+        def files_dir
+          "/data/helm/#{name}/config"
+        end
+
         def namespace
           Gitlab::Kubernetes::Helm::NAMESPACE
         end
+
+        def service_account_name
+          Gitlab::Kubernetes::Helm::SERVICE_ACCOUNT
+        end
       end
     end
   end
diff --git a/lib/gitlab/kubernetes/helm/certificate.rb b/lib/gitlab/kubernetes/helm/certificate.rb
new file mode 100644
index 00000000000..598714e0874
--- /dev/null
+++ b/lib/gitlab/kubernetes/helm/certificate.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+module Gitlab
+  module Kubernetes
+    module Helm
+      class Certificate
+        INFINITE_EXPIRY = 1000.years
+        SHORT_EXPIRY = 30.minutes
+
+        attr_reader :key, :cert
+
+        def key_string
+          @key.to_s
+        end
+
+        def cert_string
+          @cert.to_pem
+        end
+
+        def self.from_strings(key_string, cert_string)
+          key = OpenSSL::PKey::RSA.new(key_string)
+          cert = OpenSSL::X509::Certificate.new(cert_string)
+          new(key, cert)
+        end
+
+        def self.generate_root
+          _issue(signed_by: nil, expires_in: INFINITE_EXPIRY, certificate_authority: true)
+        end
+
+        def issue(expires_in: SHORT_EXPIRY)
+          self.class._issue(signed_by: self, expires_in: expires_in, certificate_authority: false)
+        end
+
+        private
+
+        def self._issue(signed_by:, expires_in:, certificate_authority:)
+          key = OpenSSL::PKey::RSA.new(4096)
+          public_key = key.public_key
+
+          subject = OpenSSL::X509::Name.parse("/C=US")
+
+          cert = OpenSSL::X509::Certificate.new
+          cert.subject = subject
+
+          cert.issuer = signed_by&.cert&.subject || subject
+
+          cert.not_before = Time.now
+          cert.not_after = expires_in.from_now
+          cert.public_key = public_key
+          cert.serial = 0x0
+          cert.version = 2
+
+          if certificate_authority
+            extension_factory = OpenSSL::X509::ExtensionFactory.new
+            extension_factory.subject_certificate = cert
+            extension_factory.issuer_certificate = cert
+            cert.add_extension(extension_factory.create_extension('subjectKeyIdentifier', 'hash'))
+            cert.add_extension(extension_factory.create_extension('basicConstraints', 'CA:TRUE', true))
+            cert.add_extension(extension_factory.create_extension('keyUsage', 'cRLSign,keyCertSign', true))
+          end
+
+          cert.sign(signed_by&.key || key, OpenSSL::Digest::SHA256.new)
+
+          new(key, cert)
+        end
+
+        def initialize(key, cert)
+          @key = key
+          @cert = cert
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/kubernetes/helm/client_command.rb b/lib/gitlab/kubernetes/helm/client_command.rb
new file mode 100644
index 00000000000..9940272a8bf
--- /dev/null
+++ b/lib/gitlab/kubernetes/helm/client_command.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    module Helm
+      module ClientCommand
+        def init_command
+          # Here we are always upgrading to the latest version of Tiller when
+          # installing an app. We ensure the helm version stored in the
+          # database is correct by also updating this after transition to
+          # :installed,:updated in Clusters::Concerns::ApplicationStatus
+          'helm init --upgrade'
+        end
+
+        def wait_for_tiller_command
+          # This is necessary to give Tiller time to restart after upgrade.
+          # Ideally we'd be able to use --wait but cannot because of
+          # https://github.com/helm/helm/issues/4855
+          'for i in $(seq 1 30); do helm version && break; sleep 1s; echo "Retrying ($i)..."; done'
+        end
+
+        def repository_command
+          ['helm', 'repo', 'add', name, repository].shelljoin if repository
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/kubernetes/helm/init_command.rb b/lib/gitlab/kubernetes/helm/init_command.rb
index a02e64561f6..88ed8572ffc 100644
--- a/lib/gitlab/kubernetes/helm/init_command.rb
+++ b/lib/gitlab/kubernetes/helm/init_command.rb
@@ -1,17 +1,81 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Kubernetes
     module Helm
-      class InitCommand < BaseCommand
+      class InitCommand
+        include BaseCommand
+
+        attr_reader :name, :files
+
+        def initialize(name:, files:, rbac:)
+          @name = name
+          @files = files
+          @rbac = rbac
+        end
+
         def generate_script
           super + [
             init_helm_command
           ].join("\n")
         end
 
+        def rbac?
+          @rbac
+        end
+
+        def service_account_resource
+          return unless rbac?
+
+          Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate
+        end
+
+        def cluster_role_binding_resource
+          return unless rbac?
+
+          subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }]
+
+          Gitlab::Kubernetes::ClusterRoleBinding.new(
+            cluster_role_binding_name,
+            cluster_role_name,
+            subjects
+          ).generate
+        end
+
         private
 
         def init_helm_command
-          "helm init >/dev/null"
+          command = %w[helm init] + init_command_flags
+
+          command.shelljoin
+        end
+
+        def init_command_flags
+          tls_flags + optional_service_account_flag
+        end
+
+        def tls_flags
+          [
+            '--tiller-tls',
+            '--tiller-tls-verify',
+            '--tls-ca-cert', "#{files_dir}/ca.pem",
+            '--tiller-tls-cert', "#{files_dir}/cert.pem",
+            '--tiller-tls-key', "#{files_dir}/key.pem"
+          ]
+        end
+
+        def optional_service_account_flag
+          return [] unless rbac?
+
+          ['--service-account', service_account_name]
+        end
+
+        def cluster_role_binding_name
+          Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING
+        end
+
+        def cluster_role_name
+          Gitlab::Kubernetes::Helm::CLUSTER_ROLE
         end
       end
     end
diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb
index 30af3e97b4a..a1ab5e048ac 100644
--- a/lib/gitlab/kubernetes/helm/install_command.rb
+++ b/lib/gitlab/kubernetes/helm/install_command.rb
@@ -1,46 +1,97 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Kubernetes
     module Helm
-      class InstallCommand < BaseCommand
-        attr_reader :name, :chart, :repository, :values
+      class InstallCommand
+        include BaseCommand
+        include ClientCommand
+
+        attr_reader :name, :files, :chart, :version, :repository, :preinstall, :postinstall
 
-        def initialize(name, chart:, values:, repository: nil)
+        def initialize(name:, chart:, files:, rbac:, version: nil, repository: nil, preinstall: nil, postinstall: nil)
           @name = name
           @chart = chart
-          @values = values
+          @version = version
+          @rbac = rbac
+          @files = files
           @repository = repository
+          @preinstall = preinstall
+          @postinstall = postinstall
         end
 
         def generate_script
           super + [
             init_command,
+            wait_for_tiller_command,
             repository_command,
-            script_command
+            repository_update_command,
+            preinstall_command,
+            install_command,
+            postinstall_command
           ].compact.join("\n")
         end
 
-        def config_map?
-          true
+        def rbac?
+          @rbac
+        end
+
+        private
+
+        def repository_update_command
+          'helm repo update' if repository
         end
 
-        def config_map_resource
-          Gitlab::Kubernetes::ConfigMap.new(name, values).generate
+        def install_command
+          command = ['helm', 'install', chart] + install_command_flags
+
+          command.shelljoin
         end
 
-        private
+        def preinstall_command
+          preinstall.join("\n") if preinstall
+        end
+
+        def postinstall_command
+          postinstall.join("\n") if postinstall
+        end
+
+        def install_command_flags
+          name_flag      = ['--name', name]
+          namespace_flag = ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE]
+          value_flag     = ['-f', "/data/helm/#{name}/config/values.yaml"]
 
-        def init_command
-          'helm init --client-only >/dev/null'
+          name_flag +
+            optional_tls_flags +
+            optional_version_flag +
+            rbac_create_flag +
+            namespace_flag +
+            value_flag
         end
 
-        def repository_command
-          "helm repo add #{name} #{repository}" if repository
+        def rbac_create_flag
+          if rbac?
+            %w[--set rbac.create=true,rbac.enabled=true]
+          else
+            %w[--set rbac.create=false,rbac.enabled=false]
+          end
         end
 
-        def script_command
-          <<~HEREDOC
-          helm install #{chart} --name #{name} --namespace #{Gitlab::Kubernetes::Helm::NAMESPACE} -f /data/helm/#{name}/config/values.yaml >/dev/null
-          HEREDOC
+        def optional_version_flag
+          return [] unless version
+
+          ['--version', version]
+        end
+
+        def optional_tls_flags
+          return [] unless files.key?(:'ca.pem')
+
+          [
+            '--tls',
+            '--tls-ca-cert', "#{files_dir}/ca.pem",
+            '--tls-cert', "#{files_dir}/cert.pem",
+            '--tls-key', "#{files_dir}/key.pem"
+          ]
         end
       end
     end
diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb
index 1e12299eefd..75484f80070 100644
--- a/lib/gitlab/kubernetes/helm/pod.rb
+++ b/lib/gitlab/kubernetes/helm/pod.rb
@@ -1,31 +1,33 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Kubernetes
     module Helm
       class Pod
-        def initialize(command, namespace_name)
+        def initialize(command, namespace_name, service_account_name: nil)
           @command = command
           @namespace_name = namespace_name
+          @service_account_name = service_account_name
         end
 
         def generate
           spec = { containers: [container_specification], restartPolicy: 'Never' }
 
-          if command.config_map?
-            spec[:volumes] = volumes_specification
-            spec[:containers][0][:volumeMounts] = volume_mounts_specification
-          end
+          spec[:volumes] = volumes_specification
+          spec[:containers][0][:volumeMounts] = volume_mounts_specification
+          spec[:serviceAccountName] = service_account_name if service_account_name
 
           ::Kubeclient::Resource.new(metadata: metadata, spec: spec)
         end
 
         private
 
-        attr_reader :command, :namespace_name, :kubeclient, :config_map
+        attr_reader :command, :namespace_name, :service_account_name
 
         def container_specification
           {
             name: 'helm',
-            image: 'alpine:3.6',
+            image: "registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/#{Gitlab::Kubernetes::Helm::HELM_VERSION}-kube-#{Gitlab::Kubernetes::Helm::KUBECTL_VERSION}",
             env: generate_pod_env(command),
             command: %w(/bin/sh),
             args: %w(-c $(COMMAND_SCRIPT))
@@ -61,7 +63,7 @@ module Gitlab
               name: 'configuration-volume',
               configMap: {
                 name: "values-content-configuration-#{command.name}",
-                items: [{ key: 'values', path: 'values.yaml' }]
+                items: command.file_names.map { |name| { key: name, path: name } }
               }
             }
           ]
diff --git a/lib/gitlab/kubernetes/helm/upgrade_command.rb b/lib/gitlab/kubernetes/helm/upgrade_command.rb
new file mode 100644
index 00000000000..9daffc138b5
--- /dev/null
+++ b/lib/gitlab/kubernetes/helm/upgrade_command.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    module Helm
+      class UpgradeCommand
+        include BaseCommand
+        include ClientCommand
+
+        attr_reader :name, :chart, :version, :repository, :files
+
+        def initialize(name, chart:, files:, rbac:, version: nil, repository: nil)
+          @name = name
+          @chart = chart
+          @rbac = rbac
+          @version = version
+          @files = files
+          @repository = repository
+        end
+
+        def generate_script
+          super + [
+            init_command,
+            wait_for_tiller_command,
+            repository_command,
+            script_command
+          ].compact.join("\n")
+        end
+
+        def rbac?
+          @rbac
+        end
+
+        def pod_name
+          "upgrade-#{name}"
+        end
+
+        private
+
+        def script_command
+          upgrade_flags = "#{optional_version_flag}#{optional_tls_flags}" \
+            " --reset-values" \
+            " --install" \
+            " --namespace #{::Gitlab::Kubernetes::Helm::NAMESPACE}" \
+            " -f /data/helm/#{name}/config/values.yaml"
+
+          "helm upgrade #{name} #{chart}#{upgrade_flags}"
+        end
+
+        def optional_version_flag
+          " --version #{version}" if version
+        end
+
+        def optional_tls_flags
+          return unless files.key?(:'ca.pem')
+
+          " --tls" \
+            " --tls-ca-cert #{files_dir}/ca.pem" \
+            " --tls-cert #{files_dir}/cert.pem" \
+            " --tls-key #{files_dir}/key.pem"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb
new file mode 100644
index 00000000000..624c2c67551
--- /dev/null
+++ b/lib/gitlab/kubernetes/kube_client.rb
@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+module Gitlab
+  module Kubernetes
+    # Wrapper around Kubeclient::Client to dispatch
+    # the right message to the client that can respond to the message.
+    # We must have a kubeclient for each ApiGroup as there is no
+    # other way to use the Kubeclient gem.
+    #
+    # See https://github.com/abonas/kubeclient/issues/348.
+    class KubeClient
+      include Gitlab::Utils::StrongMemoize
+
+      SUPPORTED_API_GROUPS = {
+        core: { group: 'api', version: 'v1' },
+        rbac: { group: 'apis/rbac.authorization.k8s.io', version: 'v1' },
+        extensions: { group: 'apis/extensions', version: 'v1beta1' },
+        knative: { group: 'apis/serving.knative.dev', version: 'v1alpha1' }
+      }.freeze
+
+      SUPPORTED_API_GROUPS.each do |name, params|
+        client_method_name = "#{name}_client".to_sym
+
+        define_method(client_method_name) do
+          strong_memoize(client_method_name) do
+            build_kubeclient(params[:group], params[:version])
+          end
+        end
+      end
+
+      # Core API methods delegates to the core api group client
+      delegate :get_pods,
+        :get_secrets,
+        :get_config_map,
+        :get_namespace,
+        :get_pod,
+        :get_secret,
+        :get_service,
+        :get_service_account,
+        :delete_pod,
+        :create_config_map,
+        :create_namespace,
+        :create_pod,
+        :create_secret,
+        :create_service_account,
+        :update_config_map,
+        :update_secret,
+        :update_service_account,
+        to: :core_client
+
+      # RBAC methods delegates to the apis/rbac.authorization.k8s.io api
+      # group client
+      delegate :create_cluster_role_binding,
+        :get_cluster_role_binding,
+        :update_cluster_role_binding,
+        to: :rbac_client
+
+      # RBAC methods delegates to the apis/rbac.authorization.k8s.io api
+      # group client
+      delegate :create_role_binding,
+        :get_role_binding,
+        :update_role_binding,
+        to: :rbac_client
+
+      # Deployments resource is currently on the apis/extensions api group
+      delegate :get_deployments,
+        to: :extensions_client
+
+      # non-entity methods that can only work with the core client
+      # as it uses the pods/log resource
+      delegate :get_pod_log,
+        :watch_pod_log,
+        to: :core_client
+
+      attr_reader :api_prefix, :kubeclient_options
+
+      # We disable redirects through 'http_max_redirects: 0',
+      # so that KubeClient does not follow redirects and
+      # expose internal services.
+      def initialize(api_prefix, **kubeclient_options)
+        @api_prefix = api_prefix
+        @kubeclient_options = kubeclient_options.merge(http_max_redirects: 0)
+      end
+
+      def create_or_update_cluster_role_binding(resource)
+        if cluster_role_binding_exists?(resource)
+          update_cluster_role_binding(resource)
+        else
+          create_cluster_role_binding(resource)
+        end
+      end
+
+      def create_or_update_role_binding(resource)
+        if role_binding_exists?(resource)
+          update_role_binding(resource)
+        else
+          create_role_binding(resource)
+        end
+      end
+
+      def create_or_update_service_account(resource)
+        if service_account_exists?(resource)
+          update_service_account(resource)
+        else
+          create_service_account(resource)
+        end
+      end
+
+      def create_or_update_secret(resource)
+        if secret_exists?(resource)
+          update_secret(resource)
+        else
+          create_secret(resource)
+        end
+      end
+
+      private
+
+      def cluster_role_binding_exists?(resource)
+        get_cluster_role_binding(resource.metadata.name)
+      rescue ::Kubeclient::ResourceNotFoundError
+        false
+      end
+
+      def role_binding_exists?(resource)
+        get_role_binding(resource.metadata.name, resource.metadata.namespace)
+      rescue ::Kubeclient::ResourceNotFoundError
+        false
+      end
+
+      def service_account_exists?(resource)
+        get_service_account(resource.metadata.name, resource.metadata.namespace)
+      rescue ::Kubeclient::ResourceNotFoundError
+        false
+      end
+
+      def secret_exists?(resource)
+        get_secret(resource.metadata.name, resource.metadata.namespace)
+      rescue ::Kubeclient::ResourceNotFoundError
+        false
+      end
+
+      def build_kubeclient(api_group, api_version)
+        ::Kubeclient::Client.new(
+          join_api_url(api_prefix, api_group),
+          api_version,
+          **kubeclient_options
+        )
+      end
+
+      def join_api_url(api_prefix, api_path)
+        url = URI.parse(api_prefix)
+        prefix = url.path.sub(%r{/+\z}, '')
+
+        url.path = [prefix, api_path].join("/")
+
+        url.to_s
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/kubernetes/logger.rb b/lib/gitlab/kubernetes/logger.rb
new file mode 100644
index 00000000000..5e59482419b
--- /dev/null
+++ b/lib/gitlab/kubernetes/logger.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    class Logger < ::Gitlab::JsonLogger
+      def self.file_name_noext
+        'kubernetes'
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/kubernetes/namespace.rb b/lib/gitlab/kubernetes/namespace.rb
index e6ff6160ab9..919f19c86d7 100644
--- a/lib/gitlab/kubernetes/namespace.rb
+++ b/lib/gitlab/kubernetes/namespace.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Kubernetes
     class Namespace
@@ -10,9 +12,7 @@ module Gitlab
 
       def exists?
         @client.get_namespace(name)
-      rescue ::Kubeclient::HttpError => ke
-        raise ke unless ke.error_code == 404
-
+      rescue ::Kubeclient::ResourceNotFoundError
         false
       end
 
diff --git a/lib/gitlab/kubernetes/pod.rb b/lib/gitlab/kubernetes/pod.rb
index f3842cdf762..81317e532b2 100644
--- a/lib/gitlab/kubernetes/pod.rb
+++ b/lib/gitlab/kubernetes/pod.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Kubernetes
     module Pod
diff --git a/lib/gitlab/kubernetes/role_binding.rb b/lib/gitlab/kubernetes/role_binding.rb
new file mode 100644
index 00000000000..cb0cb42d007
--- /dev/null
+++ b/lib/gitlab/kubernetes/role_binding.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    class RoleBinding
+      def initialize(name:, role_name:, namespace:, service_account_name:)
+        @name = name
+        @role_name = role_name
+        @namespace = namespace
+        @service_account_name = service_account_name
+      end
+
+      def generate
+        ::Kubeclient::Resource.new.tap do |resource|
+          resource.metadata = metadata
+          resource.roleRef  = role_ref
+          resource.subjects = subjects
+        end
+      end
+
+      private
+
+      attr_reader :name, :role_name, :namespace, :service_account_name
+
+      def metadata
+        { name: name, namespace: namespace }
+      end
+
+      def role_ref
+        {
+          apiGroup: 'rbac.authorization.k8s.io',
+          kind: 'ClusterRole',
+          name: role_name
+        }
+      end
+
+      def subjects
+        [
+          {
+            kind: 'ServiceAccount',
+            name: service_account_name,
+            namespace: namespace
+          }
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/kubernetes/service_account.rb b/lib/gitlab/kubernetes/service_account.rb
new file mode 100644
index 00000000000..d58fc1c3976
--- /dev/null
+++ b/lib/gitlab/kubernetes/service_account.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    class ServiceAccount
+      attr_reader :name, :namespace_name
+
+      def initialize(name, namespace_name)
+        @name = name
+        @namespace_name = namespace_name
+      end
+
+      def generate
+        ::Kubeclient::Resource.new(metadata: metadata)
+      end
+
+      private
+
+      def metadata
+        {
+          name: name,
+          namespace: namespace_name
+        }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/kubernetes/service_account_token.rb b/lib/gitlab/kubernetes/service_account_token.rb
new file mode 100644
index 00000000000..2e912b26c09
--- /dev/null
+++ b/lib/gitlab/kubernetes/service_account_token.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    class ServiceAccountToken
+      attr_reader :name, :service_account_name, :namespace_name
+
+      def initialize(name, service_account_name, namespace_name)
+        @name = name
+        @service_account_name = service_account_name
+        @namespace_name = namespace_name
+      end
+
+      def generate
+        ::Kubeclient::Resource.new(metadata: metadata, type: service_acount_token_type)
+      end
+
+      private
+
+      # as per https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/#to-create-additional-api-tokens
+      def service_acount_token_type
+        'kubernetes.io/service-account-token'
+      end
+
+      def metadata
+        {
+          name: name,
+          namespace: namespace_name,
+          annotations: {
+            "kubernetes.io/service-account.name": service_account_name
+          }
+        }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/language_data.rb b/lib/gitlab/language_data.rb
new file mode 100644
index 00000000000..bfdd7175198
--- /dev/null
+++ b/lib/gitlab/language_data.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module LanguageData
+    EXTENSION_MUTEX = Mutex.new
+
+    class << self
+      include Gitlab::Utils::StrongMemoize
+
+      def extensions
+        EXTENSION_MUTEX.synchronize do
+          strong_memoize(:extensions) do
+            Set.new.tap do |set|
+              YAML.load_file(Rails.root.join('vendor', 'languages.yml')).each do |_name, details|
+                details['extensions']&.each do |ext|
+                  next unless ext.start_with?('.')
+
+                  set << ext.downcase
+                end
+              end
+            end
+          end
+        end
+      end
+
+      def clear_extensions!
+        EXTENSION_MUTEX.synchronize do
+          clear_memoization(:extensions)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/language_detection.rb b/lib/gitlab/language_detection.rb
new file mode 100644
index 00000000000..7600e60b904
--- /dev/null
+++ b/lib/gitlab/language_detection.rb
@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class LanguageDetection
+    MAX_LANGUAGES = 5
+
+    def initialize(repository, repository_languages)
+      @repository = repository
+      @repository_languages = repository_languages
+    end
+
+    def languages
+      detection.keys
+    end
+
+    def language_color(name)
+      detection.dig(name, :color)
+    end
+
+    # Newly detected languages, returned in a structure accepted by
+    # Gitlab::Database.bulk_insert
+    def insertions(programming_languages)
+      lang_to_id = programming_languages.map { |p| [p.name, p.id] }.to_h
+
+      (languages - previous_language_names).map do |new_lang|
+        {
+          project_id: @repository.project.id,
+          share: detection[new_lang][:value],
+          programming_language_id: lang_to_id[new_lang]
+        }
+      end
+    end
+
+    # updates analyses which records only require updating of their share
+    def updates
+      to_update = @repository_languages.select do |lang|
+        detection.key?(lang.name) && detection[lang.name][:value] != lang.share
+      end
+
+      to_update.map do |lang|
+        { programming_language_id: lang.programming_language_id, share: detection[lang.name][:value] }
+      end
+    end
+
+    # Returns the ids of the programming languages that do not occur in the detection
+    # as current repository languages
+    def deletions
+      @repository_languages.map do |repo_lang|
+        next if detection.key?(repo_lang.name)
+
+        repo_lang.programming_language_id
+      end.compact
+    end
+
+    private
+
+    def previous_language_names
+      @previous_language_names ||= @repository_languages.map(&:name)
+    end
+
+    def detection
+      @detection ||=
+        @repository
+        .languages
+        .first(MAX_LANGUAGES)
+        .map { |l| [l[:label], l] }
+        .to_h
+    end
+  end
+end
diff --git a/lib/gitlab/lazy.rb b/lib/gitlab/lazy.rb
index 99594577141..d7a22aa339e 100644
--- a/lib/gitlab/lazy.rb
+++ b/lib/gitlab/lazy.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   # A class that can be wrapped around an expensive method call so it's only
   # executed when actually needed.
diff --git a/lib/gitlab/legacy_github_import/base_formatter.rb b/lib/gitlab/legacy_github_import/base_formatter.rb
index 2f07fde406c..0b19cf742ed 100644
--- a/lib/gitlab/legacy_github_import/base_formatter.rb
+++ b/lib/gitlab/legacy_github_import/base_formatter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module LegacyGithubImport
     class BaseFormatter
@@ -10,6 +12,7 @@ module Gitlab
         @formatter = Gitlab::ImportFormatter.new
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def create!
         association = project.public_send(project_association) # rubocop:disable GitlabSecurity/PublicSend
 
@@ -17,6 +20,7 @@ module Gitlab
           record.attributes = attributes
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def url
         raw_data.url || ''
diff --git a/lib/gitlab/legacy_github_import/branch_formatter.rb b/lib/gitlab/legacy_github_import/branch_formatter.rb
index 80fe1d67209..1177751457f 100644
--- a/lib/gitlab/legacy_github_import/branch_formatter.rb
+++ b/lib/gitlab/legacy_github_import/branch_formatter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module LegacyGithubImport
     class BranchFormatter < BaseFormatter
diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb
index d8ed0ebca9d..bc952147667 100644
--- a/lib/gitlab/legacy_github_import/client.rb
+++ b/lib/gitlab/legacy_github_import/client.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module LegacyGithubImport
     class Client
diff --git a/lib/gitlab/legacy_github_import/comment_formatter.rb b/lib/gitlab/legacy_github_import/comment_formatter.rb
index d2c7a8ae9f4..d83cc4f6b3c 100644
--- a/lib/gitlab/legacy_github_import/comment_formatter.rb
+++ b/lib/gitlab/legacy_github_import/comment_formatter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module LegacyGithubImport
     class CommentFormatter < BaseFormatter
diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb
index b04d678cf98..c526d31a591 100644
--- a/lib/gitlab/legacy_github_import/importer.rb
+++ b/lib/gitlab/legacy_github_import/importer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module LegacyGithubImport
     class Importer
@@ -78,8 +80,7 @@ module Gitlab
       def handle_errors
         return unless errors.any?
 
-        project.ensure_import_state
-        project.import_state&.update_column(:last_error, {
+        project.import_state.update_column(:last_error, {
           message: 'The remote data could not be fully imported.',
           errors: errors
         }.to_json)
@@ -113,6 +114,7 @@ module Gitlab
         end
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def import_issues
         fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues|
           issues.each do |raw|
@@ -133,6 +135,7 @@ module Gitlab
           end
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def import_pull_requests
         fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests|
@@ -193,6 +196,7 @@ module Gitlab
         issuable.update_attribute(:label_ids, label_ids)
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def import_comments(issuable_type)
         resource_type = "#{issuable_type}_comments".to_sym
 
@@ -213,7 +217,9 @@ module Gitlab
           create_comments(comments)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def create_comments(comments)
         ActiveRecord::Base.no_touching do
           comments.each do |raw|
@@ -238,6 +244,7 @@ module Gitlab
           end
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def discard_inserted_comments(comments, last_note)
         last_note_attrs = nil
diff --git a/lib/gitlab/legacy_github_import/issuable_formatter.rb b/lib/gitlab/legacy_github_import/issuable_formatter.rb
index de55382d3ad..1a0aefbbd62 100644
--- a/lib/gitlab/legacy_github_import/issuable_formatter.rb
+++ b/lib/gitlab/legacy_github_import/issuable_formatter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module LegacyGithubImport
     class IssuableFormatter < BaseFormatter
@@ -55,12 +57,14 @@ module Gitlab
         end
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def milestone
         if raw_data.milestone.present?
           milestone = MilestoneFormatter.new(project, raw_data.milestone)
           project.milestones.find_by(milestone.find_condition)
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/gitlab/legacy_github_import/issue_formatter.rb b/lib/gitlab/legacy_github_import/issue_formatter.rb
index 4c8825ccf19..2f46e2e30d1 100644
--- a/lib/gitlab/legacy_github_import/issue_formatter.rb
+++ b/lib/gitlab/legacy_github_import/issue_formatter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module LegacyGithubImport
     class IssueFormatter < IssuableFormatter
diff --git a/lib/gitlab/legacy_github_import/label_formatter.rb b/lib/gitlab/legacy_github_import/label_formatter.rb
index c3eed12e739..89200e794d8 100644
--- a/lib/gitlab/legacy_github_import/label_formatter.rb
+++ b/lib/gitlab/legacy_github_import/label_formatter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module LegacyGithubImport
     class LabelFormatter < BaseFormatter
@@ -13,6 +15,7 @@ module Gitlab
         :labels
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def create!
         params  = attributes.except(:project)
         service = ::Labels::FindOrCreateService.new(nil, project, params)
@@ -22,6 +25,7 @@ module Gitlab
 
         label
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       private
 
diff --git a/lib/gitlab/legacy_github_import/milestone_formatter.rb b/lib/gitlab/legacy_github_import/milestone_formatter.rb
index a565294384d..2fe1b4258d3 100644
--- a/lib/gitlab/legacy_github_import/milestone_formatter.rb
+++ b/lib/gitlab/legacy_github_import/milestone_formatter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module LegacyGithubImport
     class MilestoneFormatter < BaseFormatter
diff --git a/lib/gitlab/legacy_github_import/project_creator.rb b/lib/gitlab/legacy_github_import/project_creator.rb
index 3ce245a8050..ca1a1b8e9bd 100644
--- a/lib/gitlab/legacy_github_import/project_creator.rb
+++ b/lib/gitlab/legacy_github_import/project_creator.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module LegacyGithubImport
     class ProjectCreator
@@ -35,7 +37,10 @@ module Gitlab
       end
 
       def visibility_level
-        repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.default_project_visibility
+        visibility_level = repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC
+        visibility_level = Gitlab::CurrentSettings.default_project_visibility if Gitlab::CurrentSettings.restricted_visibility_levels.include?(visibility_level)
+
+        visibility_level
       end
 
       #
diff --git a/lib/gitlab/legacy_github_import/pull_request_formatter.rb b/lib/gitlab/legacy_github_import/pull_request_formatter.rb
index 94c2e99066a..5b847f13d4a 100644
--- a/lib/gitlab/legacy_github_import/pull_request_formatter.rb
+++ b/lib/gitlab/legacy_github_import/pull_request_formatter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module LegacyGithubImport
     class PullRequestFormatter < IssuableFormatter
diff --git a/lib/gitlab/legacy_github_import/release_formatter.rb b/lib/gitlab/legacy_github_import/release_formatter.rb
index 3ed9d4f76da..8c0c17780ca 100644
--- a/lib/gitlab/legacy_github_import/release_formatter.rb
+++ b/lib/gitlab/legacy_github_import/release_formatter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module LegacyGithubImport
     class ReleaseFormatter < BaseFormatter
diff --git a/lib/gitlab/legacy_github_import/user_formatter.rb b/lib/gitlab/legacy_github_import/user_formatter.rb
index 6d8055622f1..ec0e221b1ff 100644
--- a/lib/gitlab/legacy_github_import/user_formatter.rb
+++ b/lib/gitlab/legacy_github_import/user_formatter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module LegacyGithubImport
     class UserFormatter
@@ -29,6 +31,7 @@ module Gitlab
             .try(:id)
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def find_by_external_uid
         return nil unless id
 
@@ -40,6 +43,7 @@ module Gitlab
             .first
             .try(:id)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/gitlab/legacy_github_import/wiki_formatter.rb b/lib/gitlab/legacy_github_import/wiki_formatter.rb
index 27f45875c7c..ea52be5ee0f 100644
--- a/lib/gitlab/legacy_github_import/wiki_formatter.rb
+++ b/lib/gitlab/legacy_github_import/wiki_formatter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module LegacyGithubImport
     class WikiFormatter
diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb
index ead5d566871..26b81847d37 100644
--- a/lib/gitlab/lfs_token.rb
+++ b/lib/gitlab/lfs_token.rb
@@ -1,9 +1,22 @@
+# frozen_string_literal: true
+
 module Gitlab
   class LfsToken
-    attr_accessor :actor
+    module LfsTokenHelper
+      def user?
+        actor.is_a?(User)
+      end
+
+      def actor_name
+        user? ? actor.username : "lfs+deploy-key-#{actor.id}"
+      end
+    end
 
-    TOKEN_LENGTH = 50
-    EXPIRY_TIME = 1800
+    include LfsTokenHelper
+
+    DEFAULT_EXPIRE_TIME = 1800
+
+    attr_accessor :actor
 
     def initialize(actor)
       @actor =
@@ -17,36 +30,108 @@ module Gitlab
         end
     end
 
-    def token
-      Gitlab::Redis::SharedState.with do |redis|
-        token = redis.get(redis_shared_state_key)
-        token ||= Devise.friendly_token(TOKEN_LENGTH)
-        redis.set(redis_shared_state_key, token, ex: EXPIRY_TIME)
+    def token(expire_time: DEFAULT_EXPIRE_TIME)
+      HMACToken.new(actor).token(expire_time)
+    end
 
-        token
-      end
+    def token_valid?(token_to_check)
+      HMACToken.new(actor).token_valid?(token_to_check) ||
+        LegacyRedisDeviseToken.new(actor).token_valid?(token_to_check)
     end
 
     def deploy_key_pushable?(project)
       actor.is_a?(DeployKey) && actor.can_push_to?(project)
     end
 
-    def user?
-      actor.is_a?(User)
-    end
-
     def type
-      actor.is_a?(User) ? :lfs_token : :lfs_deploy_token
+      user? ? :lfs_token : :lfs_deploy_token
     end
 
-    def actor_name
-      actor.is_a?(User) ? actor.username : "lfs+deploy-key-#{actor.id}"
+    private # rubocop:disable Lint/UselessAccessModifier
+
+    class HMACToken
+      include LfsTokenHelper
+
+      def initialize(actor)
+        @actor = actor
+      end
+
+      def token(expire_time)
+        hmac_token = JSONWebToken::HMACToken.new(secret)
+        hmac_token.expire_time = Time.now + expire_time
+        hmac_token[:data] = { actor: actor_name }
+        hmac_token.encoded
+      end
+
+      def token_valid?(token_to_check)
+        decoded_token = JSONWebToken::HMACToken.decode(token_to_check, secret).first
+        decoded_token.dig('data', 'actor') == actor_name
+      rescue JWT::DecodeError
+        false
+      end
+
+      private
+
+      attr_reader :actor
+
+      def secret
+        salt + key
+      end
+
+      def salt
+        case actor
+        when DeployKey, Key
+          actor.fingerprint.delete(':').first(16)
+        when User
+          # Take the last 16 characters as they're more unique than the first 16
+          actor.id.to_s + actor.encrypted_password.last(16)
+        end
+      end
+
+      def key
+        # Take 16 characters of attr_encrypted_db_key_base, as that's what the
+        # cipher needs exactly
+        Settings.attr_encrypted_db_key_base.first(16)
+      end
     end
 
-    private
+    # TODO: LegacyRedisDeviseToken and references need to be removed after
+    # next released milestone
+    #
+    class LegacyRedisDeviseToken
+      TOKEN_LENGTH = 50
+      DEFAULT_EXPIRY_TIME = 1800 * 1000 # 30 mins
+
+      def initialize(actor)
+        @actor = actor
+      end
 
-    def redis_shared_state_key
-      "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor
+      def token_valid?(token_to_check)
+        Devise.secure_compare(stored_token, token_to_check)
+      end
+
+      def stored_token
+        Gitlab::Redis::SharedState.with { |redis| redis.get(state_key) }
+      end
+
+      # This method exists purely to facilitate legacy testing to ensure the
+      # same redis key is used.
+      #
+      def store_new_token(expiry_time_in_ms = DEFAULT_EXPIRY_TIME)
+        Gitlab::Redis::SharedState.with do |redis|
+          new_token = Devise.friendly_token(TOKEN_LENGTH)
+          redis.set(state_key, new_token, px: expiry_time_in_ms)
+          new_token
+        end
+      end
+
+      private
+
+      attr_reader :actor
+
+      def state_key
+        "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}"
+      end
     end
   end
 end
diff --git a/lib/gitlab/logger.rb b/lib/gitlab/logger.rb
index a42e312b5d3..128a5dd8936 100644
--- a/lib/gitlab/logger.rb
+++ b/lib/gitlab/logger.rb
@@ -1,13 +1,23 @@
+# frozen_string_literal: true
+
 module Gitlab
   class Logger < ::Logger
     def self.file_name
       file_name_noext + '.log'
     end
 
+    def self.debug(message)
+      build.debug(message)
+    end
+
     def self.error(message)
       build.error(message)
     end
 
+    def self.warn(message)
+      build.warn(message)
+    end
+
     def self.info(message)
       build.info(message)
     end
@@ -22,7 +32,7 @@ module Gitlab
     end
 
     def self.build
-      RequestStore[self.cache_key] ||= new(self.full_log_path)
+      Gitlab::SafeRequestStore[self.cache_key] ||= new(self.full_log_path)
     end
 
     def self.full_log_path
diff --git a/lib/gitlab/loop_helpers.rb b/lib/gitlab/loop_helpers.rb
new file mode 100644
index 00000000000..3873156a3b0
--- /dev/null
+++ b/lib/gitlab/loop_helpers.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module LoopHelpers
+    ##
+    # This helper method repeats the same task until it's expired.
+    #
+    # Note: ExpiredLoopError does not happen until the given block finished.
+    #       Please do not use this method for heavy or asynchronous operations.
+    def loop_until(timeout: nil, limit: 1_000_000)
+      raise ArgumentError unless limit
+
+      start = Time.now
+
+      limit.times do
+        return true unless yield
+
+        return false if timeout && (Time.now - start) > timeout
+      end
+
+      false
+    end
+  end
+end
diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb
index 344784c866f..78f2d83c1af 100644
--- a/lib/gitlab/mail_room.rb
+++ b/lib/gitlab/mail_room.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'yaml'
 require 'json'
 require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues)
@@ -53,7 +55,7 @@ module Gitlab
       end
 
       def config_file
-        ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] || File.expand_path('../../../config/gitlab.yml', __FILE__)
+        ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] || File.expand_path('../../config/gitlab.yml', __dir__)
       end
     end
   end
diff --git a/lib/gitlab/manifest_import/manifest.rb b/lib/gitlab/manifest_import/manifest.rb
new file mode 100644
index 00000000000..7208fe5bbc5
--- /dev/null
+++ b/lib/gitlab/manifest_import/manifest.rb
@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+# Class to parse manifest file and build a list of repositories for import
+#
+# <manifest>
+#   <remote review="https://android-review.googlesource.com/" />
+#   <project path="platform-common" name="platform" />
+#   <project path="platform/art" name="platform/art" />
+#   <project path="platform/device" name="platform/device" />
+# </manifest>
+#
+# 1. Project path must be uniq and can't be part of other project path.
+#    For example, you can't have projects with 'foo' and 'foo/bar' paths.
+# 2. Remote must be present with review attribute so GitLab knows
+#    where to fetch source code
+module Gitlab
+  module ManifestImport
+    class Manifest
+      attr_reader :parsed_xml, :errors
+
+      def initialize(file)
+        @parsed_xml = Nokogiri::XML(file) { |config| config.strict }
+        @errors = []
+      rescue Nokogiri::XML::SyntaxError
+        @errors = ['The uploaded file is not a valid XML file.']
+      end
+
+      def projects
+        raw_projects.each_with_index.map do |project, i|
+          {
+            id: i,
+            name: project['name'],
+            path: project['path'],
+            url: repository_url(project['name'])
+          }
+        end
+      end
+
+      def valid?
+        return false if @errors.any?
+
+        unless validate_remote
+          @errors << 'Make sure a <remote> tag is present and is valid.'
+        end
+
+        unless validate_projects
+          @errors << 'Make sure every <project> tag has name and path attributes.'
+        end
+
+        @errors.empty?
+      end
+
+      private
+
+      def validate_remote
+        remote.present? && URI.parse(remote).host
+      rescue URI::Error
+        false
+      end
+
+      def validate_projects
+        raw_projects.all? do |project|
+          project['name'] && project['path']
+        end
+      end
+
+      def repository_url(name)
+        Gitlab::Utils.append_path(remote, name)
+      end
+
+      def remote
+        return @remote if defined?(@remote)
+
+        remote_tag = parsed_xml.css('manifest > remote').first
+        @remote = remote_tag['review'] if remote_tag
+      end
+
+      def raw_projects
+        @raw_projects ||= parsed_xml.css('manifest > project')
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/manifest_import/project_creator.rb b/lib/gitlab/manifest_import/project_creator.rb
new file mode 100644
index 00000000000..837d65e5f7c
--- /dev/null
+++ b/lib/gitlab/manifest_import/project_creator.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ManifestImport
+    class ProjectCreator
+      attr_reader :repository, :destination, :current_user
+
+      def initialize(repository, destination, current_user)
+        @repository = repository
+        @destination = destination
+        @current_user = current_user
+      end
+
+      def execute
+        group_full_path, _, project_path = repository[:path].rpartition('/')
+        group_full_path = File.join(destination.full_path, group_full_path) if destination
+        group = create_group_with_parents(group_full_path)
+
+        params = {
+          import_url: repository[:url],
+          import_type: 'manifest',
+          namespace_id: group.id,
+          path: project_path,
+          name: project_path,
+          visibility_level: destination.visibility_level
+        }
+
+        Projects::CreateService.new(current_user, params).execute
+      end
+
+      private
+
+      def create_group_with_parents(full_path)
+        params = {
+          group_path: full_path,
+          visibility_level: destination.visibility_level
+        }
+
+        Groups::NestedCreateService.new(current_user, params).execute
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb
index 49285e35251..d419fa66e57 100644
--- a/lib/gitlab/markup_helper.rb
+++ b/lib/gitlab/markup_helper.rb
@@ -1,11 +1,14 @@
+# frozen_string_literal: true
+
 module Gitlab
   module MarkupHelper
     extend self
 
-    MARKDOWN_EXTENSIONS = %w(mdown mkd mkdn md markdown).freeze
-    ASCIIDOC_EXTENSIONS = %w(adoc ad asciidoc).freeze
-    OTHER_EXTENSIONS = %w(textile rdoc org creole wiki mediawiki rst).freeze
+    MARKDOWN_EXTENSIONS = %w[mdown mkd mkdn md markdown].freeze
+    ASCIIDOC_EXTENSIONS = %w[adoc ad asciidoc].freeze
+    OTHER_EXTENSIONS = %w[textile rdoc org creole wiki mediawiki rst].freeze
     EXTENSIONS = MARKDOWN_EXTENSIONS + ASCIIDOC_EXTENSIONS + OTHER_EXTENSIONS
+    PLAIN_FILENAMES = %w[readme index].freeze
 
     # Public: Determines if a given filename is compatible with GitHub::Markup.
     #
@@ -41,7 +44,7 @@ module Gitlab
     #
     # Returns boolean
     def plain?(filename)
-      extension(filename) == 'txt' || filename.casecmp('readme').zero?
+      extension(filename) == 'txt' || plain_filename?(filename)
     end
 
     def previewable?(filename)
@@ -53,5 +56,9 @@ module Gitlab
     def extension(filename)
       File.extname(filename).downcase.delete('.')
     end
+
+    def plain_filename?(filename)
+      PLAIN_FILENAMES.include?(filename.downcase)
+    end
   end
 end
diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 7d63ca5627d..61ed20ad623 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     include Gitlab::Metrics::InfluxDb
diff --git a/lib/gitlab/metrics/background_transaction.rb b/lib/gitlab/metrics/background_transaction.rb
index 5919ebb1493..fe1722b1095 100644
--- a/lib/gitlab/metrics/background_transaction.rb
+++ b/lib/gitlab/metrics/background_transaction.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     class BackgroundTransaction < Transaction
diff --git a/lib/gitlab/metrics/delta.rb b/lib/gitlab/metrics/delta.rb
index bcf28eed84d..ab2d9e46390 100644
--- a/lib/gitlab/metrics/delta.rb
+++ b/lib/gitlab/metrics/delta.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     # Class for calculating the difference between two numeric values.
diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb
index 66f30e3b397..0b04340fbb5 100644
--- a/lib/gitlab/metrics/influx_db.rb
+++ b/lib/gitlab/metrics/influx_db.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     module InfluxDb
@@ -86,7 +88,7 @@ module Gitlab
         # Example:
         #
         #     Gitlab::Metrics.measure(:find_by_username_duration) do
-        #       User.find_by_username(some_username)
+        #       UserFinder.new(some_username).find_by_username
         #     end
         #
         # name - The name of the field to store the execution time in.
@@ -145,9 +147,7 @@ module Gitlab
         #
         # See `Gitlab::Metrics::Transaction#add_event` for more details.
         def add_event(*args)
-          trans = current_transaction
-
-          trans&.add_event(*args)
+          current_transaction&.add_event(*args)
         end
 
         # Returns the prefix to use for the name of a series.
@@ -162,7 +162,6 @@ module Gitlab
 
         # When enabled this should be set before being used as the usual pattern
         # "@foo ||= bar" is _not_ thread-safe.
-        # rubocop:disable Gitlab/ModuleWithInstanceVariables
         def pool
           if influx_metrics_enabled?
             if @pool.nil?
@@ -180,7 +179,6 @@ module Gitlab
             @pool
           end
         end
-        # rubocop:enable Gitlab/ModuleWithInstanceVariables
       end
     end
   end
diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index 023e9963493..651e241362c 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     # Module for instrumenting methods.
diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb
index b11520a79bb..85438011cb9 100644
--- a/lib/gitlab/metrics/method_call.rb
+++ b/lib/gitlab/metrics/method_call.rb
@@ -1,4 +1,4 @@
-# rubocop:disable Style/ClassVars
+# frozen_string_literal: true
 
 module Gitlab
   module Metrics
diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb
index f79eb0cd1bf..447d03bfca4 100644
--- a/lib/gitlab/metrics/methods.rb
+++ b/lib/gitlab/metrics/methods.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # rubocop:disable Style/ClassVars
 
 module Gitlab
diff --git a/lib/gitlab/metrics/methods/metric_options.rb b/lib/gitlab/metrics/methods/metric_options.rb
index 70e122d4e15..8e6ceb74c09 100644
--- a/lib/gitlab/metrics/methods/metric_options.rb
+++ b/lib/gitlab/metrics/methods/metric_options.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     module Methods
diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb
index bd0afe53c51..9e4d70a71ff 100644
--- a/lib/gitlab/metrics/metric.rb
+++ b/lib/gitlab/metrics/metric.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     # Class for storing details of a single metric (label, value, etc).
diff --git a/lib/gitlab/metrics/null_metric.rb b/lib/gitlab/metrics/null_metric.rb
index aabada5c21a..7dbd2a1f8e3 100644
--- a/lib/gitlab/metrics/null_metric.rb
+++ b/lib/gitlab/metrics/null_metric.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     # Mocks ::Prometheus::Client::Metric and all derived metrics
diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb
index d41a855bff1..cab1edab48f 100644
--- a/lib/gitlab/metrics/prometheus.rb
+++ b/lib/gitlab/metrics/prometheus.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'prometheus/client'
 
 module Gitlab
diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb
index 2d45765df3f..9aa97515961 100644
--- a/lib/gitlab/metrics/rack_middleware.rb
+++ b/lib/gitlab/metrics/rack_middleware.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     # Rack middleware for tracking Rails and Grape requests.
diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb
index 0dc19f31d03..74c956ab5af 100644
--- a/lib/gitlab/metrics/requests_rack_middleware.rb
+++ b/lib/gitlab/metrics/requests_rack_middleware.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     class RequestsRackMiddleware
diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb
index 37f90c4673d..6a062e93f0f 100644
--- a/lib/gitlab/metrics/samplers/base_sampler.rb
+++ b/lib/gitlab/metrics/samplers/base_sampler.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'logger'
 
 module Gitlab
diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb
index 5a0f7f28fc8..c4c38b23a55 100644
--- a/lib/gitlab/metrics/samplers/influx_sampler.rb
+++ b/lib/gitlab/metrics/samplers/influx_sampler.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     module Samplers
@@ -16,12 +18,6 @@ module Gitlab
 
           @last_minor_gc = Delta.new(GC.stat[:minor_gc_count])
           @last_major_gc = Delta.new(GC.stat[:major_gc_count])
-
-          if Gitlab::Metrics.mri?
-            require 'allocations'
-
-            Allocations.start
-          end
         end
 
         def sample
diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb
index 4e1ea62351f..232a58a7d69 100644
--- a/lib/gitlab/metrics/samplers/ruby_sampler.rb
+++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'prometheus/client/support/unicorn'
 
 module Gitlab
@@ -20,39 +22,29 @@ module Gitlab
           {}
         end
 
-        def initialize(interval)
-          super(interval)
-
-          if Metrics.mri?
-            require 'allocations'
-
-            Allocations.start
-          end
-        end
-
         def init_metrics
           metrics = {}
-          metrics[:sampler_duration] = Metrics.histogram(with_prefix(:sampler_duration, :seconds), 'Sampler time', { worker: nil })
-          metrics[:total_time] = Metrics.gauge(with_prefix(:gc, :time_total), 'Total GC time', labels, :livesum)
+          metrics[:sampler_duration] = Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels)
+          metrics[:total_time] = Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels)
           GC.stat.keys.each do |key|
-            metrics[key] = Metrics.gauge(with_prefix(:gc, key), to_doc_string(key), labels, :livesum)
+            metrics[key] = Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum)
           end
 
-          metrics[:objects_total] = Metrics.gauge(with_prefix(:objects, :total), 'Objects total', labels.merge(class: nil), :livesum)
-          metrics[:memory_usage] = Metrics.gauge(with_prefix(:memory, :usage_total), 'Memory used total', labels, :livesum)
-          metrics[:file_descriptors] = Metrics.gauge(with_prefix(:file, :descriptors_total), 'File descriptors total', labels, :livesum)
+          metrics[:memory_usage] = Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum)
+          metrics[:file_descriptors] = Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum)
 
           metrics
         end
 
         def sample
           start_time = System.monotonic_time
-          sample_gc
 
-          metrics[:memory_usage].set(labels, System.memory_usage)
-          metrics[:file_descriptors].set(labels, System.file_descriptor_count)
+          metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage)
+          metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count)
+
+          sample_gc
 
-          metrics[:sampler_duration].observe(labels.merge(worker_label), System.monotonic_time - start_time)
+          metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time)
         ensure
           GC::Profiler.clear
         end
@@ -60,11 +52,13 @@ module Gitlab
         private
 
         def sample_gc
-          metrics[:total_time].set(labels, GC::Profiler.total_time * 1000)
-
+          # Collect generic GC stats.
           GC.stat.each do |key, value|
             metrics[key].set(labels, value)
           end
+
+          # Collect the GC time since last sample in float seconds.
+          metrics[:total_time].increment(labels, GC::Profiler.total_time)
         end
 
         def worker_label
diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb
index ea325651fbb..4c4ec026823 100644
--- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb
+++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     module Samplers
diff --git a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb
index 47b4af5d649..56e106b9612 100644
--- a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb
+++ b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'webrick'
 require 'prometheus/client/rack/exporter'
 
diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb
index df4bdf16847..0b4485feea9 100644
--- a/lib/gitlab/metrics/sidekiq_middleware.rb
+++ b/lib/gitlab/metrics/sidekiq_middleware.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     # Sidekiq middleware for tracking jobs.
diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb
index b600e8a2a50..c068f8017fd 100644
--- a/lib/gitlab/metrics/subscribers/action_view.rb
+++ b/lib/gitlab/metrics/subscribers/action_view.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     module Subscribers
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index 4b3e8d0a6a0..a02dd850582 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     module Subscribers
@@ -6,9 +8,15 @@ module Gitlab
         include Gitlab::Metrics::Methods
         attach_to :active_record
 
+        IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze
+
         def sql(event)
           return unless current_transaction
 
+          payload = event.payload
+
+          return if payload[:name] == 'SCHEMA' || IGNORABLE_SQL.include?(payload[:sql])
+
           self.class.gitlab_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0)
 
           current_transaction.increment(:sql_duration, event.duration, false)
@@ -20,7 +28,7 @@ module Gitlab
         define_histogram :gitlab_sql_duration_seconds do
           docstring 'SQL time'
           base_labels Transaction::BASE_LABELS
-          buckets [0.001, 0.01, 0.1, 1.0, 10.0]
+          buckets [0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0]
         end
 
         def current_transaction
diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb
index 250897a79c2..f633e1a9d7c 100644
--- a/lib/gitlab/metrics/subscribers/rails_cache.rb
+++ b/lib/gitlab/metrics/subscribers/rails_cache.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     module Subscribers
diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb
index e60e245cf89..426496855e3 100644
--- a/lib/gitlab/metrics/system.rb
+++ b/lib/gitlab/metrics/system.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     # Module for gathering system/process statistics such as the memory usage.
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index f3e48083c19..468d7cb56fc 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     # Class for storing metrics information of a single transaction.
@@ -140,7 +142,7 @@ module Gitlab
       define_histogram :gitlab_transaction_duration_seconds do
         docstring 'Transaction duration'
         base_labels BASE_LABELS
-        buckets [0.001, 0.01, 0.1, 1.0, 10.0]
+        buckets [0.1, 0.25, 0.5, 1.0, 2.5, 5.0]
       end
 
       define_histogram :gitlab_transaction_allocated_memory_bytes do
diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb
index 3799aaebf1c..b2a43d46fb2 100644
--- a/lib/gitlab/metrics/web_transaction.rb
+++ b/lib/gitlab/metrics/web_transaction.rb
@@ -1,8 +1,11 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Metrics
     class WebTransaction < Transaction
       CONTROLLER_KEY = 'action_controller.instance'.freeze
       ENDPOINT_KEY = 'api.endpoint'.freeze
+      ALLOWED_SUFFIXES = Set.new(%w[json js atom rss xml zip])
 
       def initialize(env)
         super()
@@ -32,10 +35,14 @@ module Gitlab
         # Devise exposes a method called "request_format" that does the below.
         # However, this method is not available to all controllers (e.g. certain
         # Doorkeeper controllers). As such we use the underlying code directly.
-        suffix = controller.request.format.try(:ref)
+        suffix = controller.request.format.try(:ref).to_s
 
-        if suffix && suffix != :html
-          action += ".#{suffix}"
+        # Sometimes the request format is set to silly data such as
+        # "application/xrds+xml" or actual URLs. To prevent such values from
+        # increasing the cardinality of our metrics, we limit the number of
+        # possible suffixes.
+        if suffix && ALLOWED_SUFFIXES.include?(suffix)
+          action = "#{action}.#{suffix}"
         end
 
         { controller: controller.class.name, action: action }
diff --git a/lib/gitlab/middleware/basic_health_check.rb b/lib/gitlab/middleware/basic_health_check.rb
new file mode 100644
index 00000000000..acf8c301b8f
--- /dev/null
+++ b/lib/gitlab/middleware/basic_health_check.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+# This middleware provides a health check that does not hit the database. Its purpose
+# is to notify the prober that the application server is handling requests, but a 200
+# response does not signify that the database or other services are ready.
+#
+# See https://thisdata.com/blog/making-a-rails-health-check-that-doesnt-hit-the-database/ for
+# more details.
+
+module Gitlab
+  module Middleware
+    class BasicHealthCheck
+      # This can't be frozen because Rails::Rack::Logger wraps the body
+      # rubocop:disable Style/MutableConstant
+      OK_RESPONSE = [200, { 'Content-Type' => 'text/plain' }, ["GitLab OK"]]
+      EMPTY_RESPONSE = [404, { 'Content-Type' => 'text/plain' }, [""]]
+      # rubocop:enable Style/MutableConstant
+      HEALTH_PATH = '/-/health'
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        return @app.call(env) unless env['PATH_INFO'] == HEALTH_PATH
+
+        request = ActionDispatch::Request.new(env)
+
+        return OK_RESPONSE if client_ip_whitelisted?(request)
+
+        EMPTY_RESPONSE
+      end
+
+      def client_ip_whitelisted?(request)
+        ip_whitelist.any? { |e| e.include?(request.ip) }
+      end
+
+      def ip_whitelist
+        @ip_whitelist ||= Settings.monitoring.ip_whitelist.map(&IPAddr.method(:new))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/middleware/correlation_id.rb b/lib/gitlab/middleware/correlation_id.rb
new file mode 100644
index 00000000000..80dddc41c12
--- /dev/null
+++ b/lib/gitlab/middleware/correlation_id.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# A dumb middleware that steals correlation id
+# and sets it as a global context for the request
+module Gitlab
+  module Middleware
+    class CorrelationId
+      include ActionView::Helpers::TagHelper
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        ::Gitlab::CorrelationId.use_id(correlation_id(env)) do
+          @app.call(env)
+        end
+      end
+
+      private
+
+      def correlation_id(env)
+        request(env).request_id
+      end
+
+      def request(env)
+        ActionDispatch::Request.new(env)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb
index 1fd8f147b44..f9efef38825 100644
--- a/lib/gitlab/middleware/go.rb
+++ b/lib/gitlab/middleware/go.rb
@@ -1,9 +1,12 @@
+# frozen_string_literal: true
+
 # A dumb middleware that returns a Go HTML document if the go-get=1 query string
 # is used irrespective if the namespace/project exists
 module Gitlab
   module Middleware
     class Go
       include ActionView::Helpers::TagHelper
+      include ActionController::HttpAuthentication::Basic
 
       PROJECT_PATH_REGEX = %r{\A(#{Gitlab::PathRegex.full_namespace_route_regex}/#{Gitlab::PathRegex.project_route_regex})/}.freeze
 
@@ -12,7 +15,7 @@ module Gitlab
       end
 
       def call(env)
-        request = Rack::Request.new(env)
+        request = ActionDispatch::Request.new(env)
 
         render_go_doc(request) || @app.call(env)
       end
@@ -38,7 +41,7 @@ module Gitlab
 
       def go_body(path)
         config = Gitlab.config
-        project_url = URI.join(config.gitlab.url, path)
+        project_url = Gitlab::Utils.append_path(config.gitlab.url, path)
         import_prefix = strip_url(project_url.to_s)
 
         repository_url = if Gitlab::CurrentSettings.enabled_git_access_protocol == 'ssh'
@@ -108,21 +111,23 @@ module Gitlab
 
       def project_for_paths(paths, request)
         project = Project.where_full_path_in(paths).first
-        return unless Ability.allowed?(current_user(request), :read_project, project)
+        return unless Ability.allowed?(current_user(request, project), :read_project, project)
 
         project
       end
 
-      def current_user(request)
-        authenticator = Gitlab::Auth::RequestAuthenticator.new(request)
-        user = authenticator.find_user_from_access_token || authenticator.find_user_from_warden
+      def current_user(request, project)
+        return unless has_basic_credentials?(request)
+
+        login, password = user_name_and_password(request)
+        auth_result = Gitlab::Auth.find_for_git_client(login, password, project: project, ip: request.ip)
+        return unless auth_result.success?
 
-        return unless user&.can?(:access_api)
+        return unless auth_result.actor&.can?(:access_git)
 
-        # Right now, the `api` scope is the only one that should be able to determine private project existence.
-        return unless authenticator.valid_access_token?(scopes: [:api])
+        return unless auth_result.authentication_abilities.include?(:read_project)
 
-        user
+        auth_result.actor
       end
     end
   end
diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb
index a5f5d719cc1..433151b80e7 100644
--- a/lib/gitlab/middleware/multipart.rb
+++ b/lib/gitlab/middleware/multipart.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Gitlab::Middleware::Multipart - a Rack::Multipart replacement
 #
 # Rack::Multipart leaves behind tempfiles in /tmp and uses valuable Ruby
@@ -30,7 +32,7 @@ module Gitlab
 
       class Handler
         def initialize(env, message)
-          @request = Rack::Request.new(env)
+          @request = ActionDispatch::Request.new(env)
           @rewritten_fields = message['rewritten_fields']
           @open_files = []
         end
@@ -42,10 +44,10 @@ module Gitlab
 
             key, value = parsed_field.first
             if value.nil?
-              value = open_file(tmp_path, @request.params["#{key}.name"])
+              value = open_file(@request.params, key)
               @open_files << value
             else
-              value = decorate_params_value(value, @request.params[key], tmp_path)
+              value = decorate_params_value(value, @request.params[key])
             end
 
             @request.update_param(key, value)
@@ -57,7 +59,7 @@ module Gitlab
         end
 
         # This function calls itself recursively
-        def decorate_params_value(path_hash, value_hash, tmp_path)
+        def decorate_params_value(path_hash, value_hash)
           unless path_hash.is_a?(Hash) && path_hash.count == 1
             raise "invalid path: #{path_hash.inspect}"
           end
@@ -70,19 +72,25 @@ module Gitlab
 
           case path_value
           when nil
-            value_hash[path_key] = open_file(tmp_path, value_hash.dig(path_key, '.name'))
+            value_hash[path_key] = open_file(value_hash.dig(path_key), '')
             @open_files << value_hash[path_key]
             value_hash
           when Hash
-            decorate_params_value(path_value, value_hash[path_key], tmp_path)
+            decorate_params_value(path_value, value_hash[path_key])
             value_hash
           else
             raise "unexpected path value: #{path_value.inspect}"
           end
         end
 
-        def open_file(path, name)
-          ::UploadedFile.new(path, filename: name || File.basename(path), content_type: 'application/octet-stream')
+        def open_file(params, key)
+          allowed_paths = [
+            ::FileUploader.root,
+            Gitlab.config.uploads.storage_path,
+            File.join(Rails.root, 'public/uploads/tmp')
+          ]
+
+          ::UploadedFile.from_params(params, key, allowed_paths)
         end
       end
 
diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb
index bc70b2459ef..96c6a0a7d28 100644
--- a/lib/gitlab/middleware/rails_queue_duration.rb
+++ b/lib/gitlab/middleware/rails_queue_duration.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # This Rack middleware is intended to measure the latency between
 # gitlab-workhorse forwarding a request to the Rails application and the
 # time this middleware is reached.
diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb
index 7f63e39b3aa..83c52a6c6e0 100644
--- a/lib/gitlab/middleware/read_only.rb
+++ b/lib/gitlab/middleware/read_only.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Middleware
     class ReadOnly
diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb
index 45b644e6510..817db12ac55 100644
--- a/lib/gitlab/middleware/read_only/controller.rb
+++ b/lib/gitlab/middleware/read_only/controller.rb
@@ -1,11 +1,23 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Middleware
     class ReadOnly
       class Controller
         DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze
         APPLICATION_JSON = 'application/json'.freeze
+        APPLICATION_JSON_TYPES = %W{#{APPLICATION_JSON} application/vnd.git-lfs+json}.freeze
         ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance'.freeze
 
+        WHITELISTED_GIT_ROUTES = {
+          'projects/git_http' => %w{git_upload_pack git_receive_pack}
+        }.freeze
+
+        WHITELISTED_GIT_LFS_ROUTES = {
+          'projects/lfs_api' => %w{batch},
+          'projects/lfs_locks_api' => %w{verify create unlock}
+        }.freeze
+
         def initialize(app, env)
           @app = app
           @env = env
@@ -36,7 +48,7 @@ module Gitlab
         end
 
         def json_request?
-          request.media_type == APPLICATION_JSON
+          APPLICATION_JSON_TYPES.include?(request.media_type)
         end
 
         def rack_flash
@@ -48,7 +60,7 @@ module Gitlab
         end
 
         def request
-          @env['rack.request'] ||= Rack::Request.new(@env)
+          @env['actionpack.request'] ||= ActionDispatch::Request.new(@env)
         end
 
         def last_visited_url
@@ -59,26 +71,40 @@ module Gitlab
           @route_hash ||= Rails.application.routes.recognize_path(request.url, { method: request.request_method }) rescue {}
         end
 
-        def whitelisted_routes
-          grack_route || ReadOnly.internal_routes.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route
+        def relative_url
+          File.join('', Gitlab.config.gitlab.relative_url_root).chomp('/')
         end
 
-        def sidekiq_route
-          request.path.start_with?('/admin/sidekiq')
+        # Overridden in EE module
+        def whitelisted_routes
+          grack_route? || internal_route? || lfs_route? || sidekiq_route?
         end
 
-        def grack_route
+        def grack_route?
           # Calling route_hash may be expensive. Only do it if we think there's a possible match
-          return false unless request.path.end_with?('.git/git-upload-pack')
+          return false unless
+            request.path.end_with?('.git/git-upload-pack', '.git/git-receive-pack')
 
-          route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack'
+          WHITELISTED_GIT_ROUTES[route_hash[:controller]]&.include?(route_hash[:action])
         end
 
-        def lfs_route
+        def internal_route?
+          ReadOnly.internal_routes.any? { |path| request.path.include?(path) }
+        end
+
+        def lfs_route?
           # Calling route_hash may be expensive. Only do it if we think there's a possible match
-          return false unless request.path.end_with?('/info/lfs/objects/batch')
+          unless request.path.end_with?('/info/lfs/objects/batch',
+            '/info/lfs/locks', '/info/lfs/locks/verify') ||
+              %r{/info/lfs/locks/\d+/unlock\z}.match?(request.path)
+            return false
+          end
+
+          WHITELISTED_GIT_LFS_ROUTES[route_hash[:controller]]&.include?(route_hash[:action])
+        end
 
-          route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch'
+        def sidekiq_route?
+          request.path.start_with?("#{relative_url}/admin/sidekiq")
         end
       end
     end
diff --git a/lib/gitlab/middleware/release_env.rb b/lib/gitlab/middleware/release_env.rb
index bfe8e113b5e..849cf8f759b 100644
--- a/lib/gitlab/middleware/release_env.rb
+++ b/lib/gitlab/middleware/release_env.rb
@@ -1,4 +1,7 @@
-module Gitlab # rubocop:disable Naming/FileName
+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+module Gitlab
   module Middleware
     # Some of middleware would hold env for no good reason even after the
     # request had already been processed, and we could not garbage collect
diff --git a/lib/gitlab/middleware/static.rb b/lib/gitlab/middleware/static.rb
index aa1e9dc0fdb..972fed2134c 100644
--- a/lib/gitlab/middleware/static.rb
+++ b/lib/gitlab/middleware/static.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Middleware
     class Static < ActionDispatch::Static
diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb
index fd5de73c526..5375077d7dc 100644
--- a/lib/gitlab/multi_collection_paginator.rb
+++ b/lib/gitlab/multi_collection_paginator.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class MultiCollectionPaginator
     attr_reader :first_collection, :second_collection, :per_page
@@ -53,6 +55,7 @@ module Gitlab
       @first_collection_page_count = first_collection_page.total_pages
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def first_collection_last_page_size
       return @first_collection_last_page_size if defined?(@first_collection_last_page_size)
 
@@ -60,5 +63,6 @@ module Gitlab
                                            .except(:select)
                                            .size
     end
+    # rubocop: enable CodeReuse/ActiveRecord
   end
 end
diff --git a/lib/gitlab/namespace_sanitizer.rb b/lib/gitlab/namespace_sanitizer.rb
new file mode 100644
index 00000000000..d755bbbcaf9
--- /dev/null
+++ b/lib/gitlab/namespace_sanitizer.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class NamespaceSanitizer
+    def self.sanitize(namespace)
+      namespace.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '')
+    end
+  end
+end
diff --git a/lib/gitlab/null_request_store.rb b/lib/gitlab/null_request_store.rb
new file mode 100644
index 00000000000..8db331dcb9f
--- /dev/null
+++ b/lib/gitlab/null_request_store.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# Used by Gitlab::SafeRequestStore
+module Gitlab
+  # The methods `begin!`, `clear!`, and `end!` are not defined because they
+  # should only be called directly on `RequestStore`.
+  class NullRequestStore
+    def store
+      {}
+    end
+
+    def active?
+    end
+
+    def read(key)
+    end
+
+    def [](key)
+    end
+
+    def write(key, value)
+      value
+    end
+
+    def []=(key, value)
+      value
+    end
+
+    def exist?(key)
+      false
+    end
+
+    def fetch(key, &block)
+      yield
+    end
+
+    def delete(key, &block)
+      yield(key) if block_given?
+    end
+  end
+end
diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/object_hierarchy.rb
index 42ded7c286f..f2772c733c7 100644
--- a/lib/gitlab/group_hierarchy.rb
+++ b/lib/gitlab/object_hierarchy.rb
@@ -1,14 +1,16 @@
+# frozen_string_literal: true
+
 module Gitlab
-  # Retrieving of parent or child groups based on a base ActiveRecord relation.
+  # Retrieving of parent or child objects based on a base ActiveRecord relation.
   #
   # This class uses recursive CTEs and as a result will only work on PostgreSQL.
-  class GroupHierarchy
+  class ObjectHierarchy
     attr_reader :ancestors_base, :descendants_base, :model
 
     # ancestors_base - An instance of ActiveRecord::Relation for which to
-    #                  get parent groups.
+    #                  get parent objects.
     # descendants_base - An instance of ActiveRecord::Relation for which to
-    #                    get child groups. If omitted, ancestors_base is used.
+    #                    get child objects. If omitted, ancestors_base is used.
     def initialize(ancestors_base, descendants_base = ancestors_base)
       raise ArgumentError.new("Model of ancestors_base does not match model of descendants_base") if ancestors_base.model != descendants_base.model
 
@@ -19,9 +21,11 @@ module Gitlab
 
     # Returns the set of descendants of a given relation, but excluding the given
     # relation
+    # rubocop: disable CodeReuse/ActiveRecord
     def descendants
       base_and_descendants.where.not(id: descendants_base.select(:id))
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     # Returns the set of ancestors of a given relation, but excluding the given
     # relation
@@ -29,32 +33,45 @@ module Gitlab
     # Passing an `upto` will stop the recursion once the specified parent_id is
     # reached. So all ancestors *lower* than the specified ancestor will be
     # included.
-    def ancestors(upto: nil)
-      base_and_ancestors(upto: upto).where.not(id: ancestors_base.select(:id))
+    # rubocop: disable CodeReuse/ActiveRecord
+    def ancestors(upto: nil, hierarchy_order: nil)
+      base_and_ancestors(upto: upto, hierarchy_order: hierarchy_order).where.not(id: ancestors_base.select(:id))
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
-    # Returns a relation that includes the ancestors_base set of groups
+    # Returns a relation that includes the ancestors_base set of objects
     # and all their ancestors (recursively).
     #
     # Passing an `upto` will stop the recursion once the specified parent_id is
     # reached. So all ancestors *lower* than the specified acestor will be
     # included.
-    def base_and_ancestors(upto: nil)
-      return ancestors_base unless Group.supports_nested_groups?
-
-      read_only(base_and_ancestors_cte(upto).apply_to(model.all))
+    #
+    # Passing a `hierarchy_order` with either `:asc` or `:desc` will cause the
+    # recursive query order from most nested object to root or from the root
+    # ancestor to most nested object respectively. This uses a `depth` column
+    # where `1` is defined as the depth for the base and increment as we go up
+    # each parent.
+    # rubocop: disable CodeReuse/ActiveRecord
+    def base_and_ancestors(upto: nil, hierarchy_order: nil)
+      return ancestors_base unless hierarchy_supported?
+
+      recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all)
+      recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order
+
+      read_only(recursive_query)
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
-    # Returns a relation that includes the descendants_base set of groups
+    # Returns a relation that includes the descendants_base set of objects
     # and all their descendants (recursively).
     def base_and_descendants
-      return descendants_base unless Group.supports_nested_groups?
+      return descendants_base unless hierarchy_supported?
 
       read_only(base_and_descendants_cte.apply_to(model.all))
     end
 
-    # Returns a relation that includes the base groups, their ancestors,
-    # and the descendants of the base groups.
+    # Returns a relation that includes the base objects, their ancestors,
+    # and the descendants of the base objects.
     #
     # The resulting query will roughly look like the following:
     #
@@ -74,46 +91,61 @@ module Gitlab
     # Using this approach allows us to further add criteria to the relation with
     # Rails thinking it's selecting data the usual way.
     #
-    # If nested groups are not supported, ancestors_base is returned.
-    def all_groups
-      return ancestors_base unless Group.supports_nested_groups?
+    # If nested objects are not supported, ancestors_base is returned.
+    # rubocop: disable CodeReuse/ActiveRecord
+    def all_objects
+      return ancestors_base unless hierarchy_supported?
 
       ancestors = base_and_ancestors_cte
       descendants = base_and_descendants_cte
 
-      ancestors_table = ancestors.alias_to(groups_table)
-      descendants_table = descendants.alias_to(groups_table)
-
-      union = SQL::Union.new([model.unscoped.from(ancestors_table),
-                              model.unscoped.from(descendants_table)])
+      ancestors_table = ancestors.alias_to(objects_table)
+      descendants_table = descendants.alias_to(objects_table)
 
       relation = model
         .unscoped
         .with
         .recursive(ancestors.to_arel, descendants.to_arel)
-        .from("(#{union.to_sql}) #{model.table_name}")
+        .from_union([
+          model.unscoped.from(ancestors_table),
+          model.unscoped.from(descendants_table)
+        ])
 
       read_only(relation)
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     private
 
-    def base_and_ancestors_cte(stop_id = nil)
+    def hierarchy_supported?
+      Gitlab::Database.postgresql?
+    end
+
+    # rubocop: disable CodeReuse/ActiveRecord
+    def base_and_ancestors_cte(stop_id = nil, hierarchy_order = nil)
       cte = SQL::RecursiveCTE.new(:base_and_ancestors)
+      depth_column = :depth
 
-      cte << ancestors_base.except(:order)
+      base_query = ancestors_base.except(:order)
+      base_query = base_query.select("1 as #{depth_column}", objects_table[Arel.star]) if hierarchy_order
+
+      cte << base_query
 
       # Recursively get all the ancestors of the base set.
       parent_query = model
-        .from([groups_table, cte.table])
-        .where(groups_table[:id].eq(cte.table[:parent_id]))
+        .from([objects_table, cte.table])
+        .where(objects_table[:id].eq(cte.table[:parent_id]))
         .except(:order)
+
+      parent_query = parent_query.select(cte.table[depth_column] + 1, objects_table[Arel.star]) if hierarchy_order
       parent_query = parent_query.where(cte.table[:parent_id].not_eq(stop_id)) if stop_id
 
       cte << parent_query
       cte
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def base_and_descendants_cte
       cte = SQL::RecursiveCTE.new(:base_and_descendants)
 
@@ -121,14 +153,15 @@ module Gitlab
 
       # Recursively get all the descendants of the base set.
       cte << model
-        .from([groups_table, cte.table])
-        .where(groups_table[:parent_id].eq(cte.table[:id]))
+        .from([objects_table, cte.table])
+        .where(objects_table[:parent_id].eq(cte.table[:id]))
         .except(:order)
 
       cte
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
-    def groups_table
+    def objects_table
       model.arel_table
     end
 
diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb
index 35ed3a5ac05..e0ac9eec1f2 100644
--- a/lib/gitlab/omniauth_initializer.rb
+++ b/lib/gitlab/omniauth_initializer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class OmniauthInitializer
     def initialize(devise_config)
@@ -6,13 +8,16 @@ module Gitlab
 
     def execute(providers)
       providers.each do |provider|
-        add_provider(provider['name'].to_sym, *arguments_for(provider))
+        name = provider['name'].to_sym
+
+        add_provider_to_devise(name, *arguments_for(provider))
+        setup_provider(name)
       end
     end
 
     private
 
-    def add_provider(*args)
+    def add_provider_to_devise(*args)
       @devise_config.omniauth(*args)
     end
 
@@ -71,5 +76,23 @@ module Gitlab
         end
       end
     end
+
+    def omniauth_customized_providers
+      @omniauth_customized_providers ||= build_omniauth_customized_providers
+    end
+
+    # We override this in EE
+    def build_omniauth_customized_providers
+      %i[bitbucket jwt]
+    end
+
+    def setup_provider(provider)
+      case provider
+      when :kerberos
+        require 'omniauth-kerberos'
+      when *omniauth_customized_providers
+        require_dependency "omni_auth/strategies/#{provider}"
+      end
+    end
   end
 end
diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb
index d09bce642b0..ce4ba9f752b 100644
--- a/lib/gitlab/optimistic_locking.rb
+++ b/lib/gitlab/optimistic_locking.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module OptimisticLocking
     module_function
diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb
index fc3f21233dd..bc467486eee 100644
--- a/lib/gitlab/other_markup.rb
+++ b/lib/gitlab/other_markup.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   # Parser/renderer for markups without other special support code.
   module OtherMarkup
diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb
index 22332474945..1d3200aa099 100644
--- a/lib/gitlab/otp_key_rotator.rb
+++ b/lib/gitlab/otp_key_rotator.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   # The +otp_key_base+ param is used to encrypt the User#otp_secret attribute.
   #
@@ -26,6 +28,7 @@ module Gitlab
       @filename = filename
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def rotate!(old_key:, new_key:)
       old_key ||= Gitlab::Application.secrets.otp_key_base
 
@@ -47,7 +50,9 @@ module Gitlab
         end
       end
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def rollback!
       ActiveRecord::Base.transaction do
         CSV.foreach(filename, headers: HEADERS, return_headers: false) do |row|
@@ -55,6 +60,7 @@ module Gitlab
         end
       end
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     private
 
diff --git a/lib/gitlab/pages.rb b/lib/gitlab/pages.rb
index 981ef8faa9a..16df0700b08 100644
--- a/lib/gitlab/pages.rb
+++ b/lib/gitlab/pages.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Pages
     VERSION = File.read(Rails.root.join("GITLAB_PAGES_VERSION")).strip.freeze
diff --git a/lib/gitlab/pages_client.rb b/lib/gitlab/pages_client.rb
index 7b358a3bd1b..d74fdba2241 100644
--- a/lib/gitlab/pages_client.rb
+++ b/lib/gitlab/pages_client.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class PagesClient
     class << self
@@ -101,7 +103,7 @@ module Gitlab
       end
 
       def write_token(new_token)
-        Tempfile.open(File.basename(token_path), File.dirname(token_path),  encoding: 'ascii-8bit') do |f|
+        Tempfile.open(File.basename(token_path), File.dirname(token_path), encoding: 'ascii-8bit') do |f|
           f.write(new_token)
           f.close
           File.link(f.path, token_path)
diff --git a/lib/gitlab/pages_transfer.rb b/lib/gitlab/pages_transfer.rb
index fb215f27cbd..a70dc826f97 100644
--- a/lib/gitlab/pages_transfer.rb
+++ b/lib/gitlab/pages_transfer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class PagesTransfer < ProjectTransfer
     def root_dir
diff --git a/lib/gitlab/patch/draw_route.rb b/lib/gitlab/patch/draw_route.rb
new file mode 100644
index 00000000000..b00244a6e04
--- /dev/null
+++ b/lib/gitlab/patch/draw_route.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# We're patching `ActionDispatch::Routing::Mapper` in
+# config/initializers/routing_draw.rb
+module Gitlab
+  module Patch
+    module DrawRoute
+      RoutesNotFound = Class.new(StandardError)
+
+      def draw(routes_name)
+        drawn_any = draw_ce(routes_name) | draw_ee(routes_name)
+
+        drawn_any || raise(RoutesNotFound.new("Cannot find #{routes_name}"))
+      end
+
+      def draw_ce(routes_name)
+        draw_route(route_path("config/routes/#{routes_name}.rb"))
+      end
+
+      def draw_ee(_)
+        true
+      end
+
+      def route_path(routes_name)
+        Rails.root.join(routes_name)
+      end
+
+      def draw_route(path)
+        if File.exist?(path)
+          instance_eval(File.read(path))
+          true
+        else
+          false
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/patch/prependable.rb b/lib/gitlab/patch/prependable.rb
new file mode 100644
index 00000000000..a9f6cfb19cb
--- /dev/null
+++ b/lib/gitlab/patch/prependable.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+# We're patching `ActiveSupport::Concern` in
+# config/initializers/0_as_concern.rb
+#
+# We want to patch `ActiveSupport::Concern` for two reasons:
+# 1. Allow defining class methods via: `class_methods` method
+# 2. Allow `prepended do; end` work like `included do; end`
+# If we don't need anything above, we don't need this patch nor the concern!
+
+# rubocop:disable Gitlab/ModuleWithInstanceVariables
+module Gitlab
+  module Patch
+    module Prependable
+      class MultiplePrependedBlocks < StandardError
+        def initialize
+          super "Cannot define multiple 'prepended' blocks for a Concern"
+        end
+      end
+
+      def prepend_features(base)
+        return false if prepended?(base)
+
+        super
+
+        if const_defined?(:ClassMethods)
+          klass_methods = const_get(:ClassMethods)
+          base.singleton_class.prepend klass_methods
+          base.instance_variable_set(:@_prepended_class_methods, klass_methods)
+        end
+
+        if instance_variable_defined?(:@_prepended_block)
+          base.class_eval(&@_prepended_block)
+        end
+
+        true
+      end
+
+      def class_methods
+        super
+
+        if instance_variable_defined?(:@_prepended_class_methods)
+          const_get(:ClassMethods).prepend @_prepended_class_methods
+        end
+      end
+
+      def prepended(base = nil, &block)
+        if base.nil?
+          raise MultiplePrependedBlocks if
+            instance_variable_defined?(:@_prepended_block)
+
+          @_prepended_block = block
+        else
+          super
+        end
+      end
+
+      def prepended?(base)
+        index = base.ancestors.index(base)
+
+        base.ancestors[0...index].index(self)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb
index 4dc38aae61e..3c888be0710 100644
--- a/lib/gitlab/path_regex.rb
+++ b/lib/gitlab/path_regex.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module PathRegex
     extend self
@@ -31,6 +33,7 @@ module Gitlab
       deploy.html
       explore
       favicon.ico
+      favicon.png
       files
       groups
       health_check
@@ -38,7 +41,7 @@ module Gitlab
       import
       invites
       jwt
-      koding
+      login
       notification_settings
       oauth
       profile
@@ -122,7 +125,8 @@ module Gitlab
     # allow non-regex validations, etc), `NAMESPACE_FORMAT_REGEX_JS` serves as a Javascript-compatible version of
     # `NAMESPACE_FORMAT_REGEX`, with the negative lookbehind assertion removed. This means that the client-side validation
     # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation.
-    PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze
+    PATH_START_CHAR = '[a-zA-Z0-9_\.]'.freeze
+    PATH_REGEX_STR = PATH_START_CHAR + '[a-zA-Z0-9_\-\.]*'.freeze
     NAMESPACE_FORMAT_REGEX_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze
 
     NO_SUFFIX_REGEX = /(?<!\.git|\.atom)/.freeze
@@ -233,7 +237,7 @@ module Gitlab
 
     def single_line_regexp(regex)
       # Turns a multiline extended regexp into a single line one,
-      # beacuse `rake routes` breaks on multiline regexes.
+      # because `rake routes` breaks on multiline regexes.
       Regexp.new(regex.source.gsub(/\(\?#.+?\)/, '').gsub(/\s*/, ''), regex.options ^ Regexp::EXTENDED).freeze
     end
   end
diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb
index 92a308a12dc..4b0c7b5c7f8 100644
--- a/lib/gitlab/performance_bar.rb
+++ b/lib/gitlab/performance_bar.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module PerformanceBar
     ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze
@@ -15,6 +17,7 @@ module Gitlab
       Gitlab::CurrentSettings.performance_bar_allowed_group_id
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def self.allowed_user_ids
       Rails.cache.fetch(ALLOWED_USER_IDS_KEY, expires_in: EXPIRY_TIME) do
         group = Group.find_by_id(allowed_group_id)
@@ -26,6 +29,7 @@ module Gitlab
         end
       end
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     def self.expire_allowed_user_ids_cache
       Rails.cache.delete(ALLOWED_USER_IDS_KEY)
diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb
index f2825db59ae..ac392432427 100644
--- a/lib/gitlab/performance_bar/peek_query_tracker.rb
+++ b/lib/gitlab/performance_bar/peek_query_tracker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Inspired by https://github.com/peek/peek-pg/blob/master/lib/peek/views/pg.rb
 # PEEK_DB_CLIENT is a constant set in config/initializers/peek.rb
 module Gitlab
@@ -23,7 +25,7 @@ module Gitlab
         end
 
         subscribe('sql.active_record') do |_, start, finish, _, data|
-          if RequestStore.active? && RequestStore.store[:peek_enabled]
+          if Gitlab::SafeRequestStore.store[:peek_enabled]
             # data[:cached] is only available starting from Rails 5.1.0
             # https://github.com/rails/rails/blob/v5.1.0/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L113
             # Before that, data[:name] was set to 'CACHE'
diff --git a/lib/gitlab/plugin.rb b/lib/gitlab/plugin.rb
index 0d1cb16b378..23353f36025 100644
--- a/lib/gitlab/plugin.rb
+++ b/lib/gitlab/plugin.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Plugin
     def self.files
diff --git a/lib/gitlab/plugin_logger.rb b/lib/gitlab/plugin_logger.rb
index c4f6ec3e21d..df3bd56fd2f 100644
--- a/lib/gitlab/plugin_logger.rb
+++ b/lib/gitlab/plugin_logger.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class PluginLogger < Gitlab::Logger
     def self.file_name_noext
diff --git a/lib/gitlab/polling_interval.rb b/lib/gitlab/polling_interval.rb
index fe4bdfe3831..0f69990df63 100644
--- a/lib/gitlab/polling_interval.rb
+++ b/lib/gitlab/polling_interval.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class PollingInterval
     HEADER_NAME = 'Poll-Interval'.freeze
diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb
index b9832a724c4..7fa00d0c68c 100644
--- a/lib/gitlab/popen.rb
+++ b/lib/gitlab/popen.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'fileutils'
 require 'open3'
 
@@ -11,7 +13,7 @@ module Gitlab
     def popen(cmd, path = nil, vars = {}, &block)
       result = popen_with_detail(cmd, path, vars, &block)
 
-      [result.stdout << result.stderr, result.status&.exitstatus]
+      ["#{result.stdout}#{result.stderr}", result.status&.exitstatus]
     end
 
     # Returns Result
@@ -34,11 +36,16 @@ module Gitlab
       start = Time.now
 
       Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr|
+        # stderr and stdout pipes can block if stderr/stdout aren't drained: https://bugs.ruby-lang.org/issues/9082
+        # Mimic what Ruby does with capture3: https://github.com/ruby/ruby/blob/1ec544695fa02d714180ef9c34e755027b6a2103/lib/open3.rb#L257-L273
+        out_reader = Thread.new { stdout.read }
+        err_reader = Thread.new { stderr.read }
+
         yield(stdin) if block_given?
         stdin.close
 
-        cmd_stdout = stdout.read
-        cmd_stderr = stderr.read
+        cmd_stdout = out_reader.value
+        cmd_stderr = err_reader.value
         cmd_status = wait_thr.value
       end
 
diff --git a/lib/gitlab/popen/runner.rb b/lib/gitlab/popen/runner.rb
index f44035a48bb..cd9ad270cd8 100644
--- a/lib/gitlab/popen/runner.rb
+++ b/lib/gitlab/popen/runner.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Popen
     class Runner
diff --git a/lib/gitlab/private_commit_email.rb b/lib/gitlab/private_commit_email.rb
new file mode 100644
index 00000000000..536fc9dae3a
--- /dev/null
+++ b/lib/gitlab/private_commit_email.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module PrivateCommitEmail
+    TOKEN = "_private".freeze
+
+    class << self
+      def regex
+        hostname_regexp = Regexp.escape(Gitlab::CurrentSettings.current_application_settings.commit_email_hostname)
+
+        /\A(?<id>([0-9]+))\-([^@]+)@#{hostname_regexp}\z/
+      end
+
+      def user_id_for_email(email)
+        match = email&.match(regex)
+        return unless match
+
+        match[:id].to_i
+      end
+
+      def user_ids_for_emails(emails)
+        emails.map { |email| user_id_for_email(email) }.compact.uniq
+      end
+
+      def for_user(user)
+        hostname = Gitlab::CurrentSettings.current_application_settings.commit_email_hostname
+
+        "#{user.id}-#{user.username}@#{hostname}"
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb
index 18540e64d4c..93a9fcf1591 100644
--- a/lib/gitlab/profiler.rb
+++ b/lib/gitlab/profiler.rb
@@ -1,4 +1,6 @@
 # coding: utf-8
+# frozen_string_literal: true
+
 module Gitlab
   module Profiler
     FILTERED_STRING = '[FILTERED]'.freeze
@@ -11,6 +13,7 @@ module Gitlab
       lib/gitlab/etag_caching/
       lib/gitlab/metrics/
       lib/gitlab/middleware/
+      ee/lib/gitlab/middleware/
       lib/gitlab/performance_bar/
       lib/gitlab/request_profiler/
       lib/gitlab/profiler.rb
@@ -43,12 +46,11 @@ module Gitlab
         headers['Content-Type'] = 'application/json'
       end
 
-      if user
-        private_token ||= user.personal_access_tokens.active.pluck(:token).first
-        raise 'Your user must have a personal_access_token' unless private_token
+      if private_token
+        headers['Private-Token'] = private_token
+        user = nil # private_token overrides user
       end
 
-      headers['Private-Token'] = private_token if private_token
       logger = create_custom_logger(logger, private_token: private_token)
 
       RequestStore.begin!
@@ -66,7 +68,9 @@ module Gitlab
       app.get('/api/v4/users')
 
       result = with_custom_logger(logger) do
-        RubyProf.profile { app.public_send(verb, url, post_data, headers) } # rubocop:disable GitlabSecurity/PublicSend
+        with_user(user) do
+          RubyProf.profile { app.public_send(verb, url, post_data, headers) } # rubocop:disable GitlabSecurity/PublicSend
+        end
       end
 
       RequestStore.end!
@@ -98,11 +102,7 @@ module Gitlab
 
             super
 
-            backtrace = Rails.backtrace_cleaner.clean(caller)
-
-            backtrace.each do |caller_line|
-              next if caller_line.match(Regexp.union(IGNORE_BACKTRACES))
-
+            Gitlab::Profiler.clean_backtrace(caller).each do |caller_line|
               stripped_caller_line = caller_line.sub("#{Rails.root}/", '')
 
               super("  ↳ #{stripped_caller_line}")
@@ -112,6 +112,12 @@ module Gitlab
       end
     end
 
+    def self.clean_backtrace(backtrace)
+      Array(Rails.backtrace_cleaner.clean(backtrace)).reject do |line|
+        line.match(Regexp.union(IGNORE_BACKTRACES))
+      end
+    end
+
     def self.with_custom_logger(logger)
       original_colorize_logging = ActiveSupport::LogSubscriber.colorize_logging
       original_activerecord_logger = ActiveRecord::Base.logger
@@ -123,15 +129,32 @@ module Gitlab
         ActionController::Base.logger = logger
       end
 
-      result = yield
+      yield.tap do
+        ActiveSupport::LogSubscriber.colorize_logging = original_colorize_logging
+        ActiveRecord::Base.logger = original_activerecord_logger
+        ActionController::Base.logger = original_actioncontroller_logger
+      end
+    end
 
-      ActiveSupport::LogSubscriber.colorize_logging = original_colorize_logging
-      ActiveRecord::Base.logger = original_activerecord_logger
-      ActionController::Base.logger = original_actioncontroller_logger
+    def self.with_user(user)
+      if user
+        API::Helpers::CommonHelpers.send(:define_method, :find_current_user!) { user } # rubocop:disable GitlabSecurity/PublicSend
+        ApplicationController.send(:define_method, :current_user) { user } # rubocop:disable GitlabSecurity/PublicSend
+        ApplicationController.send(:define_method, :authenticate_user!) { } # rubocop:disable GitlabSecurity/PublicSend
+      end
 
-      result
+      yield.tap do
+        remove_method(API::Helpers::CommonHelpers, :find_current_user!)
+        remove_method(ApplicationController, :current_user)
+        remove_method(ApplicationController, :authenticate_user!)
+      end
     end
 
+    def self.remove_method(klass, meth)
+      klass.send(:remove_method, meth) if klass.instance_methods(false).include?(meth) # rubocop:disable GitlabSecurity/PublicSend
+    end
+
+    # rubocop: disable CodeReuse/ActiveRecord
     def self.log_load_times_by_model(logger)
       return unless logger.respond_to?(:load_times_by_model)
 
@@ -143,5 +166,12 @@ module Gitlab
         logger.info("#{model} total (#{query_count}): #{time.round(2)}ms")
       end
     end
+    # rubocop: enable CodeReuse/ActiveRecord
+
+    def self.print_by_total_time(result, options = {})
+      default_options = { sort_method: :total_time }
+
+      Gitlab::Profiler::TotalTimeFlatPrinter.new(result).print(STDOUT, default_options.merge(options))
+    end
   end
 end
diff --git a/lib/gitlab/profiler/total_time_flat_printer.rb b/lib/gitlab/profiler/total_time_flat_printer.rb
new file mode 100644
index 00000000000..2c105d2722b
--- /dev/null
+++ b/lib/gitlab/profiler/total_time_flat_printer.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Profiler
+    class TotalTimeFlatPrinter < RubyProf::FlatPrinter
+      def max_percent
+        @options[:max_percent] || 100
+      end
+
+      # Copied from:
+      #   <https://github.com/ruby-prof/ruby-prof/blob/master/lib/ruby-prof/printers/flat_printer.rb>
+      #
+      # The changes are just to filter by total time, not self time, and add a
+      # max_percent option as well.
+      def print_methods(thread)
+        total_time = thread.total_time
+        methods = thread.methods.sort_by(&sort_method).reverse
+
+        sum = 0
+        methods.each do |method|
+          total_percent = (method.total_time / total_time) * 100
+          next if total_percent < min_percent
+          next if total_percent > max_percent
+
+          sum += method.self_time
+
+          @output << "%6.2f  %9.3f %9.3f %9.3f %9.3f %8d  %s%s\n" % [
+            method.self_time / total_time * 100, # %self
+            method.total_time,                   # total
+            method.self_time,                    # self
+            method.wait_time,                    # wait
+            method.children_time,                # children
+            method.called,                       # calls
+            method.recursive? ? "*" : " ",       # cycle
+            method_name(method)                  # name
+          ]
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb
index 15b8beacf60..2372a316ab0 100644
--- a/lib/gitlab/project_authorizations/with_nested_groups.rb
+++ b/lib/gitlab/project_authorizations/with_nested_groups.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ProjectAuthorizations
     # Calculating new project authorizations when supporting nested groups.
@@ -24,7 +26,7 @@ module Gitlab
           user.projects.select_for_project_authorization,
 
           # The personal projects of the user.
-          user.personal_projects.select_as_master_for_project_authorization,
+          user.personal_projects.select_as_maintainer_for_project_authorization,
 
           # Projects that belong directly to any of the groups the user has
           # access to.
@@ -49,13 +51,11 @@ module Gitlab
             .where('p_ns.share_with_group_lock IS FALSE')
         ]
 
-        union = Gitlab::SQL::Union.new(relations)
-
         ProjectAuthorization
           .unscoped
           .with
           .recursive(cte.to_arel)
-          .select_from_union(union)
+          .select_from_union(relations)
       end
 
       private
diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb
index ad87540e6c2..50b41b17649 100644
--- a/lib/gitlab/project_authorizations/without_nested_groups.rb
+++ b/lib/gitlab/project_authorizations/without_nested_groups.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ProjectAuthorizations
     # Calculating new project authorizations when not supporting nested groups.
@@ -15,7 +17,7 @@ module Gitlab
           user.projects.select_for_project_authorization,
 
           # Personal projects
-          user.personal_projects.select_as_master_for_project_authorization,
+          user.personal_projects.select_as_maintainer_for_project_authorization,
 
           # Projects of groups the user is a member of
           user.groups_projects.select_for_project_authorization,
@@ -24,11 +26,9 @@ module Gitlab
           user.groups.joins(:shared_projects).select_for_project_authorization
         ]
 
-        union = Gitlab::SQL::Union.new(relations)
-
         ProjectAuthorization
           .unscoped
-          .select_from_union(union)
+          .select_from_union(relations)
       end
     end
   end
diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index 2e9b6e302f5..a68f8801c2a 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class ProjectSearchResults < SearchResults
     attr_reader :project, :repository_ref
@@ -5,7 +7,7 @@ module Gitlab
     def initialize(current_user, project, query, repository_ref = nil, per_page: 20)
       @current_user = current_user
       @project = project
-      @repository_ref = repository_ref.presence || project.default_branch
+      @repository_ref = repository_ref.presence
       @query = query
       @per_page = per_page
     end
@@ -15,9 +17,9 @@ module Gitlab
       when 'notes'
         notes.page(page).per(per_page)
       when 'blobs'
-        Kaminari.paginate_array(blobs).page(page).per(per_page)
+        paginated_blobs(blobs, page)
       when 'wiki_blobs'
-        Kaminari.paginate_array(wiki_blobs).page(page).per(per_page)
+        paginated_blobs(wiki_blobs, page)
       when 'commits'
         Kaminari.paginate_array(commits).page(page).per(per_page)
       else
@@ -29,6 +31,7 @@ module Gitlab
       @blobs_count ||= blobs.count
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def limited_notes_count
       return @limited_notes_count if defined?(@limited_notes_count)
 
@@ -42,6 +45,7 @@ module Gitlab
 
       @limited_notes_count
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     def wiki_blobs_count
       @wiki_blobs_count ||= wiki_blobs.count
@@ -51,36 +55,6 @@ module Gitlab
       @commits_count ||= commits.count
     end
 
-    def self.parse_search_result(result, project = nil)
-      ref = nil
-      filename = nil
-      basename = nil
-      data = ""
-      startline = 0
-
-      result.each_line.each_with_index do |line, index|
-        prefix ||= line.match(/^(?<ref>[^:]*):(?<filename>[^\x00]*)\x00(?<startline>\d+)\x00/)&.tap do |matches|
-          ref = matches[:ref]
-          filename = matches[:filename]
-          startline = matches[:startline]
-          startline = startline.to_i - index
-          extname = Regexp.escape(File.extname(filename))
-          basename = filename.sub(/#{extname}$/, '')
-        end
-
-        data << line.sub(prefix.to_s, '')
-      end
-
-      FoundBlob.new(
-        filename: filename,
-        basename: basename,
-        ref: ref,
-        startline: startline,
-        data: data,
-        project_id: project ? project.id : nil
-      )
-    end
-
     def single_commit_result?
       return false if commits_count != 1
 
@@ -92,10 +66,18 @@ module Gitlab
 
     private
 
+    def paginated_blobs(blobs, page)
+      results = Kaminari.paginate_array(blobs).page(page).per(per_page)
+
+      Gitlab::Search::FoundBlob.preload_blobs(results)
+
+      results
+    end
+
     def blobs
       return [] unless Ability.allowed?(@current_user, :download_code, @project)
 
-      @blobs ||= Gitlab::FileFinder.new(project, repository_ref).find(query)
+      @blobs ||= Gitlab::FileFinder.new(project, repository_project_ref).find(query)
     end
 
     def wiki_blobs
@@ -103,10 +85,8 @@ module Gitlab
 
       @wiki_blobs ||= begin
         if project.wiki_enabled? && query.present?
-          project_wiki = ProjectWiki.new(project)
-
-          unless project_wiki.empty?
-            project_wiki.search_files(query)
+          unless project.wiki.empty?
+            Gitlab::WikiFileFinder.new(project, repository_wiki_ref).find(query)
           else
             []
           end
@@ -120,9 +100,11 @@ module Gitlab
       @notes ||= notes_finder(nil)
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def notes_finder(type)
       NotesFinder.new(project, @current_user, search: query, target_type: type).execute.user.order('updated_at DESC')
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     def commits
       @commits ||= find_commits(query)
@@ -149,5 +131,13 @@ module Gitlab
     def project_ids_relation
       project
     end
+
+    def repository_project_ref
+      @repository_project_ref ||= repository_ref || project.default_branch
+    end
+
+    def repository_wiki_ref
+      @repository_wiki_ref ||= repository_ref || project.wiki.default_branch
+    end
   end
 end
diff --git a/lib/gitlab/project_service_logger.rb b/lib/gitlab/project_service_logger.rb
new file mode 100644
index 00000000000..9b0357d3161
--- /dev/null
+++ b/lib/gitlab/project_service_logger.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class ProjectServiceLogger < Gitlab::JsonLogger
+    def self.file_name_noext
+      'integrations_json'
+    end
+  end
+end
diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb
index 08f6a54776f..3bfd6ee892c 100644
--- a/lib/gitlab/project_template.rb
+++ b/lib/gitlab/project_template.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class ProjectTemplate
     attr_reader :title, :name, :description, :preview
diff --git a/lib/gitlab/project_transfer.rb b/lib/gitlab/project_transfer.rb
index 690c38737c0..d8f1d1e2316 100644
--- a/lib/gitlab/project_transfer.rb
+++ b/lib/gitlab/project_transfer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   # This class is used to move local, unhashed files owned by projects to their new location
   class ProjectTransfer
diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb
index bb1172f82a1..bd4ca578840 100644
--- a/lib/gitlab/prometheus/additional_metrics_parser.rb
+++ b/lib/gitlab/prometheus/additional_metrics_parser.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Prometheus
     module AdditionalMetricsParser
@@ -5,7 +7,7 @@ module Gitlab
       MUTEX = Mutex.new
       extend self
 
-      def load_groups_from_yaml(file_name = 'additional_metrics.yml')
+      def load_groups_from_yaml(file_name)
         yaml_metrics_raw(file_name).map(&method(:group_from_entry))
       end
 
diff --git a/lib/gitlab/prometheus/metric.rb b/lib/gitlab/prometheus/metric.rb
index f54b2c6aaff..7ebfc2e25a9 100644
--- a/lib/gitlab/prometheus/metric.rb
+++ b/lib/gitlab/prometheus/metric.rb
@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Prometheus
     class Metric
       include ActiveModel::Model
 
-      attr_accessor :title, :required_metrics, :weight, :y_label, :queries
+      attr_accessor :id, :title, :required_metrics, :weight, :y_label, :queries
 
       validates :title, :required_metrics, :weight, :y_label, :queries, presence: true
 
diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb
index e91c6fb2e27..394556e8708 100644
--- a/lib/gitlab/prometheus/metric_group.rb
+++ b/lib/gitlab/prometheus/metric_group.rb
@@ -1,13 +1,24 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Prometheus
     class MetricGroup
       include ActiveModel::Model
 
       attr_accessor :name, :priority, :metrics
+
       validates :name, :priority, :metrics, presence: true
 
       def self.common_metrics
-        AdditionalMetricsParser.load_groups_from_yaml
+        all_groups = ::PrometheusMetric.common.group_by(&:group_title).map do |name, metrics|
+          MetricGroup.new(
+            name: name,
+            priority: metrics.map(&:priority).max,
+            metrics: metrics.map(&:to_query_metric)
+          )
+        end
+
+        all_groups.sort_by(&:priority).reverse
       end
 
       # EE only
diff --git a/lib/gitlab/prometheus/parsing_error.rb b/lib/gitlab/prometheus/parsing_error.rb
index 49cc0e16080..20b5ef5ce55 100644
--- a/lib/gitlab/prometheus/parsing_error.rb
+++ b/lib/gitlab/prometheus/parsing_error.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Prometheus
     ParsingError = Class.new(StandardError)
diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb
index e677ec84cd4..ab6ef7d5466 100644
--- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb
+++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb
@@ -1,13 +1,17 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Prometheus
     module Queries
       class AdditionalMetricsDeploymentQuery < BaseQuery
         include QueryAdditionalMetrics
 
+        # rubocop: disable CodeReuse/ActiveRecord
         def query(deployment_id)
           Deployment.find_by(id: deployment_id).try do |deployment|
             query_metrics(
               deployment.project,
+              deployment.environment,
               common_query_context(
                 deployment.environment,
                 timeframe_start: (deployment.created_at - 30.minutes).to_f,
@@ -16,6 +20,7 @@ module Gitlab
             )
           end
         end
+        # rubocop: enable CodeReuse/ActiveRecord
       end
     end
   end
diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb
index 9273e69e158..34b705138ba 100644
--- a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb
+++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb
@@ -1,17 +1,22 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Prometheus
     module Queries
       class AdditionalMetricsEnvironmentQuery < BaseQuery
         include QueryAdditionalMetrics
 
+        # rubocop: disable CodeReuse/ActiveRecord
         def query(environment_id)
           ::Environment.find_by(id: environment_id).try do |environment|
             query_metrics(
               environment.project,
+              environment,
               common_query_context(environment, timeframe_start: 8.hours.ago.to_f, timeframe_end: Time.now.to_f)
             )
           end
         end
+        # rubocop: enable CodeReuse/ActiveRecord
       end
     end
   end
diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb
index 29cab6e9c15..9ff414d5236 100644
--- a/lib/gitlab/prometheus/queries/base_query.rb
+++ b/lib/gitlab/prometheus/queries/base_query.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Prometheus
     module Queries
diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb
index c2626581897..fc32c4353f0 100644
--- a/lib/gitlab/prometheus/queries/deployment_query.rb
+++ b/lib/gitlab/prometheus/queries/deployment_query.rb
@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Prometheus
     module Queries
       class DeploymentQuery < BaseQuery
+        # rubocop: disable CodeReuse/ActiveRecord
         def query(deployment_id)
           Deployment.find_by(id: deployment_id).try do |deployment|
             environment_slug = deployment.environment.slug
@@ -25,6 +28,7 @@ module Gitlab
             }
           end
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         def self.transform_reactive_result(result)
           result[:metrics] = result.delete :data
diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb
index b62910c8de6..56195f85a70 100644
--- a/lib/gitlab/prometheus/queries/environment_query.rb
+++ b/lib/gitlab/prometheus/queries/environment_query.rb
@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Prometheus
     module Queries
       class EnvironmentQuery < BaseQuery
+        # rubocop: disable CodeReuse/ActiveRecord
         def query(environment_id)
           ::Environment.find_by(id: environment_id).try do |environment|
             environment_slug = environment.slug
@@ -19,6 +22,7 @@ module Gitlab
             }
           end
         end
+        # rubocop: enable CodeReuse/ActiveRecord
 
         def self.transform_reactive_result(result)
           result[:metrics] = result.delete :data
diff --git a/lib/gitlab/prometheus/queries/matched_metric_query.rb b/lib/gitlab/prometheus/queries/matched_metric_query.rb
index d920e9a749f..32294756aa2 100644
--- a/lib/gitlab/prometheus/queries/matched_metric_query.rb
+++ b/lib/gitlab/prometheus/queries/matched_metric_query.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Prometheus
     module Queries
diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb
index f5879de1e94..960d3536ec0 100644
--- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb
+++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb
@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Prometheus
     module Queries
       module QueryAdditionalMetrics
-        def query_metrics(project, query_context)
+        def query_metrics(project, environment, query_context)
           matched_metrics(project).map(&query_group(query_context))
             .select(&method(:group_with_any_metrics))
         end
@@ -14,12 +16,16 @@ module Gitlab
 
           lambda do |group|
             metrics = group.metrics.map do |metric|
-              {
+              metric_hsh = {
                 title: metric.title,
                 weight: metric.weight,
                 y_label: metric.y_label,
                 queries: metric.queries.map(&query_processor).select(&method(:query_with_result))
               }
+
+              metric_hsh[:id] = metric.id if metric.id
+
+              metric_hsh
             end
 
             {
@@ -77,11 +83,8 @@ module Gitlab
         end
 
         def common_query_context(environment, timeframe_start:, timeframe_end:)
-          base_query_context(timeframe_start, timeframe_end).merge({
-            ci_environment_slug: environment.slug,
-            kube_namespace: environment.deployment_platform&.actual_namespace || '',
-            environment_filter: %{container_name!="POD",environment="#{environment.slug}"}
-          })
+          base_query_context(timeframe_start, timeframe_end)
+            .merge(QueryVariables.call(environment))
         end
 
         def base_query_context(timeframe_start, timeframe_end)
diff --git a/lib/gitlab/prometheus/query_variables.rb b/lib/gitlab/prometheus/query_variables.rb
new file mode 100644
index 00000000000..1cc85d4b4a6
--- /dev/null
+++ b/lib/gitlab/prometheus/query_variables.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Prometheus
+    module QueryVariables
+      def self.call(environment)
+        {
+          ci_environment_slug: environment.slug,
+          kube_namespace: environment.deployment_platform&.actual_namespace || '',
+          environment_filter: %{container_name!="POD",environment="#{environment.slug}"}
+        }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb
index b66253a10e0..45828c77a33 100644
--- a/lib/gitlab/prometheus_client.rb
+++ b/lib/gitlab/prometheus_client.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   # Helper methods to interact with Prometheus network services & resources
   class PrometheusClient
diff --git a/lib/gitlab/protocol_access.rb b/lib/gitlab/protocol_access.rb
index 2819c7d062c..efeb1e07d49 100644
--- a/lib/gitlab/protocol_access.rb
+++ b/lib/gitlab/protocol_access.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module ProtocolAccess
     def self.allowed?(protocol)
diff --git a/lib/gitlab/proxy_http_connection_adapter.rb b/lib/gitlab/proxy_http_connection_adapter.rb
index d682289b632..a64cb47e77e 100644
--- a/lib/gitlab/proxy_http_connection_adapter.rb
+++ b/lib/gitlab/proxy_http_connection_adapter.rb
@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 # This class is part of the Gitlab::HTTP wrapper. Depending on the value
 # of the global setting allow_local_requests_from_hooks_and_services this adapter
 # will allow/block connection to internal IPs and/or urls.
 #
-# This functionality can be overriden by providing the setting the option
+# This functionality can be overridden by providing the setting the option
 # allow_local_requests = true in the request. For example:
 # Gitlab::HTTP.get('http://www.gitlab.com', allow_local_requests: true)
 #
diff --git a/lib/gitlab/query_limiting.rb b/lib/gitlab/query_limiting.rb
index 9f69a9e4a39..31e6b120e45 100644
--- a/lib/gitlab/query_limiting.rb
+++ b/lib/gitlab/query_limiting.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module QueryLimiting
     # Returns true if we should enable tracking of query counts.
diff --git a/lib/gitlab/query_limiting/active_support_subscriber.rb b/lib/gitlab/query_limiting/active_support_subscriber.rb
index 3c4ff5d1928..065862174bb 100644
--- a/lib/gitlab/query_limiting/active_support_subscriber.rb
+++ b/lib/gitlab/query_limiting/active_support_subscriber.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module QueryLimiting
     class ActiveSupportSubscriber < ActiveSupport::Subscriber
diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb
index 66d7d9275cf..e8fad067fa6 100644
--- a/lib/gitlab/query_limiting/transaction.rb
+++ b/lib/gitlab/query_limiting/transaction.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module QueryLimiting
     class Transaction
@@ -68,7 +70,7 @@ module Gitlab
 
       def error_message
         header = 'Too many SQL queries were executed'
-        header += " in #{action}" if action
+        header = "#{header} in #{action}" if action
 
         "#{header}: a maximum of #{THRESHOLD} is allowed but #{count} SQL queries were executed"
       end
diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb
index 96415271316..259345b8a9a 100644
--- a/lib/gitlab/quick_actions/command_definition.rb
+++ b/lib/gitlab/quick_actions/command_definition.rb
@@ -1,14 +1,17 @@
+# frozen_string_literal: true
+
 module Gitlab
   module QuickActions
     class CommandDefinition
       attr_accessor :name, :aliases, :description, :explanation, :params,
-        :condition_block, :parse_params_block, :action_block
+        :condition_block, :parse_params_block, :action_block, :warning
 
       def initialize(name, attributes = {})
         @name = name
 
         @aliases = attributes[:aliases] || []
         @description = attributes[:description] || ''
+        @warning = attributes[:warning] || ''
         @explanation = attributes[:explanation] || ''
         @params = attributes[:params] || []
         @condition_block = attributes[:condition_block]
@@ -33,11 +36,13 @@ module Gitlab
       def explain(context, arg)
         return unless available?(context)
 
-        if explanation.respond_to?(:call)
-          execute_block(explanation, context, arg)
-        else
-          explanation
-        end
+        message = if explanation.respond_to?(:call)
+                    execute_block(explanation, context, arg)
+                  else
+                    explanation
+                  end
+
+        warning.empty? ? message : "#{message} (#{warning})"
       end
 
       def execute(context, arg)
@@ -61,6 +66,7 @@ module Gitlab
           name: name,
           aliases: aliases,
           description: desc,
+          warning: warning,
           params: prms
         }
       end
diff --git a/lib/gitlab/quick_actions/dsl.rb b/lib/gitlab/quick_actions/dsl.rb
index d82dccd0db5..a3aab92061b 100644
--- a/lib/gitlab/quick_actions/dsl.rb
+++ b/lib/gitlab/quick_actions/dsl.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module QuickActions
     module Dsl
@@ -31,6 +33,10 @@ module Gitlab
           @description = block_given? ? block : text
         end
 
+        def warning(message = '')
+          @warning = message
+        end
+
         # Allows to define params for the next quick action.
         # These params are shown in the autocomplete menu.
         #
@@ -133,6 +139,7 @@ module Gitlab
             name,
             aliases: aliases,
             description: @description,
+            warning: @warning,
             explanation: @explanation,
             params: @params,
             condition_block: @condition_block,
@@ -150,6 +157,7 @@ module Gitlab
           @explanation = nil
           @params = nil
           @condition_block = nil
+          @warning = nil
           @parse_params_block = nil
         end
       end
diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb
index 075ff91700c..ff9bb293b47 100644
--- a/lib/gitlab/quick_actions/extractor.rb
+++ b/lib/gitlab/quick_actions/extractor.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module QuickActions
     # This class takes an array of commands that should be extracted from a
@@ -29,7 +31,7 @@ module Gitlab
       # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']]
       # msg #=> "hello\nworld"
       # ```
-      def extract_commands(content)
+      def extract_commands(content, only: nil)
         return [content, []] unless content
 
         content = content.dup
@@ -37,9 +39,9 @@ module Gitlab
         commands = []
 
         content.delete!("\r")
-        content.gsub!(commands_regex) do
+        content.gsub!(commands_regex(only: only)) do
           if $~[:cmd]
-            commands << [$~[:cmd], $~[:arg]].reject(&:blank?)
+            commands << [$~[:cmd].downcase, $~[:arg]].reject(&:blank?)
             ''
           else
             $~[0]
@@ -60,8 +62,8 @@ module Gitlab
       # It looks something like:
       #
       #   /^\/(?<cmd>close|reopen|...)(?:( |$))(?<arg>[^\/\n]*)(?:\n|$)/
-      def commands_regex
-        names = command_names.map(&:to_s)
+      def commands_regex(only:)
+        names = command_names(limit_to_commands: only).map(&:to_s)
 
         @commands_regex ||= %r{
             (?<code>
@@ -102,14 +104,14 @@ module Gitlab
               # /close
 
               ^\/
-              (?<cmd>#{Regexp.union(names)})
+              (?<cmd>#{Regexp.new(Regexp.union(names).source, Regexp::IGNORECASE)})
               (?:
                 [ ]
                 (?<arg>[^\n]*)
               )?
               (?:\n|$)
             )
-        }mx
+        }mix
       end
 
       def perform_substitutions(content, commands)
@@ -120,7 +122,7 @@ module Gitlab
         end
 
         substitution_definitions.each do |substitution|
-          match_data = substitution.match(content)
+          match_data = substitution.match(content.downcase)
           if match_data
             command = [substitution.name.to_s]
             command << match_data[1] unless match_data[1].empty?
@@ -133,10 +135,14 @@ module Gitlab
         [content, commands]
       end
 
-      def command_names
+      def command_names(limit_to_commands:)
         command_definitions.flat_map do |command|
           next if command.noop?
 
+          if limit_to_commands && (command.all_names & limit_to_commands).empty?
+            next
+          end
+
           command.all_names
         end.compact
       end
diff --git a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb
index 7328c517a30..f5176376a60 100644
--- a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb
+++ b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module QuickActions
     # This class takes spend command argument
diff --git a/lib/gitlab/quick_actions/substitution_definition.rb b/lib/gitlab/quick_actions/substitution_definition.rb
index 032c49ed159..2f78ea05cf0 100644
--- a/lib/gitlab/quick_actions/substitution_definition.rb
+++ b/lib/gitlab/quick_actions/substitution_definition.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module QuickActions
     class SubstitutionDefinition < CommandDefinition
@@ -15,7 +17,7 @@ module Gitlab
         return unless content
 
         all_names.each do |a_name|
-          content.gsub!(%r{/#{a_name} ?(.*)$}, execute_block(action_block, context, '\1'))
+          content.gsub!(%r{/#{a_name} ?(.*)$}i, execute_block(action_block, context, '\1'))
         end
         content
       end
diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb
index c9efa28d7e7..6559c3e3c57 100644
--- a/lib/gitlab/recaptcha.rb
+++ b/lib/gitlab/recaptcha.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Recaptcha
     def self.load_configurations!
diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb
index a991933e910..6e8403ad878 100644
--- a/lib/gitlab/redis/cache.rb
+++ b/lib/gitlab/redis/cache.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # please require all dependencies below:
 require_relative 'wrapper' unless defined?(::Rails) && ::Rails.root.present?
 
diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb
index e1695aafbeb..8b42c269dd0 100644
--- a/lib/gitlab/redis/queues.rb
+++ b/lib/gitlab/redis/queues.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # please require all dependencies below:
 require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper)
 
diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb
index e5a0fdae7ef..9066606ca21 100644
--- a/lib/gitlab/redis/shared_state.rb
+++ b/lib/gitlab/redis/shared_state.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # please require all dependencies below:
 require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper)
 
diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb
index 4178b436acf..07a1e20b076 100644
--- a/lib/gitlab/redis/wrapper.rb
+++ b/lib/gitlab/redis/wrapper.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # This file should only be used by sub-classes, not directly by any clients of the sub-classes
 # please require all dependencies below:
 require 'active_support/core_ext/hash/keys'
diff --git a/lib/gitlab/reference_counter.rb b/lib/gitlab/reference_counter.rb
index bb26f1b610a..d2dbc6f5ef5 100644
--- a/lib/gitlab/reference_counter.rb
+++ b/lib/gitlab/reference_counter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class ReferenceCounter
     REFERENCE_EXPIRE_TIME = 600
diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb
index 9ff82d628c0..00f817c2399 100644
--- a/lib/gitlab/reference_extractor.rb
+++ b/lib/gitlab/reference_extractor.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   # Extract possible GFM references from an arbitrary String for further processing.
   class ReferenceExtractor < Banzai::ReferenceExtractor
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index ac3de2a8f71..7a1a2eaf6c0 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Regex
     extend self
@@ -73,5 +75,35 @@ module Gitlab
     def build_trace_section_regex
       @build_trace_section_regexp ||= /section_((?:start)|(?:end)):(\d+):([a-zA-Z0-9_.-]+)\r\033\[0K/.freeze
     end
+
+    def markdown_code_or_html_blocks
+      @markdown_code_or_html_blocks ||= %r{
+          (?<code>
+            # Code blocks:
+            # ```
+            # Anything, including `>>>` blocks which are ignored by this filter
+            # ```
+
+            ^```
+            .+?
+            \n```\ *$
+          )
+        |
+          (?<html>
+            # HTML block:
+            # <tag>
+            # Anything, including `>>>` blocks which are ignored by this filter
+            # </tag>
+
+            ^<[^>]+?>\ *\n
+            .+?
+            \n<\/[^>]+?>\ *$
+          )
+      }mx
+    end
+
+    def jira_transition_id_regex
+      @jira_transition_id_regex ||= /\d+/
+    end
   end
 end
diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb
index 4888184403c..202d310e237 100644
--- a/lib/gitlab/repo_path.rb
+++ b/lib/gitlab/repo_path.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module RepoPath
     NotFoundError = Class.new(StandardError)
diff --git a/lib/gitlab/repository_cache.rb b/lib/gitlab/repository_cache.rb
index b1bf3ca4143..56007574b1b 100644
--- a/lib/gitlab/repository_cache.rb
+++ b/lib/gitlab/repository_cache.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Interface to the Redis-backed cache store
 module Gitlab
   class RepositoryCache
@@ -6,7 +8,7 @@ module Gitlab
     def initialize(repository, extra_namespace: nil, backend: Rails.cache)
       @repository = repository
       @namespace = "#{repository.full_path}:#{repository.project.id}"
-      @namespace += ":#{extra_namespace}" if extra_namespace
+      @namespace = "#{@namespace}:#{extra_namespace}" if extra_namespace
       @backend = backend
     end
 
@@ -29,5 +31,21 @@ module Gitlab
     def read(key)
       backend.read(cache_key(key))
     end
+
+    def write(key, value)
+      backend.write(cache_key(key), value)
+    end
+
+    def fetch_without_caching_false(key, &block)
+      value = read(key)
+      return value if value
+
+      value = yield
+
+      # Don't cache false values
+      write(key, value) if value
+
+      value
+    end
   end
 end
diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb
index 7f64a8c9e46..931298b5117 100644
--- a/lib/gitlab/repository_cache_adapter.rb
+++ b/lib/gitlab/repository_cache_adapter.rb
@@ -1,23 +1,82 @@
+# frozen_string_literal: true
+
 module Gitlab
   module RepositoryCacheAdapter
     extend ActiveSupport::Concern
+    include Gitlab::Utils::StrongMemoize
 
     class_methods do
-      # Wraps around the given method and caches its output in Redis and an instance
-      # variable.
+      # Caches and strongly memoizes the method.
       #
       # This only works for methods that do not take any arguments.
-      def cache_method(name, fallback: nil, memoize_only: false)
-        original = :"_uncached_#{name}"
+      #
+      # name     - The name of the method to be cached.
+      # fallback - A value to fall back to if the repository does not exist, or
+      #            in case of a Git error. Defaults to nil.
+      def cache_method(name, fallback: nil)
+        uncached_name = alias_uncached_method(name)
 
-        alias_method(original, name)
+        define_method(name) do
+          cache_method_output(name, fallback: fallback) do
+            __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend
+          end
+        end
+      end
+
+      # Caches truthy values from the method. All values are strongly memoized,
+      # and cached in RequestStore.
+      #
+      # Currently only used to cache `exists?` since stale false values are
+      # particularly troublesome. This can occur, for example, when an NFS mount
+      # is temporarily down.
+      #
+      # This only works for methods that do not take any arguments.
+      #
+      # name - The name of the method to be cached.
+      def cache_method_asymmetrically(name)
+        uncached_name = alias_uncached_method(name)
 
         define_method(name) do
-          cache_method_output(name, fallback: fallback, memoize_only: memoize_only) do
-            __send__(original) # rubocop:disable GitlabSecurity/PublicSend
+          cache_method_output_asymmetrically(name) do
+            __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend
           end
         end
       end
+
+      # Strongly memoizes the method.
+      #
+      # This only works for methods that do not take any arguments.
+      #
+      # name     - The name of the method to be memoized.
+      # fallback - A value to fall back to if the repository does not exist, or
+      #            in case of a Git error. Defaults to nil. The fallback value
+      #            is not memoized.
+      def memoize_method(name, fallback: nil)
+        uncached_name = alias_uncached_method(name)
+
+        define_method(name) do
+          memoize_method_output(name, fallback: fallback) do
+            __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend
+          end
+        end
+      end
+
+      # Prepends "_uncached_" to the target method name
+      #
+      # Returns the uncached method name
+      def alias_uncached_method(name)
+        uncached_name = :"_uncached_#{name}"
+
+        alias_method(uncached_name, name)
+
+        uncached_name
+      end
+    end
+
+    # RequestStore-backed RepositoryCache to be used. Should be overridden by
+    # the including class
+    def request_store_cache
+      raise NotImplementedError
     end
 
     # RepositoryCache to be used. Should be overridden by the including class
@@ -25,60 +84,98 @@ module Gitlab
       raise NotImplementedError
     end
 
-    # Caches the supplied block both in a cache and in an instance variable.
+    # List of cached methods. Should be overridden by the including class
+    def cached_methods
+      raise NotImplementedError
+    end
+
+    # Caches and strongly memoizes the supplied block.
     #
-    # The cache key and instance variable are named the same way as the value of
-    # the `key` argument.
+    # name     - The name of the method to be cached.
+    # fallback - A value to fall back to if the repository does not exist, or
+    #            in case of a Git error. Defaults to nil.
+    def cache_method_output(name, fallback: nil, &block)
+      memoize_method_output(name, fallback: fallback) do
+        cache.fetch(name, &block)
+      end
+    end
+
+    # Caches truthy values from the supplied block. All values are strongly
+    # memoized, and cached in RequestStore.
     #
-    # This method will return `nil` if the corresponding instance variable is also
-    # set to `nil`. This ensures we don't keep yielding the block when it returns
-    # `nil`.
+    # Currently only used to cache `exists?` since stale false values are
+    # particularly troublesome. This can occur, for example, when an NFS mount
+    # is temporarily down.
     #
-    # key - The name of the key to cache the data in.
-    # fallback - A value to fall back to in the event of a Git error.
-    def cache_method_output(key, fallback: nil, memoize_only: false, &block)
-      ivar = cache_instance_variable_name(key)
-
-      if instance_variable_defined?(ivar)
-        instance_variable_get(ivar)
-      else
-        # If the repository doesn't exist and a fallback was specified we return
-        # that value inmediately. This saves us Rugged/gRPC invocations.
-        return fallback unless fallback.nil? || cache.repository.exists?
-
-        begin
-          value =
-            if memoize_only
-              yield
-            else
-              cache.fetch(key, &block)
-            end
-
-          instance_variable_set(ivar, value)
-        rescue Gitlab::Git::Repository::NoRepository
-          # Even if the above `#exists?` check passes these errors might still
-          # occur (for example because of a non-existing HEAD). We want to
-          # gracefully handle this and not cache anything
-          fallback
+    # name - The name of the method to be cached.
+    def cache_method_output_asymmetrically(name, &block)
+      memoize_method_output(name) do
+        request_store_cache.fetch(name) do
+          cache.fetch_without_caching_false(name, &block)
         end
       end
     end
 
+    # Strongly memoizes the supplied block.
+    #
+    # name     - The name of the method to be memoized.
+    # fallback - A value to fall back to if the repository does not exist, or
+    #            in case of a Git error. Defaults to nil. The fallback value is
+    #            not memoized.
+    def memoize_method_output(name, fallback: nil, &block)
+      no_repository_fallback(name, fallback: fallback) do
+        strong_memoize(memoizable_name(name), &block)
+      end
+    end
+
+    # Returns the fallback value if the repository does not exist
+    def no_repository_fallback(name, fallback: nil, &block)
+      # Avoid unnecessary gRPC invocations
+      return fallback if fallback && fallback_early?(name)
+
+      yield
+    rescue Gitlab::Git::Repository::NoRepository
+      # Even if the `#exists?` check in `fallback_early?` passes, these errors
+      # might still occur (for example because of a non-existing HEAD). We
+      # want to gracefully handle this and not memoize anything.
+      fallback
+    end
+
     # Expires the caches of a specific set of methods
     def expire_method_caches(methods)
-      methods.each do |key|
-        cache.expire(key)
+      methods.each do |name|
+        unless cached_methods.include?(name.to_sym)
+          Rails.logger.error "Requested to expire non-existent method '#{name}' for Repository"
+          next
+        end
 
-        ivar = cache_instance_variable_name(key)
+        cache.expire(name)
 
-        remove_instance_variable(ivar) if instance_variable_defined?(ivar)
+        clear_memoization(memoizable_name(name))
       end
+
+      expire_request_store_method_caches(methods)
     end
 
     private
 
-    def cache_instance_variable_name(key)
-      :"@#{key.to_s.tr('?!', '')}"
+    def memoizable_name(name)
+      "#{name.to_s.tr('?!', '')}"
+    end
+
+    def expire_request_store_method_caches(methods)
+      methods.each do |name|
+        request_store_cache.expire(name)
+      end
+    end
+
+    # All cached repository methods depend on the existence of a Git repository,
+    # so if the repository doesn't exist, we already know not to call it.
+    def fallback_early?(method_name)
+      # Avoid infinite loop
+      return false if method_name == :exists?
+
+      !exists?
     end
   end
 end
diff --git a/lib/gitlab/repository_check_logger.rb b/lib/gitlab/repository_check_logger.rb
index 485b596ca57..e90b0a002af 100644
--- a/lib/gitlab/repository_check_logger.rb
+++ b/lib/gitlab/repository_check_logger.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class RepositoryCheckLogger < Gitlab::Logger
     def self.file_name_noext
diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb
index fef536ecb0b..d9811e036d3 100644
--- a/lib/gitlab/request_context.rb
+++ b/lib/gitlab/request_context.rb
@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 module Gitlab
   class RequestContext
     class << self
       def client_ip
-        RequestStore[:client_ip]
+        Gitlab::SafeRequestStore[:client_ip]
       end
     end
 
@@ -11,9 +13,9 @@ module Gitlab
     end
 
     def call(env)
-      req = Rack::Request.new(env)
+      req = ActionDispatch::Request.new(env)
 
-      RequestStore[:client_ip] = req.ip
+      Gitlab::SafeRequestStore[:client_ip] = req.ip
 
       @app.call(env)
     end
diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb
index ccfe0d6bed3..b1e478093d3 100644
--- a/lib/gitlab/request_forgery_protection.rb
+++ b/lib/gitlab/request_forgery_protection.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # A module to check CSRF tokens in requests.
 # It's used in API helpers and OmniAuth.
 # Usage: GitLab::RequestForgeryProtection.call(env)
@@ -5,7 +7,7 @@
 module Gitlab
   module RequestForgeryProtection
     class Controller < ActionController::Base
-      protect_from_forgery with: :exception
+      protect_from_forgery with: :exception, prepend: true
 
       rescue_from ActionController::InvalidAuthenticityToken do |e|
         logger.warn "This CSRF token verification failure is handled internally by `GitLab::RequestForgeryProtection`"
diff --git a/lib/gitlab/request_profiler.rb b/lib/gitlab/request_profiler.rb
index 0c9ab759e81..64593153686 100644
--- a/lib/gitlab/request_profiler.rb
+++ b/lib/gitlab/request_profiler.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'fileutils'
 
 module Gitlab
diff --git a/lib/gitlab/request_profiler/middleware.rb b/lib/gitlab/request_profiler/middleware.rb
index ef42b0557e0..7615f6f443b 100644
--- a/lib/gitlab/request_profiler/middleware.rb
+++ b/lib/gitlab/request_profiler/middleware.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'ruby-prof'
 
 module Gitlab
diff --git a/lib/gitlab/request_profiler/profile.rb b/lib/gitlab/request_profiler/profile.rb
index f89d56903ef..46996ef8c51 100644
--- a/lib/gitlab/request_profiler/profile.rb
+++ b/lib/gitlab/request_profiler/profile.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module RequestProfiler
     class Profile
diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb
index f3952657983..a555bf1d812 100644
--- a/lib/gitlab/route_map.rb
+++ b/lib/gitlab/route_map.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class RouteMap
     FormatError = Class.new(StandardError)
diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb
index 2c994536060..3b05f181ed2 100644
--- a/lib/gitlab/routing.rb
+++ b/lib/gitlab/routing.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Routing
     extend ActiveSupport::Concern
@@ -47,7 +49,7 @@ module Gitlab
         #
         # `request.fullpath` includes the querystring
         new_path = request.path.sub(%r{/#{path}(/*)(?!.*#{path})}, "/-/#{path}\\1")
-        new_path << "?#{request.query_string}" if request.query_string.present?
+        new_path = "#{new_path}?#{request.query_string}" if request.query_string.present?
 
         new_path
       end
diff --git a/lib/gitlab/safe_request_store.rb b/lib/gitlab/safe_request_store.rb
new file mode 100644
index 00000000000..d146913bdb3
--- /dev/null
+++ b/lib/gitlab/safe_request_store.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SafeRequestStore
+    NULL_STORE = Gitlab::NullRequestStore.new
+
+    class << self
+      # These methods should always run directly against RequestStore
+      delegate :clear!, :begin!, :end!, :active?, to: :RequestStore
+
+      # These methods will run against NullRequestStore if RequestStore is disabled
+      delegate :read, :[], :write, :[]=, :exist?, :fetch, :delete, to: :store
+    end
+
+    def self.store
+      if RequestStore.active?
+        RequestStore
+      else
+        NULL_STORE
+      end
+    end
+
+    # This method accept an options hash to be compatible with
+    # ActiveSupport::Cache::Store#write method. The options are
+    # not passed to the underlying cache implementation because
+    # RequestStore#write accepts only a key, and value params.
+    def self.write(key, value, options = nil)
+      store.write(key, value)
+    end
+  end
+end
diff --git a/lib/gitlab/sanitizers/svg.rb b/lib/gitlab/sanitizers/svg.rb
index 8304b9a482c..0d4e6be2129 100644
--- a/lib/gitlab/sanitizers/svg.rb
+++ b/lib/gitlab/sanitizers/svg.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Sanitizers
     module SVG
diff --git a/lib/gitlab/sanitizers/svg/whitelist.rb b/lib/gitlab/sanitizers/svg/whitelist.rb
index d50f826f924..b4da24b3215 100644
--- a/lib/gitlab/sanitizers/svg/whitelist.rb
+++ b/lib/gitlab/sanitizers/svg/whitelist.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Generated from:
 # SVG element list: https://www.w3.org/TR/SVG/eltindex.html
 # SVG Attribute list: https://www.w3.org/TR/SVG/attindex.html
diff --git a/lib/gitlab/search/found_blob.rb b/lib/gitlab/search/found_blob.rb
new file mode 100644
index 00000000000..a62ab1521a7
--- /dev/null
+++ b/lib/gitlab/search/found_blob.rb
@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Search
+    class FoundBlob
+      include EncodingHelper
+      include Presentable
+      include BlobLanguageFromGitAttributes
+      include Gitlab::Utils::StrongMemoize
+
+      attr_reader :project, :content_match, :blob_filename
+
+      FILENAME_REGEXP = /\A(?<ref>[^:]*):(?<filename>[^\x00]*)\x00/.freeze
+      CONTENT_REGEXP = /^(?<ref>[^:]*):(?<filename>[^\x00]*)\x00(?<startline>\d+)\x00/.freeze
+
+      def self.preload_blobs(blobs)
+        to_fetch = blobs.select { |blob| blob.is_a?(self) && blob.blob_filename }
+
+        to_fetch.each { |blob| blob.fetch_blob }
+      end
+
+      def initialize(opts = {})
+        @id = opts.fetch(:id, nil)
+        @binary_filename = opts.fetch(:filename, nil)
+        @binary_basename = opts.fetch(:basename, nil)
+        @ref = opts.fetch(:ref, nil)
+        @startline = opts.fetch(:startline, nil)
+        @binary_data = opts.fetch(:data, nil)
+        @per_page = opts.fetch(:per_page, 20)
+        @project = opts.fetch(:project, nil)
+        # Some caller does not have project object (e.g. elastic search),
+        # yet they can trigger many calls in one go,
+        # causing duplicated queries.
+        # Allow those to just pass project_id instead.
+        @project_id = opts.fetch(:project_id, nil)
+        @content_match = opts.fetch(:content_match, nil)
+        @blob_filename = opts.fetch(:blob_filename, nil)
+        @repository = opts.fetch(:repository, nil)
+      end
+
+      def id
+        @id ||= parsed_content[:id]
+      end
+
+      def ref
+        @ref ||= parsed_content[:ref]
+      end
+
+      def startline
+        @startline ||= parsed_content[:startline]
+      end
+
+      # binary_filename is used for running filters on all matches,
+      # for grepped results (which use content_match), we get
+      # filename from the beginning of the grepped result which is faster
+      # then parsing whole snippet
+      def binary_filename
+        @binary_filename ||= content_match ? search_result_filename : parsed_content[:binary_filename]
+      end
+
+      def filename
+        @filename ||= encode_utf8(@binary_filename || parsed_content[:binary_filename])
+      end
+
+      def basename
+        @basename ||= encode_utf8(@binary_basename || parsed_content[:binary_basename])
+      end
+
+      def data
+        @data ||= encode_utf8(@binary_data || parsed_content[:binary_data])
+      end
+
+      def path
+        filename
+      end
+
+      def project_id
+        @project_id || @project&.id
+      end
+
+      def present
+        super(presenter_class: BlobPresenter)
+      end
+
+      def fetch_blob
+        path = [ref, blob_filename]
+        missing_blob = { binary_filename: blob_filename }
+
+        BatchLoader.for(path).batch(default_value: missing_blob) do |refs, loader|
+          Gitlab::Git::Blob.batch(repository, refs, blob_size_limit: 1024).each do |blob|
+            # if the blob couldn't be fetched for some reason,
+            # show at least the blob filename
+            data = {
+              id: blob.id,
+              binary_filename: blob.path,
+              binary_basename: File.basename(blob.path, File.extname(blob.path)),
+              ref: ref,
+              startline: 1,
+              binary_data: blob.data,
+              project: project
+            }
+
+            loader.call([ref, blob.path], data)
+          end
+        end
+      end
+
+      private
+
+      def search_result_filename
+        content_match.match(FILENAME_REGEXP) { |matches| matches[:filename] }
+      end
+
+      def parsed_content
+        strong_memoize(:parsed_content) do
+          if content_match
+            parse_search_result
+          elsif blob_filename
+            fetch_blob
+          else
+            {}
+          end
+        end
+      end
+
+      def parse_search_result
+        ref = nil
+        filename = nil
+        basename = nil
+
+        data = []
+        startline = 0
+
+        content_match.each_line.each_with_index do |line, index|
+          prefix ||= line.match(CONTENT_REGEXP)&.tap do |matches|
+            ref = matches[:ref]
+            filename = matches[:filename]
+            startline = matches[:startline]
+            startline = startline.to_i - index
+            extname = Regexp.escape(File.extname(filename))
+            basename = filename.sub(/#{extname}$/, '')
+          end
+
+          data << line.sub(prefix.to_s, '')
+        end
+
+        {
+          binary_filename: filename,
+          binary_basename: basename,
+          ref: ref,
+          startline: startline,
+          binary_data: data.join,
+          project: project
+        }
+      end
+
+      def repository
+        @repository ||= project.repository
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/search/parsed_query.rb b/lib/gitlab/search/parsed_query.rb
new file mode 100644
index 00000000000..c4fb0199558
--- /dev/null
+++ b/lib/gitlab/search/parsed_query.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Search
+    class ParsedQuery
+      attr_reader :term, :filters
+
+      def initialize(term, filters)
+        @term = term
+        @filters = filters
+      end
+
+      def filter_results(results)
+        filters = @filters.reject { |filter| filter[:matcher].nil? }
+        return unless filters
+
+        results.select do |result|
+          filters.all? do |filter|
+            filter[:matcher].call(filter, result)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/search/query.rb b/lib/gitlab/search/query.rb
new file mode 100644
index 00000000000..ba0e16607a6
--- /dev/null
+++ b/lib/gitlab/search/query.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Search
+    class Query < SimpleDelegator
+      include EncodingHelper
+
+      def initialize(query, filter_opts = {}, &block)
+        @raw_query = query.dup
+        @filters = []
+        @filter_options = { default_parser: :downcase.to_proc }.merge(filter_opts)
+
+        self.instance_eval(&block) if block_given?
+
+        @query = Gitlab::Search::ParsedQuery.new(*extract_filters)
+        # set the ParsedQuery as our default delegator thanks to SimpleDelegator
+        super(@query)
+      end
+
+      private
+
+      def filter(name, **attributes)
+        filter = { parser: @filter_options[:default_parser], name: name }.merge(attributes)
+
+        @filters << filter
+      end
+
+      def filter_options(**options)
+        @filter_options.merge!(options)
+      end
+
+      def extract_filters
+        fragments = []
+
+        filters = @filters.each_with_object([]) do |filter, parsed_filters|
+          match = @raw_query.split.find { |part| part =~ /\A#{filter[:name]}:/ }
+          next unless match
+
+          input = match.split(':')[1..-1].join
+          next if input.empty?
+
+          filter[:value] = parse_filter(filter, input)
+          filter[:regex_value] = Regexp.escape(filter[:value]).gsub('\*', '.*?')
+          fragments << match
+
+          parsed_filters << filter
+        end
+
+        query = (@raw_query.split - fragments).join(' ')
+
+        [query, filters]
+      end
+
+      def parse_filter(filter, input)
+        result = filter[:parser].call(input)
+
+        @filter_options[:encode_binary] ? encode_binary(result) : result
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb
index 1e45d074e0a..491148ec1a6 100644
--- a/lib/gitlab/search_results.rb
+++ b/lib/gitlab/search_results.rb
@@ -1,30 +1,7 @@
+# frozen_string_literal: true
+
 module Gitlab
   class SearchResults
-    class FoundBlob
-      include EncodingHelper
-
-      attr_reader :id, :filename, :basename, :ref, :startline, :data, :project_id
-
-      def initialize(opts = {})
-        @id = opts.fetch(:id, nil)
-        @filename = encode_utf8(opts.fetch(:filename, nil))
-        @basename = encode_utf8(opts.fetch(:basename, nil))
-        @ref = opts.fetch(:ref, nil)
-        @startline = opts.fetch(:startline, nil)
-        @data = encode_utf8(opts.fetch(:data, nil))
-        @per_page = opts.fetch(:per_page, 20)
-        @project_id = opts.fetch(:project_id, nil)
-      end
-
-      def path
-        filename
-      end
-
-      def no_highlighting?
-        false
-      end
-    end
-
     attr_reader :current_user, :query, :per_page
 
     # Limit search results by passed projects
@@ -62,10 +39,13 @@ module Gitlab
       without_count ? collection.without_count : collection
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def limited_projects_count
       @limited_projects_count ||= projects.limit(count_limit).count
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def limited_issues_count
       return @limited_issues_count if @limited_issues_count
 
@@ -77,14 +57,19 @@ module Gitlab
       sum = issues(public_only: true).limit(count_limit).count
       @limited_issues_count = sum < count_limit ? issues.limit(count_limit).count : sum
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def limited_merge_requests_count
       @limited_merge_requests_count ||= merge_requests.limit(count_limit).count
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def limited_milestones_count
       @limited_milestones_count ||= milestones.limit(count_limit).count
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     def single_commit_result?
       false
@@ -100,6 +85,7 @@ module Gitlab
       limit_projects.search(query)
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def issues(finder_params = {})
       issues = IssuesFinder.new(current_user, finder_params).execute
       unless default_project_filter
@@ -115,13 +101,17 @@ module Gitlab
 
       issues.reorder('updated_at DESC')
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def milestones
       milestones = Milestone.where(project_id: project_ids_relation)
       milestones = milestones.search(query)
       milestones.reorder('updated_at DESC')
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def merge_requests
       merge_requests = MergeRequestsFinder.new(current_user).execute
       unless default_project_filter
@@ -137,13 +127,16 @@ module Gitlab
 
       merge_requests.reorder('updated_at DESC')
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     def default_scope
       'projects'
     end
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def project_ids_relation
       limit_projects.select(:id).reorder(nil)
     end
+    # rubocop: enable CodeReuse/ActiveRecord
   end
 end
diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb
index 98f005cb61b..8e2f16271eb 100644
--- a/lib/gitlab/seeder.rb
+++ b/lib/gitlab/seeder.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # :nocov:
 module DeliverNever
   def deliver_later
@@ -24,6 +26,19 @@ module Gitlab
       puts "\nOK".color(:green)
     end
 
+    def self.without_gitaly_timeout
+      # Remove Gitaly timeout
+      old_timeout = Gitlab::CurrentSettings.current_application_settings.gitaly_timeout_default
+      Gitlab::CurrentSettings.current_application_settings.update_columns(gitaly_timeout_default: 0)
+      # Otherwise we still see the default value when running seed_fu
+      ApplicationSetting.expire
+
+      yield
+    ensure
+      Gitlab::CurrentSettings.current_application_settings.update_columns(gitaly_timeout_default: old_timeout)
+      ApplicationSetting.expire
+    end
+
     def self.mute_notifications
       NotificationService.prepend(MuteNotifications)
     end
diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb
index 6381e94c1d2..956c16117f5 100644
--- a/lib/gitlab/sentry.rb
+++ b/lib/gitlab/sentry.rb
@@ -1,11 +1,14 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Sentry
     def self.enabled?
-      Rails.env.production? && Gitlab::CurrentSettings.sentry_enabled?
+      (Rails.env.production? || Rails.env.development?) &&
+        Gitlab::CurrentSettings.sentry_enabled?
     end
 
     def self.context(current_user = nil)
-      return unless self.enabled?
+      return unless enabled?
 
       Raven.tags_context(locale: I18n.locale)
 
@@ -27,25 +30,29 @@ module Gitlab
     #
     # Provide an issue URL for follow up.
     def self.track_exception(exception, issue_url: nil, extra: {})
+      track_acceptable_exception(exception, issue_url: issue_url, extra: extra)
+
+      raise exception if should_raise_for_dev?
+    end
+
+    # This should be used when you do not want to raise an exception in
+    # development and test. If you need development and test to behave
+    # just the same as production you can use this instead of
+    # track_exception.
+    def self.track_acceptable_exception(exception, issue_url: nil, extra: {})
       if enabled?
         extra[:issue_url] = issue_url if issue_url
         context # Make sure we've set everything we know in the context
 
-        Raven.capture_exception(exception, extra: extra)
-      end
-
-      raise exception if should_raise?
-    end
+        tags = {
+          Gitlab::CorrelationId::LOG_KEY.to_sym => Gitlab::CorrelationId.current_id
+        }
 
-    def self.program_context
-      if Sidekiq.server?
-        'sidekiq'
-      else
-        'rails'
+        Raven.capture_exception(exception, tags: tags, extra: extra)
       end
     end
 
-    def self.should_raise?
+    def self.should_raise_for_dev?
       Rails.env.development? || Rails.env.test?
     end
   end
diff --git a/lib/gitlab/serializer/ci/variables.rb b/lib/gitlab/serializer/ci/variables.rb
index c059c454eac..9abf3a54f37 100644
--- a/lib/gitlab/serializer/ci/variables.rb
+++ b/lib/gitlab/serializer/ci/variables.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Serializer
     module Ci
@@ -13,8 +15,9 @@ module Gitlab
           object = YAML.safe_load(string, [Symbol])
 
           object.map do |variable|
-            variable[:key] = variable[:key].to_s
-            variable
+            variable.symbolize_keys.tap do |variable|
+              variable[:key] = variable[:key].to_s
+            end
           end
         end
 
diff --git a/lib/gitlab/serializer/pagination.rb b/lib/gitlab/serializer/pagination.rb
index 6bb00d8ae21..eb242cc7c20 100644
--- a/lib/gitlab/serializer/pagination.rb
+++ b/lib/gitlab/serializer/pagination.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Serializer
     class Pagination
diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb
index e5c02dd8ecc..2b7e12639be 100644
--- a/lib/gitlab/setup_helper.rb
+++ b/lib/gitlab/setup_helper.rb
@@ -1,3 +1,7 @@
+# frozen_string_literal: true
+
+require 'toml-rb'
+
 module Gitlab
   module SetupHelper
     class << self
@@ -9,7 +13,7 @@ module Gitlab
       # because it uses a Unix socket.
       # For development and testing purposes, an extra storage is added to gitaly,
       # which is not known to Rails, but must be explicitly stubbed.
-      def gitaly_configuration_toml(gitaly_dir, gitaly_ruby: true)
+      def gitaly_configuration_toml(gitaly_dir, storage_paths, gitaly_ruby: true)
         storages = []
         address = nil
 
@@ -24,11 +28,14 @@ module Gitlab
             address = val['gitaly_address']
           end
 
-          storages << { name: key, path: val.legacy_disk_path }
+          storages << { name: key, path: storage_paths[key] }
         end
 
         if Rails.env.test?
-          storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s }
+          storage_path = Rails.root.join('tmp', 'tests', 'second_storage').to_s
+
+          FileUtils.mkdir(storage_path) unless File.exist?(storage_path)
+          storages << { name: 'test_second_storage', path: storage_path }
         end
 
         config = { socket_path: address.sub(/\Aunix:/, ''), storage: storages }
@@ -41,12 +48,12 @@ module Gitlab
       end
 
       # rubocop:disable Rails/Output
-      def create_gitaly_configuration(dir, force: false)
+      def create_gitaly_configuration(dir, storage_paths, force: false)
         config_path = File.join(dir, 'config.toml')
         FileUtils.rm_f(config_path) if force
 
         File.open(config_path, File::WRONLY | File::CREAT | File::EXCL) do |f|
-          f.puts gitaly_configuration_toml(dir)
+          f.puts gitaly_configuration_toml(dir, storage_paths)
         end
       rescue Errno::EEXIST
         puts "Skipping config.toml generation:"
diff --git a/lib/gitlab/shard_health_cache.rb b/lib/gitlab/shard_health_cache.rb
new file mode 100644
index 00000000000..6612347438e
--- /dev/null
+++ b/lib/gitlab/shard_health_cache.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class ShardHealthCache
+    HEALTHY_SHARDS_KEY = 'gitlab-healthy-shards'.freeze
+    HEALTHY_SHARDS_TIMEOUT = 300
+
+    # Clears the Redis set storing the list of healthy shards
+    def self.clear
+      Gitlab::Redis::Cache.with { |redis| redis.del(HEALTHY_SHARDS_KEY) }
+    end
+
+    # Updates the list of healthy shards using a Redis set
+    #
+    # shards - An array of shard names to store
+    def self.update(shards)
+      Gitlab::Redis::Cache.with do |redis|
+        redis.multi do |m|
+          m.del(HEALTHY_SHARDS_KEY)
+          shards.each { |shard_name| m.sadd(HEALTHY_SHARDS_KEY, shard_name) }
+          m.expire(HEALTHY_SHARDS_KEY, HEALTHY_SHARDS_TIMEOUT)
+        end
+      end
+    end
+
+    # Returns an array of strings of healthy shards
+    def self.cached_healthy_shards
+      Gitlab::Redis::Cache.with { |redis| redis.smembers(HEALTHY_SHARDS_KEY) }
+    end
+
+    # Checks whether the given shard name is in the list of healthy shards.
+    #
+    # shard_name - The string to check
+    def self.healthy_shard?(shard_name)
+      Gitlab::Redis::Cache.with { |redis| redis.sismember(HEALTHY_SHARDS_KEY, shard_name) }
+    end
+
+    # Returns the number of healthy shards in the Redis set
+    def self.healthy_shard_count
+      Gitlab::Redis::Cache.with { |redis| redis.scard(HEALTHY_SHARDS_KEY) }
+    end
+  end
+end
diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb
index 4a691d640b3..bdf21cf3134 100644
--- a/lib/gitlab/shell.rb
+++ b/lib/gitlab/shell.rb
@@ -1,5 +1,6 @@
-# Gitaly note: JV: two sets of straightforward RPC's. 1 Hard RPC: fork_repository.
-# SSH key operations are not part of Gitaly so will never be migrated.
+# frozen_string_literal: true
+
+# Gitaly note: SSH key operations are not part of Gitaly so will never be migrated.
 
 require 'securerandom'
 
@@ -75,17 +76,10 @@ module Gitlab
       relative_path = name.dup
       relative_path << '.git' unless relative_path.end_with?('.git')
 
-      gitaly_migrate(:create_repository,
-                     status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-        if is_enabled
-          repository = Gitlab::Git::Repository.new(storage, relative_path, '')
-          repository.gitaly_repository_client.create_repository
-          true
-        else
-          repo_path = File.join(Gitlab.config.repositories.storages[storage].legacy_disk_path, relative_path)
-          Gitlab::Git::Repository.create(repo_path, bare: true, symlink_hooks_to: gitlab_shell_hooks_path)
-        end
-      end
+      repository = Gitlab::Git::Repository.new(storage, relative_path, '')
+      wrapped_gitaly_errors { repository.gitaly_repository_client.create_repository }
+
+      true
     rescue => err # Once the Rugged codes gets removes this can be improved
       Rails.logger.error("Failed to add repository #{storage}/#{name}: #{err}")
       false
@@ -100,42 +94,20 @@ module Gitlab
     # Ex.
     #   import_repository("nfs-file06", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git")
     #
-    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874
     def import_repository(storage, name, url)
       if url.start_with?('.', '/')
         raise Error.new("don't use disk paths with import_repository: #{url.inspect}")
       end
 
-      # The timeout ensures the subprocess won't hang forever
-      cmd = gitlab_projects(storage, "#{name}.git")
-      success = cmd.import_project(url, git_timeout)
+      relative_path = "#{name}.git"
+      cmd = GitalyGitlabProjects.new(storage, relative_path)
 
+      success = cmd.import_project(url, git_timeout)
       raise Error, cmd.output unless success
 
       success
     end
 
-    # Fetch remote for repository
-    #
-    # repository - an instance of Git::Repository
-    # remote - remote name
-    # ssh_auth - SSH known_hosts data and a private key to use for public-key authentication
-    # forced - should we use --force flag?
-    # no_tags - should we use --no-tags flag?
-    #
-    # Ex.
-    #   fetch_remote(my_repo, "upstream")
-    #
-    def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true)
-      gitaly_migrate(:fetch_remote) do |is_enabled|
-        if is_enabled
-          repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune)
-        else
-          local_fetch_remote(repository.storage, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune)
-        end
-      end
-    end
-
     # Move repository reroutes to mv_directory which is an alias for
     # mv_namespace. Given the underlying implementation is a move action,
     # indescriminate of what the folders might be.
@@ -146,8 +118,6 @@ module Gitlab
     #
     # Ex.
     #   mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
-    #
-    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873
     def mv_repository(storage, path, new_path)
       return false if path.empty? || new_path.empty?
 
@@ -162,11 +132,11 @@ module Gitlab
     #
     # Ex.
     #  fork_repository("nfs-file06", "gitlab/gitlab-ci", "nfs-file07", "new-namespace/gitlab-ci")
-    #
-    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817
     def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path)
-      gitlab_projects(forked_from_storage, "#{forked_from_disk_path}.git")
-        .fork_repository(forked_to_storage, "#{forked_to_disk_path}.git")
+      forked_from_relative_path = "#{forked_from_disk_path}.git"
+      fork_args = [forked_to_storage, "#{forked_to_disk_path}.git"]
+
+      GitalyGitlabProjects.new(forked_from_storage, forked_from_relative_path).fork_repository(*fork_args)
     end
 
     # Removes a repository from file system, using rm_diretory which is an alias
@@ -178,8 +148,6 @@ module Gitlab
     #
     # Ex.
     #   remove_repository("/path/to/storage", "gitlab/gitlab-ci")
-    #
-    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873
     def remove_repository(storage, name)
       return false if name.empty?
 
@@ -242,6 +210,7 @@ module Gitlab
     # Ex.
     #   remove_keys_not_found_in_db
     #
+    # rubocop: disable CodeReuse/ActiveRecord
     def remove_keys_not_found_in_db
       return unless self.authorized_keys_enabled?
 
@@ -260,6 +229,7 @@ module Gitlab
         end
       end
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     # Iterate over all ssh key IDs from gitlab shell, in batches
     #
@@ -319,10 +289,12 @@ module Gitlab
     #
     def mv_namespace(storage, old_name, new_name)
       Gitlab::GitalyClient::NamespaceService.new(storage).rename(old_name, new_name)
-    rescue GRPC::InvalidArgument
+    rescue GRPC::InvalidArgument => e
+      Gitlab::Sentry.track_acceptable_exception(e, extra: { old_name: old_name, new_name: new_name, storage: storage })
+
       false
     end
-    alias_method :mv_directory, :mv_namespace
+    alias_method :mv_directory, :mv_namespace # Note: ShellWorker uses this alias
 
     def url_to_repo(path)
       Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git"
@@ -343,9 +315,11 @@ module Gitlab
     #   exists?(storage, 'gitlab')
     #   exists?(storage, 'gitlab/cookies.git')
     #
+    # rubocop: disable CodeReuse/ActiveRecord
     def exists?(storage, dir_name)
       Gitlab::GitalyClient::NamespaceService.new(storage).exists?(dir_name)
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     protected
 
@@ -385,37 +359,6 @@ module Gitlab
 
     private
 
-    def gitlab_projects(shard_name, disk_path)
-      Gitlab::Git::GitlabProjects.new(
-        shard_name,
-        disk_path,
-        global_hooks_path: Gitlab.config.gitlab_shell.hooks_path,
-        logger: Rails.logger
-      )
-    end
-
-    def local_fetch_remote(storage_name, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true)
-      vars = { force: forced, tags: !no_tags, prune: prune }
-
-      if ssh_auth&.ssh_import?
-        if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
-          vars[:ssh_key] = ssh_auth.ssh_private_key
-        end
-
-        if ssh_auth.ssh_known_hosts.present?
-          vars[:known_hosts] = ssh_auth.ssh_known_hosts
-        end
-      end
-
-      cmd = gitlab_projects(storage_name, repository_relative_path)
-
-      success = cmd.fetch_remote(remote, git_timeout, vars)
-
-      raise Error, cmd.output unless success
-
-      success
-    end
-
     def gitlab_shell_fast_execute(cmd)
       output, status = gitlab_shell_fast_execute_helper(cmd)
 
@@ -445,12 +388,46 @@ module Gitlab
       Gitlab.config.gitlab_shell.git_timeout
     end
 
-    def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
-      Gitlab::GitalyClient.migrate(method, status: status, &block)
+    def wrapped_gitaly_errors
+      yield
     rescue GRPC::NotFound, GRPC::BadStatus => e
       # Old Popen code returns [Error, output] to the caller, so we
       # need to do the same here...
       raise Error, e
     end
+
+    class GitalyGitlabProjects
+      attr_reader :shard_name, :repository_relative_path, :output
+
+      def initialize(shard_name, repository_relative_path)
+        @shard_name = shard_name
+        @repository_relative_path = repository_relative_path
+        @output = ''
+      end
+
+      def import_project(source, _timeout)
+        raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil)
+
+        Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source)
+        true
+      rescue GRPC::BadStatus => e
+        @output = e.message
+        false
+      end
+
+      def fork_repository(new_shard_name, new_repository_relative_path)
+        target_repository = Gitlab::Git::Repository.new(new_shard_name, new_repository_relative_path, nil)
+        raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil)
+
+        Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository)
+      rescue GRPC::BadStatus => e
+        logger.error "fork-repository failed: #{e.message}"
+        false
+      end
+
+      def logger
+        Rails.logger
+      end
+    end
   end
 end
diff --git a/lib/gitlab/shell_adapter.rb b/lib/gitlab/shell_adapter.rb
index 053dd4ab9e0..59fc6ee8dc8 100644
--- a/lib/gitlab/shell_adapter.rb
+++ b/lib/gitlab/shell_adapter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # == GitLab Shell mixin
 #
 # Provide a shortcut to Gitlab::Shell instance by gitlab_shell
diff --git a/lib/gitlab/sherlock.rb b/lib/gitlab/sherlock.rb
index 6360527a7aa..a1471c9de47 100644
--- a/lib/gitlab/sherlock.rb
+++ b/lib/gitlab/sherlock.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'securerandom'
 
 module Gitlab
diff --git a/lib/gitlab/sherlock/collection.rb b/lib/gitlab/sherlock/collection.rb
index 66bd6258521..ce3a376cf75 100644
--- a/lib/gitlab/sherlock/collection.rb
+++ b/lib/gitlab/sherlock/collection.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Sherlock
     # A collection of transactions recorded by Sherlock.
diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb
index 89072b01f2e..604b6df12cc 100644
--- a/lib/gitlab/sherlock/file_sample.rb
+++ b/lib/gitlab/sherlock/file_sample.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Sherlock
     class FileSample
diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb
index b5f9d040047..209ba784f9c 100644
--- a/lib/gitlab/sherlock/line_profiler.rb
+++ b/lib/gitlab/sherlock/line_profiler.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Sherlock
     # Class for profiling code on a per line basis.
diff --git a/lib/gitlab/sherlock/line_sample.rb b/lib/gitlab/sherlock/line_sample.rb
index eb1948eb6d6..c92fa9ea1ff 100644
--- a/lib/gitlab/sherlock/line_sample.rb
+++ b/lib/gitlab/sherlock/line_sample.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Sherlock
     class LineSample
diff --git a/lib/gitlab/sherlock/location.rb b/lib/gitlab/sherlock/location.rb
index 5ac265618ad..4bba60f3490 100644
--- a/lib/gitlab/sherlock/location.rb
+++ b/lib/gitlab/sherlock/location.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Sherlock
     class Location
diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb
index 4c88e33699a..747cb0f9142 100644
--- a/lib/gitlab/sherlock/middleware.rb
+++ b/lib/gitlab/sherlock/middleware.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Sherlock
     # Rack middleware used for tracking request metrics.
diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb
index 02ddc3f47eb..11561eec32a 100644
--- a/lib/gitlab/sherlock/query.rb
+++ b/lib/gitlab/sherlock/query.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Sherlock
     class Query
@@ -48,7 +50,7 @@ module Gitlab
         end
 
         unless @query.end_with?(';')
-          @query += ';'
+          @query = "#{@query};"
         end
       end
 
diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb
index 400a552bf99..d04624977dc 100644
--- a/lib/gitlab/sherlock/transaction.rb
+++ b/lib/gitlab/sherlock/transaction.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Sherlock
     class Transaction
diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb
index c3d7814551c..3b8de64913b 100644
--- a/lib/gitlab/sidekiq_config.rb
+++ b/lib/gitlab/sidekiq_config.rb
@@ -1,13 +1,22 @@
+# frozen_string_literal: true
+
 require 'yaml'
 require 'set'
 
 module Gitlab
   module SidekiqConfig
+    QUEUE_CONFIG_PATHS = %w[app/workers/all_queues.yml ee/app/workers/all_queues.yml].freeze
+
     # This method is called by `bin/sidekiq-cluster` in EE, which runs outside
     # of bundler/Rails context, so we cannot use any gem or Rails methods.
     def self.worker_queues(rails_path = Rails.root.to_s)
       @worker_queues ||= {}
-      @worker_queues[rails_path] ||= YAML.load_file(File.join(rails_path, 'app/workers/all_queues.yml'))
+
+      @worker_queues[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path|
+        full_path = File.join(rails_path, path)
+
+        File.exist?(full_path) ? YAML.load_file(full_path) : []
+      end
     end
 
     # This method is called by `bin/sidekiq-cluster` in EE, which runs outside
diff --git a/lib/gitlab/sidekiq_logger.rb b/lib/gitlab/sidekiq_logger.rb
index c1dab87a432..ce82a6f04bb 100644
--- a/lib/gitlab/sidekiq_logger.rb
+++ b/lib/gitlab/sidekiq_logger.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class SidekiqLogger < Gitlab::Logger
     def self.file_name_noext
diff --git a/lib/gitlab/sidekiq_logging/json_formatter.rb b/lib/gitlab/sidekiq_logging/json_formatter.rb
index 98f8222fd03..88888c5994e 100644
--- a/lib/gitlab/sidekiq_logging/json_formatter.rb
+++ b/lib/gitlab/sidekiq_logging/json_formatter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SidekiqLogging
     class JSONFormatter
diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb
index 9a89ae70b98..fdc0d518c59 100644
--- a/lib/gitlab/sidekiq_logging/structured_logger.rb
+++ b/lib/gitlab/sidekiq_logging/structured_logger.rb
@@ -1,8 +1,11 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SidekiqLogging
     class StructuredLogger
       START_TIMESTAMP_FIELDS = %w[created_at enqueued_at].freeze
       DONE_TIMESTAMP_FIELDS = %w[started_at retried_at failed_at completed_at].freeze
+      MAXIMUM_JOB_ARGUMENTS_LENGTH = 10.kilobytes
 
       def call(job, queue)
         started_at = current_time
@@ -62,6 +65,7 @@ module Gitlab
         job['pid'] = ::Process.pid
 
         job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS']
+        job['args'] = limited_job_args(job['args']) if job['args']
 
         convert_to_iso8601(job, START_TIMESTAMP_FIELDS)
 
@@ -91,6 +95,21 @@ module Gitlab
 
         Time.at(timestamp).utc.iso8601(3)
       end
+
+      def limited_job_args(args)
+        return unless args.is_a?(Array)
+
+        total_length = 0
+        limited_args = args.take_while do |arg|
+          total_length += arg.to_json.length
+
+          total_length <= MAXIMUM_JOB_ARGUMENTS_LENGTH
+        end
+
+        limited_args.push('...') if total_length > MAXIMUM_JOB_ARGUMENTS_LENGTH
+
+        limited_args
+      end
     end
   end
 end
diff --git a/lib/gitlab/sidekiq_middleware/arguments_logger.rb b/lib/gitlab/sidekiq_middleware/arguments_logger.rb
index 82a59a7a87e..2859aa5f4a6 100644
--- a/lib/gitlab/sidekiq_middleware/arguments_logger.rb
+++ b/lib/gitlab/sidekiq_middleware/arguments_logger.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SidekiqMiddleware
     class ArgumentsLogger
diff --git a/lib/gitlab/sidekiq_middleware/batch_loader.rb b/lib/gitlab/sidekiq_middleware/batch_loader.rb
new file mode 100644
index 00000000000..75c4efc3042
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/batch_loader.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqMiddleware
+    class BatchLoader
+      def call(worker, job, queue)
+        yield
+      ensure
+        ::BatchLoader::Executor.clear_current
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_middleware/correlation_injector.rb b/lib/gitlab/sidekiq_middleware/correlation_injector.rb
new file mode 100644
index 00000000000..b807b3a03ed
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/correlation_injector.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqMiddleware
+    class CorrelationInjector
+      def call(worker_class, job, queue, redis_pool)
+        job[Gitlab::CorrelationId::LOG_KEY] ||=
+          Gitlab::CorrelationId.current_or_new_id
+
+        yield
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_middleware/correlation_logger.rb b/lib/gitlab/sidekiq_middleware/correlation_logger.rb
new file mode 100644
index 00000000000..cb8ff4a6284
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/correlation_logger.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqMiddleware
+    class CorrelationLogger
+      def call(worker, job, queue)
+        correlation_id = job[Gitlab::CorrelationId::LOG_KEY]
+
+        Gitlab::CorrelationId.use_id(correlation_id) do
+          yield
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_middleware/request_store_middleware.rb b/lib/gitlab/sidekiq_middleware/request_store_middleware.rb
index b1fa0e3cb4e..8824f81e8e3 100644
--- a/lib/gitlab/sidekiq_middleware/request_store_middleware.rb
+++ b/lib/gitlab/sidekiq_middleware/request_store_middleware.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SidekiqMiddleware
     class RequestStoreMiddleware
diff --git a/lib/gitlab/sidekiq_middleware/shutdown.rb b/lib/gitlab/sidekiq_middleware/shutdown.rb
index b232ac4da33..19f3be83bce 100644
--- a/lib/gitlab/sidekiq_middleware/shutdown.rb
+++ b/lib/gitlab/sidekiq_middleware/shutdown.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'mutex_m'
 
 module Gitlab
diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb
index a1f689d94d9..583a970bf4e 100644
--- a/lib/gitlab/sidekiq_status.rb
+++ b/lib/gitlab/sidekiq_status.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   # The SidekiqStatus module and its child classes can be used for checking if a
   # Sidekiq job has been processed or not.
diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb
index 00983b3284a..bfd5038557d 100644
--- a/lib/gitlab/sidekiq_status/client_middleware.rb
+++ b/lib/gitlab/sidekiq_status/client_middleware.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SidekiqStatus
     class ClientMiddleware
diff --git a/lib/gitlab/sidekiq_status/server_middleware.rb b/lib/gitlab/sidekiq_status/server_middleware.rb
index ceab10b8301..01bc58fd2be 100644
--- a/lib/gitlab/sidekiq_status/server_middleware.rb
+++ b/lib/gitlab/sidekiq_status/server_middleware.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SidekiqStatus
     class ServerMiddleware
diff --git a/lib/gitlab/sidekiq_throttler.rb b/lib/gitlab/sidekiq_throttler.rb
deleted file mode 100644
index 5512afa45a8..00000000000
--- a/lib/gitlab/sidekiq_throttler.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-module Gitlab
-  class SidekiqThrottler
-    class << self
-      def execute!
-        if Gitlab::CurrentSettings.sidekiq_throttling_enabled?
-          require 'sidekiq-limit_fetch'
-
-          Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_queues.each do |queue|
-            Sidekiq::Queue[queue].limit = queue_limit
-          end
-        end
-      end
-
-      private
-
-      def queue_limit
-        @queue_limit ||=
-          begin
-            factor = Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_factor
-            (factor * Sidekiq.options[:concurrency]).ceil
-          end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/sidekiq_versioning.rb b/lib/gitlab/sidekiq_versioning.rb
index 9683214ec18..8164a5a9d7a 100644
--- a/lib/gitlab/sidekiq_versioning.rb
+++ b/lib/gitlab/sidekiq_versioning.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SidekiqVersioning
     def self.install!
diff --git a/lib/gitlab/sidekiq_versioning/manager.rb b/lib/gitlab/sidekiq_versioning/manager.rb
index 308be0fdf76..e5852b43003 100644
--- a/lib/gitlab/sidekiq_versioning/manager.rb
+++ b/lib/gitlab/sidekiq_versioning/manager.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SidekiqVersioning
     module Manager
diff --git a/lib/gitlab/slash_commands/base_command.rb b/lib/gitlab/slash_commands/base_command.rb
index 466554e398c..fcc120112f2 100644
--- a/lib/gitlab/slash_commands/base_command.rb
+++ b/lib/gitlab/slash_commands/base_command.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SlashCommands
     class BaseCommand
@@ -40,9 +42,11 @@ module Gitlab
 
       private
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def find_by_iid(iid)
         collection.find_by(iid: iid)
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/gitlab/slash_commands/command.rb b/lib/gitlab/slash_commands/command.rb
index bb778f37096..474c09b9c4d 100644
--- a/lib/gitlab/slash_commands/command.rb
+++ b/lib/gitlab/slash_commands/command.rb
@@ -1,13 +1,17 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SlashCommands
     class Command < BaseCommand
-      COMMANDS = [
-        Gitlab::SlashCommands::IssueShow,
-        Gitlab::SlashCommands::IssueNew,
-        Gitlab::SlashCommands::IssueSearch,
-        Gitlab::SlashCommands::IssueMove,
-        Gitlab::SlashCommands::Deploy
-      ].freeze
+      def self.commands
+        [
+          Gitlab::SlashCommands::IssueShow,
+          Gitlab::SlashCommands::IssueNew,
+          Gitlab::SlashCommands::IssueSearch,
+          Gitlab::SlashCommands::IssueMove,
+          Gitlab::SlashCommands::Deploy
+        ]
+      end
 
       def execute
         command, match = match_command
@@ -37,7 +41,7 @@ module Gitlab
       private
 
       def available_commands
-        COMMANDS.select do |klass|
+        self.class.commands.keep_if do |klass|
           klass.available?(project)
         end
       end
diff --git a/lib/gitlab/slash_commands/deploy.rb b/lib/gitlab/slash_commands/deploy.rb
index 93e00ab75a1..157d924f99f 100644
--- a/lib/gitlab/slash_commands/deploy.rb
+++ b/lib/gitlab/slash_commands/deploy.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SlashCommands
     class Deploy < BaseCommand
@@ -36,6 +38,7 @@ module Gitlab
 
       private
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def find_action(from, to)
         environment = project.environments.find_by(name: from)
         return unless environment
@@ -50,6 +53,7 @@ module Gitlab
           actions.first
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/gitlab/slash_commands/help.rb b/lib/gitlab/slash_commands/help.rb
index 81f3707e03e..dbe15baa3d7 100644
--- a/lib/gitlab/slash_commands/help.rb
+++ b/lib/gitlab/slash_commands/help.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SlashCommands
     class Help < BaseCommand
diff --git a/lib/gitlab/slash_commands/issue_command.rb b/lib/gitlab/slash_commands/issue_command.rb
index 3d96982b820..4c8dc4b1784 100644
--- a/lib/gitlab/slash_commands/issue_command.rb
+++ b/lib/gitlab/slash_commands/issue_command.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SlashCommands
     class IssueCommand < BaseCommand
diff --git a/lib/gitlab/slash_commands/issue_move.rb b/lib/gitlab/slash_commands/issue_move.rb
index 3985e635983..d2f1f130b38 100644
--- a/lib/gitlab/slash_commands/issue_move.rb
+++ b/lib/gitlab/slash_commands/issue_move.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SlashCommands
     class IssueMove < IssueCommand
diff --git a/lib/gitlab/slash_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb
index 25f965e843d..48379031537 100644
--- a/lib/gitlab/slash_commands/issue_new.rb
+++ b/lib/gitlab/slash_commands/issue_new.rb
@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SlashCommands
     class IssueNew < IssueCommand
       def self.match(text)
         # we can not match \n with the dot by passing the m modifier as than
-        # the title and description are not seperated
+        # the title and description are not separated
         /\Aissue\s+(new|create)\s+(?<title>[^\n]*)\n*(?<description>(.|\n)*)/.match(text)
       end
 
diff --git a/lib/gitlab/slash_commands/issue_search.rb b/lib/gitlab/slash_commands/issue_search.rb
index acba84b54b4..0a705de4484 100644
--- a/lib/gitlab/slash_commands/issue_search.rb
+++ b/lib/gitlab/slash_commands/issue_search.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SlashCommands
     class IssueSearch < IssueCommand
@@ -9,6 +11,7 @@ module Gitlab
         "issue search <your query>"
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def execute(match)
         issues = collection.search(match[:query]).limit(QUERY_LIMIT)
 
@@ -18,6 +21,7 @@ module Gitlab
           Presenters::Access.new(issues).not_found
         end
       end
+      # rubocop: enable CodeReuse/ActiveRecord
     end
   end
 end
diff --git a/lib/gitlab/slash_commands/issue_show.rb b/lib/gitlab/slash_commands/issue_show.rb
index ffa5184e5cb..5f5fa32ff20 100644
--- a/lib/gitlab/slash_commands/issue_show.rb
+++ b/lib/gitlab/slash_commands/issue_show.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SlashCommands
     class IssueShow < IssueCommand
diff --git a/lib/gitlab/slash_commands/presenters/access.rb b/lib/gitlab/slash_commands/presenters/access.rb
index 1a817eb735b..fa163cb098e 100644
--- a/lib/gitlab/slash_commands/presenters/access.rb
+++ b/lib/gitlab/slash_commands/presenters/access.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SlashCommands
     module Presenters
@@ -15,7 +17,7 @@ module Gitlab
             if @resource
               ":wave: Hi there! Before I do anything for you, please [connect your GitLab account](#{@resource})."
             else
-              ":sweat_smile: Couldn't identify you, nor can I autorize you!"
+              ":sweat_smile: Couldn't identify you, nor can I authorize you!"
             end
 
           ephemeral_response(text: message)
diff --git a/lib/gitlab/slash_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb
index e13808a2720..73814aa180f 100644
--- a/lib/gitlab/slash_commands/presenters/base.rb
+++ b/lib/gitlab/slash_commands/presenters/base.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SlashCommands
     module Presenters
diff --git a/lib/gitlab/slash_commands/presenters/deploy.rb b/lib/gitlab/slash_commands/presenters/deploy.rb
index ebae0f57f9b..7d852eb1f9a 100644
--- a/lib/gitlab/slash_commands/presenters/deploy.rb
+++ b/lib/gitlab/slash_commands/presenters/deploy.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SlashCommands
     module Presenters
diff --git a/lib/gitlab/slash_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb
index ab855319077..480d7aa6a30 100644
--- a/lib/gitlab/slash_commands/presenters/help.rb
+++ b/lib/gitlab/slash_commands/presenters/help.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SlashCommands
     module Presenters
diff --git a/lib/gitlab/slash_commands/presenters/issue_base.rb b/lib/gitlab/slash_commands/presenters/issue_base.rb
index 31c1e97efba..b6db103b82b 100644
--- a/lib/gitlab/slash_commands/presenters/issue_base.rb
+++ b/lib/gitlab/slash_commands/presenters/issue_base.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SlashCommands
     module Presenters
diff --git a/lib/gitlab/slash_commands/presenters/issue_move.rb b/lib/gitlab/slash_commands/presenters/issue_move.rb
index 03921729941..ca0644ede95 100644
--- a/lib/gitlab/slash_commands/presenters/issue_move.rb
+++ b/lib/gitlab/slash_commands/presenters/issue_move.rb
@@ -1,4 +1,6 @@
 # coding: utf-8
+# frozen_string_literal: true
+
 module Gitlab
   module SlashCommands
     module Presenters
diff --git a/lib/gitlab/slash_commands/presenters/issue_new.rb b/lib/gitlab/slash_commands/presenters/issue_new.rb
index 5964bfe9960..ac78745ae70 100644
--- a/lib/gitlab/slash_commands/presenters/issue_new.rb
+++ b/lib/gitlab/slash_commands/presenters/issue_new.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SlashCommands
     module Presenters
diff --git a/lib/gitlab/slash_commands/presenters/issue_search.rb b/lib/gitlab/slash_commands/presenters/issue_search.rb
index 4e27d668685..0d497efec0e 100644
--- a/lib/gitlab/slash_commands/presenters/issue_search.rb
+++ b/lib/gitlab/slash_commands/presenters/issue_search.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SlashCommands
     module Presenters
diff --git a/lib/gitlab/slash_commands/presenters/issue_show.rb b/lib/gitlab/slash_commands/presenters/issue_show.rb
index 562f15f403c..5a2c79a928e 100644
--- a/lib/gitlab/slash_commands/presenters/issue_show.rb
+++ b/lib/gitlab/slash_commands/presenters/issue_show.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SlashCommands
     module Presenters
@@ -38,10 +40,10 @@ module Gitlab
         end
 
         def text
-          message = "**#{status_text(@resource)}**"
+          message = ["**#{status_text(@resource)}**"]
 
           if @resource.upvotes.zero? && @resource.downvotes.zero? && @resource.user_notes_count.zero?
-            return message
+            return message.join
           end
 
           message << " · "
@@ -49,7 +51,7 @@ module Gitlab
           message << ":-1: #{@resource.downvotes} " unless @resource.downvotes.zero?
           message << ":speech_balloon: #{@resource.user_notes_count}" unless @resource.user_notes_count.zero?
 
-          message
+          message.join
         end
 
         def pretext
diff --git a/lib/gitlab/slash_commands/result.rb b/lib/gitlab/slash_commands/result.rb
index 3669dedf0fe..607c9c8dec1 100644
--- a/lib/gitlab/slash_commands/result.rb
+++ b/lib/gitlab/slash_commands/result.rb
@@ -1,4 +1,7 @@
-module Gitlab # rubocop:disable Naming/FileName
+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+module Gitlab
   module SlashCommands
     Result = Struct.new(:type, :message)
   end
diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb
index 4f86b3e8f73..e360b552f89 100644
--- a/lib/gitlab/snippet_search_results.rb
+++ b/lib/gitlab/snippet_search_results.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class SnippetSearchResults < SearchResults
     include SnippetsHelper
@@ -30,13 +32,17 @@ module Gitlab
 
     private
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def snippet_titles
       limit_snippets.search(query).order('updated_at DESC').includes(:author)
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
+    # rubocop: disable CodeReuse/ActiveRecord
     def snippet_blobs
       limit_snippets.search_code(query).order('updated_at DESC').includes(:author)
     end
+    # rubocop: enable CodeReuse/ActiveRecord
 
     def default_scope
       'snippet_blobs'
diff --git a/lib/gitlab/sql/cte.rb b/lib/gitlab/sql/cte.rb
new file mode 100644
index 00000000000..7817a2a1ce2
--- /dev/null
+++ b/lib/gitlab/sql/cte.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SQL
+    # Class for easily building CTE statements.
+    #
+    # Example:
+    #
+    #     cte = CTE.new(:my_cte_name)
+    #     ns = Arel::Table.new(:namespaces)
+    #
+    #     cte << Namespace.
+    #       where(ns[:parent_id].eq(some_namespace_id))
+    #
+    #     Namespace
+    #       with(cte.to_arel).
+    #       from(cte.alias_to(ns))
+    class CTE
+      attr_reader :table, :query
+
+      # name - The name of the CTE as a String or Symbol.
+      def initialize(name, query)
+        @table = Arel::Table.new(name)
+        @query = query
+      end
+
+      # Returns the Arel relation for this CTE.
+      def to_arel
+        sql = Arel::Nodes::SqlLiteral.new("(#{query.to_sql})")
+
+        Arel::Nodes::As.new(table, sql)
+      end
+
+      # Returns an "AS" statement that aliases the CTE name as the given table
+      # name. This allows one to trick ActiveRecord into thinking it's selecting
+      # from an actual table, when in reality it's selecting from a CTE.
+      #
+      # alias_table - The Arel table to use as the alias.
+      def alias_to(alias_table)
+        Arel::Nodes::As.new(table, alias_table)
+      end
+
+      # Applies the CTE to the given relation, returning a new one that will
+      # query from it.
+      def apply_to(relation)
+        relation.except(:where)
+          .with(to_arel)
+          .from(alias_to(relation.model.arel_table))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sql/glob.rb b/lib/gitlab/sql/glob.rb
index 5e89e12b2b1..f3421bd95d2 100644
--- a/lib/gitlab/sql/glob.rb
+++ b/lib/gitlab/sql/glob.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SQL
     module Glob
diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb
index 53744bad1f4..92388262035 100644
--- a/lib/gitlab/sql/pattern.rb
+++ b/lib/gitlab/sql/pattern.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SQL
     module Pattern
diff --git a/lib/gitlab/sql/recursive_cte.rb b/lib/gitlab/sql/recursive_cte.rb
index 16ec002f139..ec1f00a3a91 100644
--- a/lib/gitlab/sql/recursive_cte.rb
+++ b/lib/gitlab/sql/recursive_cte.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SQL
     # Class for easily building recursive CTE statements.
diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb
index c99b262f1ca..f05592fc3a3 100644
--- a/lib/gitlab/sql/union.rb
+++ b/lib/gitlab/sql/union.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module SQL
     # Class for building SQL UNION statements.
@@ -7,7 +9,7 @@ module Gitlab
     #
     # Example usage:
     #
-    #     union = Gitlab::SQL::Union.new(user.personal_projects, user.projects)
+    #     union = Gitlab::SQL::Union.new([user.personal_projects, user.projects])
     #     sql   = union.to_sql
     #
     #     Project.where("id IN (#{sql})")
diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb
index 6f63ea91ae8..6df54852d02 100644
--- a/lib/gitlab/ssh_public_key.rb
+++ b/lib/gitlab/ssh_public_key.rb
@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 module Gitlab
   class SSHPublicKey
     Technology = Struct.new(:name, :key_class, :supported_sizes)
 
-    Technologies = [
+    TECHNOLOGIES = [
       Technology.new(:rsa, OpenSSL::PKey::RSA, [1024, 2048, 3072, 4096]),
       Technology.new(:dsa, OpenSSL::PKey::DSA, [1024, 2048, 3072]),
       Technology.new(:ecdsa, OpenSSL::PKey::EC, [256, 384, 521]),
@@ -10,11 +12,11 @@ module Gitlab
     ].freeze
 
     def self.technology(name)
-      Technologies.find { |tech| tech.name.to_s == name.to_s }
+      TECHNOLOGIES.find { |tech| tech.name.to_s == name.to_s }
     end
 
     def self.technology_for_key(key)
-      Technologies.find { |tech| key.is_a?(tech.key_class) }
+      TECHNOLOGIES.find { |tech| key.is_a?(tech.key_class) }
     end
 
     def self.supported_sizes(name)
@@ -26,7 +28,7 @@ module Gitlab
 
       return key_content if parts.empty?
 
-      parts.each_with_object("#{ssh_type} ").with_index do |(part, content), index|
+      parts.each_with_object(+"#{ssh_type} ").with_index do |(part, content), index|
         content << part
 
         if Gitlab::SSHPublicKey.new(content).valid?
diff --git a/lib/gitlab/storage_check.rb b/lib/gitlab/storage_check.rb
deleted file mode 100644
index fe81513c9ec..00000000000
--- a/lib/gitlab/storage_check.rb
+++ /dev/null
@@ -1,11 +0,0 @@
-require_relative 'storage_check/cli'
-require_relative 'storage_check/gitlab_caller'
-require_relative 'storage_check/option_parser'
-require_relative 'storage_check/response'
-
-module Gitlab
-  module StorageCheck
-    ENDPOINT = '/-/storage_check'.freeze
-    Options = Struct.new(:target, :token, :interval, :dryrun)
-  end
-end
diff --git a/lib/gitlab/storage_check/cli.rb b/lib/gitlab/storage_check/cli.rb
deleted file mode 100644
index 9b64c8e033a..00000000000
--- a/lib/gitlab/storage_check/cli.rb
+++ /dev/null
@@ -1,71 +0,0 @@
-module Gitlab
-  module StorageCheck
-    class CLI
-      def self.start!(args)
-        runner = new(Gitlab::StorageCheck::OptionParser.parse!(args))
-        runner.start_loop
-      end
-
-      attr_reader :logger, :options
-
-      def initialize(options)
-        @options = options
-        @logger = Logger.new(STDOUT)
-      end
-
-      def start_loop
-        logger.info "Checking #{options.target} every #{options.interval} seconds"
-
-        if options.dryrun
-          logger.info "Dryrun, exiting..."
-          return
-        end
-
-        begin
-          loop do
-            response = GitlabCaller.new(options).call!
-            log_response(response)
-            update_settings(response)
-
-            sleep options.interval
-          end
-        rescue Interrupt
-          logger.info "Ending storage-check"
-        end
-      end
-
-      def update_settings(response)
-        previous_interval = options.interval
-
-        if response.valid?
-          options.interval = response.check_interval || previous_interval
-        end
-
-        if previous_interval != options.interval
-          logger.info "Interval changed: #{options.interval} seconds"
-        end
-      end
-
-      def log_response(response)
-        unless response.valid?
-          return logger.error("Invalid response checking nfs storage: #{response.http_response.inspect}")
-        end
-
-        if response.responsive_shards.any?
-          logger.debug("Responsive shards: #{response.responsive_shards.join(', ')}")
-        end
-
-        warnings = []
-        if response.skipped_shards.any?
-          warnings << "Skipped shards: #{response.skipped_shards.join(', ')}"
-        end
-
-        if response.failing_shards.any?
-          warnings << "Failing shards: #{response.failing_shards.join(', ')}"
-        end
-
-        logger.warn(warnings.join(' - ')) if warnings.any?
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/storage_check/gitlab_caller.rb b/lib/gitlab/storage_check/gitlab_caller.rb
deleted file mode 100644
index 44952b68844..00000000000
--- a/lib/gitlab/storage_check/gitlab_caller.rb
+++ /dev/null
@@ -1,39 +0,0 @@
-require 'excon'
-
-module Gitlab
-  module StorageCheck
-    class GitlabCaller
-      def initialize(options)
-        @options = options
-      end
-
-      def call!
-        Gitlab::StorageCheck::Response.new(get_response)
-      rescue Errno::ECONNREFUSED, Excon::Error
-        # Server not ready, treated as invalid response.
-        Gitlab::StorageCheck::Response.new(nil)
-      end
-
-      def get_response
-        scheme, *other_parts = URI.split(@options.target)
-        socket_path = if  scheme == 'unix'
-                        other_parts.compact.join
-                      end
-
-        connection = Excon.new(@options.target, socket: socket_path)
-        connection.post(path: Gitlab::StorageCheck::ENDPOINT,
-                        headers: headers)
-      end
-
-      def headers
-        @headers ||= begin
-                       headers = {}
-                       headers['Content-Type'] = headers['Accept'] = 'application/json'
-                       headers['TOKEN'] = @options.token if @options.token
-
-                       headers
-                     end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/storage_check/option_parser.rb b/lib/gitlab/storage_check/option_parser.rb
deleted file mode 100644
index 66ed7906f97..00000000000
--- a/lib/gitlab/storage_check/option_parser.rb
+++ /dev/null
@@ -1,39 +0,0 @@
-module Gitlab
-  module StorageCheck
-    class OptionParser
-      def self.parse!(args)
-        # Start out with some defaults
-        options = Gitlab::StorageCheck::Options.new(nil, nil, 1, false)
-
-        parser = ::OptionParser.new do |opts|
-          opts.banner = "Usage: bin/storage_check [options]"
-
-          opts.on('-t=string', '--target string', 'URL or socket to trigger storage check') do |value|
-            options.target = value
-          end
-
-          opts.on('-T=string', '--token string', 'Health token to use') { |value| options.token = value }
-
-          opts.on('-i=n', '--interval n', ::OptionParser::DecimalInteger, 'Seconds between checks') do |value|
-            options.interval = value
-          end
-
-          opts.on('-d', '--dryrun', "Output what will be performed, but don't start the process") do |value|
-            options.dryrun = value
-          end
-        end
-        parser.parse!(args)
-
-        unless options.target
-          raise ::OptionParser::InvalidArgument.new('Provide a URI to provide checks')
-        end
-
-        if URI.parse(options.target).scheme.nil?
-          raise ::OptionParser::InvalidArgument.new('Add the scheme to the target, `unix://`, `https://` or `http://` are supported')
-        end
-
-        options
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/storage_check/response.rb b/lib/gitlab/storage_check/response.rb
deleted file mode 100644
index 326ab236e3e..00000000000
--- a/lib/gitlab/storage_check/response.rb
+++ /dev/null
@@ -1,77 +0,0 @@
-require 'json'
-
-module Gitlab
-  module StorageCheck
-    class Response
-      attr_reader :http_response
-
-      def initialize(http_response)
-        @http_response = http_response
-      end
-
-      def valid?
-        @http_response && (200...299).cover?(@http_response.status) &&
-          @http_response.headers['Content-Type'].include?('application/json') &&
-          parsed_response
-      end
-
-      def check_interval
-        return nil unless parsed_response
-
-        parsed_response['check_interval']
-      end
-
-      def responsive_shards
-        divided_results[:responsive_shards]
-      end
-
-      def skipped_shards
-        divided_results[:skipped_shards]
-      end
-
-      def failing_shards
-        divided_results[:failing_shards]
-      end
-
-      private
-
-      def results
-        return [] unless parsed_response
-
-        parsed_response['results']
-      end
-
-      def divided_results
-        return @divided_results if @divided_results
-
-        @divided_results = {}
-        @divided_results[:responsive_shards] = []
-        @divided_results[:skipped_shards] = []
-        @divided_results[:failing_shards] = []
-
-        results.each do |info|
-          name = info['storage']
-
-          case info['success']
-          when true
-            @divided_results[:responsive_shards] << name
-          when false
-            @divided_results[:failing_shards] << name
-          else
-            @divided_results[:skipped_shards] << name
-          end
-        end
-
-        @divided_results
-      end
-
-      def parsed_response
-        return @parsed_response if defined?(@parsed_response)
-
-        @parsed_response = JSON.parse(@http_response.body)
-      rescue JSON::JSONError
-        @parsed_response = nil
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/string_placeholder_replacer.rb b/lib/gitlab/string_placeholder_replacer.rb
index 9a2219b7d77..62621255a53 100644
--- a/lib/gitlab/string_placeholder_replacer.rb
+++ b/lib/gitlab/string_placeholder_replacer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class StringPlaceholderReplacer
     # This method accepts the following paras
diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb
index c6ad997a4d4..780fe4c7725 100644
--- a/lib/gitlab/string_range_marker.rb
+++ b/lib/gitlab/string_range_marker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class StringRangeMarker
     attr_accessor :raw_line, :rich_line, :html_escaped
diff --git a/lib/gitlab/string_regex_marker.rb b/lib/gitlab/string_regex_marker.rb
index b19aa6dea35..f1982ff914c 100644
--- a/lib/gitlab/string_regex_marker.rb
+++ b/lib/gitlab/string_regex_marker.rb
@@ -1,5 +1,8 @@
+# frozen_string_literal: true
+
 module Gitlab
   class StringRegexMarker < StringRangeMarker
+    # rubocop: disable CodeReuse/ActiveRecord
     def mark(regex, group: 0, &block)
       ranges = []
 
@@ -11,5 +14,6 @@ module Gitlab
 
       super(ranges, &block)
     end
+    # rubocop: enable CodeReuse/ActiveRecord
   end
 end
diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb
index 42be301fd9b..224bb648d8f 100644
--- a/lib/gitlab/task_helpers.rb
+++ b/lib/gitlab/task_helpers.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rainbow/ext/string'
 require 'gitlab/utils/strong_memoize'
 
@@ -39,7 +41,7 @@ module Gitlab
           File.read('/etc/os-release').match(/PRETTY_NAME=\"(.+)\"/)[1]
         end
 
-      os_name.try(:squish!)
+      os_name.try(:squish)
     end
 
     # Prompt the user to input something
@@ -128,17 +130,21 @@ module Gitlab
     end
 
     def all_repos
-      Gitlab.config.repositories.storages.each_value do |repository_storage|
-        IO.popen(%W(find #{repository_storage.legacy_disk_path} -mindepth 2 -type d -name *.git)) do |find|
-          find.each_line do |path|
-            yield path.chomp
+      Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+        Gitlab.config.repositories.storages.each_value do |repository_storage|
+          IO.popen(%W(find #{repository_storage.legacy_disk_path} -mindepth 2 -type d -name *.git)) do |find|
+            find.each_line do |path|
+              yield path.chomp
+            end
           end
         end
       end
     end
 
     def repository_storage_paths_args
-      Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path }
+      Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+        Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path }
+      end
     end
 
     def user_home
diff --git a/lib/gitlab/tcp_checker.rb b/lib/gitlab/tcp_checker.rb
index 6e24e46d0ea..f37a044b607 100644
--- a/lib/gitlab/tcp_checker.rb
+++ b/lib/gitlab/tcp_checker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class TcpChecker
     attr_reader :remote_host, :remote_port, :local_host, :local_port, :error
diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb
index 7393574ac13..0b4cc571dc0 100644
--- a/lib/gitlab/template/base_template.rb
+++ b/lib/gitlab/template/base_template.rb
@@ -1,21 +1,34 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Template
     class BaseTemplate
-      def initialize(path, project = nil)
+      attr_accessor :category
+
+      def initialize(path, project = nil, category: nil)
         @path = path
+        @category = category
         @finder = self.class.finder(project)
       end
 
       def name
         File.basename(@path, self.class.extension)
       end
+      alias_method :key, :name
 
       def content
         @finder.read(@path)
       end
 
+      # Present for compatibility with license templates, which can replace text
+      # like `[fullname]` with a user-specified string. This is a no-op for
+      # other templates
+      def resolve!(_placeholders = {})
+        self
+      end
+
       def to_json
-        { name: name, content: content }
+        { key: key, name: name, content: content }
       end
 
       def <=>(other)
@@ -62,7 +75,7 @@ module Gitlab
           directory = category_directory(category)
           files = finder(project).list_files_for(directory)
 
-          files.map { |f| new(f, project) }.sort
+          files.map { |f| new(f, project, category: category) }.sort
         end
 
         def category_directory(category)
diff --git a/lib/gitlab/template/dockerfile_template.rb b/lib/gitlab/template/dockerfile_template.rb
index 20b054b0bd8..3b516bb862a 100644
--- a/lib/gitlab/template/dockerfile_template.rb
+++ b/lib/gitlab/template/dockerfile_template.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Template
     class DockerfileTemplate < BaseTemplate
diff --git a/lib/gitlab/template/finders/base_template_finder.rb b/lib/gitlab/template/finders/base_template_finder.rb
index 473b05257c6..93c229af143 100644
--- a/lib/gitlab/template/finders/base_template_finder.rb
+++ b/lib/gitlab/template/finders/base_template_finder.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Template
     module Finders
@@ -21,7 +23,7 @@ module Gitlab
         def category_directory(category)
           return @base_dir unless category.present?
 
-          @base_dir + @categories[category]
+          File.join(@base_dir, @categories[category])
         end
 
         class << self
diff --git a/lib/gitlab/template/finders/global_template_finder.rb b/lib/gitlab/template/finders/global_template_finder.rb
index 831da45191f..2dd4b7a4092 100644
--- a/lib/gitlab/template/finders/global_template_finder.rb
+++ b/lib/gitlab/template/finders/global_template_finder.rb
@@ -1,4 +1,6 @@
-# Searches and reads file present on Gitlab installation directory
+# frozen_string_literal: true
+
+# Searches and reads file present on GitLab installation directory
 module Gitlab
   module Template
     module Finders
@@ -16,12 +18,16 @@ module Gitlab
         def find(key)
           file_name = "#{key}#{@extension}"
 
+          # The key is untrusted input, so ensure we can't be directed outside
+          # of base_dir
+          Gitlab::Utils.check_path_traversal!(file_name)
+
           directory = select_directory(file_name)
           directory ? File.join(category_directory(directory), file_name) : nil
         end
 
         def list_files_for(dir)
-          dir << '/' unless dir.end_with?('/')
+          dir = "#{dir}/" unless dir.end_with?('/')
           Dir.glob(File.join(dir, "*#{@extension}")).select { |f| f =~ self.class.filter_regex(@extension) }
         end
 
diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb
index 33f07fa0120..8e234148a63 100644
--- a/lib/gitlab/template/finders/repo_template_finder.rb
+++ b/lib/gitlab/template/finders/repo_template_finder.rb
@@ -1,4 +1,6 @@
-# Searches and reads files present on each Gitlab project repository
+# frozen_string_literal: true
+
+# Searches and reads files present on each GitLab project repository
 module Gitlab
   module Template
     module Finders
@@ -24,21 +26,26 @@ module Gitlab
 
         def find(key)
           file_name = "#{key}#{@extension}"
+
+          # The key is untrusted input, so ensure we can't be directed outside
+          # of base_dir inside the repository
+          Gitlab::Utils.check_path_traversal!(file_name)
+
           directory = select_directory(file_name)
           raise FileNotFoundError if directory.nil?
 
-          category_directory(directory) + file_name
+          File.join(category_directory(directory), file_name)
         end
 
         def list_files_for(dir)
           return [] unless @commit
 
-          dir << '/' unless dir.end_with?('/')
+          dir = "#{dir}/" unless dir.end_with?('/')
 
           entries = @repository.tree(:head, dir).entries
 
-          names = entries.map(&:name)
-          names.select { |f| f =~ self.class.filter_regex(@extension) }
+          paths = entries.map(&:path)
+          paths.select { |f| f =~ self.class.filter_regex(@extension) }
         end
 
         private
@@ -47,10 +54,10 @@ module Gitlab
           return [] unless @commit
 
           # Insert root as directory
-          directories = ["", @categories.keys]
+          directories = ["", *@categories.keys]
 
           directories.find do |category|
-            path = category_directory(category) + file_name
+            path = File.join(category_directory(category), file_name)
             @repository.blob_at(@commit.id, path)
           end
         end
diff --git a/lib/gitlab/template/gitignore_template.rb b/lib/gitlab/template/gitignore_template.rb
index 8d2a9d2305c..72a1b7460c2 100644
--- a/lib/gitlab/template/gitignore_template.rb
+++ b/lib/gitlab/template/gitignore_template.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Template
     class GitignoreTemplate < BaseTemplate
diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb
index fd040148a1e..fbefb5f7f0e 100644
--- a/lib/gitlab/template/gitlab_ci_yml_template.rb
+++ b/lib/gitlab/template/gitlab_ci_yml_template.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Template
     class GitlabCiYmlTemplate < BaseTemplate
@@ -20,7 +22,7 @@ module Gitlab
         end
 
         def base_dir
-          Rails.root.join('vendor/gitlab-ci-yml')
+          Rails.root.join('lib/gitlab/ci/templates')
         end
 
         def finder(project = nil)
diff --git a/lib/gitlab/template/issue_template.rb b/lib/gitlab/template/issue_template.rb
index c6fa8d3eafc..01b191733d4 100644
--- a/lib/gitlab/template/issue_template.rb
+++ b/lib/gitlab/template/issue_template.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Template
     class IssueTemplate < BaseTemplate
diff --git a/lib/gitlab/template/merge_request_template.rb b/lib/gitlab/template/merge_request_template.rb
index f826c02f3b5..357b31cd82e 100644
--- a/lib/gitlab/template/merge_request_template.rb
+++ b/lib/gitlab/template/merge_request_template.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Template
     class MergeRequestTemplate < BaseTemplate
diff --git a/lib/gitlab/template_helper.rb b/lib/gitlab/template_helper.rb
new file mode 100644
index 00000000000..b0e01697a66
--- /dev/null
+++ b/lib/gitlab/template_helper.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module TemplateHelper
+    def prepare_template_environment(file)
+      return unless file
+
+      params[:import_export_upload] = ImportExportUpload.new(import_file: file)
+    end
+
+    def tmp_filename
+      SecureRandom.hex
+    end
+  end
+end
diff --git a/lib/gitlab/temporarily_allow.rb b/lib/gitlab/temporarily_allow.rb
index 880e55f71df..000f8ca699d 100644
--- a/lib/gitlab/temporarily_allow.rb
+++ b/lib/gitlab/temporarily_allow.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module TemporarilyAllow
     TEMPORARILY_ALLOW_MUTEX = Mutex.new
@@ -10,7 +12,7 @@ module Gitlab
     end
 
     def temporarily_allowed?(key)
-      if RequestStore.active?
+      if Gitlab::SafeRequestStore.active?
         temporarily_allow_request_store[key] > 0
       else
         TEMPORARILY_ALLOW_MUTEX.synchronize do
@@ -26,11 +28,11 @@ module Gitlab
     end
 
     def temporarily_allow_request_store
-      RequestStore[:temporarily_allow] ||= Hash.new(0)
+      Gitlab::SafeRequestStore[:temporarily_allow] ||= Hash.new(0)
     end
 
     def temporarily_allow_add(key, value)
-      if RequestStore.active?
+      if Gitlab::SafeRequestStore.active?
         temporarily_allow_request_store[key] += value
       else
         TEMPORARILY_ALLOW_MUTEX.synchronize do
diff --git a/lib/gitlab/testing/request_blocker_middleware.rb b/lib/gitlab/testing/request_blocker_middleware.rb
index 53333b9b06b..513cbe839ba 100644
--- a/lib/gitlab/testing/request_blocker_middleware.rb
+++ b/lib/gitlab/testing/request_blocker_middleware.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # rubocop:disable Style/ClassVars
 
 # This is inspired by http://www.salsify.com/blog/engineering/tearing-capybara-ajax-tests
diff --git a/lib/gitlab/testing/request_inspector_middleware.rb b/lib/gitlab/testing/request_inspector_middleware.rb
index e387667480d..36cdfebcc28 100644
--- a/lib/gitlab/testing/request_inspector_middleware.rb
+++ b/lib/gitlab/testing/request_inspector_middleware.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # rubocop:disable Style/ClassVars
 
 module Gitlab
@@ -35,11 +37,15 @@ module Gitlab
         request_headers = env_http_headers(env)
         status, headers, body = @app.call(env)
 
+        full_body = +''
+        body.each { |b| full_body << b }
+
         request = OpenStruct.new(
           url: url,
           status_code: status,
           request_headers: request_headers,
-          response_headers: headers
+          response_headers: headers,
+          body: full_body
         )
         log_request request
 
diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb
index d43eff5ba4a..63860b9cb26 100644
--- a/lib/gitlab/themes.rb
+++ b/lib/gitlab/themes.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   # Module containing GitLab's application theme definitions and helper methods
   # for accessing them.
@@ -12,11 +14,16 @@ module Gitlab
 
     # All available Themes
     THEMES = [
-      Theme.new(1, 'Indigo', 'ui_indigo'),
-      Theme.new(2, 'Dark', 'ui_dark'),
-      Theme.new(3, 'Light', 'ui_light'),
-      Theme.new(4, 'Blue', 'ui_blue'),
-      Theme.new(5, 'Green', 'ui_green')
+      Theme.new(1, 'Indigo', 'ui-indigo'),
+      Theme.new(6, 'Light Indigo', 'ui-light-indigo'),
+      Theme.new(4, 'Blue', 'ui-blue'),
+      Theme.new(7, 'Light Blue', 'ui-light-blue'),
+      Theme.new(5, 'Green', 'ui-green'),
+      Theme.new(8, 'Light Green', 'ui-light-green'),
+      Theme.new(9, 'Red', 'ui-red'),
+      Theme.new(10, 'Light Red', 'ui-light-red'),
+      Theme.new(2, 'Dark', 'ui-dark'),
+      Theme.new(3, 'Light', 'ui-light')
     ].freeze
 
     # Convenience method to get a space-separated String of all the theme
diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb
index d615c24149a..cc206010e74 100644
--- a/lib/gitlab/time_tracking_formatter.rb
+++ b/lib/gitlab/time_tracking_formatter.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module TimeTrackingFormatter
     extend self
diff --git a/lib/gitlab/timeless.rb b/lib/gitlab/timeless.rb
index 76a1808c8ac..4f974c98c71 100644
--- a/lib/gitlab/timeless.rb
+++ b/lib/gitlab/timeless.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Timeless
     def self.timeless(model, &block)
diff --git a/lib/gitlab/tracing.rb b/lib/gitlab/tracing.rb
new file mode 100644
index 00000000000..3c4db42ac06
--- /dev/null
+++ b/lib/gitlab/tracing.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Tracing
+    # Only enable tracing when the `GITLAB_TRACING` env var is configured. Note that we avoid using ApplicationSettings since
+    # the same environment variable needs to be configured for Workhorse, Gitaly and any other components which
+    # emit tracing. Since other components may start before Rails, and may not have access to ApplicationSettings,
+    # an env var makes more sense.
+    def self.enabled?
+      connection_string.present?
+    end
+
+    def self.connection_string
+      ENV['GITLAB_TRACING']
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/common.rb b/lib/gitlab/tracing/common.rb
new file mode 100644
index 00000000000..3a08ede8138
--- /dev/null
+++ b/lib/gitlab/tracing/common.rb
@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'opentracing'
+
+module Gitlab
+  module Tracing
+    module Common
+      def tracer
+        OpenTracing.global_tracer
+      end
+
+      # Convience method for running a block with a span
+      def in_tracing_span(operation_name:, tags:, child_of: nil)
+        scope = tracer.start_active_span(
+          operation_name,
+          child_of: child_of,
+          tags: tags
+        )
+        span = scope.span
+
+        # Add correlation details to the span if we have them
+        correlation_id = Gitlab::CorrelationId.current_id
+        if correlation_id
+          span.set_tag('correlation_id', correlation_id)
+        end
+
+        begin
+          yield span
+        rescue => e
+          log_exception_on_span(span, e)
+          raise e
+        ensure
+          scope.close
+        end
+      end
+
+      def postnotify_span(operation_name, start_time, end_time, tags: nil, child_of: nil, exception: nil)
+        span = OpenTracing.start_span(operation_name, start_time: start_time, tags: tags, child_of: child_of)
+
+        log_exception_on_span(span, exception) if exception
+
+        span.finish(end_time: end_time)
+      end
+
+      def log_exception_on_span(span, exception)
+        span.set_tag('error', true)
+        span.log_kv(kv_tags_for_exception(exception))
+      end
+
+      def kv_tags_for_exception(exception)
+        case exception
+        when Exception
+          {
+            'event':      'error',
+            'error.kind': exception.class.to_s,
+            'message':    Gitlab::UrlSanitizer.sanitize(exception.message),
+            'stack':      exception.backtrace&.join("\n")
+          }
+        else
+          {
+            'event':        'error',
+            'error.kind':   exception.class.to_s,
+            'error.object': Gitlab::UrlSanitizer.sanitize(exception.to_s)
+          }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/factory.rb b/lib/gitlab/tracing/factory.rb
new file mode 100644
index 00000000000..fc714164353
--- /dev/null
+++ b/lib/gitlab/tracing/factory.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "cgi"
+
+module Gitlab
+  module Tracing
+    class Factory
+      OPENTRACING_SCHEME = "opentracing"
+
+      def self.create_tracer(service_name, connection_string)
+        return unless connection_string.present?
+
+        begin
+          opentracing_details = parse_connection_string(connection_string)
+          driver_name = opentracing_details[:driver_name]
+
+          case driver_name
+          when "jaeger"
+            JaegerFactory.create_tracer(service_name, opentracing_details[:options])
+          else
+            raise "Unknown driver: #{driver_name}"
+          end
+        rescue => e
+          # Can't create the tracer? Warn and continue sans tracer
+          warn "Unable to instantiate tracer: #{e}"
+          nil
+        end
+      end
+
+      def self.parse_connection_string(connection_string)
+        parsed = URI.parse(connection_string)
+
+        unless valid_uri?(parsed)
+          raise "Invalid tracing connection string"
+        end
+
+        {
+          driver_name: parsed.host,
+          options: parse_query(parsed.query)
+        }
+      end
+      private_class_method :parse_connection_string
+
+      def self.parse_query(query)
+        return {} unless query
+
+        CGI.parse(query).symbolize_keys.transform_values(&:first)
+      end
+      private_class_method :parse_query
+
+      def self.valid_uri?(uri)
+        return false unless uri
+
+        uri.scheme == OPENTRACING_SCHEME &&
+          uri.host.to_s =~ /^[a-z0-9_]+$/ &&
+          uri.path.empty?
+      end
+      private_class_method :valid_uri?
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/grpc_interceptor.rb b/lib/gitlab/tracing/grpc_interceptor.rb
new file mode 100644
index 00000000000..6c2aab73125
--- /dev/null
+++ b/lib/gitlab/tracing/grpc_interceptor.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'opentracing'
+require 'grpc'
+
+module Gitlab
+  module Tracing
+    class GRPCInterceptor < GRPC::ClientInterceptor
+      include Common
+      include Singleton
+
+      def request_response(request:, call:, method:, metadata:)
+        wrap_with_tracing(method, 'unary', metadata) do
+          yield
+        end
+      end
+
+      def client_streamer(requests:, call:, method:, metadata:)
+        wrap_with_tracing(method, 'client_stream', metadata) do
+          yield
+        end
+      end
+
+      def server_streamer(request:, call:, method:, metadata:)
+        wrap_with_tracing(method, 'server_stream', metadata) do
+          yield
+        end
+      end
+
+      def bidi_streamer(requests:, call:, method:, metadata:)
+        wrap_with_tracing(method, 'bidi_stream', metadata) do
+          yield
+        end
+      end
+
+      private
+
+      def wrap_with_tracing(method, grpc_type, metadata)
+        tags = {
+          'component' =>  'grpc',
+          'span.kind' =>  'client',
+          'grpc.method' => method,
+          'grpc.type' =>   grpc_type
+        }
+
+        in_tracing_span(operation_name: "grpc:#{method}", tags: tags) do |span|
+          OpenTracing.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, metadata)
+
+          yield
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/jaeger_factory.rb b/lib/gitlab/tracing/jaeger_factory.rb
new file mode 100644
index 00000000000..2682007302a
--- /dev/null
+++ b/lib/gitlab/tracing/jaeger_factory.rb
@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require 'jaeger/client'
+
+module Gitlab
+  module Tracing
+    class JaegerFactory
+      # When the probabilistic sampler is used, by default 0.1% of requests will be traced
+      DEFAULT_PROBABILISTIC_RATE = 0.001
+
+      # The default port for the Jaeger agent UDP listener
+      DEFAULT_UDP_PORT = 6831
+
+      # Reduce this from default of 10 seconds as the Ruby jaeger
+      # client doesn't have overflow control, leading to very large
+      # messages which fail to send over UDP (max packet = 64k)
+      # Flush more often, with smaller packets
+      FLUSH_INTERVAL = 5
+
+      def self.create_tracer(service_name, options)
+        kwargs = {
+          service_name: service_name,
+          sampler: get_sampler(options[:sampler], options[:sampler_param]),
+          reporter: get_reporter(service_name, options[:http_endpoint], options[:udp_endpoint])
+        }.compact
+
+        extra_params = options.except(:sampler, :sampler_param, :http_endpoint, :udp_endpoint, :strict_parsing, :debug) # rubocop: disable CodeReuse/ActiveRecord
+        if extra_params.present?
+          message = "jaeger tracer: invalid option: #{extra_params.keys.join(", ")}"
+
+          if options[:strict_parsing]
+            raise message
+          else
+            warn message
+          end
+        end
+
+        Jaeger::Client.build(kwargs)
+      end
+
+      def self.get_sampler(sampler_type, sampler_param)
+        case sampler_type
+        when "probabilistic"
+          sampler_rate = sampler_param ? sampler_param.to_f : DEFAULT_PROBABILISTIC_RATE
+          Jaeger::Samplers::Probabilistic.new(rate: sampler_rate)
+        when "const"
+          const_value = sampler_param == "1"
+          Jaeger::Samplers::Const.new(const_value)
+        else
+          nil
+        end
+      end
+      private_class_method :get_sampler
+
+      def self.get_reporter(service_name, http_endpoint, udp_endpoint)
+        encoder = Jaeger::Encoders::ThriftEncoder.new(service_name: service_name)
+
+        if http_endpoint.present?
+          sender = get_http_sender(encoder, http_endpoint)
+        elsif udp_endpoint.present?
+          sender = get_udp_sender(encoder, udp_endpoint)
+        else
+          return nil
+        end
+
+        Jaeger::Reporters::RemoteReporter.new(
+          sender: sender,
+          flush_interval: FLUSH_INTERVAL
+        )
+      end
+      private_class_method :get_reporter
+
+      def self.get_http_sender(encoder, address)
+        Jaeger::HttpSender.new(
+          url: address,
+          encoder: encoder,
+          logger: Logger.new(STDOUT)
+        )
+      end
+      private_class_method :get_http_sender
+
+      def self.get_udp_sender(encoder, address)
+        pair = address.split(":", 2)
+        host = pair[0]
+        port = pair[1] ? pair[1].to_i : DEFAULT_UDP_PORT
+
+        Jaeger::UdpSender.new(
+          host: host,
+          port: port,
+          encoder: encoder,
+          logger: Logger.new(STDOUT)
+        )
+      end
+      private_class_method :get_udp_sender
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/rack_middleware.rb b/lib/gitlab/tracing/rack_middleware.rb
new file mode 100644
index 00000000000..e6a31293f7b
--- /dev/null
+++ b/lib/gitlab/tracing/rack_middleware.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'opentracing'
+
+module Gitlab
+  module Tracing
+    class RackMiddleware
+      include Common
+
+      REQUEST_METHOD = 'REQUEST_METHOD'
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        method = env[REQUEST_METHOD]
+
+        context = tracer.extract(OpenTracing::FORMAT_RACK, env)
+        tags = {
+          'component' =>   'rack',
+          'span.kind' =>   'server',
+          'http.method' => method,
+          'http.url' =>    self.class.build_sanitized_url_from_env(env)
+        }
+
+        in_tracing_span(operation_name: "http:#{method}", child_of: context, tags: tags) do |span|
+          @app.call(env).tap do |status_code, _headers, _body|
+            span.set_tag('http.status_code', status_code)
+          end
+        end
+      end
+
+      # Generate a sanitized (safe) request URL from the rack environment
+      def self.build_sanitized_url_from_env(env)
+        request = ActionDispatch::Request.new(env)
+
+        original_url = request.original_url
+        uri = URI.parse(original_url)
+        uri.query = request.filtered_parameters.to_query if uri.query.present?
+
+        uri.to_s
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/rails/action_view_subscriber.rb b/lib/gitlab/tracing/rails/action_view_subscriber.rb
new file mode 100644
index 00000000000..88816e1fb32
--- /dev/null
+++ b/lib/gitlab/tracing/rails/action_view_subscriber.rb
@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Tracing
+    module Rails
+      class ActionViewSubscriber
+        include RailsCommon
+
+        COMPONENT_TAG = 'ActionView'
+        RENDER_TEMPLATE_NOTIFICATION_TOPIC = 'render_template.action_view'
+        RENDER_COLLECTION_NOTIFICATION_TOPIC = 'render_collection.action_view'
+        RENDER_PARTIAL_NOTIFICATION_TOPIC = 'render_partial.action_view'
+
+        # Instruments Rails ActionView events for opentracing.
+        # Returns a lambda, which, when called will unsubscribe from the notifications
+        def self.instrument
+          subscriber = new
+
+          subscriptions = [
+            ActiveSupport::Notifications.subscribe(RENDER_TEMPLATE_NOTIFICATION_TOPIC) do |_, start, finish, _, payload|
+              subscriber.notify_render_template(start, finish, payload)
+            end,
+            ActiveSupport::Notifications.subscribe(RENDER_COLLECTION_NOTIFICATION_TOPIC) do |_, start, finish, _, payload|
+              subscriber.notify_render_collection(start, finish, payload)
+            end,
+            ActiveSupport::Notifications.subscribe(RENDER_PARTIAL_NOTIFICATION_TOPIC) do |_, start, finish, _, payload|
+              subscriber.notify_render_partial(start, finish, payload)
+            end
+          ]
+
+          create_unsubscriber subscriptions
+        end
+
+        # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
+        def notify_render_template(start, finish, payload)
+          generate_span_for_notification("render_template", start, finish, payload, tags_for_render_template(payload))
+        end
+
+        def notify_render_collection(start, finish, payload)
+          generate_span_for_notification("render_collection", start, finish, payload, tags_for_render_collection(payload))
+        end
+
+        def notify_render_partial(start, finish, payload)
+          generate_span_for_notification("render_partial", start, finish, payload, tags_for_render_partial(payload))
+        end
+
+        private
+
+        def tags_for_render_template(payload)
+          {
+            'component' =>       COMPONENT_TAG,
+            'template.id' =>     payload[:identifier],
+            'template.layout' => payload[:layout]
+          }
+        end
+
+        def tags_for_render_collection(payload)
+          {
+            'component' =>            COMPONENT_TAG,
+            'template.id' =>          payload[:identifier],
+            'template.count' =>       payload[:count] || 0,
+            'template.cache.hits' =>  payload[:cache_hits] || 0
+          }
+        end
+
+        def tags_for_render_partial(payload)
+          {
+            'component' =>            COMPONENT_TAG,
+            'template.id' =>          payload[:identifier]
+          }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/rails/active_record_subscriber.rb b/lib/gitlab/tracing/rails/active_record_subscriber.rb
new file mode 100644
index 00000000000..32f5658e57e
--- /dev/null
+++ b/lib/gitlab/tracing/rails/active_record_subscriber.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Tracing
+    module Rails
+      class ActiveRecordSubscriber
+        include RailsCommon
+
+        ACTIVE_RECORD_NOTIFICATION_TOPIC = 'sql.active_record'
+        OPERATION_NAME_PREFIX = 'active_record:'
+        DEFAULT_OPERATION_NAME = 'sqlquery'
+
+        # Instruments Rails ActiveRecord events for opentracing.
+        # Returns a lambda, which, when called will unsubscribe from the notifications
+        def self.instrument
+          subscriber = new
+
+          subscription = ActiveSupport::Notifications.subscribe(ACTIVE_RECORD_NOTIFICATION_TOPIC) do |_, start, finish, _, payload|
+            subscriber.notify(start, finish, payload)
+          end
+
+          create_unsubscriber [subscription]
+        end
+
+        # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
+        def notify(start, finish, payload)
+          generate_span_for_notification(notification_name(payload), start, finish, payload, tags_for_notification(payload))
+        end
+
+        private
+
+        def notification_name(payload)
+          OPERATION_NAME_PREFIX + (payload[:name].presence || DEFAULT_OPERATION_NAME)
+        end
+
+        def tags_for_notification(payload)
+          {
+            'component' =>        'ActiveRecord',
+            'span.kind' =>        'client',
+            'db.type' =>          'sql',
+            'db.connection_id' => payload[:connection_id],
+            'db.cached' =>        payload[:cached] || false,
+            'db.statement' =>     payload[:sql]
+          }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/rails/rails_common.rb b/lib/gitlab/tracing/rails/rails_common.rb
new file mode 100644
index 00000000000..88e914f62f8
--- /dev/null
+++ b/lib/gitlab/tracing/rails/rails_common.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Tracing
+    module Rails
+      module RailsCommon
+        extend ActiveSupport::Concern
+        include Gitlab::Tracing::Common
+
+        class_methods do
+          def create_unsubscriber(subscriptions)
+            -> { subscriptions.each { |subscriber| ActiveSupport::Notifications.unsubscribe(subscriber) } }
+          end
+        end
+
+        def generate_span_for_notification(operation_name, start, finish, payload, tags)
+          exception = payload[:exception]
+
+          postnotify_span(operation_name, start, finish, tags: tags, exception: exception)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/sidekiq/client_middleware.rb b/lib/gitlab/tracing/sidekiq/client_middleware.rb
new file mode 100644
index 00000000000..2b71c1ea21e
--- /dev/null
+++ b/lib/gitlab/tracing/sidekiq/client_middleware.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'opentracing'
+
+module Gitlab
+  module Tracing
+    module Sidekiq
+      class ClientMiddleware
+        include SidekiqCommon
+
+        SPAN_KIND = 'client'
+
+        def call(worker_class, job, queue, redis_pool)
+          in_tracing_span(
+            operation_name: "sidekiq:#{job['class']}",
+            tags: tags_from_job(job, SPAN_KIND)) do |span|
+            # Inject the details directly into the job
+            tracer.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, job)
+
+            yield
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/sidekiq/server_middleware.rb b/lib/gitlab/tracing/sidekiq/server_middleware.rb
new file mode 100644
index 00000000000..5b43c4310e6
--- /dev/null
+++ b/lib/gitlab/tracing/sidekiq/server_middleware.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'opentracing'
+
+module Gitlab
+  module Tracing
+    module Sidekiq
+      class ServerMiddleware
+        include SidekiqCommon
+
+        SPAN_KIND = 'server'
+
+        def call(worker, job, queue)
+          context = tracer.extract(OpenTracing::FORMAT_TEXT_MAP, job)
+
+          in_tracing_span(
+            operation_name: "sidekiq:#{job['class']}",
+            child_of: context,
+            tags: tags_from_job(job, SPAN_KIND)) do |span|
+            yield
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/sidekiq/sidekiq_common.rb b/lib/gitlab/tracing/sidekiq/sidekiq_common.rb
new file mode 100644
index 00000000000..a911a29d773
--- /dev/null
+++ b/lib/gitlab/tracing/sidekiq/sidekiq_common.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Tracing
+    module Sidekiq
+      module SidekiqCommon
+        include Gitlab::Tracing::Common
+
+        def tags_from_job(job, kind)
+          {
+            'component' =>     'sidekiq',
+            'span.kind' =>     kind,
+            'sidekiq.queue' => job['queue'],
+            'sidekiq.jid' =>   job['jid'],
+            'sidekiq.retry' => job['retry'].to_s,
+            'sidekiq.args' =>  job['args']&.join(", ")
+          }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb
new file mode 100644
index 00000000000..453d78e2f7b
--- /dev/null
+++ b/lib/gitlab/tree_summary.rb
@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class TreeSummary
+    include ::Gitlab::Utils::StrongMemoize
+
+    attr_reader :commit, :project, :path, :offset, :limit
+
+    attr_reader :resolved_commits
+    private :resolved_commits
+
+    def initialize(commit, project, params = {})
+      @commit = commit
+      @project = project
+
+      @path = params.fetch(:path, nil).presence
+      @offset = params.fetch(:offset, 0).to_i
+      @limit = (params.fetch(:limit, 25) || 25).to_i
+
+      # Ensure that if multiple tree entries share the same last commit, they share
+      # a ::Commit instance. This prevents us from rendering the same commit title
+      # multiple times
+      @resolved_commits = {}
+    end
+
+    # Creates a summary of the tree entries for a commit, within the window of
+    # entries defined by the offset and limit parameters. This consists of two
+    # return values:
+    #
+    #     - An Array of Hashes containing the following keys:
+    #         - file_name:   The full path of the tree entry
+    #         - type:        One of :blob, :tree, or :submodule
+    #         - commit:      The last ::Commit to touch this entry in the tree
+    #         - commit_path: URI of the commit in the web interface
+    #     - An Array of the unique ::Commit objects in the first value
+    def summarize
+      summary = contents
+        .map { |content| build_entry(content) }
+        .tap { |summary| fill_last_commits!(summary) }
+
+      [summary, commits]
+    end
+
+    # Does the tree contain more entries after the given offset + limit?
+    def more?
+      all_contents[next_offset].present?
+    end
+
+    # The offset of the next batch of tree entries. If more? returns false, this
+    # batch will be empty
+    def next_offset
+      [all_contents.size + 1, offset + limit].min
+    end
+
+    private
+
+    def contents
+      all_contents[offset, limit]
+    end
+
+    def commits
+      resolved_commits.values
+    end
+
+    def repository
+      project.repository
+    end
+
+    def entry_path(entry)
+      File.join(*[path, entry[:file_name]].compact)
+    end
+
+    def build_entry(entry)
+      { file_name: entry.name, type: entry.type }
+    end
+
+    def fill_last_commits!(entries)
+      # Ensure the path is in "path/" format
+      ensured_path =
+        if path
+          File.join(*[path, ""])
+        end
+
+      commits_hsh = repository.list_last_commits_for_tree(commit.id, ensured_path, offset: offset, limit: limit)
+
+      entries.each do |entry|
+        path_key = entry_path(entry)
+        commit = cache_commit(commits_hsh[path_key])
+
+        if commit
+          entry[:commit] = commit
+          entry[:commit_path] = commit_path(commit)
+        end
+      end
+    end
+
+    def cache_commit(commit)
+      return nil unless commit.present?
+
+      resolved_commits[commit.id] ||= commit
+    end
+
+    def commit_path(commit)
+      Gitlab::Routing.url_helpers.project_commit_path(project, commit)
+    end
+
+    def all_contents
+      strong_memoize(:all_contents) do
+        [
+          *tree.trees,
+          *tree.blobs,
+          *tree.submodules
+        ]
+      end
+    end
+
+    def tree
+      strong_memoize(:tree) { repository.tree(commit.id, path) }
+    end
+  end
+end
diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb
index dc2d91dfa23..ba1137313d8 100644
--- a/lib/gitlab/untrusted_regexp.rb
+++ b/lib/gitlab/untrusted_regexp.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   # An untrusted regular expression is any regexp containing patterns sourced
   # from user input.
diff --git a/lib/gitlab/update_path_error.rb b/lib/gitlab/update_path_error.rb
index 8947ecfb92e..bc066bf4143 100644
--- a/lib/gitlab/update_path_error.rb
+++ b/lib/gitlab/update_path_error.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   UpdatePathError = Class.new(StandardError)
 end
diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb
deleted file mode 100644
index 024be6aca44..00000000000
--- a/lib/gitlab/upgrader.rb
+++ /dev/null
@@ -1,109 +0,0 @@
-module Gitlab
-  class Upgrader
-    def execute
-      puts "GitLab #{current_version.major} upgrade tool"
-      puts "Your version is #{current_version}"
-      puts "Latest available version for GitLab #{current_version.major} is #{latest_version}"
-
-      if latest_version?
-        puts "You are using the latest GitLab version"
-      else
-        puts "Newer GitLab version is available"
-
-        answer = if ARGV.first == "-y"
-                   "yes"
-                 else
-                   prompt("Do you want to upgrade (yes/no)? ", %w{yes no})
-                 end
-
-        if answer == "yes"
-          upgrade
-        else
-          exit 0
-        end
-      end
-    end
-
-    def latest_version?
-      current_version >= latest_version
-    end
-
-    def current_version
-      @current_version ||= Gitlab::VersionInfo.parse(current_version_raw)
-    end
-
-    def latest_version
-      @latest_version ||= Gitlab::VersionInfo.parse(latest_version_raw)
-    end
-
-    def current_version_raw
-      File.read(File.join(gitlab_path, "VERSION")).strip
-    end
-
-    def latest_version_raw
-      git_tags = fetch_git_tags
-      git_tags = git_tags.select { |version| version =~ /v\d+\.\d+\.\d+\Z/ }
-      git_versions = git_tags.map { |tag| Gitlab::VersionInfo.parse(tag.match(/v\d+\.\d+\.\d+/).to_s) }
-      "v#{git_versions.sort.last}"
-    end
-
-    def fetch_git_tags
-      remote_tags, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git))
-      remote_tags.split("\n").grep(%r{tags/v#{current_version.major}})
-    end
-
-    def update_commands
-      {
-        "Stash changed files" => %W(#{Gitlab.config.git.bin_path} stash),
-        "Get latest code" => %W(#{Gitlab.config.git.bin_path} fetch),
-        "Switch to new version" => %W(#{Gitlab.config.git.bin_path} checkout v#{latest_version}),
-        "Install gems" => %w(bundle),
-        "Migrate DB" => %w(bundle exec rake db:migrate),
-        "Recompile assets" => %w(bundle exec rake yarn:install gitlab:assets:clean gitlab:assets:compile),
-        "Clear cache" => %w(bundle exec rake cache:clear)
-      }
-    end
-
-    def env
-      {
-        'RAILS_ENV' => 'production',
-        'NODE_ENV' => 'production'
-      }
-    end
-
-    def upgrade
-      update_commands.each do |title, cmd|
-        puts title
-        puts " -> #{cmd.join(' ')}"
-
-        if system(env, *cmd)
-          puts " -> OK"
-        else
-          puts " -> FAILED"
-          puts "Failed to upgrade. Try to repeat task or proceed with upgrade manually "
-          exit 1
-        end
-      end
-
-      puts "Done"
-    end
-
-    def gitlab_path
-      File.expand_path(File.join(File.dirname(__FILE__), '../..'))
-    end
-
-    # Prompt the user to input something
-    #
-    # message - the message to display before input
-    # choices - array of strings of acceptable answers or nil for any answer
-    #
-    # Returns the user's answer
-    def prompt(message, choices = nil)
-      begin
-        print(message)
-        answer = STDIN.gets.chomp
-      end while !choices.include?(answer)
-      answer
-    end
-  end
-end
diff --git a/lib/gitlab/uploads_transfer.rb b/lib/gitlab/uploads_transfer.rb
index 7d7400bdabf..e0e7084e27e 100644
--- a/lib/gitlab/uploads_transfer.rb
+++ b/lib/gitlab/uploads_transfer.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class UploadsTransfer < ProjectTransfer
     def root_dir
diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb
index 20be193ea0c..9b7b0db9525 100644
--- a/lib/gitlab/url_blocker.rb
+++ b/lib/gitlab/url_blocker.rb
@@ -1,36 +1,43 @@
+# frozen_string_literal: true
+
 require 'resolv'
+require 'ipaddress'
 
 module Gitlab
   class UrlBlocker
     BlockedUrlError = Class.new(StandardError)
 
     class << self
-      def validate!(url, allow_localhost: false, allow_local_network: true, ports: [], protocols: [])
+      def validate!(url, ports: [], protocols: [], allow_localhost: false, allow_local_network: true, ascii_only: false, enforce_user: false, enforce_sanitization: false)
         return true if url.nil?
 
-        begin
-          uri = Addressable::URI.parse(url)
-        rescue Addressable::URI::InvalidURIError
-          raise BlockedUrlError, "URI is invalid"
-        end
+        # Param url can be a string, URI or Addressable::URI
+        uri = parse_url(url)
+
+        validate_html_tags!(uri) if enforce_sanitization
 
         # Allow imports from the GitLab instance itself but only from the configured ports
         return true if internal?(uri)
 
-        port = uri.port || uri.default_port
+        port = get_port(uri)
         validate_protocol!(uri.scheme, protocols)
         validate_port!(port, ports) if ports.any?
-        validate_user!(uri.user)
+        validate_user!(uri.user) if enforce_user
         validate_hostname!(uri.hostname)
+        validate_unicode_restriction!(uri) if ascii_only
 
         begin
-          addrs_info = Addrinfo.getaddrinfo(uri.hostname, port, nil, :STREAM)
+          addrs_info = Addrinfo.getaddrinfo(uri.hostname, port, nil, :STREAM).map do |addr|
+            addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr
+          end
         rescue SocketError
           return true
         end
 
         validate_localhost!(addrs_info) unless allow_localhost
+        validate_loopback!(addrs_info) unless allow_localhost
         validate_local_network!(addrs_info) unless allow_local_network
+        validate_link_local!(addrs_info) unless allow_local_network
 
         true
       end
@@ -45,6 +52,30 @@ module Gitlab
 
       private
 
+      def get_port(uri)
+        uri.port || uri.default_port
+      end
+
+      def validate_html_tags!(uri)
+        uri_str = uri.to_s
+        sanitized_uri = ActionController::Base.helpers.sanitize(uri_str, tags: [])
+        if sanitized_uri != uri_str
+          raise BlockedUrlError, 'HTML/CSS/JS tags are not allowed'
+        end
+      end
+
+      def parse_url(url)
+        raise Addressable::URI::InvalidURIError if multiline?(url)
+
+        Addressable::URI.parse(url)
+      rescue Addressable::URI::InvalidURIError, URI::InvalidURIError
+        raise BlockedUrlError, 'URI is invalid'
+      end
+
+      def multiline?(url)
+        CGI.unescape(url.to_s) =~ /\n|\r/
+      end
+
       def validate_port!(port, ports)
         return if port.blank?
         # Only ports under 1024 are restricted
@@ -69,13 +100,20 @@ module Gitlab
 
       def validate_hostname!(value)
         return if value.blank?
+        return if IPAddress.valid?(value)
         return if value =~ /\A\p{Alnum}/
 
-        raise BlockedUrlError, "Hostname needs to start with an alphanumeric character"
+        raise BlockedUrlError, "Hostname or IP address invalid"
+      end
+
+      def validate_unicode_restriction!(uri)
+        return if uri.to_s.ascii_only?
+
+        raise BlockedUrlError, "URI must be ascii only #{uri.to_s.dump}"
       end
 
       def validate_localhost!(addrs_info)
-        local_ips = ["127.0.0.1", "::1", "0.0.0.0"]
+        local_ips = ["::", "0.0.0.0"]
         local_ips.concat(Socket.ip_address_list.map(&:ip_address))
 
         return if (local_ips & addrs_info.map(&:ip_address)).empty?
@@ -83,23 +121,38 @@ module Gitlab
         raise BlockedUrlError, "Requests to localhost are not allowed"
       end
 
+      def validate_loopback!(addrs_info)
+        return unless addrs_info.any? { |addr| addr.ipv4_loopback? || addr.ipv6_loopback? }
+
+        raise BlockedUrlError, "Requests to loopback addresses are not allowed"
+      end
+
       def validate_local_network!(addrs_info)
-        return unless addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? }
+        return unless addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? || addr.ipv6_unique_local? }
 
         raise BlockedUrlError, "Requests to the local network are not allowed"
       end
 
+      def validate_link_local!(addrs_info)
+        netmask = IPAddr.new('169.254.0.0/16')
+        return unless addrs_info.any? { |addr| addr.ipv6_linklocal? || netmask.include?(addr.ip_address) }
+
+        raise BlockedUrlError, "Requests to the link local network are not allowed"
+      end
+
       def internal?(uri)
         internal_web?(uri) || internal_shell?(uri)
       end
 
       def internal_web?(uri)
-        uri.hostname == config.gitlab.host &&
+        uri.scheme == config.gitlab.protocol &&
+          uri.hostname == config.gitlab.host &&
           (uri.port.blank? || uri.port == config.gitlab.port)
       end
 
       def internal_shell?(uri)
-        uri.hostname == config.gitlab_shell.ssh_host &&
+        uri.scheme == 'ssh' &&
+          uri.hostname == config.gitlab_shell.ssh_host &&
           (uri.port.blank? || uri.port == config.gitlab_shell.ssh_port)
       end
 
diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb
index 824e2d7251f..f86d599e4cb 100644
--- a/lib/gitlab/url_builder.rb
+++ b/lib/gitlab/url_builder.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class UrlBuilder
     include Gitlab::Routing
@@ -26,6 +28,8 @@ module Gitlab
         project_snippet_url(object.project, object)
       when Snippet
         snippet_url(object)
+      when Milestone
+        milestone_url(object)
       else
         raise NotImplementedError.new("No URL builder defined for #{object.class}")
       end
diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb
index 59331c827af..880712de5fe 100644
--- a/lib/gitlab/url_sanitizer.rb
+++ b/lib/gitlab/url_sanitizer.rb
@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 module Gitlab
   class UrlSanitizer
     ALLOWED_SCHEMES = %w[http https ssh git].freeze
 
     def self.sanitize(content)
-      regexp = URI::Parser.new.make_regexp(ALLOWED_SCHEMES)
+      regexp = URI::DEFAULT_PARSER.make_regexp(ALLOWED_SCHEMES)
 
       content.gsub(regexp) { |url| new(url).masked_url }
     rescue Addressable::URI::InvalidURIError
@@ -12,6 +14,7 @@ module Gitlab
 
     def self.valid?(url)
       return false unless url.present?
+      return false unless url.is_a?(String)
 
       uri = Addressable::URI.parse(url.strip)
 
@@ -58,7 +61,7 @@ module Gitlab
       if raw_credentials.present?
         url.sub!("#{raw_credentials}@", '')
 
-        user, password = raw_credentials.split(':')
+        user, _, password = raw_credentials.partition(':')
         @credentials ||= { user: user.presence, password: password.presence }
       end
 
@@ -71,12 +74,10 @@ module Gitlab
     def generate_full_url
       return @url unless valid_credentials?
 
-      @full_url = @url.dup
-
-      @full_url.password = credentials[:password] if credentials[:password].present?
-      @full_url.user = credentials[:user] if credentials[:user].present?
-
-      @full_url
+      @url.dup.tap do |generated|
+        generated.password = encode_percent(credentials[:password]) if credentials[:password].present?
+        generated.user = encode_percent(credentials[:user]) if credentials[:user].present?
+      end
     end
 
     def safe_url
@@ -89,5 +90,10 @@ module Gitlab
     def valid_credentials?
       credentials && credentials.is_a?(Hash) && credentials.any?
     end
+
+    def encode_percent(string)
+      # CGI.escape converts spaces to +, but this doesn't work for git clone
+      CGI.escape(string).gsub('+', '%20')
+    end
   end
 end
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index e294f3c4ebc..6bfcf83f388 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -1,5 +1,9 @@
+# frozen_string_literal: true
+
 module Gitlab
   class UsageData
+    APPROXIMATE_COUNT_MODELS = [Label, MergeRequest, Note, Todo].freeze
+
     class << self
       def data(force_refresh: false)
         Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data }
@@ -10,6 +14,7 @@ module Gitlab
                           .merge(features_usage_data)
                           .merge(components_usage_data)
                           .merge(cycle_analytics_usage_data)
+                          .merge(usage_counters)
       end
 
       def to_json(force_refresh: false)
@@ -21,9 +26,9 @@ module Gitlab
           uuid: Gitlab::CurrentSettings.uuid,
           hostname: Gitlab.config.gitlab.host,
           version: Gitlab::VERSION,
-          active_user_count: User.active.count,
+          installation_type: Gitlab::INSTALLATION_TYPE,
+          active_user_count: count(User.active),
           recorded_at: Time.now,
-          mattermost_enabled: Gitlab.config.mattermost.enabled,
           edition: 'CE'
         }
 
@@ -31,54 +36,64 @@ module Gitlab
       end
 
       # rubocop:disable Metrics/AbcSize
+      # rubocop: disable CodeReuse/ActiveRecord
       def system_usage_data
         {
           counts: {
-            boards: Board.count,
-            ci_builds: ::Ci::Build.count,
-            ci_internal_pipelines: ::Ci::Pipeline.internal.count,
-            ci_external_pipelines: ::Ci::Pipeline.external.count,
-            ci_pipeline_config_auto_devops: ::Ci::Pipeline.auto_devops_source.count,
-            ci_pipeline_config_repository: ::Ci::Pipeline.repository_source.count,
-            ci_runners: ::Ci::Runner.count,
-            ci_triggers: ::Ci::Trigger.count,
-            ci_pipeline_schedules: ::Ci::PipelineSchedule.count,
-            auto_devops_enabled: ::ProjectAutoDevops.enabled.count,
-            auto_devops_disabled: ::ProjectAutoDevops.disabled.count,
-            deploy_keys: DeployKey.count,
-            deployments: Deployment.count,
-            environments: ::Environment.count,
-            clusters: ::Clusters::Cluster.count,
-            clusters_enabled: ::Clusters::Cluster.enabled.count,
-            clusters_disabled: ::Clusters::Cluster.disabled.count,
-            clusters_platforms_gke: ::Clusters::Cluster.gcp_installed.enabled.count,
-            clusters_platforms_user: ::Clusters::Cluster.user_provided.enabled.count,
-            clusters_applications_helm: ::Clusters::Applications::Helm.installed.count,
-            clusters_applications_ingress: ::Clusters::Applications::Ingress.installed.count,
-            clusters_applications_prometheus: ::Clusters::Applications::Prometheus.installed.count,
-            clusters_applications_runner: ::Clusters::Applications::Runner.installed.count,
-            in_review_folder: ::Environment.in_review_folder.count,
-            groups: Group.count,
-            issues: Issue.count,
-            keys: Key.count,
-            labels: Label.count,
-            lfs_objects: LfsObject.count,
-            merge_requests: MergeRequest.count,
-            milestones: Milestone.count,
-            notes: Note.count,
-            pages_domains: PagesDomain.count,
-            projects: Project.count,
-            projects_imported_from_github: Project.where(import_type: 'github').count,
-            protected_branches: ProtectedBranch.count,
-            releases: Release.count,
-            remote_mirrors: RemoteMirror.count,
-            snippets: Snippet.count,
-            todos: Todo.count,
-            uploads: Upload.count,
-            web_hooks: WebHook.count
-          }.merge(services_usage)
+            assignee_lists: count(List.assignee),
+            boards: count(Board),
+            ci_builds: count(::Ci::Build),
+            ci_internal_pipelines: count(::Ci::Pipeline.internal),
+            ci_external_pipelines: count(::Ci::Pipeline.external),
+            ci_pipeline_config_auto_devops: count(::Ci::Pipeline.auto_devops_source),
+            ci_pipeline_config_repository: count(::Ci::Pipeline.repository_source),
+            ci_runners: count(::Ci::Runner),
+            ci_triggers: count(::Ci::Trigger),
+            ci_pipeline_schedules: count(::Ci::PipelineSchedule),
+            auto_devops_enabled: count(::ProjectAutoDevops.enabled),
+            auto_devops_disabled: count(::ProjectAutoDevops.disabled),
+            deploy_keys: count(DeployKey),
+            deployments: count(Deployment),
+            environments: count(::Environment),
+            clusters: count(::Clusters::Cluster),
+            clusters_enabled: count(::Clusters::Cluster.enabled),
+            project_clusters_enabled: count(::Clusters::Cluster.enabled.project_type),
+            group_clusters_enabled: count(::Clusters::Cluster.enabled.group_type),
+            clusters_disabled: count(::Clusters::Cluster.disabled),
+            project_clusters_disabled: count(::Clusters::Cluster.disabled.project_type),
+            group_clusters_disabled: count(::Clusters::Cluster.disabled.group_type),
+            clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled),
+            clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled),
+            clusters_applications_helm: count(::Clusters::Applications::Helm.installed),
+            clusters_applications_ingress: count(::Clusters::Applications::Ingress.installed),
+            clusters_applications_cert_managers: count(::Clusters::Applications::CertManager.installed),
+            clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.installed),
+            clusters_applications_runner: count(::Clusters::Applications::Runner.installed),
+            clusters_applications_knative: count(::Clusters::Applications::Knative.installed),
+            in_review_folder: count(::Environment.in_review_folder),
+            groups: count(Group),
+            issues: count(Issue),
+            keys: count(Key),
+            label_lists: count(List.label),
+            lfs_objects: count(LfsObject),
+            milestone_lists: count(List.milestone),
+            milestones: count(Milestone),
+            pages_domains: count(PagesDomain),
+            projects: count(Project),
+            projects_imported_from_github: count(Project.where(import_type: 'github')),
+            projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)),
+            protected_branches: count(ProtectedBranch),
+            releases: count(Release),
+            remote_mirrors: count(RemoteMirror),
+            snippets: count(Snippet),
+            suggestions: count(Suggestion),
+            todos: count(Todo),
+            uploads: count(Upload),
+            web_hooks: count(WebHook)
+          }.merge(services_usage).merge(approximate_counts)
         }
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def cycle_analytics_usage_data
         Gitlab::CycleAnalytics::UsageData.new.to_json
@@ -90,13 +105,20 @@ module Gitlab
 
       def features_usage_data_ce
         {
-          signup: Gitlab::CurrentSettings.allow_signup?,
-          ldap: Gitlab.config.ldap.enabled,
-          gravatar: Gitlab::CurrentSettings.gravatar_enabled?,
-          omniauth: Gitlab.config.omniauth.enabled,
-          reply_by_email: Gitlab::IncomingEmail.enabled?,
-          container_registry: Gitlab.config.registry.enabled,
-          gitlab_shared_runners: Gitlab.config.gitlab_ci.shared_runners_enabled
+          container_registry_enabled: Gitlab.config.registry.enabled,
+          gitlab_shared_runners_enabled: Gitlab.config.gitlab_ci.shared_runners_enabled,
+          gravatar_enabled: Gitlab::CurrentSettings.gravatar_enabled?,
+          ldap_enabled: Gitlab.config.ldap.enabled,
+          mattermost_enabled: Gitlab.config.mattermost.enabled,
+          omniauth_enabled: Gitlab::Auth.omniauth_enabled?,
+          reply_by_email_enabled: Gitlab::IncomingEmail.enabled?,
+          signup_enabled: Gitlab::CurrentSettings.allow_signup?
+        }
+      end
+
+      def usage_counters
+        {
+          web_ide_commits: Gitlab::WebIdeCommitsCounter.total_count
         }
       end
 
@@ -108,16 +130,50 @@ module Gitlab
         }
       end
 
+      # rubocop: disable CodeReuse/ActiveRecord
       def services_usage
         types = {
-          JiraService: :projects_jira_active,
           SlackService: :projects_slack_notifications_active,
           SlackSlashCommandsService: :projects_slack_slash_active,
           PrometheusService: :projects_prometheus_active
         }
 
-        results = Service.unscoped.where(type: types.keys, active: true).group(:type).count
-        results.each_with_object({}) { |(key, value), response| response[types[key.to_sym]] = value  }
+        results = count(Service.unscoped.where(type: types.keys, active: true).group(:type), fallback: Hash.new(-1))
+        types.each_with_object({}) { |(klass, key), response| response[key] = results[klass.to_s] || 0 }
+          .merge(jira_usage)
+      end
+
+      def jira_usage
+        # Jira Cloud does not support custom domains as per https://jira.atlassian.com/browse/CLOUD-6999
+        # so we can just check for subdomains of atlassian.net
+        services = count(
+          Service.unscoped.where(type: :JiraService, active: true)
+            .group("CASE WHEN properties LIKE '%.atlassian.net%' THEN 'cloud' ELSE 'server' END"),
+          fallback: Hash.new(-1)
+        )
+
+        {
+          projects_jira_server_active: services['server'] || 0,
+          projects_jira_cloud_active: services['cloud'] || 0,
+          projects_jira_active: services['server'] == -1 ? -1 : services.values.sum
+        }
+      end
+
+      def count(relation, fallback: -1)
+        relation.count
+      rescue ActiveRecord::StatementInvalid
+        fallback
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      def approximate_counts
+        approx_counts = Gitlab::Database::Count.approximate_counts(APPROXIMATE_COUNT_MODELS)
+
+        APPROXIMATE_COUNT_MODELS.each_with_object({}) do |model, result|
+          key = model.name.underscore.pluralize.to_sym
+
+          result[key] = approx_counts[model] || -1
+        end
       end
     end
   end
diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb
index 8cf5d636743..980a8014409 100644
--- a/lib/gitlab/user_access.rb
+++ b/lib/gitlab/user_access.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class UserAccess
     extend Gitlab::Cache::RequestCache
@@ -65,7 +67,7 @@ module Gitlab
       return false unless can_access_git?
       return false unless project
 
-      return false if !user.can?(:push_code, project) && !project.branch_allows_maintainer_push?(user, ref)
+      return false if !user.can?(:push_code, project) && !project.branch_allows_collaboration?(user, ref)
 
       if protected?(ProtectedBranch, project, ref)
         protected_branch_accessible_to?(ref, action: :push)
diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb
deleted file mode 100644
index 125488536e1..00000000000
--- a/lib/gitlab/user_activities.rb
+++ /dev/null
@@ -1,34 +0,0 @@
-module Gitlab
-  class UserActivities
-    include Enumerable
-
-    KEY = 'users:activities'.freeze
-    BATCH_SIZE = 500
-
-    def self.record(key, time = Time.now)
-      Gitlab::Redis::SharedState.with do |redis|
-        redis.hset(KEY, key, time.to_i)
-      end
-    end
-
-    def delete(*keys)
-      Gitlab::Redis::SharedState.with do |redis|
-        redis.hdel(KEY, keys)
-      end
-    end
-
-    def each
-      cursor = 0
-      loop do
-        cursor, pairs =
-          Gitlab::Redis::SharedState.with do |redis|
-            redis.hscan(KEY, cursor, count: BATCH_SIZE)
-          end
-
-        Hash[pairs].each { |pair| yield pair }
-
-        break if cursor == '0'
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/user_extractor.rb b/lib/gitlab/user_extractor.rb
new file mode 100644
index 00000000000..874599688bb
--- /dev/null
+++ b/lib/gitlab/user_extractor.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# This class extracts all users found in a piece of text by the username or the
+# email address
+
+module Gitlab
+  class UserExtractor
+    # Not using `Devise.email_regexp` to filter out any chars that an email
+    # does not end with and not pinning the email to a start of end of a string.
+    EMAIL_REGEXP = /(?<email>([^@\s]+@[^@\s]+(?<!\W)))/
+    USERNAME_REGEXP = User.reference_pattern
+
+    def initialize(text)
+      @text = text
+    end
+
+    def users
+      return User.none unless @text.present?
+
+      @users ||= User.from_union(union_relations)
+    end
+
+    def usernames
+      matches[:usernames]
+    end
+
+    def emails
+      matches[:emails]
+    end
+
+    def references
+      @references ||= matches.values.flatten
+    end
+
+    def matches
+      @matches ||= {
+        emails: @text.scan(EMAIL_REGEXP).flatten.uniq,
+        usernames: @text.scan(USERNAME_REGEXP).flatten.uniq
+      }
+    end
+
+    private
+
+    def union_relations
+      relations = []
+
+      relations << User.by_any_email(emails) if emails.any?
+      relations << User.by_username(usernames) if usernames.any?
+
+      relations
+    end
+  end
+end
diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb
index aeda66763e8..99fa65e0e90 100644
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@@ -1,7 +1,18 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Utils
     extend self
 
+    # Ensure that the relative path will not traverse outside the base directory
+    def check_path_traversal!(path)
+      raise StandardError.new("Invalid path") if path.start_with?("..#{File::SEPARATOR}") ||
+          path.include?("#{File::SEPARATOR}..#{File::SEPARATOR}") ||
+          path.end_with?("#{File::SEPARATOR}..")
+
+      path
+    end
+
     # Run system command without outputting to stdout.
     #
     # @param  cmd [Array<String>]
@@ -14,6 +25,26 @@ module Gitlab
       str.force_encoding(Encoding::UTF_8)
     end
 
+    def ensure_utf8_size(str, bytes:)
+      raise ArgumentError, 'Empty string provided!' if str.empty?
+      raise ArgumentError, 'Negative string size provided!' if bytes.negative?
+
+      truncated = str.each_char.each_with_object(+'') do |char, object|
+        if object.bytesize + char.bytesize > bytes
+          break object
+        else
+          object.concat(char)
+        end
+      end
+
+      truncated + ('0' * (bytes - truncated.bytesize))
+    end
+
+    # Append path to host, making sure there's one single / in between
+    def append_path(host, path)
+      "#{host.to_s.sub(%r{\/+$}, '')}/#{path.to_s.sub(%r{^\/+}, '')}"
+    end
+
     # A slugified version of the string, suitable for inclusion in URLs and
     # domain names. Rules:
     #
@@ -29,7 +60,7 @@ module Gitlab
 
     # Converts newlines into HTML line break elements
     def nlbr(str)
-      ActionView::Base.full_sanitizer.sanitize(str, tags: []).gsub(/\r?\n/, '<br>').html_safe
+      ActionView::Base.full_sanitizer.sanitize(+str, tags: []).gsub(/\r?\n/, '<br>').html_safe
     end
 
     def remove_line_breaks(str)
@@ -84,5 +115,15 @@ module Gitlab
 
       string_or_array.split(',').map(&:strip)
     end
+
+    def deep_indifferent_access(data)
+      if data.is_a?(Array)
+        data.map(&method(:deep_indifferent_access))
+      elsif data.is_a?(Hash)
+        data.with_indifferent_access
+      else
+        data
+      end
+    end
   end
 end
diff --git a/lib/gitlab/utils/merge_hash.rb b/lib/gitlab/utils/merge_hash.rb
index 385141d44d0..fc237861e2f 100644
--- a/lib/gitlab/utils/merge_hash.rb
+++ b/lib/gitlab/utils/merge_hash.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Utils
     module MergeHash
diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb
index 8bf6bcb1fe2..c87e97d0213 100644
--- a/lib/gitlab/utils/override.rb
+++ b/lib/gitlab/utils/override.rb
@@ -1,17 +1,14 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Utils
     module Override
       class Extension
-        def self.verify_class!(klass, method_name)
-          instance_method_defined?(klass, method_name) ||
-            raise(
-              NotImplementedError.new(
-                "#{klass}\##{method_name} doesn't exist!"))
-        end
-
-        def self.instance_method_defined?(klass, name, include_super: true)
-          klass.instance_methods(include_super).include?(name) ||
-            klass.private_instance_methods(include_super).include?(name)
+        def self.verify_class!(klass, method_name, arity)
+          extension = new(klass)
+          parents = extension.parents_for(klass)
+          extension.verify_method!(
+            klass: klass, parents: parents, method_name: method_name, sub_method_arity: arity)
         end
 
         attr_reader :subject
@@ -20,35 +17,77 @@ module Gitlab
           @subject = subject
         end
 
-        def add_method_name(method_name)
-          method_names << method_name
-        end
-
-        def add_class(klass)
-          classes << klass
+        def parents_for(klass)
+          index = klass.ancestors.index(subject)
+          klass.ancestors.drop(index + 1)
         end
 
         def verify!
           classes.each do |klass|
-            index = klass.ancestors.index(subject)
-            parents = klass.ancestors.drop(index + 1)
-
-            method_names.each do |method_name|
-              parents.any? do |parent|
-                self.class.instance_method_defined?(
-                  parent, method_name, include_super: false)
-              end ||
-                raise(
-                  NotImplementedError.new(
-                    "#{klass}\##{method_name} doesn't exist!"))
+            parents = parents_for(klass)
+
+            method_names.each_pair do |method_name, arity|
+              verify_method!(
+                klass: klass,
+                parents: parents,
+                method_name: method_name,
+                sub_method_arity: arity)
             end
           end
         end
 
+        def verify_method!(klass:, parents:, method_name:, sub_method_arity:)
+          overridden_parent = parents.find do |parent|
+            instance_method_defined?(parent, method_name)
+          end
+
+          raise NotImplementedError.new("#{klass}\##{method_name} doesn't exist!") unless overridden_parent
+
+          super_method_arity = find_direct_method(overridden_parent, method_name).arity
+
+          unless arity_compatible?(sub_method_arity, super_method_arity)
+            raise NotImplementedError.new("#{subject}\##{method_name} has arity of #{sub_method_arity}, but #{overridden_parent}\##{method_name} has arity of #{super_method_arity}")
+          end
+        end
+
+        def add_method_name(method_name, arity = nil)
+          method_names[method_name] = arity
+        end
+
+        def add_class(klass)
+          classes << klass
+        end
+
+        def verify_override?(method_name)
+          method_names.has_key?(method_name)
+        end
+
         private
 
+        def instance_method_defined?(klass, name)
+          klass.instance_methods(false).include?(name) ||
+            klass.private_instance_methods(false).include?(name)
+        end
+
+        def find_direct_method(klass, name)
+          method = klass.instance_method(name)
+          method = method.super_method until method && klass == method.owner
+          method
+        end
+
+        def arity_compatible?(sub_method_arity, super_method_arity)
+          if sub_method_arity >= 0 && super_method_arity >= 0
+            # Regular arguments
+            sub_method_arity == super_method_arity
+          else
+            # It's too complex to check this case, just allow sub-method having negative arity
+            # But we don't allow sub_method_arity > 0 yet super_method_arity < 0
+            sub_method_arity < 0
+          end
+        end
+
         def method_names
-          @method_names ||= []
+          @method_names ||= {}
         end
 
         def classes
@@ -78,27 +117,51 @@ module Gitlab
       def override(method_name)
         return unless ENV['STATIC_VERIFICATION']
 
+        Override.extensions[self] ||= Extension.new(self)
+        Override.extensions[self].add_method_name(method_name)
+      end
+
+      def method_added(method_name)
+        super
+
+        return unless ENV['STATIC_VERIFICATION']
+        return unless Override.extensions[self]&.verify_override?(method_name)
+
+        method_arity = instance_method(method_name).arity
         if is_a?(Class)
-          Extension.verify_class!(self, method_name)
+          Extension.verify_class!(self, method_name, method_arity)
         else # We delay the check for modules
-          Override.extensions[self] ||= Extension.new(self)
-          Override.extensions[self].add_method_name(method_name)
+          Override.extensions[self].add_method_name(method_name, method_arity)
         end
       end
 
       def included(base = nil)
-        return super if base.nil? # Rails concern, ignoring it
+        super
 
+        queue_verification(base) if base
+      end
+
+      def prepended(base = nil)
+        super
+
+        queue_verification(base) if base
+      end
+
+      def extended(mod = nil)
         super
 
+        queue_verification(mod.singleton_class) if mod
+      end
+
+      def queue_verification(base)
+        return unless ENV['STATIC_VERIFICATION']
+
         if base.is_a?(Class) # We could check for Class in `override`
           # This could be `nil` if `override` was never called
           Override.extensions[self]&.add_class(base)
         end
       end
 
-      alias_method :prepended, :included
-
       def self.extensions
         @extensions ||= {}
       end
diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb
index fe091f4611b..aa1f8e2fdda 100644
--- a/lib/gitlab/utils/strong_memoize.rb
+++ b/lib/gitlab/utils/strong_memoize.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Utils
     module StrongMemoize
diff --git a/lib/gitlab/verify/batch_verifier.rb b/lib/gitlab/verify/batch_verifier.rb
index 1ef369a4b67..dbda19a4a66 100644
--- a/lib/gitlab/verify/batch_verifier.rb
+++ b/lib/gitlab/verify/batch_verifier.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Verify
     class BatchVerifier
@@ -7,13 +9,15 @@ module Gitlab
         @batch_size = batch_size
         @start = start
         @finish = finish
+
+        fix_google_api_logger
       end
 
       # Yields a Range of IDs and a Hash of failed verifications (object => error)
       def run_batches(&blk)
-        relation.in_batches(of: batch_size, start: start, finish: finish) do |relation| # rubocop: disable Cop/InBatches
-          range = relation.first.id..relation.last.id
-          failures = run_batch(relation)
+        all_relation.in_batches(of: batch_size, start: start, finish: finish) do |batch| # rubocop: disable Cop/InBatches
+          range = batch.first.id..batch.last.id
+          failures = run_batch_for(batch)
 
           yield(range, failures)
         end
@@ -29,24 +33,56 @@ module Gitlab
 
       private
 
-      def run_batch(relation)
-        relation.map { |upload| verify(upload) }.compact.to_h
+      def run_batch_for(batch)
+        batch.map { |upload| verify(upload) }.compact.to_h
       end
 
       def verify(object)
+        local?(object) ? verify_local(object) : verify_remote(object)
+      rescue => err
+        failure(object, err.inspect)
+      end
+
+      def verify_local(object)
         expected = expected_checksum(object)
         actual = actual_checksum(object)
 
-        raise 'Checksum missing' unless expected.present?
-        raise 'Checksum mismatch' unless expected == actual
+        return failure(object, 'Checksum missing') unless expected.present?
+        return failure(object, 'Checksum mismatch') unless expected == actual
 
+        success
+      end
+
+      # We don't calculate checksum for remote objects, so just check existence
+      def verify_remote(object)
+        return failure(object, 'Remote object does not exist') unless remote_object_exists?(object)
+
+        success
+      end
+
+      def success
         nil
-      rescue => err
-        [object, err]
+      end
+
+      def failure(object, message)
+        [object, message]
+      end
+
+      # It's already set to Logger::INFO, but acts as if it is set to
+      # Logger::DEBUG, and this fixes it...
+      def fix_google_api_logger
+        if Object.const_defined?('Google::Apis')
+          Google::Apis.logger.level = Logger::INFO
+        end
       end
 
       # This should return an ActiveRecord::Relation suitable for calling #in_batches on
-      def relation
+      def all_relation
+        raise NotImplementedError.new
+      end
+
+      # Should return true if the object is stored locally
+      def local?(_object)
         raise NotImplementedError.new
       end
 
@@ -59,6 +95,11 @@ module Gitlab
       def actual_checksum(_object)
         raise NotImplementedError.new
       end
+
+      # Be sure to perform a hard check of the remote object (don't just check DB value)
+      def remote_object_exists?(object)
+        raise NotImplementedError.new
+      end
     end
   end
 end
diff --git a/lib/gitlab/verify/job_artifacts.rb b/lib/gitlab/verify/job_artifacts.rb
index 03500a61074..3b90c8b1a8e 100644
--- a/lib/gitlab/verify/job_artifacts.rb
+++ b/lib/gitlab/verify/job_artifacts.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Verify
     class JobArtifacts < BatchVerifier
@@ -11,10 +13,14 @@ module Gitlab
 
       private
 
-      def relation
+      def all_relation
         ::Ci::JobArtifact.all
       end
 
+      def local?(artifact)
+        artifact.local_store?
+      end
+
       def expected_checksum(artifact)
         artifact.file_sha256
       end
@@ -22,6 +28,10 @@ module Gitlab
       def actual_checksum(artifact)
         Digest::SHA256.file(artifact.file.path).hexdigest
       end
+
+      def remote_object_exists?(artifact)
+        artifact.file.file.exists?
+      end
     end
   end
 end
diff --git a/lib/gitlab/verify/lfs_objects.rb b/lib/gitlab/verify/lfs_objects.rb
index 970e2a7b718..20dbb7addff 100644
--- a/lib/gitlab/verify/lfs_objects.rb
+++ b/lib/gitlab/verify/lfs_objects.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Verify
     class LfsObjects < BatchVerifier
@@ -11,8 +13,12 @@ module Gitlab
 
       private
 
-      def relation
-        LfsObject.with_files_stored_locally
+      def all_relation
+        LfsObject.all
+      end
+
+      def local?(lfs_object)
+        lfs_object.local_store?
       end
 
       def expected_checksum(lfs_object)
@@ -22,6 +28,10 @@ module Gitlab
       def actual_checksum(lfs_object)
         LfsObject.calculate_oid(lfs_object.file.path)
       end
+
+      def remote_object_exists?(lfs_object)
+        lfs_object.file.file.exists?
+      end
     end
   end
 end
diff --git a/lib/gitlab/verify/rake_task.rb b/lib/gitlab/verify/rake_task.rb
index dd138e6b92b..3efed311237 100644
--- a/lib/gitlab/verify/rake_task.rb
+++ b/lib/gitlab/verify/rake_task.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Verify
     class RakeTask
@@ -45,7 +47,7 @@ module Gitlab
         return unless verbose?
 
         failures.each do |object, error|
-          say "  - #{verifier.describe(object)}: #{error.inspect}".color(:red)
+          say "  - #{verifier.describe(object)}: #{error}".color(:red)
         end
       end
     end
diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb
index 0ffa71a6d72..875e8a120e9 100644
--- a/lib/gitlab/verify/uploads.rb
+++ b/lib/gitlab/verify/uploads.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module Verify
     class Uploads < BatchVerifier
@@ -11,8 +13,14 @@ module Gitlab
 
       private
 
-      def relation
-        Upload.with_files_stored_locally
+      # rubocop: disable CodeReuse/ActiveRecord
+      def all_relation
+        Upload.all.preload(:model)
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      def local?(upload)
+        upload.local?
       end
 
       def expected_checksum(upload)
@@ -22,6 +30,10 @@ module Gitlab
       def actual_checksum(upload)
         Upload.hexdigest(upload.absolute_path)
       end
+
+      def remote_object_exists?(upload)
+        upload.build_uploader.file.exists?
+      end
     end
   end
 end
diff --git a/lib/gitlab/version_info.rb b/lib/gitlab/version_info.rb
index 6ee41e85cc9..aa6d5310161 100644
--- a/lib/gitlab/version_info.rb
+++ b/lib/gitlab/version_info.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   class VersionInfo
     include Comparable
diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb
index 36162faa1eb..5e70afe730a 100644
--- a/lib/gitlab/view/presenter/base.rb
+++ b/lib/gitlab/view/presenter/base.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module View
     module Presenter
@@ -11,8 +13,8 @@ module Gitlab
 
         attr_reader :subject
 
-        def can?(user, action, overriden_subject = nil)
-          super(user, action, overriden_subject || subject)
+        def can?(user, action, overridden_subject = nil)
+          super(user, action, overridden_subject || subject)
         end
 
         # delegate all #can? queries to the subject
diff --git a/lib/gitlab/view/presenter/delegated.rb b/lib/gitlab/view/presenter/delegated.rb
index 387ff0f5d43..4a90ab758fb 100644
--- a/lib/gitlab/view/presenter/delegated.rb
+++ b/lib/gitlab/view/presenter/delegated.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module View
     module Presenter
diff --git a/lib/gitlab/view/presenter/factory.rb b/lib/gitlab/view/presenter/factory.rb
index 570f0723e39..302697ff8eb 100644
--- a/lib/gitlab/view/presenter/factory.rb
+++ b/lib/gitlab/view/presenter/factory.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module View
     module Presenter
diff --git a/lib/gitlab/view/presenter/simple.rb b/lib/gitlab/view/presenter/simple.rb
index b7653a0f3cc..31dcd1d4c4c 100644
--- a/lib/gitlab/view/presenter/simple.rb
+++ b/lib/gitlab/view/presenter/simple.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module View
     module Presenter
diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb
index 2612208a927..a3c7de87765 100644
--- a/lib/gitlab/visibility_level.rb
+++ b/lib/gitlab/visibility_level.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Gitlab::VisibilityLevel module
 #
 # Define allowed public modes that can be used for
diff --git a/lib/gitlab/web_ide_commits_counter.rb b/lib/gitlab/web_ide_commits_counter.rb
new file mode 100644
index 00000000000..1cd9b5295b9
--- /dev/null
+++ b/lib/gitlab/web_ide_commits_counter.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module WebIdeCommitsCounter
+    WEB_IDE_COMMITS_KEY = "WEB_IDE_COMMITS_COUNT".freeze
+
+    class << self
+      def increment
+        Gitlab::Redis::SharedState.with { |redis| redis.incr(WEB_IDE_COMMITS_KEY) }
+      end
+
+      def total_count
+        Gitlab::Redis::SharedState.with { |redis| redis.get(WEB_IDE_COMMITS_KEY).to_i }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/webpack/dev_server_middleware.rb b/lib/gitlab/webpack/dev_server_middleware.rb
index 529f7d6a8d6..fda41da5a94 100644
--- a/lib/gitlab/webpack/dev_server_middleware.rb
+++ b/lib/gitlab/webpack/dev_server_middleware.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # This Rack middleware is intended to proxy the webpack assets directory to the
 # webpack-dev-server.  It is only intended for use in development.
 
diff --git a/lib/gitlab/webpack/manifest.rb b/lib/gitlab/webpack/manifest.rb
index 0c343e5bc1d..1d2aff5e5b4 100644
--- a/lib/gitlab/webpack/manifest.rb
+++ b/lib/gitlab/webpack/manifest.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'webpack/rails/manifest'
 
 module Gitlab
diff --git a/lib/gitlab/wiki_file_finder.rb b/lib/gitlab/wiki_file_finder.rb
new file mode 100644
index 00000000000..5303b3582ab
--- /dev/null
+++ b/lib/gitlab/wiki_file_finder.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class WikiFileFinder < FileFinder
+    BATCH_SIZE = 100
+
+    attr_reader :repository
+
+    def initialize(project, ref)
+      @project = project
+      @ref = ref
+      @repository = project.wiki.repository
+    end
+
+    private
+
+    def search_filenames(query)
+      safe_query = Regexp.escape(query.tr(' ', '-'))
+      safe_query = Regexp.new(safe_query, Regexp::IGNORECASE)
+      filenames = repository.ls_files(ref)
+
+      filenames.grep(safe_query).first(BATCH_SIZE)
+    end
+  end
+end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index e893e46ee86..265f6213a99 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'base64'
 require 'json'
 require 'securerandom'
@@ -11,6 +13,7 @@ module Gitlab
     INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request'.freeze
     NOTIFICATION_CHANNEL = 'workhorse:notifications'.freeze
     ALLOWED_GIT_HTTP_ACTIONS = %w[git_receive_pack git_upload_pack info_refs].freeze
+    DETECT_HEADER = 'Gitlab-Workhorse-Detect-Content-Type'.freeze
 
     # Supposedly the effective key size for HMAC-SHA256 is 256 bits, i.e. 32
     # bytes https://tools.ietf.org/html/rfc4868#section-2.6
@@ -22,36 +25,37 @@ module Gitlab
 
         project = repository.project
 
-        {
+        attrs = {
           GL_ID: Gitlab::GlId.gl_id(user),
           GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki),
           GL_USERNAME: user&.username,
           ShowAllRefs: show_all_refs,
           Repository: repository.gitaly_repository.to_h,
-          RepoPath: 'ignored but not allowed to be empty in gitlab-workhorse',
+          GitConfigOptions: [],
           GitalyServer: {
             address: Gitlab::GitalyClient.address(project.repository_storage),
             token: Gitlab::GitalyClient.token(project.repository_storage)
           }
         }
+
+        # Custom option for git-receive-pack command
+        receive_max_input_size = Gitlab::CurrentSettings.receive_max_input_size.to_i
+        if receive_max_input_size > 0
+          attrs[:GitConfigOptions] << "receive.maxInputSize=#{receive_max_input_size.megabytes}"
+        end
+
+        attrs
       end
 
       def send_git_blob(repository, blob)
-        params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT)
-                   {
-                     'GitalyServer' => gitaly_server_hash(repository),
-                     'GetBlobRequest' => {
-                       repository: repository.gitaly_repository.to_h,
-                       oid: blob.id,
-                       limit: -1
-                     }
-                   }
-                 else
-                   {
-                     'RepoPath' => repository.path_to_repo,
-                     'BlobId' => blob.id
-                   }
-                 end
+        params = {
+          'GitalyServer' => gitaly_server_hash(repository),
+          'GetBlobRequest' => {
+            repository: repository.gitaly_repository.to_h,
+            oid: blob.id,
+            limit: -1
+          }
+        }
 
         [
           SEND_DATA_HEADER,
@@ -61,7 +65,7 @@ module Gitlab
 
       def send_git_archive(repository, ref:, format:, append_sha:)
         format ||= 'tar.gz'
-        format.downcase!
+        format = format.downcase
         params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format, append_sha: append_sha)
         raise "Repository or ref not found" if params.empty?
 
@@ -91,16 +95,12 @@ module Gitlab
       end
 
       def send_git_diff(repository, diff_refs)
-        params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT)
-                   {
-                     'GitalyServer' => gitaly_server_hash(repository),
-                     'RawDiffRequest' => Gitaly::RawDiffRequest.new(
-                       gitaly_diff_or_patch_hash(repository, diff_refs)
-                     ).to_json
-                   }
-                 else
-                   workhorse_diff_or_patch_hash(repository, diff_refs)
-                 end
+        params = {
+          'GitalyServer' => gitaly_server_hash(repository),
+          'RawDiffRequest' => Gitaly::RawDiffRequest.new(
+            gitaly_diff_or_patch_hash(repository, diff_refs)
+          ).to_json
+        }
 
         [
           SEND_DATA_HEADER,
@@ -109,16 +109,12 @@ module Gitlab
       end
 
       def send_git_patch(repository, diff_refs)
-        params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_patch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT)
-                   {
-                     'GitalyServer' => gitaly_server_hash(repository),
-                     'RawPatchRequest' => Gitaly::RawPatchRequest.new(
-                       gitaly_diff_or_patch_hash(repository, diff_refs)
-                     ).to_json
-                   }
-                 else
-                   workhorse_diff_or_patch_hash(repository, diff_refs)
-                 end
+        params = {
+          'GitalyServer' => gitaly_server_hash(repository),
+          'RawPatchRequest' => Gitaly::RawPatchRequest.new(
+            gitaly_diff_or_patch_hash(repository, diff_refs)
+          ).to_json
+        }
 
         [
           SEND_DATA_HEADER,
@@ -231,14 +227,6 @@ module Gitlab
         }
       end
 
-      def workhorse_diff_or_patch_hash(repository, diff_refs)
-        {
-          'RepoPath'  => repository.path_to_repo,
-          'ShaFrom'   => diff_refs.base_sha,
-          'ShaTo'     => diff_refs.head_sha
-        }
-      end
-
       def gitaly_diff_or_patch_hash(repository, diff_refs)
         {
           repository: repository.gitaly_repository,
diff --git a/lib/google_api/auth.rb b/lib/google_api/auth.rb
index 1aeaa387a49..56f056fd869 100644
--- a/lib/google_api/auth.rb
+++ b/lib/google_api/auth.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GoogleApi
   class Auth
     attr_reader :access_token, :redirect_uri, :state
@@ -14,7 +16,7 @@ module GoogleApi
       client.auth_code.authorize_url(
         redirect_uri: redirect_uri,
         scope: scope,
-        state: state # This is used for arbitary redirection
+        state: state # This is used for arbitrary redirection
       )
     end
 
diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb
index 36859b4d025..e74ff6a9129 100644
--- a/lib/google_api/cloud_platform/client.rb
+++ b/lib/google_api/cloud_platform/client.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'google/apis/compute_v1'
 require 'google/apis/container_v1'
 require 'google/apis/cloudbilling_v1'
@@ -50,7 +52,7 @@ module GoogleApi
         service.get_zone_cluster(project_id, zone, cluster_id, options: user_agent_header)
       end
 
-      def projects_zones_clusters_create(project_id, zone, cluster_name, cluster_size, machine_type:)
+      def projects_zones_clusters_create(project_id, zone, cluster_name, cluster_size, machine_type:, legacy_abac:)
         service = Google::Apis::ContainerV1::ContainerService.new
         service.authorization = access_token
 
@@ -63,7 +65,7 @@ module GoogleApi
                 "machine_type": machine_type
               },
               "legacy_abac": {
-                "enabled": true
+                "enabled": legacy_abac
               }
             }
           }
diff --git a/lib/gt_one_coercion.rb b/lib/gt_one_coercion.rb
index ef2dc09767c..99be51bc8c6 100644
--- a/lib/gt_one_coercion.rb
+++ b/lib/gt_one_coercion.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class GtOneCoercion < Virtus::Attribute
   def coerce(value)
     [1, value.to_i].max
diff --git a/lib/haml_lint/inline_javascript.rb b/lib/haml_lint/inline_javascript.rb
index adbed20f152..2e98227a05e 100644
--- a/lib/haml_lint/inline_javascript.rb
+++ b/lib/haml_lint/inline_javascript.rb
@@ -1,7 +1,10 @@
-unless Rails.env.production? # rubocop:disable Naming/FileName
-  require 'haml_lint/haml_visitor'
-  require 'haml_lint/linter'
-  require 'haml_lint/linter_registry'
+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+unless Rails.env.production?
+  require_dependency 'haml_lint/haml_visitor'
+  require_dependency 'haml_lint/linter'
+  require_dependency 'haml_lint/linter_registry'
 
   module HamlLint
     class Linter::InlineJavaScript < Linter
diff --git a/lib/json_web_token/hmac_token.rb b/lib/json_web_token/hmac_token.rb
new file mode 100644
index 00000000000..ec0917ab49d
--- /dev/null
+++ b/lib/json_web_token/hmac_token.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'jwt'
+
+module JSONWebToken
+  class HMACToken < Token
+    IAT_LEEWAY = 60
+    JWT_ALGORITHM = 'HS256'
+
+    def initialize(secret)
+      super()
+
+      @secret = secret
+    end
+
+    def self.decode(token, secret, leeway: IAT_LEEWAY, verify_iat: true)
+      JWT.decode(token, secret, true, leeway: leeway, verify_iat: verify_iat, algorithm: JWT_ALGORITHM)
+    end
+
+    def encoded
+      JWT.encode(payload, secret, JWT_ALGORITHM, { typ: 'JWT' })
+    end
+
+    private
+
+    attr_reader :secret
+  end
+end
diff --git a/lib/json_web_token/rsa_token.rb b/lib/json_web_token/rsa_token.rb
index d6d6af7089c..bcce811cd28 100644
--- a/lib/json_web_token/rsa_token.rb
+++ b/lib/json_web_token/rsa_token.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module JSONWebToken
   class RSAToken < Token
     attr_reader :key_file
@@ -9,7 +11,8 @@ module JSONWebToken
 
     def encoded
       headers = {
-        kid: kid
+        kid: kid,
+        typ: 'JWT'
       }
       JWT.encode(payload, key, 'RS256', headers)
     end
diff --git a/lib/json_web_token/token.rb b/lib/json_web_token/token.rb
index 5b67715b0b2..c59beef02c9 100644
--- a/lib/json_web_token/token.rb
+++ b/lib/json_web_token/token.rb
@@ -1,15 +1,22 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
 module JSONWebToken
   class Token
     attr_accessor :issuer, :subject, :audience, :id
     attr_accessor :issued_at, :not_before, :expire_time
 
+    DEFAULT_NOT_BEFORE_TIME = 5
+    DEFAULT_EXPIRE_TIME = 60
+
     def initialize
       @id = SecureRandom.uuid
       @issued_at = Time.now
       # we give a few seconds for time shift
-      @not_before = issued_at - 5.seconds
+      @not_before = issued_at - DEFAULT_NOT_BEFORE_TIME
       # default 60 seconds should be more than enough for this authentication token
-      @expire_time = issued_at + 1.minute
+      @expire_time = issued_at + DEFAULT_EXPIRE_TIME
       @custom_payload = {}
     end
 
diff --git a/lib/mattermost/client.rb b/lib/mattermost/client.rb
index d80cd7d2a4e..293d0c563c5 100644
--- a/lib/mattermost/client.rb
+++ b/lib/mattermost/client.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mattermost
   ClientError = Class.new(Mattermost::Error)
 
diff --git a/lib/mattermost/command.rb b/lib/mattermost/command.rb
index 704813dfdf0..a02745486d6 100644
--- a/lib/mattermost/command.rb
+++ b/lib/mattermost/command.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mattermost
   class Command < Client
     def create(params)
diff --git a/lib/mattermost/error.rb b/lib/mattermost/error.rb
index dee6deb7974..054bd5457bd 100644
--- a/lib/mattermost/error.rb
+++ b/lib/mattermost/error.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mattermost
   Error = Class.new(StandardError)
 end
diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb
index 2aa7a2f64d8..e2083848a8d 100644
--- a/lib/mattermost/session.rb
+++ b/lib/mattermost/session.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mattermost
   class NoSessionError < Mattermost::Error
     def message
diff --git a/lib/mattermost/team.rb b/lib/mattermost/team.rb
index 95c2f6f9d6b..58120178f50 100644
--- a/lib/mattermost/team.rb
+++ b/lib/mattermost/team.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mattermost
   class Team < Client
     # Returns all teams that the current user is a member of
diff --git a/lib/microsoft_teams/activity.rb b/lib/microsoft_teams/activity.rb
index d2c420efdaf..207e90d2638 100644
--- a/lib/microsoft_teams/activity.rb
+++ b/lib/microsoft_teams/activity.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module MicrosoftTeams
   class Activity
     def initialize(title:, subtitle:, text:, image:)
diff --git a/lib/microsoft_teams/notifier.rb b/lib/microsoft_teams/notifier.rb
index c08d3e933a8..c7dec09ba6b 100644
--- a/lib/microsoft_teams/notifier.rb
+++ b/lib/microsoft_teams/notifier.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module MicrosoftTeams
   class Notifier
     def initialize(webhook)
@@ -30,7 +32,7 @@ module MicrosoftTeams
       result = { 'sections' => [] }
 
       result['title'] = options[:title]
-      result['summary'] = options[:pretext]
+      result['summary'] = options[:summary]
       result['sections'] << MicrosoftTeams::Activity.new(options[:activity]).prepare
 
       attachments = options[:attachments]
diff --git a/lib/milestone_array.rb b/lib/milestone_array.rb
index 4ed8485b36a..461e73e9670 100644
--- a/lib/milestone_array.rb
+++ b/lib/milestone_array.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module MilestoneArray
   class << self
     def sort(array, sort_method)
diff --git a/lib/mysql_zero_date.rb b/lib/mysql_zero_date.rb
new file mode 100644
index 00000000000..f36610abf8f
--- /dev/null
+++ b/lib/mysql_zero_date.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Disable NO_ZERO_DATE mode for mysql in rails 5.
+# We use zero date as a default value
+# (config/initializers/active_record_mysql_timestamp.rb), in
+# Rails 5 using zero date fails by default (https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/75450216)
+# and NO_ZERO_DATE has to be explicitly disabled. Disabling strict mode
+# is not sufficient.
+
+require 'active_record/connection_adapters/abstract_mysql_adapter'
+
+module MysqlZeroDate
+  def configure_connection
+    super
+
+    @connection.query "SET @@SESSION.sql_mode = REPLACE(@@SESSION.sql_mode, 'NO_ZERO_DATE', '');" # rubocop:disable Gitlab/ModuleWithInstanceVariables
+  end
+end
+
+ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.prepend(MysqlZeroDate)
diff --git a/lib/object_storage/direct_upload.rb b/lib/object_storage/direct_upload.rb
new file mode 100644
index 00000000000..fd26663fef0
--- /dev/null
+++ b/lib/object_storage/direct_upload.rb
@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+module ObjectStorage
+  #
+  # The DirectUpload c;ass generates a set of presigned URLs
+  # that can be used to upload data to object storage from untrusted component: Workhorse, Runner?
+  #
+  # For Google it assumes that the platform supports variable Content-Length.
+  #
+  # For AWS it initiates Multipart Upload and presignes a set of part uploads.
+  #   Class calculates the best part size to be able to upload up to asked maximum size.
+  #   The number of generated parts will never go above 100,
+  #   but we will always try to reduce amount of generated parts.
+  #   The part size is rounded-up to 5MB.
+  #
+  class DirectUpload
+    include Gitlab::Utils::StrongMemoize
+
+    TIMEOUT = 4.hours
+    EXPIRE_OFFSET = 15.minutes
+
+    MAXIMUM_MULTIPART_PARTS = 100
+    MINIMUM_MULTIPART_SIZE = 5.megabytes
+
+    attr_reader :credentials, :bucket_name, :object_name
+    attr_reader :has_length, :maximum_size
+
+    def initialize(credentials, bucket_name, object_name, has_length:, maximum_size: nil)
+      unless has_length
+        raise ArgumentError, 'maximum_size has to be specified if length is unknown' unless maximum_size
+      end
+
+      @credentials = credentials
+      @bucket_name = bucket_name
+      @object_name = object_name
+      @has_length = has_length
+      @maximum_size = maximum_size
+    end
+
+    def to_hash
+      {
+        Timeout: TIMEOUT,
+        GetURL: get_url,
+        StoreURL: store_url,
+        DeleteURL: delete_url,
+        MultipartUpload: multipart_upload_hash,
+        CustomPutHeaders: true,
+        PutHeaders: upload_options
+      }.compact
+    end
+
+    def multipart_upload_hash
+      return unless requires_multipart_upload?
+
+      {
+        PartSize: rounded_multipart_part_size,
+        PartURLs: multipart_part_urls,
+        CompleteURL: multipart_complete_url,
+        AbortURL: multipart_abort_url
+      }
+    end
+
+    def provider
+      credentials[:provider].to_s
+    end
+
+    # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html
+    def get_url
+      connection.get_object_url(bucket_name, object_name, expire_at)
+    end
+
+    # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html
+    def delete_url
+      connection.delete_object_url(bucket_name, object_name, expire_at)
+    end
+
+    # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
+    def store_url
+      connection.put_object_url(bucket_name, object_name, expire_at, upload_options)
+    end
+
+    def multipart_part_urls
+      Array.new(number_of_multipart_parts) do |part_index|
+        multipart_part_upload_url(part_index + 1)
+      end
+    end
+
+    # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
+    def multipart_part_upload_url(part_number)
+      connection.signed_url({
+        method: 'PUT',
+        bucket_name: bucket_name,
+        object_name: object_name,
+        query: { 'uploadId' => upload_id, 'partNumber' => part_number },
+        headers: upload_options
+      }, expire_at)
+    end
+
+    # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
+    def multipart_complete_url
+      connection.signed_url({
+        method: 'POST',
+        bucket_name: bucket_name,
+        object_name: object_name,
+        query: { 'uploadId' => upload_id },
+        headers: { 'Content-Type' => 'application/xml' }
+      }, expire_at)
+    end
+
+    # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadAbort.html
+    def multipart_abort_url
+      connection.signed_url({
+        method: 'DELETE',
+        bucket_name: bucket_name,
+        object_name: object_name,
+        query: { 'uploadId' => upload_id }
+      }, expire_at)
+    end
+
+    private
+
+    def rounded_multipart_part_size
+      # round multipart_part_size up to minimum_mulitpart_size
+      (multipart_part_size + MINIMUM_MULTIPART_SIZE - 1) / MINIMUM_MULTIPART_SIZE * MINIMUM_MULTIPART_SIZE
+    end
+
+    def multipart_part_size
+      maximum_size / number_of_multipart_parts
+    end
+
+    def number_of_multipart_parts
+      [
+        # round maximum_size up to minimum_mulitpart_size
+        (maximum_size + MINIMUM_MULTIPART_SIZE - 1) / MINIMUM_MULTIPART_SIZE,
+        MAXIMUM_MULTIPART_PARTS
+      ].min
+    end
+
+    def aws?
+      provider == 'AWS'
+    end
+
+    def requires_multipart_upload?
+      aws? && !has_length
+    end
+
+    def upload_id
+      return unless requires_multipart_upload?
+
+      strong_memoize(:upload_id) do
+        new_upload = connection.initiate_multipart_upload(bucket_name, object_name)
+        new_upload.body["UploadId"]
+      end
+    end
+
+    def expire_at
+      strong_memoize(:expire_at) do
+        Time.now + TIMEOUT + EXPIRE_OFFSET
+      end
+    end
+
+    def upload_options
+      {}
+    end
+
+    def connection
+      @connection ||= ::Fog::Storage.new(credentials)
+    end
+  end
+end
diff --git a/lib/omni_auth/strategies/bitbucket.rb b/lib/omni_auth/strategies/bitbucket.rb
index ce1bdfe6ee4..6c914b4222a 100644
--- a/lib/omni_auth/strategies/bitbucket.rb
+++ b/lib/omni_auth/strategies/bitbucket.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'omniauth-oauth2'
 
 module OmniAuth
diff --git a/lib/omni_auth/strategies/jwt.rb b/lib/omni_auth/strategies/jwt.rb
index ebdb5c7faf0..2f3d477a591 100644
--- a/lib/omni_auth/strategies/jwt.rb
+++ b/lib/omni_auth/strategies/jwt.rb
@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 require 'omniauth'
+require 'openssl'
 require 'jwt'
 
 module OmniAuth
@@ -35,7 +38,19 @@ module OmniAuth
       end
 
       def decoded
-        @decoded ||= ::JWT.decode(request.params['jwt'], options.secret, options.algorithm).first
+        secret =
+          case options.algorithm
+          when *%w[RS256 RS384 RS512]
+            OpenSSL::PKey::RSA.new(options.secret).public_key
+          when *%w[ES256 ES384 ES512]
+            OpenSSL::PKey::EC.new(options.secret).tap { |key| key.private_key = nil }
+          when *%w(HS256 HS384 HS512)
+            options.secret
+          else
+            raise NotImplementedError, "Unsupported algorithm: #{options.algorithm}"
+          end
+
+        @decoded ||= ::JWT.decode(request.params['jwt'], secret, true, { algorithm: options.algorithm }).first
 
         (options.required_claims || []).each do |field|
           raise ClaimInvalid, "Missing required '#{field}' claim" unless @decoded.key?(field.to_s)
@@ -43,7 +58,7 @@ module OmniAuth
 
         raise ClaimInvalid, "Missing required 'iat' claim" if options.valid_within && !@decoded["iat"]
 
-        if options.valid_within && (Time.now.to_i - @decoded["iat"]).abs > options.valid_within
+        if options.valid_within && (Time.now.to_i - @decoded["iat"]).abs > options.valid_within.to_i
           raise ClaimInvalid, "'iat' timestamp claim is too skewed from present"
         end
 
diff --git a/lib/peek/rblineprof/custom_controller_helpers.rb b/lib/peek/rblineprof/custom_controller_helpers.rb
index da24a36603e..581cc6a37b4 100644
--- a/lib/peek/rblineprof/custom_controller_helpers.rb
+++ b/lib/peek/rblineprof/custom_controller_helpers.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Peek
   module Rblineprof
     module CustomControllerHelpers
@@ -41,7 +43,7 @@ module Peek
             ]
           end.sort_by{ |a,b,c,d,e,f| -f }
 
-          output = "<div class='modal-dialog modal-lg'><div class='modal-content'>"
+          output = ["<div class='modal-dialog modal-xl'><div class='modal-content'>"]
           output << "<div class='modal-header'>"
           output << "<h4>Line profiling: #{human_description(params[:lineprofiler])}</h4>"
           output << "<button class='close' type='button' data-dismiss='modal' aria-label='close'><span aria-hidden='true'>&times;</span></button>"
@@ -93,7 +95,7 @@ module Peek
 
           output << "</div></div></div>"
 
-          response.body += "<div class='modal' id='modal-peek-line-profile' tabindex=-1>#{output}</div>".html_safe
+          response.body += "<div class='modal' id='modal-peek-line-profile' tabindex=-1>#{output.join}</div>".html_safe
         end
 
         ret
diff --git a/lib/peek/views/gitaly.rb b/lib/peek/views/gitaly.rb
index ab35f7a2258..30f95a10024 100644
--- a/lib/peek/views/gitaly.rb
+++ b/lib/peek/views/gitaly.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Peek
   module Views
     class Gitaly < View
@@ -21,7 +23,6 @@ module Peek
 
       def details
         ::Gitlab::GitalyClient.list_call_details
-          .values
           .sort { |a, b| b[:duration] <=> a[:duration] }
           .map(&method(:format_call_details))
       end
diff --git a/lib/peek/views/host.rb b/lib/peek/views/host.rb
index 43c8a35c7ea..b77355ea11b 100644
--- a/lib/peek/views/host.rb
+++ b/lib/peek/views/host.rb
@@ -1,8 +1,13 @@
+# frozen_string_literal: true
+
 module Peek
   module Views
     class Host < View
       def results
-        { hostname: Gitlab::Environment.hostname }
+        {
+          hostname: Gitlab::Environment.hostname,
+          canary: Gitlab::Utils.to_boolean(ENV['CANARY'])
+        }
       end
     end
   end
diff --git a/lib/quality/helm_client.rb b/lib/quality/helm_client.rb
new file mode 100644
index 00000000000..cf1f03b35b5
--- /dev/null
+++ b/lib/quality/helm_client.rb
@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require 'time'
+require_relative '../gitlab/popen' unless defined?(Gitlab::Popen)
+
+module Quality
+  class HelmClient
+    CommandFailedError = Class.new(StandardError)
+
+    attr_reader :namespace
+
+    RELEASE_JSON_ATTRIBUTES = %w[Name Revision Updated Status Chart AppVersion Namespace].freeze
+
+    Release = Struct.new(:name, :revision, :last_update, :status, :chart, :app_version, :namespace) do
+      def revision
+        @revision ||= self[:revision].to_i
+      end
+
+      def last_update
+        @last_update ||= Time.parse(self[:last_update])
+      end
+    end
+
+    # A single page of data and the corresponding page number.
+    Page = Struct.new(:releases, :number)
+
+    def initialize(namespace:)
+      @namespace = namespace
+    end
+
+    def releases(args: [])
+      each_release(args)
+    end
+
+    def delete(release_name:)
+      run_command([
+        'delete',
+        %(--tiller-namespace "#{namespace}"),
+        '--purge',
+        release_name
+      ])
+    end
+
+    private
+
+    def run_command(command)
+      final_command = ['helm', *command].join(' ')
+      puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output
+
+      result = Gitlab::Popen.popen_with_detail([final_command])
+
+      if result.status.success?
+        result.stdout.chomp.freeze
+      else
+        raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}"
+      end
+    end
+
+    def raw_releases(args = [])
+      command = [
+        'list',
+        %(--namespace "#{namespace}"),
+        %(--tiller-namespace "#{namespace}" --output json),
+        *args
+      ]
+      json = JSON.parse(run_command(command))
+
+      releases = json['Releases'].map do |json_release|
+        Release.new(*json_release.values_at(*RELEASE_JSON_ATTRIBUTES))
+      end
+
+      [releases, json['Next']]
+    rescue JSON::ParserError => ex
+      puts "Ignoring this JSON parsing error: #{ex}" # rubocop:disable Rails/Output
+      [[], nil]
+    end
+
+    # Fetches data from Helm and yields a Page object for every page
+    # of data, without loading all of them into memory.
+    #
+    # method - The Octokit method to use for getting the data.
+    # args - Arguments to pass to the `helm list` command.
+    def each_releases_page(args, &block)
+      return to_enum(__method__, args) unless block_given?
+
+      page = 1
+      offset = ''
+
+      loop do
+        final_args = args.dup
+        final_args << "--offset #{offset}" unless offset.to_s.empty?
+        collection, offset = raw_releases(final_args)
+
+        yield Page.new(collection, page += 1)
+
+        break if offset.to_s.empty?
+      end
+    end
+
+    # Iterates over all of the releases.
+    #
+    # args - Any arguments to pass to the `helm list` command.
+    def each_release(args, &block)
+      return to_enum(__method__, args) unless block_given?
+
+      each_releases_page(args) do |page|
+        page.releases.each do |release|
+          yield release
+        end
+      end
+    end
+  end
+end
diff --git a/lib/quality/kubernetes_client.rb b/lib/quality/kubernetes_client.rb
new file mode 100644
index 00000000000..2ff9e811425
--- /dev/null
+++ b/lib/quality/kubernetes_client.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative '../gitlab/popen' unless defined?(Gitlab::Popen)
+
+module Quality
+  class KubernetesClient
+    CommandFailedError = Class.new(StandardError)
+
+    attr_reader :namespace
+
+    def initialize(namespace:)
+      @namespace = namespace
+    end
+
+    def cleanup(release_name:)
+      command = [
+        %(--namespace "#{namespace}"),
+        'delete',
+        'ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa',
+        '--now',
+        %(-l release="#{release_name}")
+      ]
+
+      run_command(command)
+    end
+
+    private
+
+    def run_command(command)
+      final_command = ['kubectl', *command].join(' ')
+      puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output
+
+      result = Gitlab::Popen.popen_with_detail([final_command])
+
+      if result.status.success?
+        result.stdout.chomp.freeze
+      else
+        raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}"
+      end
+    end
+  end
+end
diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb
index be0d97370d0..e2a7d3ef5ba 100644
--- a/lib/rouge/formatters/html_gitlab.rb
+++ b/lib/rouge/formatters/html_gitlab.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rouge
   module Formatters
     class HTMLGitlab < Rouge::Formatters::HTML
@@ -11,7 +13,7 @@ module Rouge
         @tag = tag
       end
 
-      def stream(tokens, &b)
+      def stream(tokens)
         is_first = true
         token_lines(tokens) do |line|
           yield "\n" unless is_first
diff --git a/lib/rouge/plugins/common_mark.rb b/lib/rouge/plugins/common_mark.rb
index 8f9de061124..d240df5a0e0 100644
--- a/lib/rouge/plugins/common_mark.rb
+++ b/lib/rouge/plugins/common_mark.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # A rouge plugin for CommonMark markdown engine.
 # Used to highlight code generated by CommonMark.
 
diff --git a/lib/rspec_flaky/config.rb b/lib/rspec_flaky/config.rb
index 06e96f969f1..55c1d4747b4 100644
--- a/lib/rspec_flaky/config.rb
+++ b/lib/rspec_flaky/config.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module RspecFlaky
   class Config
     def self.generate_report?
diff --git a/lib/rspec_flaky/example.rb b/lib/rspec_flaky/example.rb
index b6e790cbbab..3c1b05257a0 100644
--- a/lib/rspec_flaky/example.rb
+++ b/lib/rspec_flaky/example.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module RspecFlaky
   # This is a wrapper class for RSpec::Core::Example
   class Example
diff --git a/lib/rspec_flaky/flaky_example.rb b/lib/rspec_flaky/flaky_example.rb
index 6be24014d89..da5dbf06bc9 100644
--- a/lib/rspec_flaky/flaky_example.rb
+++ b/lib/rspec_flaky/flaky_example.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module RspecFlaky
   # This represents a flaky RSpec example and is mainly meant to be saved in a JSON file
   class FlakyExample < OpenStruct
diff --git a/lib/rspec_flaky/flaky_examples_collection.rb b/lib/rspec_flaky/flaky_examples_collection.rb
index dea23c325be..290a51766e9 100644
--- a/lib/rspec_flaky/flaky_examples_collection.rb
+++ b/lib/rspec_flaky/flaky_examples_collection.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'active_support/hash_with_indifferent_access'
 
 require_relative 'flaky_example'
diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb
index 9cd0c38cb55..19cc0baa2d3 100644
--- a/lib/rspec_flaky/listener.rb
+++ b/lib/rspec_flaky/listener.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'json'
 
 require_dependency 'rspec_flaky/config'
diff --git a/lib/rspec_flaky/report.rb b/lib/rspec_flaky/report.rb
index 1c362fdd20d..9a0fb88c424 100644
--- a/lib/rspec_flaky/report.rb
+++ b/lib/rspec_flaky/report.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'json'
 require 'time'
 
diff --git a/lib/safe_zip/entry.rb b/lib/safe_zip/entry.rb
new file mode 100644
index 00000000000..664e2f52f91
--- /dev/null
+++ b/lib/safe_zip/entry.rb
@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module SafeZip
+  class Entry
+    attr_reader :zip_archive, :zip_entry
+    attr_reader :path, :params
+
+    def initialize(zip_archive, zip_entry, params)
+      @zip_archive = zip_archive
+      @zip_entry = zip_entry
+      @params = params
+      @path = ::File.expand_path(zip_entry.name, params.extract_path)
+    end
+
+    def path_dir
+      ::File.dirname(path)
+    end
+
+    def real_path_dir
+      ::File.realpath(path_dir)
+    end
+
+    def exist?
+      ::File.exist?(path)
+    end
+
+    def extract
+      # do not extract if file is not part of target directory
+      return false unless matching_target_directory
+
+      # do not overwrite existing file
+      raise SafeZip::Extract::AlreadyExistsError, "File already exists #{zip_entry.name}" if exist?
+
+      create_path_dir
+
+      if zip_entry.file?
+        extract_file
+      elsif zip_entry.directory?
+        extract_dir
+      elsif zip_entry.symlink?
+        extract_symlink
+      else
+        raise SafeZip::Extract::UnsupportedEntryError, "File #{zip_entry.name} cannot be extracted"
+      end
+    rescue SafeZip::Extract::Error
+      raise
+    rescue => e
+      raise SafeZip::Extract::ExtractError, e.message
+    end
+
+    private
+
+    def extract_file
+      zip_archive.extract(zip_entry, path)
+    end
+
+    def extract_dir
+      FileUtils.mkdir(path)
+    end
+
+    def extract_symlink
+      source_path = read_symlink
+      real_source_path = expand_symlink(source_path)
+
+      # ensure that source path of symlink is within target directories
+      unless real_source_path.start_with?(matching_target_directory)
+        raise SafeZip::Extract::PermissionDeniedError, "Symlink cannot be created targeting: #{source_path}"
+      end
+
+      ::File.symlink(source_path, path)
+    end
+
+    def create_path_dir
+      # Create all directories, but ignore permissions
+      FileUtils.mkdir_p(path_dir)
+
+      # disallow to make path dirs to point to another directories
+      unless path_dir == real_path_dir
+        raise SafeZip::Extract::PermissionDeniedError, "Directory of #{zip_entry.name} points to another directory"
+      end
+    end
+
+    def matching_target_directory
+      params.matching_target_directory(path)
+    end
+
+    def read_symlink
+      zip_archive.read(zip_entry)
+    end
+
+    def expand_symlink(source_path)
+      ::File.realpath(source_path, path_dir)
+    rescue
+      raise SafeZip::Extract::SymlinkSourceDoesNotExistError, "Symlink source #{source_path} does not exist"
+    end
+  end
+end
diff --git a/lib/safe_zip/extract.rb b/lib/safe_zip/extract.rb
new file mode 100644
index 00000000000..679c021c730
--- /dev/null
+++ b/lib/safe_zip/extract.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module SafeZip
+  class Extract
+    Error = Class.new(StandardError)
+    PermissionDeniedError = Class.new(Error)
+    SymlinkSourceDoesNotExistError = Class.new(Error)
+    UnsupportedEntryError = Class.new(Error)
+    AlreadyExistsError = Class.new(Error)
+    NoMatchingError = Class.new(Error)
+    ExtractError = Class.new(Error)
+
+    attr_reader :archive_path
+
+    def initialize(archive_file)
+      @archive_path = archive_file
+    end
+
+    def extract(opts = {})
+      params = SafeZip::ExtractParams.new(**opts)
+
+      if Feature.enabled?(:safezip_use_rubyzip, default_enabled: true)
+        extract_with_ruby_zip(params)
+      else
+        legacy_unsafe_extract_with_system_zip(params)
+      end
+    end
+
+    private
+
+    def extract_with_ruby_zip(params)
+      ::Zip::File.open(archive_path) do |zip_archive|
+        # Extract all files in the following order:
+        # 1. Directories first,
+        # 2. Files next,
+        # 3. Symlinks last (or anything else)
+        extracted = extract_all_entries(zip_archive, params,
+          zip_archive.lazy.select(&:directory?))
+
+        extracted += extract_all_entries(zip_archive, params,
+          zip_archive.lazy.select(&:file?))
+
+        extracted += extract_all_entries(zip_archive, params,
+          zip_archive.lazy.reject(&:directory?).reject(&:file?))
+
+        raise NoMatchingError, 'No entries extracted' unless extracted > 0
+      end
+    end
+
+    def extract_all_entries(zip_archive, params, entries)
+      entries.count do |zip_entry|
+        SafeZip::Entry.new(zip_archive, zip_entry, params)
+          .extract
+      end
+    end
+
+    def legacy_unsafe_extract_with_system_zip(params)
+      # Requires UnZip at least 6.00 Info-ZIP.
+      # -n  never overwrite existing files
+      args = %W(unzip -n -qq #{archive_path})
+
+      # We add * to end of directory, because we want to extract directory and all subdirectories
+      args += params.directories_wildcard
+
+      # Target directory where we extract
+      args += %W(-d #{params.extract_path})
+
+      unless system(*args)
+        raise Error, 'archive failed to extract'
+      end
+    end
+  end
+end
diff --git a/lib/safe_zip/extract_params.rb b/lib/safe_zip/extract_params.rb
new file mode 100644
index 00000000000..bd3b788bac9
--- /dev/null
+++ b/lib/safe_zip/extract_params.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module SafeZip
+  class ExtractParams
+    include Gitlab::Utils::StrongMemoize
+
+    attr_reader :directories, :extract_path
+
+    def initialize(directories:, to:)
+      @directories = directories
+      @extract_path = ::File.realpath(to)
+    end
+
+    def matching_target_directory(path)
+      target_directories.find do |directory|
+        path.start_with?(directory)
+      end
+    end
+
+    def target_directories
+      strong_memoize(:target_directories) do
+        directories.map do |directory|
+          ::File.join(::File.expand_path(directory, extract_path), '')
+        end
+      end
+    end
+
+    def directories_wildcard
+      strong_memoize(:directories_wildcard) do
+        directories.map do |directory|
+          ::File.join(directory, '*')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/sentry/client.rb b/lib/sentry/client.rb
new file mode 100644
index 00000000000..4187014d49e
--- /dev/null
+++ b/lib/sentry/client.rb
@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module Sentry
+  class Client
+    Error = Class.new(StandardError)
+    SentryError = Class.new(StandardError)
+
+    attr_accessor :url, :token
+
+    def initialize(api_url, token)
+      @url = api_url
+      @token = token
+    end
+
+    def list_issues(issue_status:, limit:)
+      issues = get_issues(issue_status: issue_status, limit: limit)
+      map_to_errors(issues)
+    end
+
+    def list_projects
+      projects = get_projects
+      map_to_projects(projects)
+    rescue KeyError => e
+      raise Client::SentryError, "Sentry API response is missing keys. #{e.message}"
+    end
+
+    private
+
+    def request_params
+      {
+        headers: {
+          'Authorization' => "Bearer #{@token}"
+        },
+        follow_redirects: false
+      }
+    end
+
+    def http_get(url, params = {})
+      resp = Gitlab::HTTP.get(url, **request_params.merge(params))
+
+      handle_response(resp)
+    end
+
+    def get_issues(issue_status:, limit:)
+      http_get(issues_api_url, query: {
+        query: "is:#{issue_status}",
+        limit: limit
+      })
+    end
+
+    def get_projects
+      http_get(projects_api_url)
+    end
+
+    def handle_response(response)
+      unless response.code == 200
+        raise Client::Error, "Sentry response error: #{response.code}"
+      end
+
+      response.as_json
+    end
+
+    def projects_api_url
+      projects_url = URI(@url)
+      projects_url.path = '/api/0/projects/'
+
+      projects_url
+    end
+
+    def issues_api_url
+      issues_url = URI(@url + '/issues/')
+      issues_url.path.squeeze!('/')
+
+      issues_url
+    end
+
+    def map_to_errors(issues)
+      issues.map(&method(:map_to_error))
+    end
+
+    def map_to_projects(projects)
+      projects.map(&method(:map_to_project))
+    end
+
+    def issue_url(id)
+      issues_url = @url + "/issues/#{id}"
+      issues_url = ErrorTracking::ProjectErrorTrackingSetting.extract_sentry_external_url(issues_url)
+
+      uri = URI(issues_url)
+      uri.path.squeeze!('/')
+
+      uri.to_s
+    end
+
+    def map_to_error(issue)
+      id = issue.fetch('id')
+      project = issue.fetch('project')
+
+      count = issue.fetch('count', nil)
+
+      frequency = issue.dig('stats', '24h')
+      message = issue.dig('metadata', 'value')
+
+      external_url = issue_url(id)
+
+      Gitlab::ErrorTracking::Error.new(
+        id: id,
+        first_seen: issue.fetch('firstSeen', nil),
+        last_seen: issue.fetch('lastSeen', nil),
+        title: issue.fetch('title', nil),
+        type: issue.fetch('type', nil),
+        user_count: issue.fetch('userCount', nil),
+        count: count,
+        message: message,
+        culprit: issue.fetch('culprit', nil),
+        external_url: external_url,
+        short_id: issue.fetch('shortId', nil),
+        status: issue.fetch('status', nil),
+        frequency: frequency,
+        project_id: project.fetch('id'),
+        project_name: project.fetch('name', nil),
+        project_slug: project.fetch('slug', nil)
+      )
+    end
+
+    def map_to_project(project)
+      organization = project.fetch('organization')
+
+      Gitlab::ErrorTracking::Project.new(
+        id: project.fetch('id'),
+        name: project.fetch('name'),
+        slug: project.fetch('slug'),
+        status: project.dig('status'),
+        organization_name: organization.fetch('name'),
+        organization_id: organization.fetch('id'),
+        organization_slug: organization.fetch('slug')
+      )
+    end
+  end
+end
diff --git a/lib/serializers/json.rb b/lib/serializers/json.rb
new file mode 100644
index 00000000000..93cb192087a
--- /dev/null
+++ b/lib/serializers/json.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Serializers
+  # This serializer exports data as JSON,
+  # it is designed to be used with interwork compatibility between MySQL and PostgreSQL
+  # implementations, as used version of MySQL does not support native json type
+  #
+  # Secondly, the loader makes the resulting hash to have deep indifferent access
+  class JSON
+    class << self
+      def dump(obj)
+        # MySQL stores data as text
+        # look at ./config/initializers/ar_mysql_jsonb_support.rb
+        if Gitlab::Database.mysql?
+          obj = ActiveSupport::JSON.encode(obj)
+        end
+
+        obj
+      end
+
+      def load(data)
+        return if data.nil?
+
+        # On MySQL we store data as text
+        # look at ./config/initializers/ar_mysql_jsonb_support.rb
+        if Gitlab::Database.mysql?
+          data = ActiveSupport::JSON.decode(data)
+        end
+
+        Gitlab::Utils.deep_indifferent_access(data)
+      end
+    end
+  end
+end
diff --git a/lib/static_model.rb b/lib/static_model.rb
index 60e2dd82e4e..86bf8d62f9a 100644
--- a/lib/static_model.rb
+++ b/lib/static_model.rb
@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 # Provides an ActiveRecord-like interface to a model whose data is not persisted to a database.
 module StaticModel
   extend ActiveSupport::Concern
 
-  module ClassMethods
+  class_methods do
     # Used by ActiveRecord's polymorphic association to set object_id
     def primary_key
       'id'
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index 72eb8adcce2..fc984d737d5 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -17,7 +17,7 @@
 ## See installation.md#using-https for additional HTTPS configuration details.
 
 upstream gitlab-workhorse {
-  # Gitlab socket file,
+  # GitLab socket file,
   # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/socket
   server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0;
 }
@@ -112,7 +112,7 @@ server {
   error_page 502 /502.html;
   error_page 503 /503.html;
   location ~ ^/(404|422|500|502|503)\.html$ {
-    # Location to the Gitlab's public directory,
+    # Location to the GitLab's public directory,
     # for Omnibus this would be: /opt/gitlab/embedded/service/gitlab-rails/public.
     root /home/git/gitlab/public;
     internal;
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index 2e3799d5e1b..ba01e250bbb 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -21,7 +21,7 @@
 ## See installation.md#using-https for additional HTTPS configuration details.
 
 upstream gitlab-workhorse {
-  # Gitlab socket file,
+  # GitLab socket file,
   # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/socket
   server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0;
 }
@@ -162,7 +162,7 @@ server {
   error_page 502 /502.html;
   error_page 503 /503.html;
   location ~ ^/(404|422|500|502|503)\.html$ {
-    # Location to the Gitlab's public directory,
+    # Location to the GitLab's public directory,
     # for Omnibus this would be: /opt/gitlab/embedded/service/gitlab-rails/public
     root /home/git/gitlab/public;
     internal;
diff --git a/lib/support/nginx/registry-ssl b/lib/support/nginx/registry-ssl
index 92511e26861..908d26a0da2 100644
--- a/lib/support/nginx/registry-ssl
+++ b/lib/support/nginx/registry-ssl
@@ -10,7 +10,7 @@ server {
   listen *:80;
   server_name  registry.gitlab.example.com;
   server_tokens off; ## Don't show the nginx version number, a security best practice
-  return 301 https://$http_host:$request_uri;
+  return 301 https://$http_host$request_uri;
   access_log  /var/log/nginx/gitlab_registry_access.log gitlab_access;
   error_log   /var/log/nginx/gitlab_registry_error.log;
 }
diff --git a/lib/system_check.rb b/lib/system_check.rb
index 466c39904fa..7ffd7c03c5b 100644
--- a/lib/system_check.rb
+++ b/lib/system_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Library to perform System Checks
 #
 # Every Check is implemented as its own class inherited from SystemCheck::BaseCheck
diff --git a/lib/system_check/app/active_users_check.rb b/lib/system_check/app/active_users_check.rb
index 1d72c8d6903..8446c2fc2c8 100644
--- a/lib/system_check/app/active_users_check.rb
+++ b/lib/system_check/app/active_users_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class ActiveUsersCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/app/database_config_exists_check.rb b/lib/system_check/app/database_config_exists_check.rb
index d1fae192350..1769145ed63 100644
--- a/lib/system_check/app/database_config_exists_check.rb
+++ b/lib/system_check/app/database_config_exists_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class DatabaseConfigExistsCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/app/git_config_check.rb b/lib/system_check/app/git_config_check.rb
index d08a81639e3..4e8d607096c 100644
--- a/lib/system_check/app/git_config_check.rb
+++ b/lib/system_check/app/git_config_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class GitConfigCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/app/git_user_default_ssh_config_check.rb b/lib/system_check/app/git_user_default_ssh_config_check.rb
index ad41760dff2..6cd53779bfd 100644
--- a/lib/system_check/app/git_user_default_ssh_config_check.rb
+++ b/lib/system_check/app/git_user_default_ssh_config_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class GitUserDefaultSSHConfigCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb
index 44ec888c197..7c3e7759dd0 100644
--- a/lib/system_check/app/git_version_check.rb
+++ b/lib/system_check/app/git_version_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class GitVersionCheck < SystemCheck::BaseCheck
@@ -5,7 +7,7 @@ module SystemCheck
       set_check_pass -> { "yes (#{self.current_version})" }
 
       def self.required_version
-        @required_version ||= Gitlab::VersionInfo.new(2, 9, 5)
+        @required_version ||= Gitlab::VersionInfo.parse('2.18.0')
       end
 
       def self.current_version
diff --git a/lib/system_check/app/gitlab_config_exists_check.rb b/lib/system_check/app/gitlab_config_exists_check.rb
index 247aa0994e4..1cc5ead0d89 100644
--- a/lib/system_check/app/gitlab_config_exists_check.rb
+++ b/lib/system_check/app/gitlab_config_exists_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class GitlabConfigExistsCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/app/gitlab_config_up_to_date_check.rb b/lib/system_check/app/gitlab_config_up_to_date_check.rb
index c609e48e133..58c7e3039c8 100644
--- a/lib/system_check/app/gitlab_config_up_to_date_check.rb
+++ b/lib/system_check/app/gitlab_config_up_to_date_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class GitlabConfigUpToDateCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/app/init_script_exists_check.rb b/lib/system_check/app/init_script_exists_check.rb
index d246e058e86..d36dbe7d67d 100644
--- a/lib/system_check/app/init_script_exists_check.rb
+++ b/lib/system_check/app/init_script_exists_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class InitScriptExistsCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/app/init_script_up_to_date_check.rb b/lib/system_check/app/init_script_up_to_date_check.rb
index 53a47eb0f42..569c41df6e4 100644
--- a/lib/system_check/app/init_script_up_to_date_check.rb
+++ b/lib/system_check/app/init_script_up_to_date_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class InitScriptUpToDateCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/app/log_writable_check.rb b/lib/system_check/app/log_writable_check.rb
index 3e0c436d6ee..e26ad143eb8 100644
--- a/lib/system_check/app/log_writable_check.rb
+++ b/lib/system_check/app/log_writable_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class LogWritableCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/app/migrations_are_up_check.rb b/lib/system_check/app/migrations_are_up_check.rb
index 5eedbacce77..b12e9ac6bba 100644
--- a/lib/system_check/app/migrations_are_up_check.rb
+++ b/lib/system_check/app/migrations_are_up_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class MigrationsAreUpCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/app/orphaned_group_members_check.rb b/lib/system_check/app/orphaned_group_members_check.rb
index 2b46d36fe51..3e6ffb8190b 100644
--- a/lib/system_check/app/orphaned_group_members_check.rb
+++ b/lib/system_check/app/orphaned_group_members_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class OrphanedGroupMembersCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/app/projects_have_namespace_check.rb b/lib/system_check/app/projects_have_namespace_check.rb
index a6ec9f7665c..2bf2529acf1 100644
--- a/lib/system_check/app/projects_have_namespace_check.rb
+++ b/lib/system_check/app/projects_have_namespace_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class ProjectsHaveNamespaceCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/app/redis_version_check.rb b/lib/system_check/app/redis_version_check.rb
index a0610e73576..890f8b44d13 100644
--- a/lib/system_check/app/redis_version_check.rb
+++ b/lib/system_check/app/redis_version_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class RedisVersionCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/app/ruby_version_check.rb b/lib/system_check/app/ruby_version_check.rb
index 57bbabece1f..60e07718338 100644
--- a/lib/system_check/app/ruby_version_check.rb
+++ b/lib/system_check/app/ruby_version_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class RubyVersionCheck < SystemCheck::BaseCheck
@@ -9,7 +11,7 @@ module SystemCheck
       end
 
       def self.current_version
-        @current_version ||= Gitlab::VersionInfo.parse(Gitlab::TaskHelpers.run_command(%w(ruby --version)))
+        @current_version ||= Gitlab::VersionInfo.parse(RUBY_VERSION)
       end
 
       def check?
diff --git a/lib/system_check/app/tmp_writable_check.rb b/lib/system_check/app/tmp_writable_check.rb
index 99a75e57abf..6687df091d3 100644
--- a/lib/system_check/app/tmp_writable_check.rb
+++ b/lib/system_check/app/tmp_writable_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class TmpWritableCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/app/uploads_directory_exists_check.rb b/lib/system_check/app/uploads_directory_exists_check.rb
index 7026d0ba075..940eff9d4cf 100644
--- a/lib/system_check/app/uploads_directory_exists_check.rb
+++ b/lib/system_check/app/uploads_directory_exists_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class UploadsDirectoryExistsCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/app/uploads_path_permission_check.rb b/lib/system_check/app/uploads_path_permission_check.rb
index 7df6c060254..4a49f3bc2bb 100644
--- a/lib/system_check/app/uploads_path_permission_check.rb
+++ b/lib/system_check/app/uploads_path_permission_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class UploadsPathPermissionCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/app/uploads_path_tmp_permission_check.rb b/lib/system_check/app/uploads_path_tmp_permission_check.rb
index b276a81eac1..ae374f4707c 100644
--- a/lib/system_check/app/uploads_path_tmp_permission_check.rb
+++ b/lib/system_check/app/uploads_path_tmp_permission_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module App
     class UploadsPathTmpPermissionCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb
index 0f5742dd67f..46aad8aa885 100644
--- a/lib/system_check/base_check.rb
+++ b/lib/system_check/base_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   # Base class for Checks. You must inherit from here
   # and implement the methods below when necessary
@@ -68,18 +70,14 @@ module SystemCheck
     # multiple reasons why a check can fail
     #
     # @param [String] reason to be displayed
-    def skip_reason=(reason)
-      @skip_reason = reason
-    end
+    attr_writer :skip_reason
 
     # Skip reason defined during runtime
     #
     # This value have precedence over the one defined in the subclass
     #
     # @return [String] the reason
-    def skip_reason
-      @skip_reason
-    end
+    attr_reader :skip_reason
 
     # Does the check support automatically repair routine?
     #
diff --git a/lib/system_check/gitaly_check.rb b/lib/system_check/gitaly_check.rb
new file mode 100644
index 00000000000..3d2517a7aca
--- /dev/null
+++ b/lib/system_check/gitaly_check.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  class GitalyCheck < BaseCheck
+    set_name 'Gitaly:'
+
+    def multi_check
+      Gitlab::HealthChecks::GitalyCheck.readiness.each do |result|
+        $stdout.print "#{result.labels[:shard]} ... "
+
+        if result.success
+          $stdout.puts 'OK'.color(:green)
+        else
+          $stdout.puts "FAIL: #{result.message}".color(:red)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/system_check/gitlab_shell_check.rb b/lib/system_check/gitlab_shell_check.rb
new file mode 100644
index 00000000000..31c4ec33247
--- /dev/null
+++ b/lib/system_check/gitlab_shell_check.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  # Used by gitlab:gitlab_shell:check rake task
+  class GitlabShellCheck < BaseCheck
+    set_name 'GitLab Shell:'
+
+    def multi_check
+      check_gitlab_shell
+      check_gitlab_shell_self_test
+    end
+
+    private
+
+    def check_gitlab_shell
+      required_version = Gitlab::VersionInfo.parse(Gitlab::Shell.version_required)
+      current_version = Gitlab::VersionInfo.parse(gitlab_shell_version)
+
+      $stdout.print "GitLab Shell version >= #{required_version} ? ... "
+      if current_version.valid? && required_version <= current_version
+        $stdout.puts "OK (#{current_version})".color(:green)
+      else
+        $stdout.puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red)
+      end
+    end
+
+    def check_gitlab_shell_self_test
+      gitlab_shell_repo_base = gitlab_shell_path
+      check_cmd = File.expand_path('bin/check', gitlab_shell_repo_base)
+      $stdout.puts "Running #{check_cmd}"
+
+      if system(check_cmd, chdir: gitlab_shell_repo_base)
+        $stdout.puts 'gitlab-shell self-check successful'.color(:green)
+      else
+        $stdout.puts 'gitlab-shell self-check failed'.color(:red)
+        try_fixing_it(
+          'Make sure GitLab is running;',
+          'Check the gitlab-shell configuration file:',
+          sudo_gitlab("editor #{File.expand_path('config.yml', gitlab_shell_repo_base)}")
+        )
+        fix_and_rerun
+      end
+    end
+
+    # Helper methods
+    ########################
+
+    def gitlab_shell_path
+      Gitlab.config.gitlab_shell.path
+    end
+
+    def gitlab_shell_version
+      Gitlab::Shell.new.version
+    end
+  end
+end
diff --git a/lib/system_check/helpers.rb b/lib/system_check/helpers.rb
index 6227e461d24..07d479848fe 100644
--- a/lib/system_check/helpers.rb
+++ b/lib/system_check/helpers.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module Helpers
     include ::Gitlab::TaskHelpers
diff --git a/lib/system_check/incoming_email/foreman_configured_check.rb b/lib/system_check/incoming_email/foreman_configured_check.rb
index 1db7bf2b782..944913087da 100644
--- a/lib/system_check/incoming_email/foreman_configured_check.rb
+++ b/lib/system_check/incoming_email/foreman_configured_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module IncomingEmail
     class ForemanConfiguredCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/incoming_email/imap_authentication_check.rb b/lib/system_check/incoming_email/imap_authentication_check.rb
index e55bea86d3f..613c2296375 100644
--- a/lib/system_check/incoming_email/imap_authentication_check.rb
+++ b/lib/system_check/incoming_email/imap_authentication_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module IncomingEmail
     class ImapAuthenticationCheck < SystemCheck::BaseCheck
@@ -7,7 +9,7 @@ module SystemCheck
         if config
           try_connect_imap
         else
-          @error = "#{mail_room_config_path} does not have mailboxes setup"
+          @error = "#{mail_room_config_path} does not have mailboxes set up"
           false
         end
       end
diff --git a/lib/system_check/incoming_email/initd_configured_check.rb b/lib/system_check/incoming_email/initd_configured_check.rb
index ea23b8ef49c..acb4b5a9e74 100644
--- a/lib/system_check/incoming_email/initd_configured_check.rb
+++ b/lib/system_check/incoming_email/initd_configured_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module IncomingEmail
     class InitdConfiguredCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/incoming_email/mail_room_running_check.rb b/lib/system_check/incoming_email/mail_room_running_check.rb
index c1807501829..b7aead4624e 100644
--- a/lib/system_check/incoming_email/mail_room_running_check.rb
+++ b/lib/system_check/incoming_email/mail_room_running_check.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module IncomingEmail
     class MailRoomRunningCheck < SystemCheck::BaseCheck
diff --git a/lib/system_check/incoming_email_check.rb b/lib/system_check/incoming_email_check.rb
new file mode 100644
index 00000000000..155b6547595
--- /dev/null
+++ b/lib/system_check/incoming_email_check.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  # Used by gitlab:incoming_email:check rake task
+  class IncomingEmailCheck < BaseCheck
+    set_name 'Incoming Email:'
+
+    def multi_check
+      if Gitlab.config.incoming_email.enabled
+        checks = [
+          SystemCheck::IncomingEmail::ImapAuthenticationCheck
+        ]
+
+        if Rails.env.production?
+          checks << SystemCheck::IncomingEmail::InitdConfiguredCheck
+          checks << SystemCheck::IncomingEmail::MailRoomRunningCheck
+        else
+          checks << SystemCheck::IncomingEmail::ForemanConfiguredCheck
+        end
+
+        SystemCheck.run('Reply by email', checks)
+      else
+        $stdout.puts 'Reply by email is disabled in config/gitlab.yml'
+      end
+    end
+  end
+end
diff --git a/lib/system_check/ldap_check.rb b/lib/system_check/ldap_check.rb
new file mode 100644
index 00000000000..619fb3cccb8
--- /dev/null
+++ b/lib/system_check/ldap_check.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  # Used by gitlab:ldap:check rake task
+  class LdapCheck < BaseCheck
+    set_name 'LDAP:'
+
+    def multi_check
+      if Gitlab::Auth::LDAP::Config.enabled?
+        # Only show up to 100 results because LDAP directories can be very big.
+        # This setting only affects the `rake gitlab:check` script.
+        limit = ENV['LDAP_CHECK_LIMIT']
+        limit = 100 if limit.blank?
+
+        check_ldap(limit)
+      else
+        $stdout.puts 'LDAP is disabled in config/gitlab.yml'
+      end
+    end
+
+    private
+
+    def check_ldap(limit)
+      servers = Gitlab::Auth::LDAP::Config.providers
+
+      servers.each do |server|
+        $stdout.puts "Server: #{server}"
+
+        begin
+          Gitlab::Auth::LDAP::Adapter.open(server) do |adapter|
+            check_ldap_auth(adapter)
+
+            $stdout.puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)"
+
+            users = adapter.users(adapter.config.uid, '*', limit)
+            users.each do |user|
+              $stdout.puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}"
+            end
+          end
+        rescue Net::LDAP::ConnectionRefusedError, Errno::ECONNREFUSED => e
+          $stdout.puts "Could not connect to the LDAP server: #{e.message}".color(:red)
+        end
+      end
+    end
+
+    def check_ldap_auth(adapter)
+      auth = adapter.config.has_auth?
+
+      message = if auth && adapter.ldap.bind
+                  'Success'.color(:green)
+                elsif auth
+                  'Failed. Check `bind_dn` and `password` configuration values'.color(:red)
+                else
+                  'Anonymous. No `bind_dn` or `password` configured'.color(:yellow)
+                end
+
+      $stdout.puts "LDAP authentication... #{message}"
+    end
+  end
+end
diff --git a/lib/system_check/orphans/namespace_check.rb b/lib/system_check/orphans/namespace_check.rb
index b5f443abe06..53b2d8fd5b3 100644
--- a/lib/system_check/orphans/namespace_check.rb
+++ b/lib/system_check/orphans/namespace_check.rb
@@ -1,16 +1,20 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module Orphans
     class NamespaceCheck < SystemCheck::BaseCheck
       set_name 'Orphaned namespaces:'
 
       def multi_check
-        Gitlab.config.repositories.storages.each do |storage_name, repository_storage|
-          $stdout.puts
-          $stdout.puts "* Storage: #{storage_name} (#{repository_storage.legacy_disk_path})".color(:yellow)
-          toplevel_namespace_dirs = disk_namespaces(repository_storage.legacy_disk_path)
-
-          orphans = (toplevel_namespace_dirs - existing_namespaces)
-          print_orphans(orphans, storage_name)
+        Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+          Gitlab.config.repositories.storages.each do |storage_name, repository_storage|
+            $stdout.puts
+            $stdout.puts "* Storage: #{storage_name} (#{repository_storage.legacy_disk_path})".color(:yellow)
+            toplevel_namespace_dirs = disk_namespaces(repository_storage.legacy_disk_path)
+
+            orphans = (toplevel_namespace_dirs - existing_namespaces)
+            print_orphans(orphans, storage_name)
+          end
         end
 
         clear_namespaces! # releases memory when check finishes
diff --git a/lib/system_check/orphans/repository_check.rb b/lib/system_check/orphans/repository_check.rb
index 5ef0b93ad08..33020417e95 100644
--- a/lib/system_check/orphans/repository_check.rb
+++ b/lib/system_check/orphans/repository_check.rb
@@ -1,20 +1,23 @@
+# frozen_string_literal: true
+
 module SystemCheck
   module Orphans
     class RepositoryCheck < SystemCheck::BaseCheck
       set_name 'Orphaned repositories:'
-      attr_accessor :orphans
 
       def multi_check
-        Gitlab.config.repositories.storages.each do |storage_name, repository_storage|
-          storage_path = repository_storage.legacy_disk_path
+        Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+          Gitlab.config.repositories.storages.each do |storage_name, repository_storage|
+            storage_path = repository_storage.legacy_disk_path
 
-          $stdout.puts
-          $stdout.puts "* Storage: #{storage_name} (#{storage_path})".color(:yellow)
+            $stdout.puts
+            $stdout.puts "* Storage: #{storage_name} (#{storage_path})".color(:yellow)
 
-          repositories = disk_repositories(storage_path)
-          orphans = (repositories - fetch_repositories(storage_name))
+            repositories = disk_repositories(storage_path)
+            orphans = (repositories - fetch_repositories(storage_name))
 
-          print_orphans(orphans, storage_name)
+            print_orphans(orphans, storage_name)
+          end
         end
       end
 
diff --git a/lib/system_check/rake_task/app_task.rb b/lib/system_check/rake_task/app_task.rb
new file mode 100644
index 00000000000..cc32feb8604
--- /dev/null
+++ b/lib/system_check/rake_task/app_task.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  module RakeTask
+    # Used by gitlab:app:check rake task
+    module AppTask
+      extend RakeTaskHelpers
+
+      def self.name
+        'GitLab App'
+      end
+
+      def self.checks
+        [
+          SystemCheck::App::GitConfigCheck,
+          SystemCheck::App::DatabaseConfigExistsCheck,
+          SystemCheck::App::MigrationsAreUpCheck,
+          SystemCheck::App::OrphanedGroupMembersCheck,
+          SystemCheck::App::GitlabConfigExistsCheck,
+          SystemCheck::App::GitlabConfigUpToDateCheck,
+          SystemCheck::App::LogWritableCheck,
+          SystemCheck::App::TmpWritableCheck,
+          SystemCheck::App::UploadsDirectoryExistsCheck,
+          SystemCheck::App::UploadsPathPermissionCheck,
+          SystemCheck::App::UploadsPathTmpPermissionCheck,
+          SystemCheck::App::InitScriptExistsCheck,
+          SystemCheck::App::InitScriptUpToDateCheck,
+          SystemCheck::App::ProjectsHaveNamespaceCheck,
+          SystemCheck::App::RedisVersionCheck,
+          SystemCheck::App::RubyVersionCheck,
+          SystemCheck::App::GitVersionCheck,
+          SystemCheck::App::GitUserDefaultSSHConfigCheck,
+          SystemCheck::App::ActiveUsersCheck
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/system_check/rake_task/gitaly_task.rb b/lib/system_check/rake_task/gitaly_task.rb
new file mode 100644
index 00000000000..0c3f694f98a
--- /dev/null
+++ b/lib/system_check/rake_task/gitaly_task.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  module RakeTask
+    # Used by gitlab:gitaly:check rake task
+    class GitalyTask
+      extend RakeTaskHelpers
+
+      def self.name
+        'Gitaly'
+      end
+
+      def self.checks
+        [SystemCheck::GitalyCheck]
+      end
+    end
+  end
+end
diff --git a/lib/system_check/rake_task/gitlab_shell_task.rb b/lib/system_check/rake_task/gitlab_shell_task.rb
new file mode 100644
index 00000000000..120e984c68b
--- /dev/null
+++ b/lib/system_check/rake_task/gitlab_shell_task.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  module RakeTask
+    # Used by gitlab:gitlab_shell:check rake task
+    class GitlabShellTask
+      extend RakeTaskHelpers
+
+      def self.name
+        'GitLab Shell'
+      end
+
+      def self.checks
+        [SystemCheck::GitlabShellCheck]
+      end
+    end
+  end
+end
diff --git a/lib/system_check/rake_task/gitlab_task.rb b/lib/system_check/rake_task/gitlab_task.rb
new file mode 100644
index 00000000000..7ff36fd6eb5
--- /dev/null
+++ b/lib/system_check/rake_task/gitlab_task.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  module RakeTask
+    # Used by gitlab:check rake task
+    class GitlabTask
+      extend RakeTaskHelpers
+
+      def self.name
+        'GitLab'
+      end
+
+      def self.manual_run_checks!
+        start_checking("#{name} subtasks")
+
+        subtasks.each(&:run_checks!)
+
+        finished_checking("#{name} subtasks")
+      end
+
+      def self.subtasks
+        [
+          SystemCheck::RakeTask::GitlabShellTask,
+          SystemCheck::RakeTask::GitalyTask,
+          SystemCheck::RakeTask::SidekiqTask,
+          SystemCheck::RakeTask::IncomingEmailTask,
+          SystemCheck::RakeTask::LdapTask,
+          SystemCheck::RakeTask::AppTask
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/system_check/rake_task/incoming_email_task.rb b/lib/system_check/rake_task/incoming_email_task.rb
new file mode 100644
index 00000000000..c296c46feab
--- /dev/null
+++ b/lib/system_check/rake_task/incoming_email_task.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  module RakeTask
+    # Used by gitlab:incoming_email:check rake task
+    class IncomingEmailTask
+      extend RakeTaskHelpers
+
+      def self.name
+        'Incoming Email'
+      end
+
+      def self.checks
+        [SystemCheck::IncomingEmailCheck]
+      end
+    end
+  end
+end
diff --git a/lib/system_check/rake_task/ldap_task.rb b/lib/system_check/rake_task/ldap_task.rb
new file mode 100644
index 00000000000..03a180b9dfb
--- /dev/null
+++ b/lib/system_check/rake_task/ldap_task.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  module RakeTask
+    # Used by gitlab:ldap:check rake task
+    class LdapTask
+      extend RakeTaskHelpers
+
+      def self.name
+        'LDAP'
+      end
+
+      def self.checks
+        [SystemCheck::LdapCheck]
+      end
+    end
+  end
+end
diff --git a/lib/system_check/rake_task/orphans/namespace_task.rb b/lib/system_check/rake_task/orphans/namespace_task.rb
new file mode 100644
index 00000000000..2822da45bc1
--- /dev/null
+++ b/lib/system_check/rake_task/orphans/namespace_task.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  module RakeTask
+    module Orphans
+      # Used by gitlab:orphans:check_namespaces rake task
+      class NamespaceTask
+        extend RakeTaskHelpers
+
+        def self.name
+          'Orphans'
+        end
+
+        def self.checks
+          [SystemCheck::Orphans::NamespaceCheck]
+        end
+      end
+    end
+  end
+end
diff --git a/lib/system_check/rake_task/orphans/repository_task.rb b/lib/system_check/rake_task/orphans/repository_task.rb
new file mode 100644
index 00000000000..f14b3af22e8
--- /dev/null
+++ b/lib/system_check/rake_task/orphans/repository_task.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  module RakeTask
+    module Orphans
+      # Used by gitlab:orphans:check_repositories rake task
+      class RepositoryTask
+        extend RakeTaskHelpers
+
+        def self.name
+          'Orphans'
+        end
+
+        def self.checks
+          [SystemCheck::Orphans::RepositoryCheck]
+        end
+      end
+    end
+  end
+end
diff --git a/lib/system_check/rake_task/orphans_task.rb b/lib/system_check/rake_task/orphans_task.rb
new file mode 100644
index 00000000000..31f8ede25e0
--- /dev/null
+++ b/lib/system_check/rake_task/orphans_task.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  module RakeTask
+    # Used by gitlab:orphans:check rake task
+    class OrphansTask
+      extend RakeTaskHelpers
+
+      def self.name
+        'Orphans'
+      end
+
+      def self.checks
+        [
+          SystemCheck::Orphans::NamespaceCheck,
+          SystemCheck::Orphans::RepositoryCheck
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/system_check/rake_task/rake_task_helpers.rb b/lib/system_check/rake_task/rake_task_helpers.rb
new file mode 100644
index 00000000000..95f2a34a719
--- /dev/null
+++ b/lib/system_check/rake_task/rake_task_helpers.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  module RakeTask
+    # Provides the run! method intended to be called from system check rake tasks
+    module RakeTaskHelpers
+      include ::SystemCheck::Helpers
+
+      def run!
+        warn_user_is_not_gitlab
+
+        if self.respond_to?(:manual_run_checks!)
+          manual_run_checks!
+        else
+          run_checks!
+        end
+      end
+
+      def run_checks!
+        SystemCheck.run(name, checks)
+      end
+
+      def name
+        raise NotImplementedError
+      end
+
+      def checks
+        raise NotImplementedError
+      end
+    end
+  end
+end
diff --git a/lib/system_check/rake_task/sidekiq_task.rb b/lib/system_check/rake_task/sidekiq_task.rb
new file mode 100644
index 00000000000..3ccf009d4b9
--- /dev/null
+++ b/lib/system_check/rake_task/sidekiq_task.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  module RakeTask
+    # Used by gitlab:sidekiq:check rake task
+    class SidekiqTask
+      extend RakeTaskHelpers
+
+      def self.name
+        'Sidekiq'
+      end
+
+      def self.checks
+        [SystemCheck::SidekiqCheck]
+      end
+    end
+  end
+end
diff --git a/lib/system_check/sidekiq_check.rb b/lib/system_check/sidekiq_check.rb
new file mode 100644
index 00000000000..2f5fc2cea89
--- /dev/null
+++ b/lib/system_check/sidekiq_check.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  # Used by gitlab:sidekiq:check rake task
+  class SidekiqCheck < BaseCheck
+    set_name 'Sidekiq:'
+
+    def multi_check
+      check_sidekiq_running
+      only_one_sidekiq_running
+    end
+
+    private
+
+    def check_sidekiq_running
+      $stdout.print "Running? ... "
+
+      if sidekiq_process_count > 0
+        $stdout.puts "yes".color(:green)
+      else
+        $stdout.puts "no".color(:red)
+        try_fixing_it(
+          sudo_gitlab("RAILS_ENV=production bin/background_jobs start")
+        )
+        for_more_information(
+          see_installation_guide_section("Install Init Script"),
+          "see log/sidekiq.log for possible errors"
+        )
+        fix_and_rerun
+      end
+    end
+
+    def only_one_sidekiq_running
+      process_count = sidekiq_process_count
+      return if process_count.zero?
+
+      $stdout.print 'Number of Sidekiq processes ... '
+
+      if process_count == 1
+        $stdout.puts '1'.color(:green)
+      else
+        $stdout.puts "#{process_count}".color(:red)
+        try_fixing_it(
+          'sudo service gitlab stop',
+          "sudo pkill -u #{gitlab_user} -f sidekiq",
+          "sleep 10 && sudo pkill -9 -u #{gitlab_user} -f sidekiq",
+          'sudo service gitlab start'
+        )
+        fix_and_rerun
+      end
+    end
+
+    def sidekiq_process_count
+      ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww))
+      ps_ux.scan(/sidekiq \d+\.\d+\.\d+/).count
+    end
+  end
+end
diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb
index d268f501b4a..11818ae54f8 100644
--- a/lib/system_check/simple_executor.rb
+++ b/lib/system_check/simple_executor.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SystemCheck
   # Simple Executor is current default executor for GitLab
   # It is a simple port from display logic in the old check.rake
@@ -43,7 +45,7 @@ module SystemCheck
     #
     # @param [SystemCheck::BaseCheck] check_klass
     def run_check(check_klass)
-      $stdout.print "#{check_klass.display_name} ... "
+      print_display_name(check_klass)
 
       check = check_klass.new
 
@@ -60,18 +62,18 @@ module SystemCheck
       end
 
       if check.check?
-        $stdout.puts check_klass.check_pass.color(:green)
+        print_check_pass(check_klass)
       else
-        $stdout.puts check_klass.check_fail.color(:red)
+        print_check_failure(check_klass)
 
         if check.can_repair?
           $stdout.print 'Trying to fix error automatically. ...'
 
           if check.repair!
-            $stdout.puts 'Success'.color(:green)
+            print_success
             return
           else
-            $stdout.puts 'Failed'.color(:red)
+            print_failure
           end
         end
 
@@ -83,6 +85,26 @@ module SystemCheck
 
     private
 
+    def print_display_name(check_klass)
+      $stdout.print "#{check_klass.display_name} ... "
+    end
+
+    def print_check_pass(check_klass)
+      $stdout.puts check_klass.check_pass.color(:green)
+    end
+
+    def print_check_failure(check_klass)
+      $stdout.puts check_klass.check_fail.color(:red)
+    end
+
+    def print_success
+      $stdout.puts 'Success'.color(:green)
+    end
+
+    def print_failure
+      $stdout.puts 'Failed'.color(:red)
+    end
+
     # Prints header content for the series of checks to be executed for this component
     #
     # @param [String] component name of the component relative to the checks being executed
diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake
deleted file mode 100644
index 4b4881cecb8..00000000000
--- a/lib/tasks/flay.rake
+++ /dev/null
@@ -1,9 +0,0 @@
-desc 'Code duplication analyze via flay'
-task :flay do
-  output = `bundle exec flay --mass 35 app/ lib/gitlab/ 2> #{File::NULL}`
-
-  if output.include?("Similar code found") || output.include?("IDENTICAL code found")
-    puts output
-    exit 1
-  end
-end
diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake
index c6942d22926..560a52053d8 100644
--- a/lib/tasks/gemojione.rake
+++ b/lib/tasks/gemojione.rake
@@ -86,7 +86,7 @@ namespace :gemojione do
     SPRITESHEET_WIDTH = 860
     SPRITESHEET_HEIGHT = 840
 
-    # Setup a map to rename image files
+    # Set up a map to rename image files
     emoji_unicode_string_to_name_map = {}
     Gitlab::Emoji.emojis.each do |name, emoji_hash|
       # Ignore aliases
diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake
index 247d7be7d78..2235a6ba194 100644
--- a/lib/tasks/gettext.rake
+++ b/lib/tasks/gettext.rake
@@ -4,7 +4,7 @@ namespace :gettext do
   # Customize list of translatable files
   # See: https://github.com/grosser/gettext_i18n_rails#customizing-list-of-translatable-files
   def files_to_translate
-    folders = %W(app lib config #{locale_path}).join(',')
+    folders = %W(ee app lib config #{locale_path}).join(',')
     exts = %w(rb erb haml slim rhtml js jsx vue handlebars hbs mustache).join(',')
 
     Dir.glob(
@@ -16,10 +16,32 @@ namespace :gettext do
     # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/33014#note_31218998
     FileUtils.touch(File.join(Rails.root, 'locale/gitlab.pot'))
 
-    Rake::Task['gettext:pack'].invoke
     Rake::Task['gettext:po_to_json'].invoke
   end
 
+  task :regenerate do
+    pot_file = 'locale/gitlab.pot'
+    # Remove all translated files, this speeds up finding
+    FileUtils.rm Dir['locale/**/gitlab.*']
+    # remove the `pot` file to ensure it's completely regenerated
+    FileUtils.rm_f pot_file
+
+    Rake::Task['gettext:find'].invoke
+
+    # leave only the required changes.
+    `git checkout -- locale/*/gitlab.po`
+
+    # Remove timestamps from the pot file
+    pot_content = File.read pot_file
+    pot_content.gsub!(/^"POT?\-(?:Creation|Revision)\-Date\:.*\n/, '')
+    File.write pot_file, pot_content
+
+    puts <<~MSG
+    All done. Please commit the changes to `locale/gitlab.pot`.
+
+    MSG
+  end
+
   desc 'Lint all po files in `locale/'
   task lint: :environment do
     require 'simple_po_parser'
@@ -50,6 +72,40 @@ namespace :gettext do
     end
   end
 
+  task :updated_check do
+    pot_file = 'locale/gitlab.pot'
+    # Removing all pre-translated files speeds up `gettext:find` as the
+    # files don't need to be merged.
+    # Having `LC_MESSAGES/gitlab.mo files present also confuses the output.
+    FileUtils.rm Dir['locale/**/gitlab.*']
+    FileUtils.rm_f pot_file
+
+    # `gettext:find` writes touches to temp files to `stderr` which would cause
+    # `static-analysis` to report failures. We can ignore these.
+    silence_stderr do
+      Rake::Task['gettext:find'].invoke
+    end
+
+    pot_diff = `git diff -- #{pot_file} | grep -E '^(\\+|-)msgid'`.strip
+
+    # reset the locale folder for potential next tasks
+    `git checkout -- locale`
+
+    if pot_diff.present?
+      raise <<~MSG
+        Newly translated strings found, please add them to `#{pot_file}` by running:
+
+          bin/rake gettext:regenerate
+
+        Then commit and push the resulting changes to `#{pot_file}`.
+
+        The diff was:
+
+        #{pot_diff}
+      MSG
+    end
+  end
+
   def report_errors_for_file(file, errors_for_file)
     puts "Errors in `#{file}`:"
 
@@ -62,4 +118,15 @@ namespace :gettext do
       end
     end
   end
+
+  def silence_stderr(&block)
+    old_stderr = $stderr.dup
+    $stderr.reopen(File::NULL)
+    $stderr.sync = true
+
+    yield
+  ensure
+    $stderr.reopen(old_stderr)
+    old_stderr.close
+  end
 end
diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake
index bfca4bfb3f7..e7634d2ed4f 100644
--- a/lib/tasks/gitlab/artifacts/migrate.rake
+++ b/lib/tasks/gitlab/artifacts/migrate.rake
@@ -15,7 +15,7 @@ namespace :gitlab do
           build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE)
           build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE)
 
-          logger.info("Transferred artifacts of #{build.id} of #{build.artifacts_size} to object storage")
+          logger.info("Transferred artifact ID #{build.id} with size #{build.artifacts_size} to object storage")
         rescue => e
           logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}")
         end
diff --git a/lib/tasks/gitlab/bulk_add_permission.rake b/lib/tasks/gitlab/bulk_add_permission.rake
index 83dd870fa31..c0d6cc8ca8e 100644
--- a/lib/tasks/gitlab/bulk_add_permission.rake
+++ b/lib/tasks/gitlab/bulk_add_permission.rake
@@ -1,6 +1,6 @@
 namespace :gitlab do
   namespace :import do
-    desc "GitLab | Add all users to all projects (admin users are added as masters)"
+    desc "GitLab | Add all users to all projects (admin users are added as maintainers)"
     task all_users_to_all_projects: :environment  do |t, args|
       user_ids = User.where(admin: false).pluck(:id)
       admin_ids = User.where(admin: true).pluck(:id)
@@ -10,11 +10,11 @@ namespace :gitlab do
       ProjectMember.add_users_to_projects(project_ids, user_ids, ProjectMember::DEVELOPER)
 
       puts "Importing #{admin_ids.size} admins into #{project_ids.size} projects"
-      ProjectMember.add_users_to_projects(project_ids, admin_ids, ProjectMember::MASTER)
+      ProjectMember.add_users_to_projects(project_ids, admin_ids, ProjectMember::MAINTAINER)
     end
 
     desc "GitLab | Add a specific user to all projects (as a developer)"
-    task :user_to_projects, [:email] => :environment  do |t, args|
+    task :user_to_projects, [:email] => :environment do |t, args|
       user = User.find_by(email: args.email)
       project_ids = Project.pluck(:id)
       puts "Importing #{user.email} users into #{project_ids.size} projects"
@@ -22,7 +22,7 @@ namespace :gitlab do
     end
 
     desc "GitLab | Add all users to all groups (admin users are added as owners)"
-    task all_users_to_all_groups: :environment  do |t, args|
+    task all_users_to_all_groups: :environment do |t, args|
       user_ids = User.where(admin: false).pluck(:id)
       admin_ids = User.where(admin: true).pluck(:id)
       groups = Group.all
@@ -36,7 +36,7 @@ namespace :gitlab do
     end
 
     desc "GitLab | Add a specific user to all groups (as a developer)"
-    task :user_to_groups, [:email] => :environment  do |t, args|
+    task :user_to_groups, [:email] => :environment do |t, args|
       user = User.find_by_email args.email
       groups = Group.all
       puts "Importing #{user.email} users into #{groups.size} groups"
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index c04dae7446f..b594f150c3b 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -1,454 +1,66 @@
 namespace :gitlab do
   desc 'GitLab | Check the configuration of GitLab and its environment'
-  task check: %w{gitlab:gitlab_shell:check
-                 gitlab:sidekiq:check
-                 gitlab:incoming_email:check
-                 gitlab:ldap:check
-                 gitlab:app:check}
+  task check: :gitlab_environment do
+    SystemCheck::RakeTask::GitlabTask.run!
+  end
 
   namespace :app do
     desc 'GitLab | Check the configuration of the GitLab Rails app'
     task check: :gitlab_environment do
-      warn_user_is_not_gitlab
-
-      checks = [
-        SystemCheck::App::GitConfigCheck,
-        SystemCheck::App::DatabaseConfigExistsCheck,
-        SystemCheck::App::MigrationsAreUpCheck,
-        SystemCheck::App::OrphanedGroupMembersCheck,
-        SystemCheck::App::GitlabConfigExistsCheck,
-        SystemCheck::App::GitlabConfigUpToDateCheck,
-        SystemCheck::App::LogWritableCheck,
-        SystemCheck::App::TmpWritableCheck,
-        SystemCheck::App::UploadsDirectoryExistsCheck,
-        SystemCheck::App::UploadsPathPermissionCheck,
-        SystemCheck::App::UploadsPathTmpPermissionCheck,
-        SystemCheck::App::InitScriptExistsCheck,
-        SystemCheck::App::InitScriptUpToDateCheck,
-        SystemCheck::App::ProjectsHaveNamespaceCheck,
-        SystemCheck::App::RedisVersionCheck,
-        SystemCheck::App::RubyVersionCheck,
-        SystemCheck::App::GitVersionCheck,
-        SystemCheck::App::GitUserDefaultSSHConfigCheck,
-        SystemCheck::App::ActiveUsersCheck
-      ]
-
-      SystemCheck.run('GitLab', checks)
+      SystemCheck::RakeTask::AppTask.run!
     end
   end
 
   namespace :gitlab_shell do
     desc "GitLab | Check the configuration of GitLab Shell"
     task check: :gitlab_environment do
-      warn_user_is_not_gitlab
-      start_checking "GitLab Shell"
-
-      check_gitlab_shell
-      check_repo_base_exists
-      check_repo_base_is_not_symlink
-      check_repo_base_user_and_group
-      check_repo_base_permissions
-      check_repos_hooks_directory_is_link
-      check_gitlab_shell_self_test
-
-      finished_checking "GitLab Shell"
-    end
-
-    # Checks
-    ########################
-
-    def check_repo_base_exists
-      puts "Repo base directory exists?"
-
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        repo_base_path = repository_storage.legacy_disk_path
-        print "#{name}... "
-
-        if File.exist?(repo_base_path)
-          puts "yes".color(:green)
-        else
-          puts "no".color(:red)
-          puts "#{repo_base_path} is missing".color(:red)
-          try_fixing_it(
-            "This should have been created when setting up GitLab Shell.",
-            "Make sure it's set correctly in config/gitlab.yml",
-            "Make sure GitLab Shell is installed correctly."
-          )
-          for_more_information(
-            see_installation_guide_section "GitLab Shell"
-          )
-          fix_and_rerun
-        end
-      end
-    end
-
-    def check_repo_base_is_not_symlink
-      puts "Repo storage directories are symlinks?"
-
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        repo_base_path = repository_storage.legacy_disk_path
-        print "#{name}... "
-
-        unless File.exist?(repo_base_path)
-          puts "can't check because of previous errors".color(:magenta)
-          break
-        end
-
-        unless File.symlink?(repo_base_path)
-          puts "no".color(:green)
-        else
-          puts "yes".color(:red)
-          try_fixing_it(
-            "Make sure it's set to the real directory in config/gitlab.yml"
-          )
-          fix_and_rerun
-        end
-      end
-    end
-
-    def check_repo_base_permissions
-      puts "Repo paths access is drwxrws---?"
-
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        repo_base_path = repository_storage.legacy_disk_path
-        print "#{name}... "
-
-        unless File.exist?(repo_base_path)
-          puts "can't check because of previous errors".color(:magenta)
-          break
-        end
-
-        if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770")
-          puts "yes".color(:green)
-        else
-          puts "no".color(:red)
-          try_fixing_it(
-            "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}",
-            "sudo chmod -R ug-s #{repo_base_path}",
-            "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s"
-          )
-          for_more_information(
-            see_installation_guide_section "GitLab Shell"
-          )
-          fix_and_rerun
-        end
-      end
-    end
-
-    def check_repo_base_user_and_group
-      gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user
-      puts "Repo paths owned by #{gitlab_shell_ssh_user}:root, or #{gitlab_shell_ssh_user}:#{Gitlab.config.gitlab_shell.owner_group}?"
-
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        repo_base_path = repository_storage.legacy_disk_path
-        print "#{name}... "
-
-        unless File.exist?(repo_base_path)
-          puts "can't check because of previous errors".color(:magenta)
-          break
-        end
-
-        user_id = uid_for(gitlab_shell_ssh_user)
-        root_group_id = gid_for('root')
-        group_ids = [root_group_id, gid_for(Gitlab.config.gitlab_shell.owner_group)]
-        if File.stat(repo_base_path).uid == user_id && group_ids.include?(File.stat(repo_base_path).gid)
-          puts "yes".color(:green)
-        else
-          puts "no".color(:red)
-          puts "  User id for #{gitlab_shell_ssh_user}: #{user_id}. Groupd id for root: #{root_group_id}".color(:blue)
-          try_fixing_it(
-            "sudo chown -R #{gitlab_shell_ssh_user}:root #{repo_base_path}"
-          )
-          for_more_information(
-            see_installation_guide_section "GitLab Shell"
-          )
-          fix_and_rerun
-        end
-      end
-    end
-
-    def check_repos_hooks_directory_is_link
-      print "hooks directories in repos are links: ... "
-
-      gitlab_shell_hooks_path = Gitlab.config.gitlab_shell.hooks_path
-
-      unless Project.count > 0
-        puts "can't check, you have no projects".color(:magenta)
-        return
-      end
-
-      puts ""
-
-      Project.find_each(batch_size: 100) do |project|
-        print sanitized_message(project)
-        project_hook_directory = File.join(project.repository.path_to_repo, "hooks")
-
-        if project.empty_repo?
-          puts "repository is empty".color(:magenta)
-        elsif File.directory?(project_hook_directory) && File.directory?(gitlab_shell_hooks_path) &&
-            (File.realpath(project_hook_directory) == File.realpath(gitlab_shell_hooks_path))
-          puts 'ok'.color(:green)
-        else
-          puts "wrong or missing hooks".color(:red)
-          try_fixing_it(
-            sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')} #{repository_storage_paths_args.join(' ')}"),
-            'Check the hooks_path in config/gitlab.yml',
-            'Check your gitlab-shell installation'
-          )
-          for_more_information(
-            see_installation_guide_section "GitLab Shell"
-          )
-          fix_and_rerun
-        end
-      end
-    end
-
-    def check_gitlab_shell_self_test
-      gitlab_shell_repo_base = gitlab_shell_path
-      check_cmd = File.expand_path('bin/check', gitlab_shell_repo_base)
-      puts "Running #{check_cmd}"
-
-      if system(check_cmd, chdir: gitlab_shell_repo_base)
-        puts 'gitlab-shell self-check successful'.color(:green)
-      else
-        puts 'gitlab-shell self-check failed'.color(:red)
-        try_fixing_it(
-          'Make sure GitLab is running;',
-          'Check the gitlab-shell configuration file:',
-          sudo_gitlab("editor #{File.expand_path('config.yml', gitlab_shell_repo_base)}")
-        )
-        fix_and_rerun
-      end
-    end
-
-    # Helper methods
-    ########################
-
-    def gitlab_shell_path
-      Gitlab.config.gitlab_shell.path
-    end
-
-    def gitlab_shell_version
-      Gitlab::Shell.new.version
-    end
-
-    def gitlab_shell_major_version
-      Gitlab::Shell.version_required.split('.')[0].to_i
-    end
-
-    def gitlab_shell_minor_version
-      Gitlab::Shell.version_required.split('.')[1].to_i
+      SystemCheck::RakeTask::GitlabShellTask.run!
     end
+  end
 
-    def gitlab_shell_patch_version
-      Gitlab::Shell.version_required.split('.')[2].to_i
+  namespace :gitaly do
+    desc 'GitLab | Check the health of Gitaly'
+    task check: :gitlab_environment do
+      SystemCheck::RakeTask::GitalyTask.run!
     end
   end
 
   namespace :sidekiq do
     desc "GitLab | Check the configuration of Sidekiq"
     task check: :gitlab_environment do
-      warn_user_is_not_gitlab
-      start_checking "Sidekiq"
-
-      check_sidekiq_running
-      only_one_sidekiq_running
-
-      finished_checking "Sidekiq"
-    end
-
-    # Checks
-    ########################
-
-    def check_sidekiq_running
-      print "Running? ... "
-
-      if sidekiq_process_count > 0
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          sudo_gitlab("RAILS_ENV=production bin/background_jobs start")
-        )
-        for_more_information(
-          see_installation_guide_section("Install Init Script"),
-          "see log/sidekiq.log for possible errors"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def only_one_sidekiq_running
-      process_count = sidekiq_process_count
-      return if process_count.zero?
-
-      print 'Number of Sidekiq processes ... '
-
-      if process_count == 1
-        puts '1'.color(:green)
-      else
-        puts "#{process_count}".color(:red)
-        try_fixing_it(
-          'sudo service gitlab stop',
-          "sudo pkill -u #{gitlab_user} -f sidekiq",
-          "sleep 10 && sudo pkill -9 -u #{gitlab_user} -f sidekiq",
-          'sudo service gitlab start'
-        )
-        fix_and_rerun
-      end
-    end
-
-    def sidekiq_process_count
-      ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww))
-      ps_ux.scan(/sidekiq \d+\.\d+\.\d+/).count
+      SystemCheck::RakeTask::SidekiqTask.run!
     end
   end
 
   namespace :incoming_email do
     desc "GitLab | Check the configuration of Reply by email"
     task check: :gitlab_environment do
-      warn_user_is_not_gitlab
-
-      if Gitlab.config.incoming_email.enabled
-        checks = [
-          SystemCheck::IncomingEmail::ImapAuthenticationCheck
-        ]
-
-        if Rails.env.production?
-          checks << SystemCheck::IncomingEmail::InitdConfiguredCheck
-          checks << SystemCheck::IncomingEmail::MailRoomRunningCheck
-        else
-          checks << SystemCheck::IncomingEmail::ForemanConfiguredCheck
-        end
-
-        SystemCheck.run('Reply by email', checks)
-      else
-        puts 'Reply by email is disabled in config/gitlab.yml'
-      end
+      SystemCheck::RakeTask::IncomingEmailTask.run!
     end
   end
 
   namespace :ldap do
     task :check, [:limit] => :gitlab_environment do |_, args|
-      # Only show up to 100 results because LDAP directories can be very big.
-      # This setting only affects the `rake gitlab:check` script.
-      args.with_defaults(limit: 100)
-      warn_user_is_not_gitlab
-      start_checking "LDAP"
-
-      if Gitlab::Auth::LDAP::Config.enabled?
-        check_ldap(args.limit)
-      else
-        puts 'LDAP is disabled in config/gitlab.yml'
-      end
-
-      finished_checking "LDAP"
-    end
-
-    def check_ldap(limit)
-      servers = Gitlab::Auth::LDAP::Config.providers
-
-      servers.each do |server|
-        puts "Server: #{server}"
+      ENV['LDAP_CHECK_LIMIT'] = args.limit if args.limit.present?
 
-        begin
-          Gitlab::Auth::LDAP::Adapter.open(server) do |adapter|
-            check_ldap_auth(adapter)
-
-            puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)"
-
-            users = adapter.users(adapter.config.uid, '*', limit)
-            users.each do |user|
-              puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}"
-            end
-          end
-        rescue Net::LDAP::ConnectionRefusedError, Errno::ECONNREFUSED => e
-          puts "Could not connect to the LDAP server: #{e.message}".color(:red)
-        end
-      end
-    end
-
-    def check_ldap_auth(adapter)
-      auth = adapter.config.has_auth?
-
-      message = if auth && adapter.ldap.bind
-                  'Success'.color(:green)
-                elsif auth
-                  'Failed. Check `bind_dn` and `password` configuration values'.color(:red)
-                else
-                  'Anonymous. No `bind_dn` or `password` configured'.color(:yellow)
-                end
-
-      puts "LDAP authentication... #{message}"
-    end
-  end
-
-  namespace :repo do
-    desc "GitLab | Check the integrity of the repositories managed by GitLab"
-    task check: :gitlab_environment do
-      puts "This task is deprecated. Please use gitlab:git:fsck instead".color(:red)
-      Rake::Task["gitlab:git:fsck"].execute
+      SystemCheck::RakeTask::LdapTask.run!
     end
   end
 
   namespace :orphans do
     desc 'Gitlab | Check for orphaned namespaces and repositories'
     task check: :gitlab_environment do
-      warn_user_is_not_gitlab
-      checks = [
-        SystemCheck::Orphans::NamespaceCheck,
-        SystemCheck::Orphans::RepositoryCheck
-      ]
-
-      SystemCheck.run('Orphans', checks)
+      SystemCheck::RakeTask::OrphansTask.run!
     end
 
     desc 'GitLab | Check for orphaned namespaces in the repositories path'
     task check_namespaces: :gitlab_environment do
-      warn_user_is_not_gitlab
-      checks = [SystemCheck::Orphans::NamespaceCheck]
-
-      SystemCheck.run('Orphans', checks)
+      SystemCheck::RakeTask::Orphans::NamespaceTask.run!
     end
 
     desc 'GitLab | Check for orphaned repositories in the repositories path'
     task check_repositories: :gitlab_environment do
-      warn_user_is_not_gitlab
-      checks = [SystemCheck::Orphans::RepositoryCheck]
-
-      SystemCheck.run('Orphans', checks)
-    end
-  end
-
-  namespace :user do
-    desc "GitLab | Check the integrity of a specific user's repositories"
-    task :check_repos, [:username] => :gitlab_environment do |t, args|
-      username = args[:username] || prompt("Check repository integrity for username? ".color(:blue))
-      user = User.find_by(username: username)
-      if user
-        repo_dirs = user.authorized_projects.map do |p|
-          p.repository.path_to_repo
-        end
-
-        repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
-      else
-        puts "\nUser '#{username}' not found".color(:red)
-      end
-    end
-  end
-
-  # Helper methods
-  ##########################
-
-  def check_gitlab_shell
-    required_version = Gitlab::VersionInfo.new(gitlab_shell_major_version, gitlab_shell_minor_version, gitlab_shell_patch_version)
-    current_version = Gitlab::VersionInfo.parse(gitlab_shell_version)
-
-    print "GitLab Shell version >= #{required_version} ? ... "
-    if current_version.valid? && required_version <= current_version
-      puts "OK (#{current_version})".color(:green)
-    else
-      puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red)
+      SystemCheck::RakeTask::Orphans::RepositoryTask.run!
     end
   end
 end
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index d6d15285489..451ba651674 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -1,41 +1,29 @@
-# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/954
-#
+# frozen_string_literal: true
+require 'set'
+
 namespace :gitlab do
   namespace :cleanup do
-    HASHED_REPOSITORY_NAME = '@hashed'.freeze
-
     desc "GitLab | Cleanup | Clean namespaces"
     task dirs: :gitlab_environment do
-      warn_user_is_not_gitlab
-      remove_flag = ENV['REMOVE']
+      namespaces = Set.new(Namespace.pluck(:path))
+      namespaces << Storage::HashedProject::REPOSITORY_PATH_PREFIX
 
-      namespaces  = Namespace.pluck(:path)
-      namespaces << HASHED_REPOSITORY_NAME  # add so that it will be ignored
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        git_base_path = repository_storage.legacy_disk_path
-        all_dirs = Dir.glob(git_base_path + '/*')
+      Gitaly::Server.all.each do |server|
+        all_dirs = Gitlab::GitalyClient::StorageService
+          .new(server.storage)
+          .list_directories(depth: 0)
+          .reject { |dir| dir.ends_with?('.git') || namespaces.include?(File.basename(dir)) }
 
-        puts git_base_path.color(:yellow)
         puts "Looking for directories to remove... "
-
-        all_dirs.reject! do |dir|
-          # skip if git repo
-          dir =~ /.git$/
-        end
-
-        all_dirs.reject! do |dir|
-          dir_name = File.basename dir
-
-          # skip if namespace present
-          namespaces.include?(dir_name)
-        end
-
         all_dirs.each do |dir_path|
-          if remove_flag
-            if FileUtils.rm_rf dir_path
-              puts "Removed...#{dir_path}".color(:red)
-            else
-              puts "Cannot remove #{dir_path}".color(:red)
+          if remove?
+            begin
+              Gitlab::GitalyClient::NamespaceService.new(server.storage)
+                .remove(dir_path)
+
+              puts "Removed...#{dir_path}"
+            rescue StandardError => e
+              puts "Cannot remove #{dir_path}: #{e.message}".color(:red)
             end
           else
             puts "Can be removed: #{dir_path}".color(:red)
@@ -43,35 +31,36 @@ namespace :gitlab do
         end
       end
 
-      unless remove_flag
+      unless remove?
         puts "To cleanup this directories run this command with REMOVE=true".color(:yellow)
       end
     end
 
     desc "GitLab | Cleanup | Clean repositories"
     task repos: :gitlab_environment do
-      warn_user_is_not_gitlab
-
       move_suffix = "+orphaned+#{Time.now.to_i}"
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        repo_root = repository_storage.legacy_disk_path
-        # Look for global repos (legacy, depth 1) and normal repos (depth 2)
-        IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
-          find.each_line do |path|
-            path.chomp!
-            repo_with_namespace = path
-              .sub(repo_root, '')
-              .sub(%r{^/*}, '')
-              .chomp('.git')
-              .chomp('.wiki')
-
-            # TODO ignoring hashed repositories for now.  But revisit to fully support
-            # possible orphaned hashed repos
-            next if repo_with_namespace.start_with?("#{HASHED_REPOSITORY_NAME}/") || Project.find_by_full_path(repo_with_namespace)
-
-            new_path = path + move_suffix
-            puts path.inspect + ' -> ' + new_path.inspect
-            File.rename(path, new_path)
+
+      Gitaly::Server.all.each do |server|
+        Gitlab::GitalyClient::StorageService
+          .new(server.storage)
+          .list_directories
+          .each do |path|
+          repo_with_namespace = path.chomp('.git').chomp('.wiki')
+
+          # TODO ignoring hashed repositories for now.  But revisit to fully support
+          # possible orphaned hashed repos
+          next if repo_with_namespace.start_with?(Storage::HashedProject::REPOSITORY_PATH_PREFIX)
+          next if Project.find_by_full_path(repo_with_namespace)
+
+          new_path = path + move_suffix
+          puts path.inspect + ' -> ' + new_path.inspect
+
+          begin
+            Gitlab::GitalyClient::NamespaceService
+              .new(server.storage)
+              .rename(path, new_path)
+          rescue StandardError => e
+            puts "Error occured while moving the repository: #{e.message}".color(:red)
           end
         end
       end
@@ -104,27 +93,46 @@ namespace :gitlab do
       end
     end
 
-    # This is a rake task which removes faulty refs. These refs where only
-    # created in the 8.13.RC cycle, and fixed in the stable builds which were
-    # released. So likely this should only be run once on gitlab.com
-    # Faulty refs are moved so they are kept around, else some features break.
-    desc 'GitLab | Cleanup | Remove faulty deployment refs'
-    task move_faulty_deployment_refs: :gitlab_environment do
-      projects = Project.where(id: Deployment.select(:project_id).distinct)
+    desc "GitLab | Cleanup | Clean orphaned project uploads"
+    task project_uploads: :gitlab_environment do
+      warn_user_is_not_gitlab
 
-      projects.find_each do |project|
-        rugged = project.repository.rugged
+      cleaner = Gitlab::Cleanup::ProjectUploads.new(logger: logger)
+      cleaner.run!(dry_run: dry_run?)
 
-        max_iid = project.deployments.maximum(:iid)
+      if dry_run?
+        logger.info "To clean up these files run this command with DRY_RUN=false".color(:yellow)
+      end
+    end
 
-        rugged.references.each('refs/environments/**/*') do |ref|
-          id = ref.name.split('/').last.to_i
-          next unless id > max_iid
+    desc 'GitLab | Cleanup | Clean orphan remote upload files that do not exist in the db'
+    task remote_upload_files: :environment do
+      cleaner = Gitlab::Cleanup::RemoteUploads.new(logger: logger)
+      cleaner.run!(dry_run: dry_run?)
 
-          project.deployments.find(id).create_ref
-          project.repository.delete_refs(ref)
-        end
+      if dry_run?
+        logger.info "To cleanup these files run this command with DRY_RUN=false".color(:yellow)
       end
     end
+
+    def remove?
+      ENV['REMOVE'] == 'true'
+    end
+
+    def dry_run?
+      ENV['DRY_RUN'] != 'false'
+    end
+
+    def logger
+      return @logger if defined?(@logger)
+
+      @logger = if Rails.env.development? || Rails.env.production?
+                  Logger.new(STDOUT).tap do |stdout_logger|
+                    stdout_logger.extend(ActiveSupport::Logger.broadcast(Rails.logger))
+                  end
+                else
+                  Rails.logger
+                end
+    end
   end
 end
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 139ab70e125..74cd70c6e9f 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -46,9 +46,13 @@ namespace :gitlab do
 
     desc 'Configures the database by running migrate, or by loading the schema and seeding if needed'
     task configure: :environment do
-      if ActiveRecord::Base.connection.tables.any?
+      # Check if we have existing db tables
+      # The schema_migrations table will still exist if drop_tables was called
+      if ActiveRecord::Base.connection.tables.count > 1
         Rake::Task['db:migrate'].invoke
       else
+        # Add post-migrate paths to ensure we mark all migrations as up
+        Gitlab::Database.add_post_migrate_path_to_rails(force: true)
         Rake::Task['db:schema:load'].invoke
         Rake::Task['db:seed_fu'].invoke
       end
diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake
index cb4f7e5c8a8..8a53b51d4fe 100644
--- a/lib/tasks/gitlab/git.rake
+++ b/lib/tasks/gitlab/git.rake
@@ -1,87 +1,24 @@
 namespace :gitlab do
   namespace :git do
-    desc "GitLab | Git | Repack"
-    task repack: :gitlab_environment do
-      failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} repack -a --quiet), "Repacking repo")
-      if failures.empty?
-        puts "Done".color(:green)
-      else
-        output_failures(failures)
-      end
-    end
-
-    desc "GitLab | Git | Run garbage collection on all repos"
-    task gc: :gitlab_environment do
-      failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} gc --auto --quiet), "Garbage Collecting")
-      if failures.empty?
-        puts "Done".color(:green)
-      else
-        output_failures(failures)
-      end
-    end
-
-    desc "GitLab | Git | Prune all repos"
-    task prune: :gitlab_environment do
-      failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} prune), "Git Prune")
-      if failures.empty?
-        puts "Done".color(:green)
-      else
-        output_failures(failures)
-      end
-    end
-
     desc 'GitLab | Git | Check all repos integrity'
     task fsck: :gitlab_environment do
-      failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} fsck --name-objects --no-progress), "Checking integrity") do |repo|
-        check_config_lock(repo)
-        check_ref_locks(repo)
-      end
-
-      if failures.empty?
-        puts "Done".color(:green)
-      else
-        output_failures(failures)
-      end
-    end
-
-    def perform_git_cmd(cmd, message)
-      puts "Starting #{message} on all repositories"
-
       failures = []
-      all_repos do |repo|
-        if system(*cmd, chdir: repo)
-          puts "Performed #{message} at #{repo}"
-        else
-          failures << repo
+      Project.find_each(batch_size: 100) do |project|
+        begin
+          project.repository.fsck
+
+        rescue => e
+          failures << "#{project.full_path} on #{project.repository_storage}: #{e}"
         end
 
-        yield(repo) if block_given?
+        puts "Performed integrity check for #{project.repository.full_path}"
       end
 
-      failures
-    end
-
-    def output_failures(failures)
-      puts "The following repositories reported errors:".color(:red)
-      failures.each { |f| puts "- #{f}" }
-    end
-
-    def check_config_lock(repo_dir)
-      config_exists = File.exist?(File.join(repo_dir, 'config.lock'))
-      config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green)
-
-      puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}"
-    end
-
-    def check_ref_locks(repo_dir)
-      lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock'))
-
-      if lock_files.present?
-        puts "Ref lock files exist:".color(:red)
-
-        lock_files.each { |lock_file| puts "  #{lock_file}" }
+      if failures.empty?
+        puts "Done".color(:green)
       else
-        puts "No ref lock files exist".color(:green)
+        puts "The following repositories reported errors:".color(:red)
+        failures.each { |f| puts "- #{f}" }
       end
     end
   end
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index e9ca6404fe8..80de3d2ef51 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -1,13 +1,12 @@
 namespace :gitlab do
   namespace :gitaly do
     desc "GitLab | Install or upgrade gitaly"
-    task :install, [:dir, :repo] => :gitlab_environment do |t, args|
-      require 'toml-rb'
-
+    task :install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args|
       warn_user_is_not_gitlab
 
-      unless args.dir.present?
-        abort %(Please specify the directory where you want to install gitaly:\n  rake "gitlab:gitaly:install[/home/git/gitaly]")
+      unless args.dir.present? && args.storage_path.present?
+        abort %(Please specify the directory where you want to install gitaly and the path for the default storage
+Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
       end
 
       args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git')
@@ -27,7 +26,8 @@ namespace :gitlab do
           "BUNDLE_PATH=#{Bundler.bundle_path}")
       end
 
-      Gitlab::SetupHelper.create_gitaly_configuration(args.dir)
+      storage_paths = { 'default' => args.storage_path }
+      Gitlab::SetupHelper.create_gitaly_configuration(args.dir, storage_paths)
       Dir.chdir(args.dir) do
         # In CI we run scripts/gitaly-test-build instead of this command
         unless ENV['CI'].present?
@@ -35,17 +35,5 @@ namespace :gitlab do
         end
       end
     end
-
-    desc "GitLab | Print storage configuration in TOML format"
-    task storage_config: :environment do
-      require 'toml-rb'
-
-      puts "# Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}"
-      puts "# This is in TOML format suitable for use in Gitaly's config.toml file."
-
-      # Exclude gitaly-ruby configuration because that depends on the gitaly
-      # installation directory.
-      puts Gitlab::SetupHelper.gitaly_configuration_toml('', gitaly_ruby: false)
-    end
   end
 end
diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake
index 44074397c05..900dbf7be24 100644
--- a/lib/tasks/gitlab/import_export.rake
+++ b/lib/tasks/gitlab/import_export.rake
@@ -10,15 +10,22 @@ namespace :gitlab do
       puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true)
     end
 
-    desc 'GitLab | Bumps the Import/Export version for test_project_export.tar.gz'
-    task bump_test_version: :environment do
-      Dir.mktmpdir do |tmp_dir|
-        system("tar -zxf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} > /dev/null")
-        File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w')
-        system("tar -zcvf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} . > /dev/null")
+    desc 'GitLab | Bumps the Import/Export version in fixtures and project templates'
+    task bump_version: :environment do
+      archives = Dir['vendor/project_templates/*.tar.gz']
+      archives.push('spec/features/projects/import_export/test_project_export.tar.gz')
+
+      archives.each do |archive|
+        raise ArgumentError unless File.exist?(archive)
+
+        Dir.mktmpdir do |tmp_dir|
+          system("tar -zxf #{archive} -C #{tmp_dir} > /dev/null")
+          File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w')
+          system("tar -zcvf #{archive} -C #{tmp_dir} . > /dev/null")
+        end
       end
 
-      puts "Updated to #{Gitlab::ImportExport.version}"
+      puts "Updated #{archives} to #{Gitlab::ImportExport.version}."
     end
   end
 end
diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake
index 289aa5d9060..e97d77d20e0 100644
--- a/lib/tasks/gitlab/info.rake
+++ b/lib/tasks/gitlab/info.rake
@@ -54,8 +54,8 @@ namespace :gitlab do
       puts "HTTP Clone URL:\t#{http_clone_url}"
       puts "SSH Clone URL:\t#{ssh_clone_url}"
       puts "Using LDAP:\t#{Gitlab.config.ldap.enabled ? "yes".color(:green) : "no"}"
-      puts "Using Omniauth:\t#{Gitlab.config.omniauth.enabled ? "yes".color(:green) : "no"}"
-      puts "Omniauth Providers: #{omniauth_providers.join(', ')}" if Gitlab.config.omniauth.enabled
+      puts "Using Omniauth:\t#{Gitlab::Auth.omniauth_enabled? ? "yes".color(:green) : "no"}"
+      puts "Omniauth Providers: #{omniauth_providers.join(', ')}" if Gitlab::Auth.omniauth_enabled?
 
       # check Gitolite version
       gitlab_shell_version_file = "#{Gitlab.config.gitlab_shell.hooks_path}/../VERSION"
@@ -67,8 +67,10 @@ namespace :gitlab do
       puts "GitLab Shell".color(:yellow)
       puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}"
       puts "Repository storage paths:"
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        puts "- #{name}: \t#{repository_storage.legacy_disk_path}"
+      Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+        Gitlab.config.repositories.storages.each do |name, repository_storage|
+          puts "- #{name}: \t#{repository_storage.legacy_disk_path}"
+        end
       end
       puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}"
       puts "Git:\t\t#{Gitlab.config.git.bin_path}"
diff --git a/lib/tasks/gitlab/ldap.rake b/lib/tasks/gitlab/ldap.rake
index c66a2a263dc..0459de27c96 100644
--- a/lib/tasks/gitlab/ldap.rake
+++ b/lib/tasks/gitlab/ldap.rake
@@ -1,7 +1,7 @@
 namespace :gitlab do
   namespace :ldap do
     desc 'GitLab | LDAP | Rename provider'
-    task :rename_provider, [:old_provider, :new_provider] => :environment  do |_, args|
+    task :rename_provider, [:old_provider, :new_provider] => :gitlab_environment do |_, args|
       old_provider = args[:old_provider] ||
         prompt('What is the old provider? Ex. \'ldapmain\': '.color(:blue))
       new_provider = args[:new_provider] ||
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index 4fcbbbf8c9d..0ebc6f00793 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -92,9 +92,11 @@ namespace :gitlab do
   def setup
     warn_user_is_not_gitlab
 
+    ensure_write_to_authorized_keys_is_enabled
+
     unless ENV['force'] == 'yes'
-      puts "This will rebuild an authorized_keys file."
-      puts "You will lose any data stored in authorized_keys file."
+      puts "This task will now rebuild the authorized_keys file."
+      puts "You will lose any data stored in the authorized_keys file."
       ask_to_continue
       puts ""
     end
@@ -118,4 +120,44 @@ namespace :gitlab do
     puts "Quitting...".color(:red)
     exit 1
   end
+
+  def ensure_write_to_authorized_keys_is_enabled
+    return if Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled
+
+    puts authorized_keys_is_disabled_warning
+
+    unless ENV['force'] == 'yes'
+      puts 'Do you want to permanently enable the "Write to authorized_keys file" setting now?'
+      ask_to_continue
+    end
+
+    puts 'Enabling the "Write to authorized_keys file" setting...'
+    Gitlab::CurrentSettings.current_application_settings.update!(authorized_keys_enabled: true)
+
+    puts 'Successfully enabled "Write to authorized_keys file"!'
+    puts ''
+  end
+
+  def authorized_keys_is_disabled_warning
+    <<-MSG.strip_heredoc
+      WARNING
+
+      The "Write to authorized_keys file" setting is disabled, which prevents
+      the file from being rebuilt!
+
+      It should be enabled for most GitLab installations. Large installations
+      may wish to disable it as part of speeding up SSH operations.
+
+      See https://docs.gitlab.com/ee/administration/operations/fast_ssh_key_lookup.html
+
+      If you did not intentionally disable this option in Admin Area > Settings,
+      then you may have been affected by the 9.3.0 bug in which the new setting
+      was disabled by default.
+
+      https://gitlab.com/gitlab-org/gitlab-ee/issues/2738
+
+      It was reverted in 9.3.1 and fixed in 9.3.3, however, if Settings were
+      saved while the setting was unchecked, then it is still disabled.
+    MSG
+  end
 end
diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake
index 68d6f9d7cb1..f9ce3e1d338 100644
--- a/lib/tasks/gitlab/storage.rake
+++ b/lib/tasks/gitlab/storage.rake
@@ -2,11 +2,34 @@ namespace :gitlab do
   namespace :storage do
     desc 'GitLab | Storage | Migrate existing projects to Hashed Storage'
     task migrate_to_hashed: :environment do
-      legacy_projects_count = Project.with_unmigrated_storage.count
+      if Gitlab::Database.read_only?
+        warn 'This task requires database write access. Exiting.'
+
+        next
+      end
+
+      storage_migrator = Gitlab::HashedStorage::Migrator.new
       helper = Gitlab::HashedStorage::RakeHelper
 
+      if helper.range_single_item?
+        project = Project.with_unmigrated_storage.find_by(id: helper.range_from)
+
+        unless project
+          warn "There are no projects requiring storage migration with ID=#{helper.range_from}"
+
+          next
+        end
+
+        puts "Enqueueing storage migration of #{project.full_path} (ID=#{project.id})..."
+        storage_migrator.migrate(project)
+
+        next
+      end
+
+      legacy_projects_count = Project.with_unmigrated_storage.count
+
       if legacy_projects_count == 0
-        puts 'There are no projects requiring storage migration. Nothing to do!'
+        warn 'There are no projects requiring storage migration. Nothing to do!'
 
         next
       end
@@ -14,7 +37,7 @@ namespace :gitlab do
       print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}"
 
       helper.project_id_batches do |start, finish|
-        StorageMigratorWorker.perform_async(start, finish)
+        storage_migrator.bulk_schedule(start: start, finish: finish)
 
         print '.'
       end
diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake
index fd2a4f2d11a..5a232091a7e 100644
--- a/lib/tasks/gitlab/traces.rake
+++ b/lib/tasks/gitlab/traces.rake
@@ -8,9 +8,7 @@ namespace :gitlab do
       logger = Logger.new(STDOUT)
       logger.info('Archiving legacy traces')
 
-      Ci::Build.finished
-        .where('NOT EXISTS (?)',
-          Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
+      Ci::Build.finished.without_archived_trace
         .order(id: :asc)
         .find_in_batches(batch_size: 1000) do |jobs|
         job_ids = jobs.map { |job| [job.id] }
@@ -20,5 +18,22 @@ namespace :gitlab do
         logger.info("Scheduled #{job_ids.count} jobs. From #{job_ids.min} to #{job_ids.max}")
       end
     end
+
+    task migrate: :environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Starting transfer of job traces')
+
+      Ci::Build.joins(:project)
+        .with_archived_trace_stored_locally
+        .find_each(batch_size: 10) do |build|
+        begin
+          build.job_artifacts_trace.file.migrate!(ObjectStorage::Store::REMOTE)
+
+          logger.info("Transferred job trace of #{build.id} to object storage")
+        rescue => e
+          logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}")
+        end
+      end
+    end
   end
 end
diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake
index a25f7ce59c7..abe10f5580e 100644
--- a/lib/tasks/gitlab/update_templates.rake
+++ b/lib/tasks/gitlab/update_templates.rake
@@ -6,6 +6,8 @@ namespace :gitlab do
 
   desc "GitLab | Update project templates"
   task :update_project_templates do
+    include Gitlab::ImportExport::CommandLineUtil
+
     if Rails.env.production?
       puts "This rake task is not meant fo production instances".red
       exit(1)
@@ -52,7 +54,7 @@ namespace :gitlab do
       end
 
       Projects::ImportExport::ExportService.new(project, admin).execute
-      FileUtils.cp(project.export_project_path, template.archive_path)
+      download_or_copy_upload(project.export_file, template.archive_path)
       Projects::DestroyService.new(admin, project).execute
       puts "Exported #{template.name}".green
     end
@@ -98,10 +100,6 @@ namespace :gitlab do
       /(\.{1,2}|LICENSE|Global|\.gitignore)\z/
     ),
     Template.new(
-      "https://gitlab.com/gitlab-org/gitlab-ci-yml.git",
-      /(\.{1,2}|LICENSE|CONTRIBUTING.md|Pages|autodeploy|\.gitlab-ci.yml)\z/
-    ),
-    Template.new(
       "https://gitlab.com/gitlab-org/Dockerfile.git",
       /(\.{1,2}|LICENSE|CONTRIBUTING.md|\.Dockerfile)\z/
     )
diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake
index 78e18992a8e..1c93609a006 100644
--- a/lib/tasks/gitlab/uploads/migrate.rake
+++ b/lib/tasks/gitlab/uploads/migrate.rake
@@ -1,6 +1,30 @@
 namespace :gitlab do
   namespace :uploads do
-    desc 'GitLab | Uploads | Migrate the uploaded files to object storage'
+    namespace :migrate do
+      desc "GitLab | Uploads | Migrate all uploaded files to object storage"
+      task all: :environment do
+        categories = [%w(AvatarUploader Project :avatar),
+                      %w(AvatarUploader Group :avatar),
+                      %w(AvatarUploader User :avatar),
+                      %w(AttachmentUploader Note :attachment),
+                      %w(AttachmentUploader Appearance :logo),
+                      %w(AttachmentUploader Appearance :header_logo),
+                      %w(FaviconUploader Appearance :favicon),
+                      %w(FileUploader Project),
+                      %w(PersonalFileUploader Snippet),
+                      %w(NamespaceFileUploader Snippet),
+                      %w(FileUploader MergeRequest)]
+
+        categories.each do |args|
+          Rake::Task["gitlab:uploads:migrate"].invoke(*args)
+          Rake::Task["gitlab:uploads:migrate"].reenable
+        end
+      end
+    end
+
+    # The following is the actual rake task that migrates uploads of specified
+    # category to object storage
+    desc 'GitLab | Uploads | Migrate the uploaded files of specified type to object storage'
     task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args|
       batch_size     = ENV.fetch('BATCH', 200).to_i
       @to_store      = ObjectStorage::Store::REMOTE
@@ -8,7 +32,7 @@ namespace :gitlab do
       @uploader_class = args.uploader_class.constantize
       @model_class    = args.model_class.constantize
 
-      uploads.each_batch(of: batch_size, &method(:enqueue_batch)) # rubocop: disable Cop/InBatches
+      uploads.each_batch(of: batch_size, &method(:enqueue_batch))
     end
 
     def enqueue_batch(batch, index)
diff --git a/lib/tasks/gitlab/web_hook.rake b/lib/tasks/gitlab/web_hook.rake
index 5a1c8006052..15cec80b6a6 100644
--- a/lib/tasks/gitlab/web_hook.rake
+++ b/lib/tasks/gitlab/web_hook.rake
@@ -25,11 +25,22 @@ namespace :gitlab do
       web_hook_url = ENV['URL']
       namespace_path = ENV['NAMESPACE']
 
-      projects = find_projects(namespace_path)
-      project_ids = projects.pluck(:id)
+      web_hooks = find_web_hooks(namespace_path)
 
       puts "Removing webhooks with the url '#{web_hook_url}' ... "
-      count = WebHook.where(url: web_hook_url, project_id: project_ids, type: 'ProjectHook').delete_all
+
+      # FIXME: Hook URLs are now encrypted, so there is no way to efficiently
+      # find them all in SQL. For now, check them in Ruby. If this is too slow,
+      # we could consider storing a hash of the URL alongside the encrypted
+      # value to speed up searches
+      count = 0
+      web_hooks.find_each do |hook|
+        next unless hook.url == web_hook_url
+
+        hook.destroy!
+        count += 1
+      end
+
       puts "#{count} webhooks were removed."
     end
 
@@ -37,29 +48,37 @@ namespace :gitlab do
     task list: :environment do
       namespace_path = ENV['NAMESPACE']
 
-      projects = find_projects(namespace_path)
-      web_hooks = projects.all.map(&:hooks).flatten
-      web_hooks.each do |hook|
+      web_hooks = find_web_hooks(namespace_path)
+      web_hooks.find_each do |hook|
         puts "#{hook.project.name.truncate(20).ljust(20)} -> #{hook.url}"
       end
 
-      puts "\n#{web_hooks.size} webhooks found."
+      puts "\n#{web_hooks.count} webhooks found."
     end
   end
 
   def find_projects(namespace_path)
     if namespace_path.blank?
       Project
-    elsif namespace_path == '/'
-      Project.in_namespace(nil)
     else
-      namespace = Namespace.where(path: namespace_path).first
-      if namespace
-        Project.in_namespace(namespace.id)
-      else
+      namespace = Namespace.find_by_full_path(namespace_path)
+
+      unless namespace
         puts "Namespace not found: #{namespace_path}".color(:red)
         exit 2
       end
+
+      Project.in_namespace(namespace.id)
+    end
+  end
+
+  def find_web_hooks(namespace_path)
+    if namespace_path.blank?
+      ProjectHook
+    else
+      project_ids = find_projects(namespace_path).select(:id)
+
+      ProjectHook.where(project_id: project_ids)
     end
   end
 end
diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake
index ad2d034b0b4..786efd14b1a 100644
--- a/lib/tasks/haml-lint.rake
+++ b/lib/tasks/haml-lint.rake
@@ -2,5 +2,16 @@ unless Rails.env.production?
   require 'haml_lint/rake_task'
   require 'haml_lint/inline_javascript'
 
+  # Workaround for warnings from parser/current
+  # Keep it even if it no longer emits any warnings,
+  # because we'll still see warnings in console/server anyway,
+  # and we don't need to break static-analysis for this.
+  task :haml_lint do
+    require 'parser'
+    def Parser.warn(*args)
+      puts(*args) # static-analysis ignores stdout if status is 0
+    end
+  end
+
   HamlLint::RakeTask.new
 end
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
index aafbe52e5f8..f912f521dfb 100644
--- a/lib/tasks/import.rake
+++ b/lib/tasks/import.rake
@@ -9,7 +9,10 @@ class GithubImport
   def initialize(token, gitlab_username, project_path, extras)
     @options = { token: token }
     @project_path = project_path
-    @current_user = User.find_by_username(gitlab_username)
+    @current_user = UserFinder.new(gitlab_username).find_by_username
+
+    raise "GitLab user #{gitlab_username} not found. Please specify a valid username." unless @current_user
+
     @github_repo = extras.empty? ? nil : extras.first
   end
 
@@ -39,7 +42,7 @@ class GithubImport
   end
 
   def import!
-    @project.force_import_start
+    @project.import_state.force_start
 
     import_success = false
 
@@ -50,11 +53,11 @@ class GithubImport
     end
 
     if import_success
-      @project.import_finish
+      @project.after_import
       puts "Import finished. Timings: #{timings}".color(:green)
     else
       puts "Import was not successful. Errors were as follows:"
-      puts @project.import_error
+      puts @project.import_state.last_error
     end
   end
 
diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake
index 8b86a5c72a5..5d673a1a285 100644
--- a/lib/tasks/lint.rake
+++ b/lib/tasks/lint.rake
@@ -17,16 +17,25 @@ unless Rails.env.production?
       Rake::Task['eslint'].invoke
     end
 
+    desc "GitLab | lint | Lint HAML files"
+    task :haml do
+      begin
+        Rake::Task['haml_lint'].invoke
+      rescue RuntimeError # The haml_lint tasks raise a RuntimeError
+        exit(1)
+      end
+    end
+
     desc "GitLab | lint | Run several lint checks"
     task :all do
       status = 0
 
       %w[
         config_lint
-        haml_lint
+        lint:haml
         scss_lint
-        flay
         gettext:lint
+        gettext:updated_check
         lint:static_verification
       ].each do |task|
         pid = Process.fork do
@@ -38,13 +47,12 @@ unless Rails.env.production?
           $stderr.reopen(wr_err)
 
           begin
-            begin
-              Rake::Task[task].invoke
-            rescue RuntimeError # The haml_lint tasks raise a RuntimeError
-              exit(1)
-            end
+            Rake::Task[task].invoke
           rescue SystemExit => ex
-            msg = "*** Rake task #{task} failed with the following error(s):"
+            msg = "*** Rake task #{task} exited:"
+            raise ex
+          rescue => ex
+            msg = "*** Rake task #{task} raised #{ex.class}:"
             raise ex
           ensure
             $stdout.reopen(stdout)
diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake
index 9b05876034c..c77fa49d586 100644
--- a/lib/tasks/migrate/add_limits_mysql.rake
+++ b/lib/tasks/migrate/add_limits_mysql.rake
@@ -3,6 +3,7 @@ require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql')
 require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql')
 require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql')
 require Rails.root.join('db/migrate/gpg_keys_limits_to_mysql')
+require Rails.root.join('db/migrate/prometheus_metrics_limits_to_mysql')
 
 desc "GitLab | Add limits to strings in mysql database"
 task add_limits_mysql: :environment do
@@ -12,4 +13,5 @@ task add_limits_mysql: :environment do
   MergeRequestDiffFileLimitsToMysql.new.up
   LimitsCiBuildTraceChunksRawDataForMysql.new.up
   IncreaseMysqlTextLimitForGpgKeys.new.up
+  PrometheusMetricsLimitsToMysql.new.up
 end
diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake
index e7aab50e42a..f69d204c579 100644
--- a/lib/tasks/migrate/setup_postgresql.rake
+++ b/lib/tasks/migrate/setup_postgresql.rake
@@ -22,3 +22,18 @@ task setup_postgresql: :environment do
   ProjectNameLowerIndex.new.up
   AddPathIndexToRedirectRoutes.new.up
 end
+
+desc 'GitLab | Generate PostgreSQL Password Hash'
+task :postgresql_md5_hash do
+  require 'digest'
+  username = ENV.fetch('USERNAME') do |missing|
+    puts "You must provide an username with '#{missing}' ENV variable"
+    exit(1)
+  end
+  password = ENV.fetch('PASSWORD') do |missing|
+    puts "You must provide a password with '#{missing}' ENV variable"
+    exit(1)
+  end
+  hash = Digest::MD5.hexdigest("#{password}#{username}")
+  puts "The MD5 hash of your database password for user: #{username} -> #{hash}"
+end
diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake
index 81829668de8..eec024f9bbb 100644
--- a/lib/tasks/tokens.rake
+++ b/lib/tasks/tokens.rake
@@ -1,4 +1,7 @@
 require_relative '../../app/models/concerns/token_authenticatable.rb'
+require_relative '../../app/models/concerns/token_authenticatable_strategies/base.rb'
+require_relative '../../app/models/concerns/token_authenticatable_strategies/insecure.rb'
+require_relative '../../app/models/concerns/token_authenticatable_strategies/digest.rb'
 
 namespace :tokens do
   desc "Reset all GitLab incoming email tokens"
@@ -26,13 +29,6 @@ class TmpUser < ActiveRecord::Base
 
   self.table_name = 'users'
 
-  def reset_incoming_email_token!
-    write_new_token(:incoming_email_token)
-    save!(validate: false)
-  end
-
-  def reset_feed_token!
-    write_new_token(:feed_token)
-    save!(validate: false)
-  end
+  add_authentication_token_field :incoming_email_token, token_generator: -> { SecureRandom.hex.to_i(16).to_s(36) }
+  add_authentication_token_field :feed_token
 end
diff --git a/lib/unfold_form.rb b/lib/unfold_form.rb
index fcd01503d1b..05bb3ed7f1c 100644
--- a/lib/unfold_form.rb
+++ b/lib/unfold_form.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'gt_one_coercion'
 
 class UnfoldForm
diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb
index 5dc85b2baea..aae542f02ac 100644
--- a/lib/uploaded_file.rb
+++ b/lib/uploaded_file.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "tempfile"
 require "tmpdir"
 require "fileutils"
@@ -21,14 +23,14 @@ class UploadedFile
     raise InvalidPathError, "#{path} file does not exist" unless ::File.exist?(path)
 
     @content_type = content_type
-    @original_filename = filename || ::File.basename(path)
+    @original_filename = sanitize_filename(filename || path)
     @content_type = content_type
     @sha256 = sha256
     @remote_id = remote_id
     @tempfile = File.new(path, 'rb')
   end
 
-  def self.from_params(params, field, upload_path)
+  def self.from_params(params, field, upload_paths)
     unless params["#{field}.path"]
       raise InvalidPathError, "file is invalid" if params["#{field}.remote_id"]
 
@@ -37,7 +39,8 @@ class UploadedFile
 
     file_path = File.realpath(params["#{field}.path"])
 
-    unless self.allowed_path?(file_path, [upload_path, Dir.tmpdir].compact)
+    paths = Array(upload_paths) << Dir.tmpdir
+    unless self.allowed_path?(file_path, paths.compact)
       raise InvalidPathError, "insecure path used '#{file_path}'"
     end
 
@@ -54,6 +57,16 @@ class UploadedFile
     end
   end
 
+  # copy-pasted from CarrierWave::SanitizedFile
+  def sanitize_filename(name)
+    name = name.tr("\\", "/") # work-around for IE
+    name = ::File.basename(name)
+    name = name.gsub(CarrierWave::SanitizedFile.sanitize_regexp, "_")
+    name = "_#{name}" if name =~ /\A\.+\z/
+    name = "unnamed" if name.empty?
+    name.mb_chars.to_s
+  end
+
   def path
     @tempfile.path
   end
diff --git a/lib/version_check.rb b/lib/version_check.rb
index 91ad07feee5..c9f102f6b19 100644
--- a/lib/version_check.rb
+++ b/lib/version_check.rb
@@ -1,18 +1,21 @@
+# frozen_string_literal: true
+
 require "base64"
 
 # This class is used to build image URL to
 # check if it is a new version for update
 class VersionCheck
-  def data
+  def self.data
     { version: Gitlab::VERSION }
   end
 
-  def url
+  def self.url
     encoded_data = Base64.urlsafe_encode64(data.to_json)
+
     "#{host}?gitlab_info=#{encoded_data}"
   end
 
-  def host
+  def self.host
     'https://version.gitlab.com/check.svg'
   end
 end