Merge branch 'master' into '42568-pipeline-empty-state'

# Conflicts:
#   app/views/projects/jobs/show.html.haml
#   lib/gitlab/ci/status/core.rb

45 files changed, 826 insertions, 133 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index 62ffebeacb0..073471b4c4d 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -78,6 +78,14 @@ module API
       rack_response({ 'message' => '404 Not found' }.to_json, 404)
     end
 
+    rescue_from UploadedFile::InvalidPathError do |e|
+      rack_response({ 'message' => e.message }.to_json, 400)
+    end
+
+    rescue_from ObjectStorage::RemoteStoreError do |e|
+      rack_response({ 'message' => e.message }.to_json, 500)
+    end
+
     # Retain 405 error rather than a 500 error for Grape 0.15.0+.
     # https://github.com/ruby-grape/grape/blob/a3a28f5b5dfbb2797442e006dbffd750b27f2a76/UPGRADING.md#changes-to-method-not-allowed-routes
     rescue_from Grape::Exceptions::MethodNotAllowed do |e|
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index e5ecd37e473..e35b1a0ff63 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -72,7 +72,7 @@ module API
 
     class ProjectHook < Hook
       expose :project_id, :issues_events, :confidential_issues_events
-      expose :note_events, :pipeline_events, :wiki_page_events
+      expose :note_events, :confidential_note_events, :pipeline_events, :wiki_page_events
       expose :job_events
     end
 
@@ -794,7 +794,7 @@ module API
       expose :id, :title, :created_at, :updated_at, :active
       expose :push_events, :issues_events, :confidential_issues_events
       expose :merge_requests_events, :tag_push_events, :note_events
-      expose :pipeline_events, :wiki_page_events
+      expose :confidential_note_events, :pipeline_events, :wiki_page_events
       expose :job_events
       # Expose serialized properties
       expose :properties do |service, options|
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index e59e8a45908..a582aa0ec2c 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -83,12 +83,13 @@ module API
     end
 
     def available_labels_for(label_parent)
-      search_params =
-        if label_parent.is_a?(Project)
-          { project_id: label_parent.id }
-        else
-          { group_id: label_parent.id, only_group_labels: true }
-        end
+      search_params = { include_ancestor_groups: true }
+
+      if label_parent.is_a?(Project)
+        search_params[:project_id] = label_parent.id
+      else
+        search_params.merge!(group_id: label_parent.id, only_group_labels: true)
+      end
 
       LabelsFinder.new(current_user, search_params).execute
     end
@@ -388,28 +389,6 @@ module API
 
     # file helpers
 
-    def uploaded_file(field, uploads_path)
-      if params[field]
-        bad_request!("#{field} is not a file") unless params[field][:filename]
-        return params[field]
-      end
-
-      return nil unless params["#{field}.path"] && params["#{field}.name"]
-
-      # sanitize file paths
-      # this requires all paths to exist
-      required_attributes! %W(#{field}.path)
-      uploads_path = File.realpath(uploads_path)
-      file_path = File.realpath(params["#{field}.path"])
-      bad_request!('Bad file path') unless file_path.start_with?(uploads_path)
-
-      UploadedFile.new(
-        file_path,
-        params["#{field}.name"],
-        params["#{field}.type"] || 'application/octet-stream'
-      )
-    end
-
     def present_disk_file!(path, filename, content_type = 'application/octet-stream')
       filename ||= File.basename(path)
       header['Content-Disposition'] = "attachment; filename=#{filename}"
diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb
new file mode 100644
index 00000000000..381d5e8968c
--- /dev/null
+++ b/lib/api/helpers/projects_helpers.rb
@@ -0,0 +1,38 @@
+module API
+  module Helpers
+    module ProjectsHelpers
+      extend ActiveSupport::Concern
+
+      included do
+        helpers do
+          params :optional_project_params_ce do
+            optional :description, type: String, desc: 'The description of the project'
+            optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`'
+            optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled'
+            optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled'
+            optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled'
+            optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled'
+            optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled'
+            optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project'
+            optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push'
+            optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project'
+            optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project'
+            optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the project.'
+            optional :public_builds, type: Boolean, desc: 'Perform public builds'
+            optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
+            optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed'
+            optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved'
+            optional :tag_list, type: Array[String], desc: 'The list of tags for a project'
+            optional :avatar, type: File, desc: 'Avatar image for project'
+            optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line'
+            optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests'
+          end
+
+          params :optional_project_params do
+            use :optional_project_params_ce
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index f74b3b26802..88e7f46c92c 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -97,7 +97,7 @@ module API
       get ":id/issues" do
         group = find_group!(params[:id])
 
-        issues = paginate(find_issues(group_id: group.id))
+        issues = paginate(find_issues(group_id: group.id, include_subgroups: true))
 
         options = {
           with: Entities::IssueBasic,
diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb
index f82241058e5..68921ae439b 100644
--- a/lib/api/project_hooks.rb
+++ b/lib/api/project_hooks.rb
@@ -14,6 +14,7 @@ module API
         optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events"
         optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events"
         optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events"
+        optional :confidential_note_events, type: Boolean, desc: "Trigger hook on confidential note(comment) events"
         optional :job_events, type: Boolean, desc: "Trigger hook on job events"
         optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events"
         optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events"
diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb
index a509c1f32c1..303b58a5942 100644
--- a/lib/api/project_import.rb
+++ b/lib/api/project_import.rb
@@ -1,6 +1,7 @@
 module API
   class ProjectImport < Grape::API
     include PaginationParams
+    include Helpers::ProjectsHelpers
 
     helpers do
       def import_params
@@ -25,6 +26,11 @@ module API
         requires :path, type: String, desc: 'The new project path and name'
         requires :file, type: File, desc: 'The project export file to be imported'
         optional :namespace, type: String, desc: "The ID or name of the namespace that the project will be imported into. Defaults to the current user's namespace."
+        optional :override_params,
+                 type: Hash,
+                 desc: 'New project params to override values in the export' do
+          use :optional_project_params
+        end
       end
       desc 'Create a new project import' do
         detail 'This feature was introduced in GitLab 10.6.'
@@ -47,7 +53,11 @@ module API
             file: import_params[:file]['tempfile']
         }
 
-        project = ::Projects::GitlabProjectsImportService.new(current_user, project_params).execute
+        override_params = import_params.delete(:override_params)
+
+        project = ::Projects::GitlabProjectsImportService.new(
+          current_user, project_params, override_params
+        ).execute
 
         render_api_error!(project.errors.full_messages&.first, 400) unless project.saved?
 
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 3d5b3c5a535..d0a4a23e074 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -4,37 +4,11 @@ module API
   class Projects < Grape::API
     include PaginationParams
     include Helpers::CustomAttributes
+    include Helpers::ProjectsHelpers
 
     before { authenticate_non_get! }
 
     helpers do
-      params :optional_params_ce do
-        optional :description, type: String, desc: 'The description of the project'
-        optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`'
-        optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled'
-        optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled'
-        optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled'
-        optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled'
-        optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled'
-        optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project'
-        optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push'
-        optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project'
-        optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project'
-        optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the project.'
-        optional :public_builds, type: Boolean, desc: 'Perform public builds'
-        optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
-        optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed'
-        optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved'
-        optional :tag_list, type: Array[String], desc: 'The list of tags for a project'
-        optional :avatar, type: File, desc: 'Avatar image for project'
-        optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line'
-        optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests'
-      end
-
-      params :optional_params do
-        use :optional_params_ce
-      end
-
       params :statistics_params do
         optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
       end
@@ -144,7 +118,7 @@ module API
         optional :name, type: String, desc: 'The name of the project'
         optional :path, type: String, desc: 'The path of the repository'
         at_least_one_of :name, :path
-        use :optional_params
+        use :optional_project_params
         use :create_params
       end
       post do
@@ -172,7 +146,7 @@ module API
         requires :user_id, type: Integer, desc: 'The ID of a user'
         optional :path, type: String, desc: 'The path of the repository'
         optional :default_branch, type: String, desc: 'The default branch of the project'
-        use :optional_params
+        use :optional_project_params
         use :create_params
       end
       post "user/:user_id" do
@@ -293,7 +267,7 @@ module API
         optional :default_branch, type: String, desc: 'The default branch of the project'
         optional :path, type: String, desc: 'The path of the repository'
 
-        use :optional_params
+        use :optional_project_params
         at_least_one_of(*at_least_one_of_ce)
       end
       put ':id' do
diff --git a/lib/api/runner.rb b/lib/api/runner.rb
index 834253d8e94..60aeb69e10a 100644
--- a/lib/api/runner.rb
+++ b/lib/api/runner.rb
@@ -186,7 +186,7 @@ module API
 
         status 200
         content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
-        Gitlab::Workhorse.artifact_upload_ok
+        JobArtifactUploader.workhorse_authorize
       end
 
       desc 'Upload artifacts for job' do
@@ -201,14 +201,15 @@ module API
         requires :id, type: Integer, desc: %q(Job's ID)
         optional :token, type: String, desc: %q(Job's authentication token)
         optional :expire_in, type: String, desc: %q(Specify when artifacts should expire)
-        optional :file, type: File, desc: %q(Artifact's file)
         optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
         optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse))
         optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse))
-        optional 'file.sha256', type: String, desc: %q(sha256 checksum of the file)
+        optional 'file.size', type: Integer, desc: %q(real size of file (generated by Workhorse))
+        optional 'file.sha256', type: String, desc: %q(sha256 checksum of the file (generated by Workhorse))
         optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
         optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse))
-        optional 'metadata.sha256', type: String, desc: %q(sha256 checksum of the file)
+        optional 'metadata.size', type: Integer, desc: %q(real size of metadata (generated by Workhorse))
+        optional 'metadata.sha256', type: String, desc: %q(sha256 checksum of metadata (generated by Workhorse))
       end
       post '/:id/artifacts' do
         not_allowed! unless Gitlab.config.artifacts.enabled
@@ -217,21 +218,34 @@ module API
         job = authenticate_job!
         forbidden!('Job is not running!') unless job.running?
 
-        workhorse_upload_path = JobArtifactUploader.workhorse_upload_path
-        artifacts = uploaded_file(:file, workhorse_upload_path)
-        metadata = uploaded_file(:metadata, workhorse_upload_path)
+        artifacts = UploadedFile.from_params(params, :file, JobArtifactUploader.workhorse_local_upload_path)
+        metadata = UploadedFile.from_params(params, :metadata, JobArtifactUploader.workhorse_local_upload_path)
 
         bad_request!('Missing artifacts file!') unless artifacts
         file_to_large! unless artifacts.size < max_artifacts_size
 
+        bad_request!("Already uploaded") if job.job_artifacts_archive
+
         expire_in = params['expire_in'] ||
           Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in
 
-        job.build_job_artifacts_archive(project: job.project, file_type: :archive, file: artifacts, file_sha256: params['file.sha256'], expire_in: expire_in)
-        job.build_job_artifacts_metadata(project: job.project, file_type: :metadata, file: metadata, file_sha256: params['metadata.sha256'], expire_in: expire_in) if metadata
-        job.artifacts_expire_in = expire_in
+        job.build_job_artifacts_archive(
+          project: job.project,
+          file: artifacts,
+          file_type: :archive,
+          file_sha256: artifacts.sha256,
+          expire_in: expire_in)
+
+        if metadata
+          job.build_job_artifacts_metadata(
+            project: job.project,
+            file: metadata,
+            file_type: :metadata,
+            file_sha256: metadata.sha256,
+            expire_in: expire_in)
+        end
 
-        if job.save
+        if job.update(artifacts_expire_in: expire_in)
           present job, with: Entities::JobRequest::Response
         else
           render_validation_error!(job)
diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb
index d8fb7705b2a..3f1e95d4cc0 100644
--- a/lib/banzai/cross_project_reference.rb
+++ b/lib/banzai/cross_project_reference.rb
@@ -4,7 +4,7 @@ module Banzai
   module CrossProjectReference
     # Given a cross-project reference string, get the Project record
     #
-    # Defaults to value of `context[:project]` if:
+    # Defaults to value of `context[:project]`, or `context[:group]` if:
     # * No reference is given OR
     # * Reference given doesn't exist
     #
@@ -12,7 +12,7 @@ module Banzai
     #
     # Returns a Project, or nil if the reference can't be found
     def parent_from_ref(ref)
-      return context[:project] unless ref
+      return context[:project] || context[:group] unless ref
 
       Project.find_by_full_path(ref)
     end
diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb
index c9e3f8ce42b..6efaed7e624 100644
--- a/lib/banzai/filter/abstract_reference_filter.rb
+++ b/lib/banzai/filter/abstract_reference_filter.rb
@@ -171,7 +171,7 @@ module Banzai
           end
 
           if object
-            title = object_link_title(object)
+            title = object_link_title(object, matches)
             klass = reference_class(object_sym)
 
             data = data_attributes_for(link_content || match, parent, object,
@@ -196,13 +196,15 @@ module Banzai
         end
       end
 
-      def data_attributes_for(text, project, object, link_content: false, link_reference: false)
+      def data_attributes_for(text, parent, object, link_content: false, link_reference: false)
+        object_parent_type = parent.is_a?(Group) ? :group : :project
+
         data_attribute(
-          original:       text,
-          link:           link_content,
-          link_reference: link_reference,
-          project:        project.id,
-          object_sym =>   object.id
+          original:             text,
+          link:                 link_content,
+          link_reference:       link_reference,
+          object_parent_type => parent.id,
+          object_sym =>         object.id
         )
       end
 
@@ -216,7 +218,7 @@ module Banzai
         extras
       end
 
-      def object_link_title(object)
+      def object_link_title(object, matches)
         object.title
       end
 
@@ -337,6 +339,12 @@ module Banzai
       def parent
         parent_type == :project ? project : group
       end
+
+      def full_group_path(group_ref)
+        return current_parent_path unless group_ref
+
+        group_ref
+      end
     end
   end
 end
diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb
index 21bcb1c5ca8..99fa2d9d8fb 100644
--- a/lib/banzai/filter/commit_range_reference_filter.rb
+++ b/lib/banzai/filter/commit_range_reference_filter.rb
@@ -34,7 +34,7 @@ module Banzai
                                         range.to_param.merge(only_path: context[:only_path]))
       end
 
-      def object_link_title(range)
+      def object_link_title(range, matches)
         nil
       end
     end
diff --git a/lib/banzai/filter/commit_trailers_filter.rb b/lib/banzai/filter/commit_trailers_filter.rb
new file mode 100644
index 00000000000..ef16df1f3ae
--- /dev/null
+++ b/lib/banzai/filter/commit_trailers_filter.rb
@@ -0,0 +1,152 @@
+module Banzai
+  module Filter
+    # HTML filter that replaces users' names and emails in commit trailers
+    # with links to their GitLab accounts or mailto links to their mentioned
+    # emails.
+    #
+    # Commit trailers are special labels in the form of `*-by:` and fall on a
+    # single line, ex:
+    #
+    #   Reported-By: John S. Doe <john.doe@foo.bar>
+    #
+    # More info about this can be found here:
+    # * https://git.wiki.kernel.org/index.php/CommitMessageConventions
+    class CommitTrailersFilter < HTML::Pipeline::Filter
+      include ActionView::Helpers::TagHelper
+      include ApplicationHelper
+      include AvatarsHelper
+
+      TRAILER_REGEXP = /(?<label>[[:alpha:]-]+-by:)/i.freeze
+      AUTHOR_REGEXP = /(?<author_name>.+)/.freeze
+      # Devise.email_regexp wouldn't work here since its designed to match
+      # against strings that only contains email addresses; the \A and \z
+      # around the expression will only match if the string being matched
+      # contains just the email nothing else.
+      MAIL_REGEXP = /&lt;(?<author_email>[^@\s]+@[^@\s]+)&gt;/.freeze
+      FILTER_REGEXP = /(?<trailer>^\s*#{TRAILER_REGEXP}\s*#{AUTHOR_REGEXP}\s+#{MAIL_REGEXP}$)/mi.freeze
+
+      def call
+        doc.xpath('descendant-or-self::text()').each do |node|
+          content = node.to_html
+
+          next unless content.match(FILTER_REGEXP)
+
+          html = trailer_filter(content)
+
+          next if html == content
+
+          node.replace(html)
+        end
+
+        doc
+      end
+
+      private
+
+      # Replace trailer lines with links to GitLab users or mailto links to
+      # non GitLab users.
+      #
+      # text - String text to replace trailers in.
+      #
+      # Returns a String with all trailer lines replaced with links to GitLab
+      # users and mailto links to non GitLab users. All links have `data-trailer`
+      # and `data-user` attributes attached.
+      def trailer_filter(text)
+        text.gsub(FILTER_REGEXP) do |author_match|
+          label = $~[:label]
+          "#{label} #{parse_user($~[:author_name], $~[:author_email], label)}"
+        end
+      end
+
+      # Find a GitLab user using the supplied email and generate
+      # a valid link to them, otherwise, generate a mailto link.
+      #
+      # name - String name used in the commit message for the user
+      # email - String email used in the commit message for the user
+      # trailer - String trailer used in the commit message
+      #
+      # Returns a String with a link to the user.
+      def parse_user(name, email, trailer)
+        link_to_user User.find_by_any_email(email),
+          name: name,
+          email: email,
+          trailer: trailer
+      end
+
+      def urls
+        Gitlab::Routing.url_helpers
+      end
+
+      def link_to_user(user, name:, email:, trailer:)
+        wrapper = link_wrapper(data: {
+          trailer: trailer,
+          user: user.try(:id)
+        })
+
+        avatar = user_avatar_without_link(
+          user: user,
+          user_email: email,
+          css_class: 'avatar-inline',
+          has_tooltip: false
+        )
+
+        link_href = user.nil? ? "mailto:#{email}" : urls.user_url(user)
+
+        avatar_link = link_tag(
+          link_href,
+          content: avatar,
+          title: email
+        )
+
+        name_link = link_tag(
+          link_href,
+          content: name,
+          title: email
+        )
+
+        email_link = link_tag(
+          "mailto:#{email}",
+          content: email,
+          title: email
+        )
+
+        wrapper << "#{avatar_link}#{name_link} <#{email_link}>"
+      end
+
+      def link_wrapper(data: {})
+        data_attributes = data_attributes_from_hash(data)
+
+        doc.document.create_element(
+          'span',
+          data_attributes
+        )
+      end
+
+      def link_tag(url, title: "", content: "", data: {})
+        data_attributes = data_attributes_from_hash(data)
+
+        attributes = data_attributes.merge(
+          href: url,
+          title: title
+        )
+
+        link = doc.document.create_element('a', attributes)
+
+        if content.html_safe?
+          link << content
+        else
+          link.content = content # make sure we escape content using nokogiri's #content=
+        end
+
+        link
+      end
+
+      def data_attributes_from_hash(data = {})
+        data.reject! {|_, value| value.nil?}
+        data.map do |key, value|
+          [%(data-#{key.to_s.dasherize}), value]
+        end.to_h
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb
index d5360ad8f68..1cbada818fb 100644
--- a/lib/banzai/filter/label_reference_filter.rb
+++ b/lib/banzai/filter/label_reference_filter.rb
@@ -32,16 +32,25 @@ module Banzai
         end
       end
 
-      def find_label(project_ref, label_id, label_name)
-        project = parent_from_ref(project_ref)
-        return unless project
+      def find_label(parent_ref, label_id, label_name)
+        parent = parent_from_ref(parent_ref)
+        return unless parent
 
         label_params = label_params(label_id, label_name)
-        find_labels(project).find_by(label_params)
+        find_labels(parent).find_by(label_params)
       end
 
-      def find_labels(project)
-        LabelsFinder.new(nil, project_id: project.id).execute(skip_authorization: true)
+      def find_labels(parent)
+        params = if parent.is_a?(Group)
+                   { group_id: parent.id,
+                     include_ancestor_groups: true,
+                     only_group_labels: true }
+                 else
+                   { project_id: parent.id,
+                     include_ancestor_groups: true }
+                 end
+
+        LabelsFinder.new(nil, params).execute(skip_authorization: true)
       end
 
       # Parameters to pass to `Label.find_by` based on the given arguments
@@ -59,25 +68,39 @@ module Banzai
         end
       end
 
-      def url_for_object(label, project)
+      def url_for_object(label, parent)
         h = Gitlab::Routing.url_helpers
-        h.project_issues_url(project, label_name: label.name, only_path: context[:only_path])
+
+        if parent.is_a?(Project)
+          h.project_issues_url(parent, label_name: label.name, only_path: context[:only_path])
+        elsif context[:label_url_method]
+          h.public_send(context[:label_url_method], parent, label_name: label.name, only_path: context[:only_path]) # rubocop:disable GitlabSecurity/PublicSend
+        end
       end
 
       def object_link_text(object, matches)
-        project_path     = full_project_path(matches[:namespace], matches[:project])
-        project_from_ref = from_ref_cached(project_path)
-        reference        = project_from_ref.to_human_reference(project)
-        label_suffix     = " <i>in #{reference}</i>" if reference.present?
+        label_suffix = ''
+
+        if project || full_path_ref?(matches)
+          project_path    = full_project_path(matches[:namespace], matches[:project])
+          parent_from_ref = from_ref_cached(project_path)
+          reference       = parent_from_ref.to_human_reference(project || group)
+
+          label_suffix = " <i>in #{reference}</i>" if reference.present?
+        end
 
         LabelsHelper.render_colored_label(object, label_suffix)
       end
 
+      def full_path_ref?(matches)
+        matches[:namespace] && matches[:project]
+      end
+
       def unescape_html_entities(text)
         CGI.unescapeHTML(text.to_s)
       end
 
-      def object_link_title(object)
+      def object_link_title(object, matches)
         # use title of wrapped element instead
         nil
       end
diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb
index b3cfa97d0e0..5cbdb01c130 100644
--- a/lib/banzai/filter/merge_request_reference_filter.rb
+++ b/lib/banzai/filter/merge_request_reference_filter.rb
@@ -17,10 +17,19 @@ module Banzai
                                             only_path: context[:only_path])
       end
 
+      def object_link_title(object, matches)
+        object_link_commit_title(object, matches) || super
+      end
+
       def object_link_text_extras(object, matches)
         extras = super
 
+        if commit_ref = object_link_commit_ref(object, matches)
+          return extras.unshift(commit_ref)
+        end
+
         path = matches[:path] if matches.names.include?("path")
+
         case path
         when '/diffs'
           extras.unshift "diffs"
@@ -38,6 +47,36 @@ module Banzai
           .where(iid: ids.to_a)
           .includes(target_project: :namespace)
       end
+
+      private
+
+      def object_link_commit_title(object, matches)
+        object_link_commit(object, matches)&.title
+      end
+
+      def object_link_commit_ref(object, matches)
+        object_link_commit(object, matches)&.short_id
+      end
+
+      def object_link_commit(object, matches)
+        return unless matches.names.include?('query') && query = matches[:query]
+
+        # Removes leading "?". CGI.parse expects "arg1&arg2&arg3"
+        params = CGI.parse(query.sub(/^\?/, ''))
+
+        return unless commit_sha = params['commit_id']&.first
+
+        if commit = find_commit_by_sha(object, commit_sha)
+          Commit.from_hash(commit.to_hash, object.project)
+        end
+      end
+
+      def find_commit_by_sha(object, commit_sha)
+        @all_commits ||= {}
+        @all_commits[object.id] ||= object.all_commits
+
+        @all_commits[object.id].find { |commit| commit.sha == commit_sha }
+      end
     end
   end
 end
diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb
index 8ec696ce5fc..1a1d7dbeb3d 100644
--- a/lib/banzai/filter/milestone_reference_filter.rb
+++ b/lib/banzai/filter/milestone_reference_filter.rb
@@ -84,7 +84,7 @@ module Banzai
         end
       end
 
-      def object_link_title(object)
+      def object_link_title(object, matches)
         nil
       end
     end
diff --git a/lib/banzai/pipeline/commit_description_pipeline.rb b/lib/banzai/pipeline/commit_description_pipeline.rb
new file mode 100644
index 00000000000..607c2731ed3
--- /dev/null
+++ b/lib/banzai/pipeline/commit_description_pipeline.rb
@@ -0,0 +1,11 @@
+module Banzai
+  module Pipeline
+    class CommitDescriptionPipeline < SingleLinePipeline
+      def self.filters
+        @filters ||= super.concat FilterArray[
+          Filter::CommitTrailersFilter,
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb
new file mode 100644
index 00000000000..e5e8837221e
--- /dev/null
+++ b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    # Ensures services which previously recieved all notes events continue
+    # to recieve confidential ones.
+    class SetConfidentialNoteEventsOnServices
+      class Service < ActiveRecord::Base
+        self.table_name = 'services'
+
+        include ::EachBatch
+
+        def self.services_to_update
+          where(confidential_note_events: nil, note_events: true)
+        end
+      end
+
+      def perform(start_id, stop_id)
+        Service.services_to_update
+               .where(id: start_id..stop_id)
+               .update_all(confidential_note_events: true)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb
new file mode 100644
index 00000000000..171c8ef21b7
--- /dev/null
+++ b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    # Ensures hooks which previously recieved all notes events continue
+    # to recieve confidential ones.
+    class SetConfidentialNoteEventsOnWebhooks
+      class WebHook < ActiveRecord::Base
+        self.table_name = 'web_hooks'
+
+        include ::EachBatch
+
+        def self.hooks_to_update
+          where(confidential_note_events: nil, note_events: true)
+        end
+      end
+
+      def perform(start_id, stop_id)
+        WebHook.hooks_to_update
+               .where(id: start_id..stop_id)
+               .update_all(confidential_note_events: true)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/checks/lfs_integrity.rb b/lib/gitlab/checks/lfs_integrity.rb
index f7276a380dc..f0e5773ec3c 100644
--- a/lib/gitlab/checks/lfs_integrity.rb
+++ b/lib/gitlab/checks/lfs_integrity.rb
@@ -15,8 +15,7 @@ module Gitlab
 
         return false unless new_lfs_pointers.present?
 
-        existing_count = @project.lfs_storage_project
-                                 .lfs_objects
+        existing_count = @project.all_lfs_objects
                                  .where(oid: new_lfs_pointers.map(&:lfs_oid))
                                  .count
 
diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb
index 9d2d4170266..2b26ebb45a1 100644
--- a/lib/gitlab/ci/status/build/factory.rb
+++ b/lib/gitlab/ci/status/build/factory.rb
@@ -12,10 +12,12 @@ module Gitlab
               Status::Build::Skipped],
              [Status::Build::Cancelable,
               Status::Build::Retryable],
+             [Status::Build::Failed],
              [Status::Build::FailedAllowed,
               Status::Build::Play,
               Status::Build::Stop],
-             [Status::Build::Action]]
+             [Status::Build::Action],
+             [Status::Build::Retried]]
           end
 
           def self.common_helpers
diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb
new file mode 100644
index 00000000000..155f4fc1343
--- /dev/null
+++ b/lib/gitlab/ci/status/build/failed.rb
@@ -0,0 +1,40 @@
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class Failed < Status::Extended
+          REASONS = {
+            'unknown_failure' => 'unknown failure',
+            'script_failure' => 'script failure',
+            'api_failure' => 'API failure',
+            'stuck_or_timeout_failure' => 'stuck or timeout failure',
+            'runner_system_failure' => 'runner system failure',
+            'missing_dependency_failure' => 'missing dependency failure'
+          }.freeze
+
+          def status_tooltip
+            base_message
+          end
+
+          def badge_tooltip
+            base_message
+          end
+
+          def self.matches?(build, user)
+            build.failed?
+          end
+
+          private
+
+          def base_message
+            "#{s_('CiStatusLabel|failed')} #{description}"
+          end
+
+          def description
+            "<br> (#{REASONS[subject.failure_reason]})"
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb
index dc90f398c7e..ca0046fb1f7 100644
--- a/lib/gitlab/ci/status/build/failed_allowed.rb
+++ b/lib/gitlab/ci/status/build/failed_allowed.rb
@@ -4,7 +4,7 @@ module Gitlab
       module Build
         class FailedAllowed < Status::Extended
           def label
-            'failed (allowed to fail)'
+            "failed #{allowed_to_fail_title}"
           end
 
           def icon
@@ -15,9 +15,19 @@ module Gitlab
             'failed_with_warnings'
           end
 
+          def status_tooltip
+            "#{@status.status_tooltip} #{allowed_to_fail_title}"
+          end
+
           def self.matches?(build, user)
             build.failed? && build.allow_failure?
           end
+
+          private
+
+          def allowed_to_fail_title
+            "(allowed to fail)"
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/status/build/retried.rb b/lib/gitlab/ci/status/build/retried.rb
new file mode 100644
index 00000000000..6e190e4ee3c
--- /dev/null
+++ b/lib/gitlab/ci/status/build/retried.rb
@@ -0,0 +1,17 @@
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class Retried < Status::Extended
+          def status_tooltip
+            @status.status_tooltip + " (retried)"
+          end
+
+          def self.matches?(build, user)
+            build.retried?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb
index 031bd4cd29c..9d6a2f51c11 100644
--- a/lib/gitlab/ci/status/core.rb
+++ b/lib/gitlab/ci/status/core.rb
@@ -65,6 +65,16 @@ module Gitlab
         def action_button_title
           raise NotImplementedError
         end
+
+        # Hint that appears on all the pipeline graph tooltips and builds on the right sidebar in Job detail view
+        def status_tooltip
+          label
+        end
+
+        # Hint that appears on the build badges
+        def badge_tooltip
+          subject.status
+        end
       end
     end
   end
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
index b3fe3ef1c4d..54894a46077 100644
--- a/lib/gitlab/ci/trace/stream.rb
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -8,7 +8,7 @@ module Gitlab
 
         attr_reader :stream
 
-        delegate :close, :tell, :seek, :size, :path, :url, :truncate, to: :stream, allow_nil: true
+        delegate :close, :tell, :seek, :size, :url, :truncate, to: :stream, allow_nil: true
 
         delegate :valid?, to: :stream, as: :present?, allow_nil: true
 
@@ -25,6 +25,10 @@ module Gitlab
           self.path.present?
         end
 
+        def path
+          self.stream.path if self.stream.respond_to?(:path)
+        end
+
         def limit(last_bytes = LIMIT_SIZE)
           if last_bytes < size
             stream.seek(-last_bytes, IO::SEEK_END)
diff --git a/lib/gitlab/data_builder/note.rb b/lib/gitlab/data_builder/note.rb
index 50fea1232af..f573368e572 100644
--- a/lib/gitlab/data_builder/note.rb
+++ b/lib/gitlab/data_builder/note.rb
@@ -9,6 +9,7 @@ module Gitlab
       #
       # data = {
       #   object_kind: "note",
+      #   event_type: "confidential_note",
       #   user: {
       #     name: String,
       #     username: String,
@@ -51,8 +52,11 @@ module Gitlab
       end
 
       def build_base_data(project, user, note)
+        event_type = note.confidential? ? 'confidential_note' : 'note'
+
         base_data = {
           object_kind: "note",
+          event_type: event_type,
           user: user.hook_attrs,
           project_id: project.id,
           project: project.hook_attrs,
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 1634fe4e9cb..77079e5e72b 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -860,7 +860,7 @@ into similar problems in the future (e.g. when new tables are created).
       # Each job is scheduled with a `delay_interval` in between.
       # If you use a small interval, then some jobs may run at the same time.
       #
-      # model_class - The table being iterated over
+      # model_class - The table or relation being iterated over
       # job_class_name - The background migration job class as a string
       # delay_interval - The duration between each job's scheduled time (must respond to `to_f`)
       # batch_size - The maximum number of rows per job
diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb
index 010b4be7b40..81e91ea0ab7 100644
--- a/lib/gitlab/diff/inline_diff_marker.rb
+++ b/lib/gitlab/diff/inline_diff_marker.rb
@@ -1,11 +1,14 @@
 module Gitlab
   module Diff
     class InlineDiffMarker < Gitlab::StringRangeMarker
+      def initialize(line, rich_line = nil)
+        super(line, rich_line || line)
+      end
+
       def mark(line_inline_diffs, mode: nil)
-        mark = super(line_inline_diffs) do |text, left:, right:|
+        super(line_inline_diffs) do |text, left:, right:|
           %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>}
         end
-        mark.html_safe
       end
 
       private
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index 93037ed8d90..0fb82441bf8 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -231,7 +231,8 @@ module Gitlab
         # relation to each other. The last 10 commits for a branch for example,
         # should go through .where
         def batch_by_oid(repo, oids)
-          repo.gitaly_migrate(:list_commits_by_oid) do |is_enabled|
+          repo.gitaly_migrate(:list_commits_by_oid,
+                              status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
             if is_enabled
               repo.gitaly_commit_client.list_commits_by_oid(oids)
             else
diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb
index 24f027d8da4..7c201c6169b 100644
--- a/lib/gitlab/git/hook.rb
+++ b/lib/gitlab/git/hook.rb
@@ -95,13 +95,13 @@ module Gitlab
         args = [ref, oldrev, newrev]
 
         stdout, stderr, status = Open3.capture3(env, path, *args, options)
-        [status.success?, (stderr.presence || stdout).gsub(/\R/, "<br>").html_safe]
+        [status.success?, Gitlab::Utils.nlbr(stderr.presence || stdout)]
       end
 
       def retrieve_error_message(stderr, stdout)
         err_message = stderr.read
         err_message = err_message.blank? ? stdout.read : err_message
-        err_message.gsub(/\R/, "<br>").html_safe
+        Gitlab::Utils.nlbr(err_message)
       end
     end
   end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index d16a096ffb9..8d97bfb0e6a 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -885,7 +885,8 @@ module Gitlab
       end
 
       def delete_refs(*ref_names)
-        gitaly_migrate(:delete_refs) do |is_enabled|
+        gitaly_migrate(:delete_refs,
+                      status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
           if is_enabled
             gitaly_delete_refs(*ref_names)
           else
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index ed0644f6cf1..6a01957184d 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -29,9 +29,9 @@ module Gitlab
     PUSH_COMMANDS = %w{ git-receive-pack }.freeze
     ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS
 
-    attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path
+    attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type
 
-    def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil)
+    def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil, auth_result_type: nil)
       @actor    = actor
       @project  = project
       @protocol = protocol
@@ -39,6 +39,7 @@ module Gitlab
       @namespace_path = namespace_path
       @project_path = project_path
       @redirected_path = redirected_path
+      @auth_result_type = auth_result_type
     end
 
     def check(cmd, changes)
@@ -78,6 +79,12 @@ module Gitlab
       authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code)
     end
 
+    def request_from_ci_build?
+      return false unless protocol == 'http'
+
+      auth_result_type == :build || auth_result_type == :ci
+    end
+
     def protocol_allowed?
       Gitlab::ProtocolAccess.allowed?(protocol)
     end
@@ -93,6 +100,8 @@ module Gitlab
     end
 
     def check_protocol!
+      return if request_from_ci_build?
+
       unless protocol_allowed?
         raise UnauthorizedError, "Git access over #{protocol.upcase} is not allowed"
       end
diff --git a/lib/gitlab/hook_data/issuable_builder.rb b/lib/gitlab/hook_data/issuable_builder.rb
index 4febb0ab430..6ab36676127 100644
--- a/lib/gitlab/hook_data/issuable_builder.rb
+++ b/lib/gitlab/hook_data/issuable_builder.rb
@@ -11,7 +11,8 @@ module Gitlab
 
       def build(user: nil, changes: {})
         hook_data = {
-          object_kind: issuable.class.name.underscore,
+          object_kind: object_kind,
+          event_type: event_type,
           user: user.hook_attrs,
           project: issuable.project.hook_attrs,
           object_attributes: issuable.hook_attrs,
@@ -36,6 +37,18 @@ module Gitlab
 
       private
 
+      def object_kind
+        issuable.class.name.underscore
+      end
+
+      def event_type
+        if issuable.try(:confidential?)
+          "confidential_#{object_kind}"
+        else
+          object_kind
+        end
+      end
+
       def issuable_builder
         case issuable
         when Issue
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 4bdd01f5e94..cd840bd5b01 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -105,6 +105,7 @@ excluded_attributes:
     - :last_repository_updated_at
     - :last_repository_check_at
     - :storage_version
+    - :description_html
   snippets:
     - :expired_at
   merge_request_diff:
@@ -124,6 +125,8 @@ excluded_attributes:
     - :trace
     - :token
     - :when
+    - :artifacts_file
+    - :artifacts_metadata
   push_event_payload:
     - :event_id
   project_badges:
@@ -144,8 +147,6 @@ methods:
     - :diff_head_sha
     - :source_branch_sha
     - :target_branch_sha
-  project:
-    - :description_html
   events:
     - :action
   push_event_payload:
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index c38df9102eb..c490bf059d2 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -13,7 +13,7 @@ module Gitlab
       end
 
       def execute
-        if import_file && check_version! && [repo_restorer, wiki_restorer, project_tree, avatar_restorer, uploads_restorer].all?(&:restore)
+        if import_file && check_version! && restorers.all?(&:restore)
           project_tree.restored_project
         else
           raise Projects::ImportService::Error.new(@shared.errors.join(', '))
@@ -24,6 +24,11 @@ module Gitlab
 
       private
 
+      def restorers
+        [repo_restorer, wiki_restorer, project_tree, avatar_restorer,
+         uploads_restorer, lfs_restorer]
+      end
+
       def import_file
         Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file,
                                                   shared: @shared)
@@ -60,6 +65,10 @@ module Gitlab
         Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared)
       end
 
+      def lfs_restorer
+        Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: @shared)
+      end
+
       def path_with_namespace
         File.join(@project.namespace.full_path, @project.path)
       end
diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb
new file mode 100644
index 00000000000..b28c3c161b7
--- /dev/null
+++ b/lib/gitlab/import_export/lfs_restorer.rb
@@ -0,0 +1,43 @@
+module Gitlab
+  module ImportExport
+    class LfsRestorer
+      def initialize(project:, shared:)
+        @project = project
+        @shared = shared
+      end
+
+      def restore
+        return true if lfs_file_paths.empty?
+
+        lfs_file_paths.each do |file_path|
+          link_or_create_lfs_object!(file_path)
+        end
+
+        true
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def link_or_create_lfs_object!(path)
+        size = File.size(path)
+        oid = LfsObject.calculate_oid(path)
+
+        lfs_object = LfsObject.find_or_initialize_by(oid: oid, size: size)
+        lfs_object.file = File.open(path) unless lfs_object.file&.exists?
+
+        @project.all_lfs_objects << lfs_object
+      end
+
+      def lfs_file_paths
+        @lfs_file_paths ||= Dir.glob("#{lfs_storage_path}/*")
+      end
+
+      def lfs_storage_path
+        File.join(@shared.export_path, 'lfs-objects')
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb
new file mode 100644
index 00000000000..29410e2331c
--- /dev/null
+++ b/lib/gitlab/import_export/lfs_saver.rb
@@ -0,0 +1,55 @@
+module Gitlab
+  module ImportExport
+    class LfsSaver
+      include Gitlab::ImportExport::CommandLineUtil
+
+      def initialize(project:, shared:)
+        @project = project
+        @shared = shared
+      end
+
+      def save
+        @project.all_lfs_objects.each do |lfs_object|
+          save_lfs_object(lfs_object)
+        end
+
+        true
+      rescue => e
+        @shared.error(e)
+
+        false
+      end
+
+      private
+
+      def save_lfs_object(lfs_object)
+        if lfs_object.local_store?
+          copy_file_for_lfs_object(lfs_object)
+        else
+          download_file_for_lfs_object(lfs_object)
+        end
+      end
+
+      def download_file_for_lfs_object(lfs_object)
+        destination = destination_path_for_object(lfs_object)
+        mkdir_p(File.dirname(destination))
+
+        File.open(destination, 'w') do |file|
+          IO.copy_stream(URI.parse(lfs_object.file.url).open, file)
+        end
+      end
+
+      def copy_file_for_lfs_object(lfs_object)
+        copy_files(lfs_object.file.path, destination_path_for_object(lfs_object))
+      end
+
+      def destination_path_for_object(lfs_object)
+        File.join(lfs_export_path, lfs_object.oid)
+      end
+
+      def lfs_export_path
+        File.join(@shared.export_path, 'lfs-objects')
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 8f5bb8f9597..2c315207298 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -77,24 +77,31 @@ module Gitlab
       end
 
       def default_relation_list
-        Gitlab::ImportExport::Reader.new(shared: @shared).tree.reject do |model|
+        reader.tree.reject do |model|
           model.is_a?(Hash) && model[:project_members]
         end
       end
 
       def restore_project
-        params = project_params
-
-        if params[:description].present?
-          params[:description_html] = nil
-        end
-
-        @project.update_columns(params)
+        @project.update_columns(project_params)
         @project
       end
 
       def project_params
-        @tree_hash.reject do |key, value|
+        @project_params ||= json_params.merge(override_params)
+      end
+
+      def override_params
+        return {} unless params = @project.import_data&.data&.fetch('override_params')
+
+        @override_params ||= params.select do |key, _value|
+          Project.column_names.include?(key.to_s) &&
+            !reader.project_tree[:except].include?(key.to_sym)
+        end
+      end
+
+      def json_params
+        @json_params ||= @tree_hash.reject do |key, value|
           # return params that are not 1 to many or 1 to 1 relations
           value.respond_to?(:each) && !Project.column_names.include?(key)
         end
@@ -181,6 +188,10 @@ module Gitlab
 
         relation_hash.merge(params)
       end
+
+      def reader
+        @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
+      end
     end
   end
 end
diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb
index d4c54049b74..a5f5d719cc1 100644
--- a/lib/gitlab/middleware/multipart.rb
+++ b/lib/gitlab/middleware/multipart.rb
@@ -82,7 +82,7 @@ module Gitlab
         end
 
         def open_file(path, name)
-          ::UploadedFile.new(path, name || File.basename(path), 'application/octet-stream')
+          ::UploadedFile.new(path, filename: name || File.basename(path), content_type: 'application/octet-stream')
         end
       end
 
diff --git a/lib/gitlab/sidekiq_logging/json_formatter.rb b/lib/gitlab/sidekiq_logging/json_formatter.rb
new file mode 100644
index 00000000000..98f8222fd03
--- /dev/null
+++ b/lib/gitlab/sidekiq_logging/json_formatter.rb
@@ -0,0 +1,21 @@
+module Gitlab
+  module SidekiqLogging
+    class JSONFormatter
+      def call(severity, timestamp, progname, data)
+        output = {
+          severity: severity,
+          time: timestamp.utc.iso8601(3)
+        }
+
+        case data
+        when String
+          output[:message] = data
+        when Hash
+          output.merge!(data)
+        end
+
+        output.to_json + "\n"
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb
new file mode 100644
index 00000000000..9a89ae70b98
--- /dev/null
+++ b/lib/gitlab/sidekiq_logging/structured_logger.rb
@@ -0,0 +1,96 @@
+module Gitlab
+  module SidekiqLogging
+    class StructuredLogger
+      START_TIMESTAMP_FIELDS = %w[created_at enqueued_at].freeze
+      DONE_TIMESTAMP_FIELDS = %w[started_at retried_at failed_at completed_at].freeze
+
+      def call(job, queue)
+        started_at = current_time
+        base_payload = parse_job(job)
+
+        Sidekiq.logger.info log_job_start(started_at, base_payload)
+
+        yield
+
+        Sidekiq.logger.info log_job_done(started_at, base_payload)
+      rescue => job_exception
+        Sidekiq.logger.warn log_job_done(started_at, base_payload, job_exception)
+
+        raise
+      end
+
+      private
+
+      def base_message(payload)
+        "#{payload['class']} JID-#{payload['jid']}"
+      end
+
+      def log_job_start(started_at, payload)
+        payload['message'] = "#{base_message(payload)}: start"
+        payload['job_status'] = 'start'
+
+        payload
+      end
+
+      def log_job_done(started_at, payload, job_exception = nil)
+        payload = payload.dup
+        payload['duration'] = elapsed(started_at)
+        payload['completed_at'] = Time.now.utc
+
+        message = base_message(payload)
+
+        if job_exception
+          payload['message'] = "#{message}: fail: #{payload['duration']} sec"
+          payload['job_status'] = 'fail'
+          payload['error_message'] = job_exception.message
+          payload['error'] = job_exception.class
+          payload['error_backtrace'] = backtrace_cleaner.clean(job_exception.backtrace)
+        else
+          payload['message'] = "#{message}: done: #{payload['duration']} sec"
+          payload['job_status'] = 'done'
+        end
+
+        convert_to_iso8601(payload, DONE_TIMESTAMP_FIELDS)
+
+        payload
+      end
+
+      def parse_job(job)
+        job = job.dup
+
+        # Add process id params
+        job['pid'] = ::Process.pid
+
+        job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS']
+
+        convert_to_iso8601(job, START_TIMESTAMP_FIELDS)
+
+        job
+      end
+
+      def convert_to_iso8601(payload, keys)
+        keys.each do |key|
+          payload[key] = format_time(payload[key]) if payload[key]
+        end
+      end
+
+      def elapsed(start)
+        (current_time - start).round(3)
+      end
+
+      def current_time
+        Gitlab::Metrics::System.monotonic_time
+      end
+
+      def backtrace_cleaner
+        @backtrace_cleaner ||= ActiveSupport::BacktraceCleaner.new
+      end
+
+      def format_time(timestamp)
+        return timestamp if timestamp.is_a?(String)
+
+        Time.at(timestamp).utc.iso8601(3)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb
index dc9391f32cf..b0a492eaa58 100644
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@@ -27,6 +27,11 @@ module Gitlab
         .gsub(/(\A-+|-+\z)/, '')
     end
 
+    # Converts newlines into HTML line break elements
+    def nlbr(str)
+      ActionView::Base.full_sanitizer.sanitize(str, tags: []).gsub(/\r?\n/, '<br>').html_safe
+    end
+
     def remove_line_breaks(str)
       str.gsub(/\r?\n/, '')
     end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index b102812ec12..2faeaf16d55 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -36,10 +36,6 @@ module Gitlab
         }
       end
 
-      def artifact_upload_ok
-        { TempPath: JobArtifactUploader.workhorse_upload_path }
-      end
-
       def send_git_blob(repository, blob)
         params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT)
                    {
diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb
index 4a3c40f88eb..5dc85b2baea 100644
--- a/lib/uploaded_file.rb
+++ b/lib/uploaded_file.rb
@@ -1,8 +1,10 @@
 require "tempfile"
+require "tmpdir"
 require "fileutils"
 
-# Taken from: Rack::Test::UploadedFile
 class UploadedFile
+  InvalidPathError = Class.new(StandardError)
+
   # The filename, *not* including the path, of the "uploaded" file
   attr_reader :original_filename
 
@@ -12,14 +14,46 @@ class UploadedFile
   # The content type of the "uploaded" file
   attr_accessor :content_type
 
-  def initialize(path, filename, content_type = "text/plain")
-    raise "#{path} file does not exist" unless ::File.exist?(path)
+  attr_reader :remote_id
+  attr_reader :sha256
+
+  def initialize(path, filename: nil, content_type: "application/octet-stream", sha256: nil, remote_id: nil)
+    raise InvalidPathError, "#{path} file does not exist" unless ::File.exist?(path)
 
     @content_type = content_type
     @original_filename = filename || ::File.basename(path)
+    @content_type = content_type
+    @sha256 = sha256
+    @remote_id = remote_id
     @tempfile = File.new(path, 'rb')
   end
 
+  def self.from_params(params, field, upload_path)
+    unless params["#{field}.path"]
+      raise InvalidPathError, "file is invalid" if params["#{field}.remote_id"]
+
+      return
+    end
+
+    file_path = File.realpath(params["#{field}.path"])
+
+    unless self.allowed_path?(file_path, [upload_path, Dir.tmpdir].compact)
+      raise InvalidPathError, "insecure path used '#{file_path}'"
+    end
+
+    UploadedFile.new(file_path,
+      filename: params["#{field}.name"],
+      content_type: params["#{field}.type"] || 'application/octet-stream',
+      sha256: params["#{field}.sha256"],
+      remote_id: params["#{field}.remote_id"])
+  end
+
+  def self.allowed_path?(file_path, paths)
+    paths.any? do |path|
+      File.exist?(path) && file_path.start_with?(File.realpath(path))
+    end
+  end
+
   def path
     @tempfile.path
   end