Merge branch 'code-freeze-20181207' into 11-6-stable-prepare-rc5

* code-freeze-20181207: (85 commits)
  Changed frontmatter filtering to support YAML, JSON, TOML, and arbitrary languages
  Disable docs lint internal_links check
  Documentation cleanup
  Allow public forks to be deduplicated
  Prettifies
  [CE] - Add milestones autocomplete for epics
  Fixes linting errors
  Reorganize Jobs Variables feature spec
  CE Port of "Web Terminal FE"
  Extract context in JobsController spec
  Add specs for TriggerVariableEntity
  Extract context in JobsController spec
  Allows user to override default issuer email for cert manager
  Add specs for TriggerVariableEntity
  Adds toggle behavior - Adds coverage for hide/reveal toggle button behavior
  Backports changes made to One notification per code review
  Further design iteration on project overview
  Fix transaction pollution in Shard.by_name
  Show primary button when all labels are prioritized
  Consistent feature name in all docs
  ...

22 files changed, 618 insertions, 55 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index a4bf0d77eb1..8abb24e6f69 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -20,7 +20,8 @@ module API
                     Gitlab::GrapeLogging::Loggers::RouteLogger.new,
                     Gitlab::GrapeLogging::Loggers::UserLogger.new,
                     Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new,
-                    Gitlab::GrapeLogging::Loggers::PerfLogger.new
+                    Gitlab::GrapeLogging::Loggers::PerfLogger.new,
+                    Gitlab::GrapeLogging::Loggers::CorrelationIdLogger.new
                   ]
 
     allow_access_with_scope :api
@@ -84,7 +85,6 @@ module API
     content_type :txt, "text/plain"
 
     # Ensure the namespace is right, otherwise we might load Grape::API::Helpers
-    helpers ::SentryHelper
     helpers ::API::Helpers
     helpers ::API::Helpers::CommonHelpers
 
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 9fda73d5b92..2cceb2ec798 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -368,10 +368,10 @@ module API
     end
 
     def handle_api_exception(exception)
-      if sentry_enabled? && report_exception?(exception)
+      if report_exception?(exception)
         define_params_for_grape_middleware
-        sentry_context
-        Raven.capture_exception(exception, extra: params)
+        Gitlab::Sentry.context(current_user)
+        Gitlab::Sentry.track_acceptable_exception(exception, extra: params)
       end
 
       # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60
diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb
index 06a57e3cd6f..3cc09f6ac3f 100644
--- a/lib/api/namespaces.rb
+++ b/lib/api/namespaces.rb
@@ -6,20 +6,35 @@ module API
 
     before { authenticate! }
 
+    helpers do
+      params :optional_list_params_ee do
+        # EE::API::Namespaces would override this helper
+      end
+
+      # EE::API::Namespaces would override this method
+      def custom_namespace_present_options
+        {}
+      end
+    end
+
     resource :namespaces do
       desc 'Get a namespaces list' do
         success Entities::Namespace
       end
       params do
         optional :search, type: String, desc: "Search query for namespaces"
+
         use :pagination
+        use :optional_list_params_ee
       end
       get do
         namespaces = current_user.admin ? Namespace.all : current_user.namespaces
 
         namespaces = namespaces.search(params[:search]) if params[:search].present?
 
-        present paginate(namespaces), with: Entities::Namespace, current_user: current_user
+        options = { with: Entities::Namespace, current_user: current_user }
+
+        present paginate(namespaces), options.reverse_merge(custom_namespace_present_options)
       end
 
       desc 'Get a namespace by ID' do
diff --git a/lib/banzai/filter/front_matter_filter.rb b/lib/banzai/filter/front_matter_filter.rb
new file mode 100644
index 00000000000..a27d18facd1
--- /dev/null
+++ b/lib/banzai/filter/front_matter_filter.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Banzai
+  module Filter
+    class FrontMatterFilter < HTML::Pipeline::Filter
+      DELIM_LANG = {
+        '---' => 'yaml',
+        '+++' => 'toml',
+        ';;;' => 'json'
+      }.freeze
+
+      DELIM = Regexp.union(DELIM_LANG.keys)
+
+      PATTERN = %r{
+        \A(?:[^\r\n]*coding:[^\r\n]*)?         # optional encoding line
+        \s*
+        ^(?<delim>#{DELIM})[ \t]*(?<lang>\S*)  # opening front matter marker (optional language specifier)
+        \s*
+        ^(?<front_matter>.*?)                  # front matter (not greedy)
+        \s*
+        ^\k<delim>                             # closing front matter marker
+        \s*
+      }mx
+
+      def call
+        html.sub(PATTERN) do |_match|
+          lang = $~[:lang].presence || DELIM_LANG[$~[:delim]]
+
+          ["```#{lang}", $~[:front_matter], "```", "\n"].join("\n")
+        end
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb
index 328c8c1803b..c70c3f0c04e 100644
--- a/lib/banzai/filter/milestone_reference_filter.rb
+++ b/lib/banzai/filter/milestone_reference_filter.rb
@@ -4,6 +4,8 @@ module Banzai
   module Filter
     # HTML filter that replaces milestone references with links.
     class MilestoneReferenceFilter < AbstractReferenceFilter
+      include Gitlab::Utils::StrongMemoize
+
       self.reference_type = :milestone
 
       def self.object_class
@@ -13,16 +15,34 @@ module Banzai
       # Links to project milestones contain the IID, but when we're handling
       # 'regular' references, we need to use the global ID to disambiguate
       # between group and project milestones.
-      def find_object(project, id)
-        return unless project.is_a?(Project)
+      def find_object(parent, id)
+        return unless valid_context?(parent)
 
-        find_milestone_with_finder(project, id: id)
+        find_milestone_with_finder(parent, id: id)
       end
 
-      def find_object_from_link(project, iid)
-        return unless project.is_a?(Project)
+      def find_object_from_link(parent, iid)
+        return unless valid_context?(parent)
+
+        find_milestone_with_finder(parent, iid: iid)
+      end
+
+      def valid_context?(parent)
+        strong_memoize(:valid_context) do
+          group_context?(parent) || project_context?(parent)
+        end
+      end
+
+      def group_context?(parent)
+        strong_memoize(:group_context) do
+          parent.is_a?(Group)
+        end
+      end
 
-        find_milestone_with_finder(project, iid: iid)
+      def project_context?(parent)
+        strong_memoize(:project_context) do
+          parent.is_a?(Project)
+        end
       end
 
       def references_in(text, pattern = Milestone.reference_pattern)
@@ -44,13 +64,15 @@ module Banzai
 
       def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name)
         project_path = full_project_path(namespace_ref, project_ref)
-        project = parent_from_ref(project_path)
 
-        return unless project && project.is_a?(Project)
+        # Returns group if project is not found by path
+        parent = parent_from_ref(project_path)
+
+        return unless parent
 
         milestone_params = milestone_params(milestone_id, milestone_name)
 
-        find_milestone_with_finder(project, milestone_params)
+        find_milestone_with_finder(parent, milestone_params)
       end
 
       def milestone_params(iid, name)
@@ -61,16 +83,28 @@ module Banzai
         end
       end
 
-      def find_milestone_with_finder(project, params)
-        finder_params = { project_ids: [project.id], order: nil, state: 'all' }
+      def find_milestone_with_finder(parent, params)
+        finder_params = milestone_finder_params(parent, params[:iid].present?)
+
+        MilestonesFinder.new(finder_params).find_by(params)
+      end
 
-        # We don't support IID lookups for group milestones, because IIDs can
-        # clash between group and project milestones.
-        if project.group && !params[:iid]
-          finder_params[:group_ids] = project.group.self_and_ancestors_ids
+      def milestone_finder_params(parent, find_by_iid)
+        { order: nil, state: 'all' }.tap do |params|
+          params[:project_ids] = parent.id if project_context?(parent)
+
+          # We don't support IID lookups because IIDs can clash between
+          # group/project milestones and group/subgroup milestones.
+          params[:group_ids] = self_and_ancestors_ids(parent) unless find_by_iid
         end
+      end
 
-        MilestonesFinder.new(finder_params).find_by(params)
+      def self_and_ancestors_ids(parent)
+        if group_context?(parent)
+          parent.self_and_ancestors_ids
+        elsif project_context?(parent)
+          parent.group&.self_and_ancestors_ids
+        end
       end
 
       def url_for_object(milestone, project)
diff --git a/lib/banzai/filter/yaml_front_matter_filter.rb b/lib/banzai/filter/yaml_front_matter_filter.rb
deleted file mode 100644
index 295964dd75d..00000000000
--- a/lib/banzai/filter/yaml_front_matter_filter.rb
+++ /dev/null
@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
-  module Filter
-    class YamlFrontMatterFilter < HTML::Pipeline::Filter
-      DELIM = '---'.freeze
-
-      # Hat-tip to Middleman: https://git.io/v2e0z
-      PATTERN = %r{
-        \A(?:[^\r\n]*coding:[^\r\n]*\r?\n)?
-        (?<start>#{DELIM})[ ]*\r?\n
-        (?<frontmatter>.*?)[ ]*\r?\n?
-        ^(?<stop>#{DELIM})[ ]*\r?\n?
-        \r?\n?
-        (?<content>.*)
-      }mx.freeze
-
-      def call
-        match = PATTERN.match(html)
-
-        return html unless match
-
-        "```yaml\n#{match['frontmatter']}\n```\n\n#{match['content']}"
-      end
-    end
-  end
-end
diff --git a/lib/banzai/pipeline/pre_process_pipeline.rb b/lib/banzai/pipeline/pre_process_pipeline.rb
index c937f783180..4c2b4ca1665 100644
--- a/lib/banzai/pipeline/pre_process_pipeline.rb
+++ b/lib/banzai/pipeline/pre_process_pipeline.rb
@@ -5,7 +5,7 @@ module Banzai
     class PreProcessPipeline < BasePipeline
       def self.filters
         FilterArray[
-          Filter::YamlFrontMatterFilter,
+          Filter::FrontMatterFilter,
           Filter::BlockquoteFenceFilter,
         ]
       end
diff --git a/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb b/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb
new file mode 100644
index 00000000000..2f76f2f7434
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb
@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Class that will create fill the project_repositories table
+    # for all projects that are on hashed storage and an entry is
+    # is missing in this table.
+    class BackfillHashedProjectRepositories
+      # Shard model
+      class Shard < ActiveRecord::Base
+        self.table_name = 'shards'
+      end
+
+      # Class that will find or create the shard by name.
+      # There is only a small set of shards, which would
+      # not change quickly, so look them up from memory
+      # instead of hitting the DB each time.
+      class ShardFinder
+        def find_shard_id(name)
+          shard_id = shards.fetch(name, nil)
+          return shard_id if shard_id.present?
+
+          Shard.transaction(requires_new: true) do
+            create!(name)
+          end
+        rescue ActiveRecord::RecordNotUnique
+          reload!
+          retry
+        end
+
+        private
+
+        def create!(name)
+          Shard.create!(name: name).tap { |shard| @shards[name] = shard.id }
+        end
+
+        def shards
+          @shards ||= reload!
+        end
+
+        def reload!
+          @shards = Hash[*Shard.all.map { |shard| [shard.name, shard.id] }.flatten]
+        end
+      end
+
+      # ProjectRegistry model
+      class ProjectRepository < ActiveRecord::Base
+        self.table_name = 'project_repositories'
+
+        belongs_to :project, inverse_of: :project_repository
+      end
+
+      # Project model
+      class Project < ActiveRecord::Base
+        self.table_name = 'projects'
+
+        HASHED_PATH_PREFIX = '@hashed'
+
+        HASHED_STORAGE_FEATURES = {
+          repository: 1,
+          attachments: 2
+        }.freeze
+
+        has_one :project_repository, inverse_of: :project
+
+        class << self
+          def on_hashed_storage
+            where(Project.arel_table[:storage_version]
+              .gteq(HASHED_STORAGE_FEATURES[:repository]))
+          end
+
+          def without_project_repository
+            joins(left_outer_join_project_repository)
+              .where(ProjectRepository.arel_table[:project_id].eq(nil))
+          end
+
+          def left_outer_join_project_repository
+            projects_table = Project.arel_table
+            repository_table = ProjectRepository.arel_table
+
+            projects_table
+              .join(repository_table, Arel::Nodes::OuterJoin)
+              .on(projects_table[:id].eq(repository_table[:project_id]))
+              .join_sources
+          end
+        end
+
+        def hashed_storage?
+          self.storage_version && self.storage_version >= 1
+        end
+
+        def hashed_disk_path
+          "#{HASHED_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}/#{disk_hash}"
+        end
+
+        def disk_hash
+          @disk_hash ||= Digest::SHA2.hexdigest(id.to_s)
+        end
+      end
+
+      def perform(start_id, stop_id)
+        Gitlab::Database.bulk_insert(:project_repositories, project_repositories(start_id, stop_id))
+      end
+
+      private
+
+      def project_repositories(start_id, stop_id)
+        Project.on_hashed_storage
+          .without_project_repository
+          .where(id: start_id..stop_id)
+          .map { |project| build_attributes_for_project(project) }
+          .compact
+      end
+
+      def build_attributes_for_project(project)
+        return unless project.hashed_storage?
+
+        {
+          project_id: project.id,
+          shard_id:   find_shard_id(project.repository_storage),
+          disk_path:  project.hashed_disk_path
+        }
+      end
+
+      def find_shard_id(repository_storage)
+        shard_finder.find_shard_id(repository_storage)
+      end
+
+      def shard_finder
+        @shard_finder ||= ShardFinder.new
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/branch_push_merge_commit_analyzer.rb b/lib/gitlab/branch_push_merge_commit_analyzer.rb
new file mode 100644
index 00000000000..a8f601f2451
--- /dev/null
+++ b/lib/gitlab/branch_push_merge_commit_analyzer.rb
@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module Gitlab
+  # Analyse a graph of commits from a push to a branch,
+  # for each commit, analyze that if it is the head of a merge request,
+  # then what should its merge_commit be, relative to the branch.
+  #
+  # A----->B----->C----->D   target branch
+  # |             ^
+  # |             |
+  # +-->E----->F--+          merged branch
+  #     |     ^
+  #     |     |
+  #     +->G--+
+  #
+  # (See merge-commit-analyze-after branch in gitlab-test)
+  #
+  # Assuming
+  # - A is already in remote
+  # - B~D are all in its own branch with its own merge request, targeting the target branch
+  #
+  # When D is finally pushed to the target branch,
+  # what are the merge commits for all the other merge requests?
+  #
+  # We can walk backwards from the HEAD commit D,
+  # and find status of its parents.
+  # First we determine if commit belongs to the target branch (i.e. A, B, C, D),
+  # and then determine its merge commit.
+  #
+  # +--------+-----------------+--------------+
+  # | Commit | Direct ancestor | Merge commit |
+  # +--------+-----------------+--------------+
+  # | D      | Y               | D            |
+  # +--------+-----------------+--------------+
+  # | C      | Y               | C            |
+  # +--------+-----------------+--------------+
+  # | F      |                 | C            |
+  # +--------+-----------------+--------------+
+  # | B      | Y               | B            |
+  # +--------+-----------------+--------------+
+  # | E      |                 | C            |
+  # +--------+-----------------+--------------+
+  # | G      |                 | C            |
+  # +--------+-----------------+--------------+
+  #
+  # By examining the result, it can be said that
+  #
+  # - If commit is direct ancestor of HEAD, its merge commit is itself.
+  # - Otherwise, the merge commit is the same as its child's merge commit.
+  #
+  class BranchPushMergeCommitAnalyzer
+    class CommitDecorator < SimpleDelegator
+      attr_accessor :merge_commit
+      attr_writer :direct_ancestor # boolean
+
+      def direct_ancestor?
+        @direct_ancestor
+      end
+
+      # @param child_commit [CommitDecorator]
+      # @param first_parent [Boolean] whether `self` is the first parent of `child_commit`
+      def set_merge_commit(child_commit:)
+        @merge_commit ||= direct_ancestor? ? self : child_commit.merge_commit
+      end
+    end
+
+    # @param commits [Array] list of commits, must be ordered from the child (tip) of the graph back to the ancestors
+    def initialize(commits, relevant_commit_ids: nil)
+      @commits = commits
+      @id_to_commit = {}
+      @commits.each do |commit|
+        @id_to_commit[commit.id] = CommitDecorator.new(commit)
+
+        if relevant_commit_ids
+          relevant_commit_ids.delete(commit.id)
+          break if relevant_commit_ids.empty? # Only limit the analyze up to relevant_commit_ids
+        end
+      end
+
+      analyze
+    end
+
+    def get_merge_commit(id)
+      get_commit(id).merge_commit.id
+    end
+
+    private
+
+    def analyze
+      head_commit = get_commit(@commits.first.id)
+      head_commit.direct_ancestor = true
+      head_commit.merge_commit = head_commit
+
+      mark_all_direct_ancestors(head_commit)
+
+      # Analyzing a commit requires its child commit be analyzed first,
+      # which is the case here since commits are ordered from child to parent.
+      @id_to_commit.each_value do |commit|
+        analyze_parents(commit)
+      end
+    end
+
+    def analyze_parents(commit)
+      commit.parent_ids.each do |parent_commit_id|
+        parent_commit = get_commit(parent_commit_id)
+
+        next unless parent_commit # parent commit may not be part of new commits
+
+        parent_commit.set_merge_commit(child_commit: commit)
+      end
+    end
+
+    # Mark all direct ancestors.
+    # If child commit is a direct ancestor, its first parent is also a direct ancestor.
+    # We assume direct ancestors matches the trail of the target branch over time,
+    # This assumption is correct most of the time, especially for gitlab managed merges,
+    # but there are exception cases which can't be solved (https://stackoverflow.com/a/49754723/474597)
+    def mark_all_direct_ancestors(commit)
+      loop do
+        commit = get_commit(commit.parent_ids.first)
+
+        break unless commit
+
+        commit.direct_ancestor = true
+      end
+    end
+
+    def get_commit(id)
+      @id_to_commit[id]
+    end
+  end
+end
diff --git a/lib/gitlab/correlation_id.rb b/lib/gitlab/correlation_id.rb
new file mode 100644
index 00000000000..0f9bde4390e
--- /dev/null
+++ b/lib/gitlab/correlation_id.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CorrelationId
+    LOG_KEY = 'correlation_id'.freeze
+
+    class << self
+      def use_id(correlation_id, &blk)
+        # always generate a id if null is passed
+        correlation_id ||= new_id
+
+        ids.push(correlation_id || new_id)
+
+        begin
+          yield(current_id)
+        ensure
+          ids.pop
+        end
+      end
+
+      def current_id
+        ids.last
+      end
+
+      def current_or_new_id
+        current_id || new_id
+      end
+
+      private
+
+      def ids
+        Thread.current[:correlation_id] ||= []
+      end
+
+      def new_id
+        SecureRandom.uuid
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 134d1e7a724..d9578852db6 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -975,9 +975,10 @@ into similar problems in the future (e.g. when new tables are created).
         raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id')
 
         jobs = []
+        table_name = model_class.quoted_table_name
 
         model_class.each_batch(of: batch_size) do |relation|
-          start_id, end_id = relation.pluck('MIN(id), MAX(id)').first
+          start_id, end_id = relation.pluck("MIN(#{table_name}.id), MAX(#{table_name}.id)").first
 
           if jobs.length >= BACKGROUND_MIGRATION_JOB_BUFFER_SIZE
             # Note: This code path generally only helps with many millions of rows
diff --git a/lib/gitlab/git/object_pool.rb b/lib/gitlab/git/object_pool.rb
new file mode 100644
index 00000000000..558699a6318
--- /dev/null
+++ b/lib/gitlab/git/object_pool.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Git
+    class ObjectPool
+      # GL_REPOSITORY has to be passed for Gitlab::Git::Repositories, but not
+      # used for ObjectPools.
+      GL_REPOSITORY = ""
+
+      delegate :exists?, :size, to: :repository
+      delegate :delete, to: :object_pool_service
+
+      attr_reader :storage, :relative_path, :source_repository
+
+      def initialize(storage, relative_path, source_repository)
+        @storage = storage
+        @relative_path = relative_path
+        @source_repository = source_repository
+      end
+
+      def create
+        object_pool_service.create(source_repository)
+      end
+
+      def link(to_link_repo)
+        remote_name = to_link_repo.object_pool_remote_name
+        repository.set_config(
+          "remote.#{remote_name}.url" => relative_path_to(to_link_repo.relative_path),
+          "remote.#{remote_name}.tagOpt" => "--no-tags",
+          "remote.#{remote_name}.fetch" => "+refs/*:refs/remotes/#{remote_name}/*"
+        )
+
+        object_pool_service.link_repository(to_link_repo)
+      end
+
+      def gitaly_object_pool
+        Gitaly::ObjectPool.new(repository: to_gitaly_repository)
+      end
+
+      def to_gitaly_repository
+        Gitlab::GitalyClient::Util.repository(storage, relative_path, GL_REPOSITORY)
+      end
+
+      # Allows for reusing other RPCs by 'tricking' Gitaly to think its a repository
+      def repository
+        @repository ||= Gitlab::Git::Repository.new(storage, relative_path, GL_REPOSITORY)
+      end
+
+      private
+
+      def object_pool_service
+        @object_pool_service ||= Gitlab::GitalyClient::ObjectPoolService.new(self)
+      end
+
+      def relative_path_to(pool_member_path)
+        pool_path = Pathname.new("#{relative_path}#{File::SEPARATOR}")
+
+        Pathname.new(pool_member_path).relative_path_from(pool_path).to_s
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 0a541031884..5bbedc9d5e3 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -69,6 +69,13 @@ module Gitlab
 
       attr_reader :storage, :gl_repository, :relative_path
 
+      # This remote name has to be stable for all types of repositories that
+      # can join an object pool. If it's structure ever changes, a migration
+      # has to be performed on the object pools to update the remote names.
+      # Else the pool can't be updated anymore and is left in an inconsistent
+      # state.
+      alias_method :object_pool_remote_name, :gl_repository
+
       # This initializer method is only used on the client side (gitlab-ce).
       # Gitaly-ruby uses a different initializer.
       def initialize(storage, relative_path, gl_repository)
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 9be553a8b86..255601382b1 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -193,6 +193,7 @@ module Gitlab
       feature = feature_stack && feature_stack[0]
       metadata['call_site'] = feature.to_s if feature
       metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage
+      metadata['correlation_id'] = Gitlab::CorrelationId.current_id if Gitlab::CorrelationId.current_id
 
       metadata.merge!(server_feature_flags)
 
diff --git a/lib/gitlab/gitaly_client/object_pool_service.rb b/lib/gitlab/gitaly_client/object_pool_service.rb
new file mode 100644
index 00000000000..272ce73ad64
--- /dev/null
+++ b/lib/gitlab/gitaly_client/object_pool_service.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GitalyClient
+    class ObjectPoolService
+      attr_reader :object_pool, :storage
+
+      def initialize(object_pool)
+        @object_pool = object_pool.gitaly_object_pool
+        @storage = object_pool.storage
+      end
+
+      def create(repository)
+        request = Gitaly::CreateObjectPoolRequest.new(
+          object_pool: object_pool,
+          origin: repository.gitaly_repository)
+
+        GitalyClient.call(storage, :object_pool_service, :create_object_pool, request)
+      end
+
+      def delete
+        request = Gitaly::DeleteObjectPoolRequest.new(object_pool: object_pool)
+
+        GitalyClient.call(storage, :object_pool_service, :delete_object_pool, request)
+      end
+
+      def link_repository(repository)
+        request = Gitaly::LinkRepositoryToObjectPoolRequest.new(
+          object_pool: object_pool,
+          repository: repository.gitaly_repository
+        )
+
+        GitalyClient.call(storage, :object_pool_service, :link_repository_to_object_pool,
+                          request, timeout: GitalyClient.fast_timeout)
+      end
+
+      def unlink_repository(repository)
+        request = Gitaly::UnlinkRepositoryFromObjectPoolRequest.new(repository: repository.gitaly_repository)
+
+        GitalyClient.call(storage, :object_pool_service, :unlink_repository_from_object_pool,
+                          request, timeout: GitalyClient.fast_timeout)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb b/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb
new file mode 100644
index 00000000000..fa4c5d86d44
--- /dev/null
+++ b/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# This module adds additional correlation id the grape logger
+module Gitlab
+  module GrapeLogging
+    module Loggers
+      class CorrelationIdLogger < ::GrapeLogging::Loggers::Base
+        def parameters(_, _)
+          { Gitlab::CorrelationId::LOG_KEY => Gitlab::CorrelationId.current_id }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index fde8561c16c..d10d4f2f746 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -143,6 +143,7 @@ excluded_attributes:
   statuses:
     - :trace
     - :token
+    - :token_encrypted
     - :when
     - :artifacts_file
     - :artifacts_metadata
diff --git a/lib/gitlab/json_logger.rb b/lib/gitlab/json_logger.rb
index 3bff77731f6..a5a5759cc89 100644
--- a/lib/gitlab/json_logger.rb
+++ b/lib/gitlab/json_logger.rb
@@ -10,6 +10,7 @@ module Gitlab
       data = {}
       data[:severity] = severity
       data[:time] = timestamp.utc.iso8601(3)
+      data[Gitlab::CorrelationId::LOG_KEY] = Gitlab::CorrelationId.current_id
 
       case message
       when String
diff --git a/lib/gitlab/middleware/correlation_id.rb b/lib/gitlab/middleware/correlation_id.rb
new file mode 100644
index 00000000000..73542dd422e
--- /dev/null
+++ b/lib/gitlab/middleware/correlation_id.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# A dumb middleware that steals correlation id
+# and sets it as a global context for the request
+module Gitlab
+  module Middleware
+    class CorrelationId
+      include ActionView::Helpers::TagHelper
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        ::Gitlab::CorrelationId.use_id(correlation_id(env)) do
+          @app.call(env)
+        end
+      end
+
+      private
+
+      def correlation_id(env)
+        if Gitlab.rails5?
+          request(env).request_id
+        else
+          request(env).uuid
+        end
+      end
+
+      def request(env)
+        ActionDispatch::Request.new(env)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb
index 8079c5882c4..46d01964eac 100644
--- a/lib/gitlab/sentry.rb
+++ b/lib/gitlab/sentry.rb
@@ -3,7 +3,8 @@
 module Gitlab
   module Sentry
     def self.enabled?
-      Rails.env.production? && Gitlab::CurrentSettings.sentry_enabled?
+      (Rails.env.production? || Rails.env.development?) &&
+        Gitlab::CurrentSettings.sentry_enabled?
     end
 
     def self.context(current_user = nil)
@@ -31,7 +32,7 @@ module Gitlab
     def self.track_exception(exception, issue_url: nil, extra: {})
       track_acceptable_exception(exception, issue_url: issue_url, extra: extra)
 
-      raise exception if should_raise?
+      raise exception if should_raise_for_dev?
     end
 
     # This should be used when you do not want to raise an exception in
@@ -43,7 +44,11 @@ module Gitlab
         extra[:issue_url] = issue_url if issue_url
         context # Make sure we've set everything we know in the context
 
-        Raven.capture_exception(exception, extra: extra)
+        tags = {
+          Gitlab::CorrelationId::LOG_KEY.to_sym => Gitlab::CorrelationId.current_id
+        }
+
+        Raven.capture_exception(exception, tags: tags, extra: extra)
       end
     end
 
@@ -55,7 +60,7 @@ module Gitlab
       end
     end
 
-    def self.should_raise?
+    def self.should_raise_for_dev?
       Rails.env.development? || Rails.env.test?
     end
   end
diff --git a/lib/gitlab/sidekiq_middleware/correlation_injector.rb b/lib/gitlab/sidekiq_middleware/correlation_injector.rb
new file mode 100644
index 00000000000..b807b3a03ed
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/correlation_injector.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqMiddleware
+    class CorrelationInjector
+      def call(worker_class, job, queue, redis_pool)
+        job[Gitlab::CorrelationId::LOG_KEY] ||=
+          Gitlab::CorrelationId.current_or_new_id
+
+        yield
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_middleware/correlation_logger.rb b/lib/gitlab/sidekiq_middleware/correlation_logger.rb
new file mode 100644
index 00000000000..cb8ff4a6284
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/correlation_logger.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqMiddleware
+    class CorrelationLogger
+      def call(worker, job, queue)
+        correlation_id = job[Gitlab::CorrelationId::LOG_KEY]
+
+        Gitlab::CorrelationId.use_id(correlation_id) do
+          yield
+        end
+      end
+    end
+  end
+end