Merge branch 'master' into backstage/gb/rename-ci-cd-processing-sidekiq-queues

* master: (115 commits)
  Use event-based waiting in Gitlab::JobWaiter
  Make sure repository's removal work for legacy and hashed storages
  Use `@hashed` prefix for hashed paths on disk, to avoid collision with existing ones
  Refactor project and storage types
  Prevent using gitlab import task when hashed storage is enabled
  Some codestyle changes and fixes for GitLab pages
  Removed some useless code, codestyle changes and removed an index
  Fix repository reloading in some specs
  Changelog
  Moving away from the "extend" based factory to a more traditional one.
  Enable automatic hashed storage for new projects by application settings
  New storage is now "Hashed" instead of "UUID"
  Add UUID Storage to Project
  Move create_repository back to project model as we can use disk_path and share it
  Codestyle: move hooks to the same place and move dependent methods to private
  Use non-i18n values for setting new group-level issue/MR button text
  indexes external issue tracker
  copyedit
  indexes user/search/ from /user/index
  Correctly encode string params for Gitaly's TreeEntries RPC
  ...

14 files changed, 149 insertions, 564 deletions

diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index 88821ae56e0..4e92be85110 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -75,7 +75,7 @@ module Backup
         path_to_project_repo = path_to_repo(project)
         path_to_project_bundle = path_to_bundle(project)
 
-        project.ensure_storage_path_exist
+        project.ensure_storage_path_exists
 
         cmd = if File.exist?(path_to_project_bundle)
                 %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo})
diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb
deleted file mode 100644
index 24bb3649a76..00000000000
--- a/lib/ci/api/api.rb
+++ /dev/null
@@ -1,39 +0,0 @@
-module Ci
-  module API
-    class API < Grape::API
-      include ::API::APIGuard
-      version 'v1', using: :path
-
-      rescue_from ActiveRecord::RecordNotFound do
-        rack_response({ 'message' => '404 Not found' }.to_json, 404)
-      end
-
-      # Retain 405 error rather than a 500 error for Grape 0.15.0+.
-      # https://github.com/ruby-grape/grape/blob/a3a28f5b5dfbb2797442e006dbffd750b27f2a76/UPGRADING.md#changes-to-method-not-allowed-routes
-      rescue_from Grape::Exceptions::MethodNotAllowed do |e|
-        error! e.message, e.status, e.headers
-      end
-
-      rescue_from Grape::Exceptions::Base do |e|
-        error! e.message, e.status, e.headers
-      end
-
-      rescue_from :all do |exception|
-        handle_api_exception(exception)
-      end
-
-      content_type :txt,  'text/plain'
-      content_type :json, 'application/json'
-      format :json
-
-      helpers ::SentryHelper
-      helpers ::Ci::API::Helpers
-      helpers ::API::Helpers
-      helpers Gitlab::CurrentSettings
-
-      mount ::Ci::API::Builds
-      mount ::Ci::API::Runners
-      mount ::Ci::API::Triggers
-    end
-  end
-end
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
deleted file mode 100644
index 79058c02ce5..00000000000
--- a/lib/ci/api/builds.rb
+++ /dev/null
@@ -1,219 +0,0 @@
-module Ci
-  module API
-    # Builds API
-    class Builds < Grape::API
-      resource :builds do
-        # Runs oldest pending build by runner - Runners only
-        #
-        # Parameters:
-        #   token (required) - The uniq token of runner
-        #
-        # Example Request:
-        #   POST /builds/register
-        post "register" do
-          authenticate_runner!
-          required_attributes! [:token]
-          not_found! unless current_runner.active?
-          update_runner_info
-
-          if current_runner.is_runner_queue_value_latest?(params[:last_update])
-            header 'X-GitLab-Last-Update', params[:last_update]
-            Gitlab::Metrics.add_event(:build_not_found_cached)
-            return build_not_found!
-          end
-
-          new_update = current_runner.ensure_runner_queue_value
-
-          result = Ci::RegisterJobService.new(current_runner).execute
-
-          if result.valid?
-            if result.build
-              Gitlab::Metrics.add_event(:build_found,
-                                        project: result.build.project.full_path)
-
-              present result.build, with: Entities::BuildDetails
-            else
-              Gitlab::Metrics.add_event(:build_not_found)
-
-              header 'X-GitLab-Last-Update', new_update
-
-              build_not_found!
-            end
-          else
-            # We received build that is invalid due to concurrency conflict
-            Gitlab::Metrics.add_event(:build_invalid)
-            conflict!
-          end
-        end
-
-        # Update an existing build - Runners only
-        #
-        # Parameters:
-        #   id (required) - The ID of a project
-        #   state (optional) - The state of a build
-        #   trace (optional) - The trace of a build
-        # Example Request:
-        #   PUT /builds/:id
-        put ":id" do
-          authenticate_runner!
-          build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id])
-          validate_build!(build)
-
-          update_runner_info
-
-          build.trace.set(params[:trace]) if params[:trace]
-
-          Gitlab::Metrics.add_event(:update_build,
-                                    project: build.project.full_path)
-
-          case params[:state].to_s
-          when 'success'
-            build.success
-          when 'failed'
-            build.drop
-          end
-        end
-
-        # Send incremental log update - Runners only
-        #
-        # Parameters:
-        #   id (required) - The ID of a build
-        # Body:
-        #   content of logs to append
-        # Headers:
-        #   Content-Range (required) - range of content that was sent
-        #   BUILD-TOKEN (required) - The build authorization token
-        # Example Request:
-        #   PATCH /builds/:id/trace.txt
-        patch ":id/trace.txt" do
-          build = authenticate_build!
-
-          error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range')
-          content_range = request.headers['Content-Range']
-          content_range = content_range.split('-')
-
-          stream_size = build.trace.append(request.body.read, content_range[0].to_i)
-          if stream_size < 0
-            return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" })
-          end
-
-          status 202
-          header 'Build-Status', build.status
-          header 'Range', "0-#{stream_size}"
-        end
-
-        # Authorize artifacts uploading for build - Runners only
-        #
-        # Parameters:
-        #   id (required) - The ID of a build
-        #   token (required) - The build authorization token
-        #   filesize (optional) - the size of uploaded file
-        # Example Request:
-        #   POST /builds/:id/artifacts/authorize
-        post ":id/artifacts/authorize" do
-          require_gitlab_workhorse!
-          Gitlab::Workhorse.verify_api_request!(headers)
-          not_allowed! unless Gitlab.config.artifacts.enabled
-          build = authenticate_build!
-          forbidden!('build is not running') unless build.running?
-
-          if params[:filesize]
-            file_size = params[:filesize].to_i
-            file_to_large! unless file_size < max_artifacts_size
-          end
-
-          status 200
-          content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
-          Gitlab::Workhorse.artifact_upload_ok
-        end
-
-        # Upload artifacts to build - Runners only
-        #
-        # Parameters:
-        #   id (required) - The ID of a build
-        #   token (required) - The build authorization token
-        #   file (required) - Artifacts file
-        #   expire_in (optional) - Specify when artifacts should expire (ex. 7d)
-        # Parameters (accelerated by GitLab Workhorse):
-        #   file.path - path to locally stored body (generated by Workhorse)
-        #   file.name - real filename as send in Content-Disposition
-        #   file.type - real content type as send in Content-Type
-        #   metadata.path - path to locally stored body (generated by Workhorse)
-        #   metadata.name - filename (generated by Workhorse)
-        # Headers:
-        #   BUILD-TOKEN (required) - The build authorization token, the same as token
-        # Body:
-        #   The file content
-        #
-        # Example Request:
-        #   POST /builds/:id/artifacts
-        post ":id/artifacts" do
-          require_gitlab_workhorse!
-          not_allowed! unless Gitlab.config.artifacts.enabled
-          build = authenticate_build!
-          forbidden!('Build is not running!') unless build.running?
-
-          artifacts_upload_path = ArtifactUploader.artifacts_upload_path
-          artifacts = uploaded_file(:file, artifacts_upload_path)
-          metadata = uploaded_file(:metadata, artifacts_upload_path)
-
-          bad_request!('Missing artifacts file!') unless artifacts
-          file_to_large! unless artifacts.size < max_artifacts_size
-
-          build.artifacts_file = artifacts
-          build.artifacts_metadata = metadata
-          build.artifacts_expire_in =
-            params['expire_in'] ||
-            Gitlab::CurrentSettings.current_application_settings
-              .default_artifacts_expire_in
-
-          if build.save
-            present(build, with: Entities::BuildDetails)
-          else
-            render_validation_error!(build)
-          end
-        end
-
-        # Download the artifacts file from build - Runners only
-        #
-        # Parameters:
-        #   id (required) - The ID of a build
-        #   token (required) - The build authorization token
-        # Headers:
-        #   BUILD-TOKEN (required) - The build authorization token, the same as token
-        # Example Request:
-        #   GET /builds/:id/artifacts
-        get ":id/artifacts" do
-          build = authenticate_build!
-          artifacts_file = build.artifacts_file
-
-          unless artifacts_file.exists?
-            not_found!
-          end
-
-          unless artifacts_file.file_storage?
-            return redirect_to build.artifacts_file.url
-          end
-
-          present_file!(artifacts_file.path, artifacts_file.filename)
-        end
-
-        # Remove the artifacts file from build - Runners only
-        #
-        # Parameters:
-        #   id (required) - The ID of a build
-        #   token (required) - The build authorization token
-        # Headers:
-        #   BUILD-TOKEN (required) - The build authorization token, the same as token
-        # Example Request:
-        #   DELETE /builds/:id/artifacts
-        delete ":id/artifacts" do
-          build = authenticate_build!
-
-          status(200)
-          build.erase_artifacts!
-        end
-      end
-    end
-  end
-end
diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb
deleted file mode 100644
index 31f66dd5a58..00000000000
--- a/lib/ci/api/entities.rb
+++ /dev/null
@@ -1,93 +0,0 @@
-module Ci
-  module API
-    module Entities
-      class Commit < Grape::Entity
-        expose :id, :sha, :project_id, :created_at
-        expose :status, :finished_at, :duration
-        expose :git_commit_message, :git_author_name, :git_author_email
-      end
-
-      class CommitWithBuilds < Commit
-        expose :builds
-      end
-
-      class ArtifactFile < Grape::Entity
-        expose :filename, :size
-      end
-
-      class BuildOptions < Grape::Entity
-        expose :image
-        expose :services
-        expose :artifacts
-        expose :cache
-        expose :dependencies
-        expose :after_script
-      end
-
-      class Build < Grape::Entity
-        expose :id, :ref, :tag, :sha, :status
-        expose :name, :token, :stage
-        expose :project_id
-        expose :project_name
-        expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? }
-      end
-
-      class BuildCredentials < Grape::Entity
-        expose :type, :url, :username, :password
-      end
-
-      class BuildDetails < Build
-        expose :commands
-        expose :repo_url
-        expose :before_sha
-        expose :allow_git_fetch
-        expose :token
-        expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? }
-
-        expose :options do |model|
-          # This part ensures that output of old API is still the same after adding support
-          # for extended docker configuration options, used by new API
-          #
-          # I'm leaving this here, not in the model, because it should be removed at the same time
-          # when old API will be removed (planned for August 2017).
-          model.options.dup.tap do |options|
-            options[:image] = options[:image][:name] if options[:image].is_a?(Hash)
-            options[:services]&.map! do |service|
-              if service.is_a?(Hash)
-                service[:name]
-              else
-                service
-              end
-            end
-          end
-        end
-
-        expose :timeout do |model|
-          model.timeout
-        end
-
-        expose :variables
-        expose :depends_on_builds, using: Build
-
-        expose :credentials, using: BuildCredentials
-      end
-
-      class Runner < Grape::Entity
-        expose :id, :token
-      end
-
-      class RunnerProject < Grape::Entity
-        expose :id, :project_id, :runner_id
-      end
-
-      class WebHook < Grape::Entity
-        expose :id, :project_id, :url
-      end
-
-      class TriggerRequest < Grape::Entity
-        expose :id, :variables
-        expose :pipeline, using: Commit, as: :commit
-      end
-    end
-  end
-end
diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb
deleted file mode 100644
index a40b6ab6c9f..00000000000
--- a/lib/ci/api/helpers.rb
+++ /dev/null
@@ -1,89 +0,0 @@
-module Ci
-  module API
-    module Helpers
-      BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN".freeze
-      BUILD_TOKEN_PARAM = :token
-      UPDATE_RUNNER_EVERY = 10 * 60
-
-      def authenticate_runners!
-        forbidden! unless runner_registration_token_valid?
-      end
-
-      def authenticate_runner!
-        forbidden! unless current_runner
-      end
-
-      def authenticate_build!
-        build = Ci::Build.find_by_id(params[:id])
-
-        validate_build!(build) do
-          forbidden! unless build_token_valid?(build)
-        end
-
-        build
-      end
-
-      def validate_build!(build)
-        not_found! unless build
-
-        yield if block_given?
-
-        project = build.project
-        forbidden!('Project has been deleted!') if project.nil? || project.pending_delete?
-        forbidden!('Build has been erased!') if build.erased?
-      end
-
-      def runner_registration_token_valid?
-        ActiveSupport::SecurityUtils.variable_size_secure_compare(
-          params[:token],
-          current_application_settings.runners_registration_token)
-      end
-
-      def build_token_valid?(build)
-        token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s
-
-        # We require to also check `runners_token` to maintain compatibility with old version of runners
-        token && (build.valid_token?(token) || build.project.valid_runners_token?(token))
-      end
-
-      def update_runner_info
-        return unless update_runner?
-
-        current_runner.contacted_at = Time.now
-        current_runner.assign_attributes(get_runner_version_from_params)
-        current_runner.save if current_runner.changed?
-      end
-
-      def update_runner?
-        # Use a random threshold to prevent beating DB updates.
-        # It generates a distribution between [40m, 80m].
-        #
-        contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
-
-        current_runner.contacted_at.nil? ||
-          (Time.now - current_runner.contacted_at) >= contacted_at_max_age
-      end
-
-      def build_not_found!
-        if headers['User-Agent'].to_s =~ /gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /
-          no_content!
-        else
-          not_found!
-        end
-      end
-
-      def current_runner
-        @runner ||= Runner.find_by_token(params[:token].to_s)
-      end
-
-      def get_runner_version_from_params
-        return unless params["info"].present?
-        attributes_for_keys(%w(name version revision platform architecture), params["info"])
-      end
-
-      def max_artifacts_size
-        current_application_settings.max_artifacts_size.megabytes.to_i
-      end
-    end
-  end
-end
diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb
deleted file mode 100644
index 45aa2adccf5..00000000000
--- a/lib/ci/api/runners.rb
+++ /dev/null
@@ -1,50 +0,0 @@
-module Ci
-  module API
-    class Runners < Grape::API
-      resource :runners do
-        desc 'Delete a runner'
-        params do
-          requires :token, type: String, desc: 'The unique token of the runner'
-        end
-        delete "delete" do
-          authenticate_runner!
-
-          status(200)
-          Ci::Runner.find_by_token(params[:token]).destroy
-        end
-
-        desc 'Register a new runner' do
-          success Entities::Runner
-        end
-        params do
-          requires :token, type: String, desc: 'The unique token of the runner'
-          optional :description, type: String, desc: 'The description of the runner'
-          optional :tag_list, type: Array[String], desc: 'A list of tags the runner should run for'
-          optional :run_untagged, type: Boolean, desc: 'Flag if the runner should execute untagged jobs'
-          optional :locked, type: Boolean, desc: 'Lock this runner for this specific project'
-        end
-        post "register" do
-          runner_params = declared(params, include_missing: false).except(:token)
-
-          runner =
-            if runner_registration_token_valid?
-              # Create shared runner. Requires admin access
-              Ci::Runner.create(runner_params.merge(is_shared: true))
-            elsif project = Project.find_by(runners_token: params[:token])
-              # Create a specific runner for project.
-              project.runners.create(runner_params)
-            end
-
-          return forbidden! unless runner
-
-          if runner.id
-            runner.update(get_runner_version_from_params)
-            present runner, with: Entities::Runner
-          else
-            not_found!
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/ci/api/triggers.rb b/lib/ci/api/triggers.rb
deleted file mode 100644
index 6225203f223..00000000000
--- a/lib/ci/api/triggers.rb
+++ /dev/null
@@ -1,39 +0,0 @@
-module Ci
-  module API
-    class Triggers < Grape::API
-      resource :projects do
-        desc 'Trigger a GitLab CI project build' do
-          success Entities::TriggerRequest
-        end
-        params do
-          requires :id, type: Integer, desc: 'The ID of a CI project'
-          requires :ref, type: String, desc: "The name of project's branch or tag"
-          requires :token, type: String, desc: 'The unique token of the trigger'
-          optional :variables, type: Hash, desc: 'Optional build variables'
-        end
-        post ":id/refs/:ref/trigger" do
-          project = Project.find_by(ci_id: params[:id])
-          trigger = Ci::Trigger.find_by_token(params[:token])
-          not_found! unless project && trigger
-          unauthorized! unless trigger.project == project
-
-          # Validate variables
-          variables = params[:variables].to_h
-          unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) }
-            render_api_error!('variables needs to be a map of key-valued strings', 400)
-          end
-
-          # create request and trigger builds
-          result = Ci::CreateTriggerRequestService.execute(project, trigger, params[:ref], variables)
-          pipeline = result.pipeline
-
-          if pipeline.persisted?
-            present result.trigger_request, with: Entities::TriggerRequest
-          else
-            render_validation_error!(pipeline)
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/background_migration/migrate_stage_status.rb b/lib/gitlab/background_migration/migrate_stage_status.rb
new file mode 100644
index 00000000000..b1ff0900709
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_stage_status.rb
@@ -0,0 +1,77 @@
+module Gitlab
+  module BackgroundMigration
+    class MigrateStageStatus
+      STATUSES = { created: 0, pending: 1, running: 2, success: 3,
+                   failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze
+
+      class Build < ActiveRecord::Base
+        self.table_name = 'ci_builds'
+
+        scope :latest, -> { where(retried: [false, nil]) }
+        scope :created, -> { where(status: 'created') }
+        scope :running, -> { where(status: 'running') }
+        scope :pending, -> { where(status: 'pending') }
+        scope :success, -> { where(status: 'success') }
+        scope :failed, -> { where(status: 'failed')  }
+        scope :canceled, -> { where(status: 'canceled')  }
+        scope :skipped, -> { where(status: 'skipped')  }
+        scope :manual, -> { where(status: 'manual')  }
+
+        scope :failed_but_allowed, -> do
+          where(allow_failure: true, status: [:failed, :canceled])
+        end
+
+        scope :exclude_ignored, -> do
+          where("allow_failure = ? OR status IN (?)",
+            false, %w[created pending running success skipped])
+        end
+
+        def self.status_sql
+          scope_relevant = latest.exclude_ignored
+          scope_warnings = latest.failed_but_allowed
+
+          builds = scope_relevant.select('count(*)').to_sql
+          created = scope_relevant.created.select('count(*)').to_sql
+          success = scope_relevant.success.select('count(*)').to_sql
+          manual = scope_relevant.manual.select('count(*)').to_sql
+          pending = scope_relevant.pending.select('count(*)').to_sql
+          running = scope_relevant.running.select('count(*)').to_sql
+          skipped = scope_relevant.skipped.select('count(*)').to_sql
+          canceled = scope_relevant.canceled.select('count(*)').to_sql
+          warnings = scope_warnings.select('count(*) > 0').to_sql
+
+          <<-SQL.strip_heredoc
+            (CASE
+              WHEN (#{builds}) = (#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]}
+              WHEN (#{builds}) = (#{skipped}) THEN #{STATUSES[:skipped]}
+              WHEN (#{builds}) = (#{success}) THEN #{STATUSES[:success]}
+              WHEN (#{builds}) = (#{created}) THEN #{STATUSES[:created]}
+              WHEN (#{builds}) = (#{success}) + (#{skipped}) THEN #{STATUSES[:success]}
+              WHEN (#{builds}) = (#{success}) + (#{skipped}) + (#{canceled}) THEN #{STATUSES[:canceled]}
+              WHEN (#{builds}) = (#{created}) + (#{skipped}) + (#{pending}) THEN #{STATUSES[:pending]}
+              WHEN (#{running}) + (#{pending}) > 0 THEN #{STATUSES[:running]}
+              WHEN (#{manual}) > 0 THEN #{STATUSES[:manual]}
+              WHEN (#{created}) > 0 THEN #{STATUSES[:running]}
+              ELSE #{STATUSES[:failed]}
+            END)
+          SQL
+        end
+      end
+
+      def perform(start_id, stop_id)
+        status_sql = Build
+          .where('ci_builds.commit_id = ci_stages.pipeline_id')
+          .where('ci_builds.stage = ci_stages.name')
+          .status_sql
+
+        sql = <<-SQL
+          UPDATE ci_stages SET status = (#{status_sql})
+            WHERE ci_stages.status IS NULL
+            AND ci_stages.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i}
+        SQL
+
+        ActiveRecord::Base.connection.execute(sql)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 53aa5b12489..eb3731ba35a 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -64,7 +64,6 @@ module Gitlab
       end
 
       delegate  :empty?,
-                :bare?,
                 to: :rugged
 
       delegate :exists?, to: :gitaly_repository_client
@@ -126,6 +125,8 @@ module Gitlab
       # This is to work around a bug in libgit2 that causes in-memory refs to
       # be stale/invalid when packed-refs is changed.
       # See https://gitlab.com/gitlab-org/gitlab-ce/issues/15392#note_14538333
+      #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/474
       def find_branch(name, force_reload = false)
         reload_rugged if force_reload
 
@@ -231,10 +232,6 @@ module Gitlab
         branch_names + tag_names
       end
 
-      def has_commits?
-        !empty?
-      end
-
       # Discovers the default branch based on the repository's available branches
       #
       # - If no branches are present, returns nil
@@ -574,6 +571,8 @@ module Gitlab
       end
 
       # Delete the specified branch from the repository
+      #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476
       def delete_branch(branch_name)
         rugged.branches.delete(branch_name)
       end
@@ -583,6 +582,8 @@ module Gitlab
       # Examples:
       #   create_branch("feature")
       #   create_branch("other-feature", "master")
+      #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476
       def create_branch(ref, start_point = "HEAD")
         rugged_ref = rugged.branches.create(ref, start_point)
         target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
@@ -592,38 +593,26 @@ module Gitlab
         raise InvalidRef.new("Invalid reference #{start_point}")
       end
 
-      # Return an array of this repository's remote names
-      def remote_names
-        rugged.remotes.each_name.to_a
-      end
-
       # Delete the specified remote from this repository.
       def remote_delete(remote_name)
         rugged.remotes.delete(remote_name)
+        nil
       end
 
-      # Add a new remote to this repository.  Returns a Rugged::Remote object
+      # Add a new remote to this repository.
       def remote_add(remote_name, url)
         rugged.remotes.create(remote_name, url)
+        nil
       end
 
       # Update the specified remote using the values in the +options+ hash
       #
       # Example
       # repo.update_remote("origin", url: "path/to/repo")
-      def remote_update(remote_name, options = {})
+      def remote_update(remote_name, url:)
         # TODO: Implement other remote options
-        rugged.remotes.set_url(remote_name, options[:url]) if options[:url]
-      end
-
-      # Fetch the specified remote
-      def fetch(remote_name)
-        rugged.remotes[remote_name].fetch
-      end
-
-      # Push +*refspecs+ to the remote identified by +remote_name+.
-      def push(remote_name, *refspecs)
-        rugged.remotes[remote_name].push(refspecs)
+        rugged.remotes.set_url(remote_name, url)
+        nil
       end
 
       AUTOCRLF_VALUES = {
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index b36e81278d6..2d58fb0186e 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -80,8 +80,8 @@ module Gitlab
       def tree_entries(repository, revision, path)
         request = Gitaly::GetTreeEntriesRequest.new(
           repository: @gitaly_repo,
-          revision: revision,
-          path: path.presence || '.'
+          revision: GitalyClient.encode(revision),
+          path: path.present? ? GitalyClient.encode(path) : '.'
         )
 
         response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request)
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 9d9ebcb389a..894950e341f 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -98,6 +98,7 @@ excluded_attributes:
     - :last_activity_at
     - :last_repository_updated_at
     - :last_repository_check_at
+    - :storage_version
   snippets:
     - :expired_at
   merge_request_diff:
diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb
index 208f0e1bbea..4d6bbda15f3 100644
--- a/lib/gitlab/job_waiter.rb
+++ b/lib/gitlab/job_waiter.rb
@@ -1,12 +1,31 @@
 module Gitlab
   # JobWaiter can be used to wait for a number of Sidekiq jobs to complete.
+  #
+  # Its use requires the cooperation of the sidekiq jobs themselves. Set up the
+  # waiter, then start the jobs, passing them its `key`. Their `perform` methods
+  # should look like:
+  #
+  #     def perform(args, notify_key)
+  #       # do work
+  #     ensure
+  #       ::Gitlab::JobWaiter.notify(notify_key, jid)
+  #     end
+  #
+  # The JobWaiter blocks popping items from a Redis array. All the sidekiq jobs
+  # push to that array when done. Once the waiter has popped `count` items, it
+  # knows all the jobs are done.
   class JobWaiter
-    # The sleep interval between checking keys, in seconds.
-    INTERVAL = 0.1
+    def self.notify(key, jid)
+      Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) }
+    end
+
+    attr_reader :key, :jobs_remaining, :finished
 
-    # jobs - The job IDs to wait for.
-    def initialize(jobs)
-      @jobs = jobs
+    # jobs_remaining - the number of jobs left to wait for
+    def initialize(jobs_remaining)
+      @key = "gitlab:job_waiter:#{SecureRandom.uuid}"
+      @jobs_remaining = jobs_remaining
+      @finished = []
     end
 
     # Waits for all the jobs to be completed.
@@ -15,13 +34,33 @@ module Gitlab
     #           ensures we don't indefinitely block a caller in case a job takes
     #           long to process, or is never processed.
     def wait(timeout = 10)
-      start = Time.current
+      deadline = Time.now.utc + timeout
+
+      Gitlab::Redis::SharedState.with do |redis|
+        # Fallback key expiry: allow a long grace period to reduce the chance of
+        # a job pushing to an expired key and recreating it
+        redis.expire(key, [timeout * 2, 10.minutes.to_i].max)
+
+        while jobs_remaining > 0
+          # Redis will not take fractional seconds. Prefer waiting too long over
+          # not waiting long enough
+          seconds_left = (deadline - Time.now.utc).ceil
 
-      while (Time.current - start) <= timeout
-        break if SidekiqStatus.all_completed?(@jobs)
+          # Redis interprets 0 as "wait forever", so skip the final `blpop` call
+          break if seconds_left <= 0
 
-        sleep(INTERVAL) # to not overload Redis too much.
+          list, jid = redis.blpop(key, timeout: seconds_left)
+          break unless list && jid # timed out
+
+          @finished << jid
+          @jobs_remaining -= 1
+        end
+
+        # All jobs have finished, so expire the key immediately
+        redis.expire(key, 0) if jobs_remaining == 0
       end
+
+      finished
     end
   end
 end
diff --git a/lib/gitlab/sidekiq_throttler.rb b/lib/gitlab/sidekiq_throttler.rb
index d4d39a888e7..5512afa45a8 100644
--- a/lib/gitlab/sidekiq_throttler.rb
+++ b/lib/gitlab/sidekiq_throttler.rb
@@ -3,6 +3,8 @@ module Gitlab
     class << self
       def execute!
         if Gitlab::CurrentSettings.sidekiq_throttling_enabled?
+          require 'sidekiq-limit_fetch'
+
           Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_queues.each do |queue|
             Sidekiq::Queue[queue].limit = queue_limit
           end
diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake
index 48bd9139ce8..6e10ba374bf 100644
--- a/lib/tasks/gitlab/import.rake
+++ b/lib/tasks/gitlab/import.rake
@@ -11,6 +11,12 @@ namespace :gitlab do
     #
     desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance"
     task repos: :environment do
+      if Project.current_application_settings.hashed_storage_enabled
+        puts 'Cannot import repositories when Hashed Storage is enabled'.color(:red)
+
+        exit 1
+      end
+
       Gitlab.config.repositories.storages.each_value do |repository_storage|
         git_base_path = repository_storage['path']
         repos_to_import = Dir.glob(git_base_path + '/**/*.git')