Merge branch 'master' into 28433-internationalise-cycle-analytics-page

Conflicts:
	db/schema.rb

22 files changed, 223 insertions, 70 deletions

diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index 66b37fd2bcc..621b9dcecd9 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -62,7 +62,7 @@ module API
       post ":id/repository/commits" do
         authorize! :push_code, user_project
 
-        attrs = declared_params.merge(start_branch: declared_params[:branch], target_branch: declared_params[:branch])
+        attrs = declared_params.merge(start_branch: declared_params[:branch], branch_name: declared_params[:branch])
 
         result = ::Files::MultiService.new(user_project, current_user, attrs).execute
 
@@ -140,7 +140,7 @@ module API
         commit_params = {
           commit: commit,
           start_branch: params[:branch],
-          target_branch: params[:branch]
+          branch_name: params[:branch]
         }
 
         result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 9919762cd82..64ab6f01eb5 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -18,6 +18,12 @@ module API
       expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization
     end
 
+    class UserActivity < Grape::Entity
+      expose :username
+      expose :last_activity_on
+      expose :last_activity_on, as: :last_activity_at # Back-compat
+    end
+
     class Identity < Grape::Entity
       expose :provider, :extern_uid
     end
@@ -25,6 +31,7 @@ module API
     class UserPublic < User
       expose :last_sign_in_at
       expose :confirmed_at
+      expose :last_activity_on
       expose :email
       expose :color_scheme_id, :projects_limit, :current_sign_in_at
       expose :identities, using: Entities::Identity
diff --git a/lib/api/files.rb b/lib/api/files.rb
index 33fc970dc09..e6ea12c5ab7 100644
--- a/lib/api/files.rb
+++ b/lib/api/files.rb
@@ -5,7 +5,7 @@ module API
         {
           file_path: attrs[:file_path],
           start_branch: attrs[:branch],
-          target_branch: attrs[:branch],
+          branch_name: attrs[:branch],
           commit_message: attrs[:commit_message],
           file_content: attrs[:content],
           file_content_encoding: attrs[:encoding],
@@ -130,7 +130,7 @@ module API
         authorize! :push_code, user_project
 
         file_params = declared_params(include_missing: false)
-        result = ::Files::DestroyService.new(user_project, current_user, commit_params(file_params)).execute
+        result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute
 
         if result[:status] != :success
           render_api_error!(result[:message], 400)
diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb
index 810e5063996..718f936a1fc 100644
--- a/lib/api/helpers/internal_helpers.rb
+++ b/lib/api/helpers/internal_helpers.rb
@@ -60,6 +60,12 @@ module API
       rescue JSON::ParserError
         {}
       end
+
+      def log_user_activity(actor)
+        commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS
+
+        ::Users::ActivityService.new(actor, 'Git SSH').execute if commands.include?(params[:action])
+      end
     end
   end
 end
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index 215bc03d0e9..5b48ee8665f 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -40,6 +40,8 @@ module API
         response = { status: access_status.status, message: access_status.message }
 
         if access_status.status
+          log_user_activity(actor)
+
           # Return the repository full path so that gitlab-shell has it when
           # handling ssh commands
           response[:repository_path] =
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 05423c17449..244725bb292 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -35,7 +35,7 @@ module API
         optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue'
         optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue'
         optional :labels, type: String, desc: 'Comma-separated list of label names'
-        optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY'
+        optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY'
         optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential'
       end
 
diff --git a/lib/api/users.rb b/lib/api/users.rb
index eedc59f8636..46f221f68fe 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -39,10 +39,13 @@ module API
       params do
         # CE
         optional :username, type: String, desc: 'Get a single user with a specific username'
+        optional :extern_uid, type: String, desc: 'Get a single user with a specific external authentication provider UID'
+        optional :provider, type: String, desc: 'The external provider'
         optional :search, type: String, desc: 'Search for a username'
         optional :active, type: Boolean, default: false, desc: 'Filters only active users'
         optional :external, type: Boolean, default: false, desc: 'Filters only external users'
         optional :blocked, type: Boolean, default: false, desc: 'Filters only blocked users'
+        all_or_none_of :extern_uid, :provider
 
         use :pagination
       end
@@ -51,14 +54,17 @@ module API
           render_api_error!("Not authorized.", 403)
         end
 
-        if params[:username].present?
-          users = User.where(username: params[:username])
-        else
-          users = User.all
-          users = users.active if params[:active]
-          users = users.search(params[:search]) if params[:search].present?
-          users = users.blocked if params[:blocked]
-          users = users.external if params[:external] && current_user.admin?
+        authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?)
+
+        users = User.all
+        users = User.where(username: params[:username]) if params[:username]
+        users = users.active if params[:active]
+        users = users.search(params[:search]) if params[:search].present?
+        users = users.blocked if params[:blocked]
+
+        if current_user.admin?
+          users = users.joins(:identities).merge(Identity.with_extern_uid(params[:provider], params[:extern_uid])) if params[:extern_uid] && params[:provider]
+          users = users.external if params[:external]
         end
 
         entity = current_user.admin? ? Entities::UserPublic : Entities::UserBasic
@@ -534,6 +540,21 @@ module API
         email.destroy
         current_user.update_secondary_emails!
       end
+
+      desc 'Get a list of user activities'
+      params do
+        optional :from, type: DateTime, default: 6.months.ago, desc: 'Date string in the format YEAR-MONTH-DAY'
+        use :pagination
+      end
+      get "activities" do
+        authenticated_as_admin!
+
+        activities = User.
+          where(User.arel_table[:last_activity_on].gteq(params[:from])).
+          reorder(last_activity_on: :asc)
+
+        present paginate(activities), with: Entities::UserActivity
+      end
     end
   end
 end
diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb
index 3414a2883e5..674de592f0a 100644
--- a/lib/api/v3/commits.rb
+++ b/lib/api/v3/commits.rb
@@ -53,7 +53,7 @@ module API
 
           attrs = declared_params.dup
           branch = attrs.delete(:branch_name)
-          attrs.merge!(branch: branch, start_branch: branch, target_branch: branch)
+          attrs.merge!(start_branch: branch, branch_name: branch)
 
           result = ::Files::MultiService.new(user_project, current_user, attrs).execute
 
@@ -131,7 +131,7 @@ module API
           commit_params = {
             commit: commit,
             start_branch: params[:branch],
-            target_branch: params[:branch]
+            branch_name: params[:branch]
           }
 
           result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute
diff --git a/lib/api/v3/files.rb b/lib/api/v3/files.rb
index 13542b0c71c..c76acc86504 100644
--- a/lib/api/v3/files.rb
+++ b/lib/api/v3/files.rb
@@ -6,7 +6,7 @@ module API
           {
             file_path: attrs[:file_path],
             start_branch: attrs[:branch],
-            target_branch: attrs[:branch],
+            branch_name: attrs[:branch],
             commit_message: attrs[:commit_message],
             file_content: attrs[:content],
             file_content_encoding: attrs[:encoding],
@@ -123,7 +123,7 @@ module API
           file_params = declared_params(include_missing: false)
           file_params[:branch] = file_params.delete(:branch_name)
 
-          result = ::Files::DestroyService.new(user_project, current_user, commit_params(file_params)).execute
+          result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute
 
           if result[:status] == :success
             status(200)
diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb
index 0b2b8bd7f4d..1a9d03beb51 100644
--- a/lib/banzai/filter/issuable_state_filter.rb
+++ b/lib/banzai/filter/issuable_state_filter.rb
@@ -9,6 +9,8 @@ module Banzai
       VISIBLE_STATES = %w(closed merged).freeze
 
       def call
+        return doc unless context[:issuable_state_filter_enabled]
+
         extractor = Banzai::IssuableExtractor.new(project, current_user)
         issuables = extractor.extract([doc])
 
diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb
index dabf71d6aeb..c2503fa2adc 100644
--- a/lib/banzai/reference_parser/base_parser.rb
+++ b/lib/banzai/reference_parser/base_parser.rb
@@ -136,7 +136,8 @@ module Banzai
 
         nodes.each_with_object({}) do |node, hash|
           if node.has_attribute?(attribute)
-            hash[node] = objects_by_id[node.attr(attribute).to_i]
+            obj = objects_by_id[node.attr(attribute).to_i]
+            hash[node] = obj if obj
           end
         end
       end
diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb
index 1020452480a..b439b0ee29b 100644
--- a/lib/ci/ansi2html.rb
+++ b/lib/ci/ansi2html.rb
@@ -172,7 +172,7 @@ module Ci
         close_open_tags()
 
         OpenStruct.new(
-          html: @out,
+          html: @out.force_encoding(Encoding.default_external),
           state: state,
           append: append,
           truncated: truncated,
diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb
index 4a585996aa5..61849a40383 100644
--- a/lib/container_registry/path.rb
+++ b/lib/container_registry/path.rb
@@ -48,7 +48,7 @@ module ContainerRegistry
     end
 
     def root_repository?
-      @path == repository_project.full_path
+      @path == project_path
     end
 
     def repository_project
@@ -60,7 +60,13 @@ module ContainerRegistry
     def repository_name
       return unless has_project?
 
-      @path.remove(%r(^#{Regexp.escape(repository_project.full_path)}/?))
+      @path.remove(%r(^#{Regexp.escape(project_path)}/?))
+    end
+
+    def project_path
+      return unless has_project?
+
+      repository_project.full_path.downcase
     end
 
     def to_s
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
index 3b335cdfd01..fa462cbe095 100644
--- a/lib/gitlab/ci/trace/stream.rb
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -4,7 +4,7 @@ module Gitlab
       # This was inspired from: http://stackoverflow.com/a/10219411/1520132
       class Stream
         BUFFER_SIZE = 4096
-        LIMIT_SIZE = 50.kilobytes
+        LIMIT_SIZE = 500.kilobytes
 
         attr_reader :stream
 
@@ -14,6 +14,7 @@ module Gitlab
 
         def initialize
           @stream = yield
+          @stream&.binmode
         end
 
         def valid?
@@ -51,7 +52,7 @@ module Gitlab
             read_last_lines(last_lines)
           else
             stream.read
-          end
+          end.force_encoding(Encoding.default_external)
         end
 
         def html_with_state(state = nil)
@@ -60,8 +61,8 @@ module Gitlab
 
         def html(last_lines: nil)
           text = raw(last_lines: last_lines)
-          stream = StringIO.new(text)
-          ::Ci::Ansi2html.convert(stream).html
+          buffer = StringIO.new(text)
+          ::Ci::Ansi2html.convert(buffer).html
         end
 
         def extract_coverage(regex)
@@ -113,7 +114,6 @@ module Gitlab
           end
 
           chunks.join.lines.last(last_lines).join
-            .force_encoding(Encoding.default_external)
         end
       end
     end
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index e56eb0d3beb..98fd4e78126 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -8,7 +8,7 @@ module Gitlab
       # the user. We load as much as we can for encoding detection
       # (Linguist) and LFS pointer parsing. All other cases where we need full
       # blob data should use load_all_data!.
-      MAX_DATA_DISPLAY_SIZE = 10485760
+      MAX_DATA_DISPLAY_SIZE = 10.megabytes
 
       attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary
 
@@ -153,7 +153,7 @@ module Gitlab
       def lfs_size
         if has_lfs_version_key?
           size = data.match(/(?<=size )([0-9]+)/)
-          return size[1] if size
+          return size[1].to_i if size
         end
 
         nil
diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb
index af1744c9c46..1add037fa5f 100644
--- a/lib/gitlab/git/index.rb
+++ b/lib/gitlab/git/index.rb
@@ -1,8 +1,12 @@
 module Gitlab
   module Git
     class Index
+      IndexError = Class.new(StandardError)
+
       DEFAULT_MODE = 0o100644
 
+      ACTIONS = %w(create create_dir update move delete).freeze
+
       attr_reader :repository, :raw_index
 
       def initialize(repository)
@@ -23,9 +27,8 @@ module Gitlab
       def create(options)
         options = normalize_options(options)
 
-        file_entry = get(options[:file_path])
-        if file_entry
-          raise Gitlab::Git::Repository::InvalidBlobName.new("Filename already exists")
+        if get(options[:file_path])
+          raise IndexError, "A file with this name already exists"
         end
 
         add_blob(options)
@@ -34,13 +37,12 @@ module Gitlab
       def create_dir(options)
         options = normalize_options(options)
 
-        file_entry = get(options[:file_path])
-        if file_entry
-          raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists as a file")
+        if get(options[:file_path])
+          raise IndexError, "A file with this name already exists"
         end
 
         if dir_exists?(options[:file_path])
-          raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists")
+          raise IndexError, "A directory with this name already exists"
         end
 
         options = options.dup
@@ -55,7 +57,7 @@ module Gitlab
 
         file_entry = get(options[:file_path])
         unless file_entry
-          raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist")
+          raise IndexError, "A file with this name doesn't exist"
         end
 
         add_blob(options, mode: file_entry[:mode])
@@ -66,7 +68,11 @@ module Gitlab
 
         file_entry = get(options[:previous_path])
         unless file_entry
-          raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist")
+          raise IndexError, "A file with this name doesn't exist"
+        end
+
+        if get(options[:file_path])
+          raise IndexError, "A file with this name already exists"
         end
 
         raw_index.remove(options[:previous_path])
@@ -77,9 +83,8 @@ module Gitlab
       def delete(options)
         options = normalize_options(options)
 
-        file_entry = get(options[:file_path])
-        unless file_entry
-          raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist")
+        unless get(options[:file_path])
+          raise IndexError, "A file with this name doesn't exist"
         end
 
         raw_index.remove(options[:file_path])
@@ -95,10 +100,20 @@ module Gitlab
       end
 
       def normalize_path(path)
+        unless path
+          raise IndexError, "You must provide a file path"
+        end
+
         pathname = Gitlab::Git::PathHelper.normalize_path(path.dup)
 
-        if pathname.each_filename.include?('..')
-          raise Gitlab::Git::Repository::InvalidBlobName.new('Invalid path')
+        pathname.each_filename do |segment|
+          if segment == '..'
+            raise IndexError, 'Path cannot include directory traversal'
+          end
+
+          unless segment =~ Gitlab::Regex.file_name_regex
+            raise IndexError, "Path #{Gitlab::Regex.file_name_regex_message}"
+          end
         end
 
         pathname.to_s
@@ -106,6 +121,10 @@ module Gitlab
 
       def add_blob(options, mode: nil)
         content = options[:content]
+        unless content
+          raise IndexError, "You must provide content"
+        end
+
         content = Base64.decode64(content) if options[:encoding] == 'base64'
 
         detect = CharlockHolmes::EncodingDetector.new.detect(content)
@@ -119,7 +138,7 @@ module Gitlab
 
         raw_index.add(path: options[:file_path], oid: oid, mode: mode || DEFAULT_MODE)
       rescue Rugged::IndexError => e
-        raise Gitlab::Git::Repository::InvalidBlobName.new(e.message)
+        raise IndexError, e.message
       end
     end
   end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index bcdf1b1faa8..c69676a1dac 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -15,7 +15,7 @@ module Gitlab
         end
 
         unless URI(address).scheme.in?(%w(tcp unix))
-          raise "Unsupported Gitaly address: #{address.inspect}"
+          raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix'"
         end
 
         @addresses[name] = address
diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb
index dda371e6554..49285e35251 100644
--- a/lib/gitlab/markup_helper.rb
+++ b/lib/gitlab/markup_helper.rb
@@ -1,6 +1,11 @@
 module Gitlab
   module MarkupHelper
-    module_function
+    extend self
+
+    MARKDOWN_EXTENSIONS = %w(mdown mkd mkdn md markdown).freeze
+    ASCIIDOC_EXTENSIONS = %w(adoc ad asciidoc).freeze
+    OTHER_EXTENSIONS = %w(textile rdoc org creole wiki mediawiki rst).freeze
+    EXTENSIONS = MARKDOWN_EXTENSIONS + ASCIIDOC_EXTENSIONS + OTHER_EXTENSIONS
 
     # Public: Determines if a given filename is compatible with GitHub::Markup.
     #
@@ -8,10 +13,7 @@ module Gitlab
     #
     # Returns boolean
     def markup?(filename)
-      gitlab_markdown?(filename) ||
-        asciidoc?(filename) ||
-        filename.downcase.end_with?(*%w(.textile .rdoc .org .creole .wiki
-                                        .mediawiki .rst))
+      EXTENSIONS.include?(extension(filename))
     end
 
     # Public: Determines if a given filename is compatible with
@@ -21,7 +23,7 @@ module Gitlab
     #
     # Returns boolean
     def gitlab_markdown?(filename)
-      filename.downcase.end_with?(*%w(.mdown .mkd .mkdn .md .markdown))
+      MARKDOWN_EXTENSIONS.include?(extension(filename))
     end
 
     # Public: Determines if the given filename has AsciiDoc extension.
@@ -30,7 +32,7 @@ module Gitlab
     #
     # Returns boolean
     def asciidoc?(filename)
-      filename.downcase.end_with?(*%w(.adoc .ad .asciidoc))
+      ASCIIDOC_EXTENSIONS.include?(extension(filename))
     end
 
     # Public: Determines if the given filename is plain text.
@@ -39,12 +41,17 @@ module Gitlab
     #
     # Returns boolean
     def plain?(filename)
-      filename.downcase.end_with?('.txt') ||
-        filename.casecmp('readme').zero?
+      extension(filename) == 'txt' || filename.casecmp('readme').zero?
     end
 
     def previewable?(filename)
       markup?(filename)
     end
+
+    private
+
+    def extension(filename)
+      File.extname(filename).downcase.delete('.')
+    end
   end
 end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index e599dd4a656..08b061d5e31 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -73,22 +73,6 @@ module Gitlab
       "can contain only letters, digits, '_', '-', '@', '+' and '.'."
     end
 
-    def file_path_regex
-      @file_path_regex ||= /\A[[[:alnum:]]_\-\.\/\@]*\z/.freeze
-    end
-
-    def file_path_regex_message
-      "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'."
-    end
-
-    def directory_traversal_regex
-      @directory_traversal_regex ||= /\.{2}/.freeze
-    end
-
-    def directory_traversal_regex_message
-      "cannot include directory traversal."
-    end
-
     def archive_formats_regex
       #                           |zip|tar|    tar.gz    |         tar.bz2         |
       @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
new file mode 100644
index 00000000000..6aca6db3123
--- /dev/null
+++ b/lib/gitlab/usage_data.rb
@@ -0,0 +1,65 @@
+module Gitlab
+  class UsageData
+    include Gitlab::CurrentSettings
+
+    class << self
+      def data(force_refresh: false)
+        Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data }
+      end
+
+      def uncached_data
+        license_usage_data.merge(system_usage_data)
+      end
+
+      def to_json(force_refresh: false)
+        data(force_refresh: force_refresh).to_json
+      end
+
+      def system_usage_data
+        {
+          counts: {
+            boards: Board.count,
+            ci_builds: ::Ci::Build.count,
+            ci_pipelines: ::Ci::Pipeline.count,
+            ci_runners: ::Ci::Runner.count,
+            ci_triggers: ::Ci::Trigger.count,
+            deploy_keys: DeployKey.count,
+            deployments: Deployment.count,
+            environments: Environment.count,
+            groups: Group.count,
+            issues: Issue.count,
+            keys: Key.count,
+            labels: Label.count,
+            lfs_objects: LfsObject.count,
+            merge_requests: MergeRequest.count,
+            milestones: Milestone.count,
+            notes: Note.count,
+            pages_domains: PagesDomain.count,
+            projects: Project.count,
+            projects_prometheus_active: PrometheusService.active.count,
+            protected_branches: ProtectedBranch.count,
+            releases: Release.count,
+            services: Service.where(active: true).count,
+            snippets: Snippet.count,
+            todos: Todo.count,
+            uploads: Upload.count,
+            web_hooks: WebHook.count
+          }
+        }
+      end
+
+      def license_usage_data
+        usage_data = {
+          uuid: current_application_settings.uuid,
+          version: Gitlab::VERSION,
+          active_user_count: User.active.count,
+          recorded_at: Time.now,
+          mattermost_enabled: Gitlab.config.mattermost.enabled,
+          edition: 'CE'
+        }
+
+        usage_data
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb
new file mode 100644
index 00000000000..eb36ab9fded
--- /dev/null
+++ b/lib/gitlab/user_activities.rb
@@ -0,0 +1,34 @@
+module Gitlab
+  class UserActivities
+    include Enumerable
+
+    KEY = 'users:activities'.freeze
+    BATCH_SIZE = 500
+
+    def self.record(key, time = Time.now)
+      Gitlab::Redis.with do |redis|
+        redis.hset(KEY, key, time.to_i)
+      end
+    end
+
+    def delete(*keys)
+      Gitlab::Redis.with do |redis|
+        redis.hdel(KEY, keys)
+      end
+    end
+
+    def each
+      cursor = 0
+      loop do
+        cursor, pairs =
+          Gitlab::Redis.with do |redis|
+            redis.hscan(KEY, cursor, count: BATCH_SIZE)
+          end
+
+        Hash[pairs].each { |pair| yield pair }
+
+        break if cursor == '0'
+      end
+    end
+  end
+end
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
index 15131fbf755..a9dad6a1bf0 100644
--- a/lib/tasks/import.rake
+++ b/lib/tasks/import.rake
@@ -52,7 +52,6 @@ class NewImporter < ::Gitlab::GithubImport::Importer
       project.repository.add_remote(project.import_type, project.import_url)
       project.repository.set_remote_as_mirror(project.import_type)
       project.repository.fetch_remote(project.import_type, forced: true)
-      project.repository.remove_remote(project.import_type)
     rescue => e
       # Expire cache to prevent scenarios such as:
       # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true