Merge branch 'master' into 'backstage/gb/rename-ci-cd-processing-sidekiq-queues'

# Conflicts:
#   db/schema.rb

19 files changed, 273 insertions, 120 deletions

diff --git a/lib/github/import.rb b/lib/github/import.rb
index 4cc01593ef4..7b848081e85 100644
--- a/lib/github/import.rb
+++ b/lib/github/import.rb
@@ -107,7 +107,7 @@ module Github
       # this means that repo has wiki enabled, but have no pages. So,
       # we can skip the import.
       if e.message !~ /repository not exported/
-        errors(:wiki, wiki_url, e.message)
+        error(:wiki, wiki_url, e.message)
       end
     end
 
diff --git a/lib/gitlab/bare_repository_importer.rb b/lib/gitlab/bare_repository_importer.rb
new file mode 100644
index 00000000000..9323bfc7fb2
--- /dev/null
+++ b/lib/gitlab/bare_repository_importer.rb
@@ -0,0 +1,96 @@
+module Gitlab
+  class BareRepositoryImporter
+    NoAdminError = Class.new(StandardError)
+
+    def self.execute
+      Gitlab.config.repositories.storages.each do |storage_name, repository_storage|
+        git_base_path = repository_storage['path']
+        repos_to_import = Dir.glob(git_base_path + '/**/*.git')
+
+        repos_to_import.each do |repo_path|
+          if repo_path.end_with?('.wiki.git')
+            log " * Skipping wiki repo"
+            next
+          end
+
+          log "Processing #{repo_path}".color(:yellow)
+
+          repo_relative_path = repo_path[repository_storage['path'].length..-1]
+                                 .sub(/^\//, '') # Remove leading `/`
+                                 .sub(/\.git$/, '') # Remove `.git` at the end
+          new(storage_name, repo_relative_path).create_project_if_needed
+        end
+      end
+    end
+
+    attr_reader :storage_name, :full_path, :group_path, :project_path, :user
+    delegate :log, to: :class
+
+    def initialize(storage_name, repo_path)
+      @storage_name = storage_name
+      @full_path = repo_path
+
+      unless @user = User.admins.order_id_asc.first
+        raise NoAdminError.new('No admin user found to import repositories')
+      end
+
+      @group_path, @project_path = File.split(repo_path)
+      @group_path = nil if @group_path == '.'
+    end
+
+    def create_project_if_needed
+      if project = Project.find_by_full_path(full_path)
+        log " * #{project.name} (#{full_path}) exists"
+        return project
+      end
+
+      create_project
+    end
+
+    private
+
+    def create_project
+      group = find_or_create_group
+
+      project_params = {
+        name: project_path,
+        path: project_path,
+        repository_storage: storage_name,
+        namespace_id: group&.id
+      }
+
+      project = Projects::CreateService.new(user, project_params).execute
+
+      if project.persisted?
+        log " * Created #{project.name} (#{full_path})".color(:green)
+        ProjectCacheWorker.perform_async(project.id)
+      else
+        log " * Failed trying to create #{project.name} (#{full_path})".color(:red)
+        log "   Errors: #{project.errors.messages}".color(:red)
+      end
+
+      project
+    end
+
+    def find_or_create_group
+      return nil unless group_path
+
+      if namespace = Namespace.find_by_full_path(group_path)
+        log " * Namespace #{group_path} exists.".color(:green)
+        return namespace
+      end
+
+      log " * Creating Group: #{group_path}"
+      Groups::NestedCreateService.new(user, group_path: group_path).execute
+    end
+
+    # This is called from within a rake task only used by Admins, so allow writing
+    # to STDOUT
+    #
+    # rubocop:disable Rails/Output
+    def self.log(message)
+      puts message
+    end
+    # rubocop:enable Rails/Output
+  end
+end
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index 6d7de52cb80..1dabd4ebdd0 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -186,7 +186,10 @@ module Gitlab
       end
 
       def content_changed?
-        old_blob && new_blob && old_blob.id != new_blob.id
+        return blobs_changed? if diff_refs
+        return false if new_file? || deleted_file? || renamed_file?
+
+        text? && diff_lines.any?
       end
 
       def different_type?
@@ -225,6 +228,10 @@ module Gitlab
 
       private
 
+      def blobs_changed?
+        old_blob && new_blob && old_blob.id != new_blob.id
+      end
+
       def simple_viewer_class
         return DiffViewer::NotDiffable unless diffable?
 
@@ -250,6 +257,8 @@ module Gitlab
           DiffViewer::Renamed
         elsif mode_changed?
           DiffViewer::ModeChanged
+        else
+          DiffViewer::NoPreview
         end
       end
 
diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb
index 093d9ed8092..10ffc345bd5 100644
--- a/lib/gitlab/file_finder.rb
+++ b/lib/gitlab/file_finder.rb
@@ -6,27 +6,48 @@ module Gitlab
 
     attr_reader :project, :ref
 
+    delegate :repository, to: :project
+
     def initialize(project, ref)
       @project = project
       @ref = ref
     end
 
     def find(query)
-      blobs = project.repository.search_files_by_content(query, ref).first(BATCH_SIZE)
-      found_file_names = Set.new
+      by_content = find_by_content(query)
 
-      results = blobs.map do |blob|
-        blob = Gitlab::ProjectSearchResults.parse_search_result(blob)
-        found_file_names << blob.filename
+      already_found = Set.new(by_content.map(&:filename))
+      by_filename = find_by_filename(query, except: already_found)
 
-        [blob.filename, blob]
-      end
+      (by_content + by_filename)
+        .sort_by(&:filename)
+        .map { |blob| [blob.filename, blob] }
+    end
 
-      project.repository.search_files_by_name(query, ref).first(BATCH_SIZE).each do |filename|
-        results << [filename, OpenStruct.new(ref: ref)] unless found_file_names.include?(filename)
-      end
+    private
 
-      results.sort_by(&:first)
+    def find_by_content(query)
+      results = repository.search_files_by_content(query, ref).first(BATCH_SIZE)
+      results.map { |result| Gitlab::ProjectSearchResults.parse_search_result(result) }
+    end
+
+    def find_by_filename(query, except: [])
+      filenames = repository.search_files_by_name(query, ref).first(BATCH_SIZE)
+      filenames.delete_if { |filename| except.include?(filename) } unless except.empty?
+
+      blob_refs = filenames.map { |filename| [ref, filename] }
+      blobs = Gitlab::Git::Blob.batch(repository, blob_refs, blob_size_limit: 1024)
+
+      blobs.map do |blob|
+        Gitlab::SearchResults::FoundBlob.new(
+          id: blob.id,
+          filename: blob.path,
+          basename: File.basename(blob.path),
+          ref: ref,
+          startline: 1,
+          data: blob.data
+        )
+      end
     end
   end
 end
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index a499bbc6266..5ee6669050c 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -271,7 +271,13 @@ module Gitlab
       #
       # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324
       def to_diff
-        rugged_diff_from_parent.patch
+        Gitlab::GitalyClient.migrate(:commit_patch) do |is_enabled|
+          if is_enabled
+            @repository.gitaly_commit_client.patch(id)
+          else
+            rugged_diff_from_parent.patch
+          end
+        end
       end
 
       # Returns a diff object for the changes from this commit's first parent.
diff --git a/lib/gitlab/git/committer.rb b/lib/gitlab/git/committer.rb
new file mode 100644
index 00000000000..1f4bcf7a3a0
--- /dev/null
+++ b/lib/gitlab/git/committer.rb
@@ -0,0 +1,21 @@
+module Gitlab
+  module Git
+    class Committer
+      attr_reader :name, :email, :gl_id
+
+      def self.from_user(user)
+        new(user.name, user.email, Gitlab::GlId.gl_id(user))
+      end
+
+      def initialize(name, email, gl_id)
+        @name = name
+        @email = email
+        @gl_id = gl_id
+      end
+
+      def ==(other)
+        [name, email, gl_id] == [other.name, other.email, other.gl_id]
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb
index 8f0c377ef4f..cc35d77c6e4 100644
--- a/lib/gitlab/git/hook.rb
+++ b/lib/gitlab/git/hook.rb
@@ -1,20 +1,23 @@
-# Gitaly note: JV: looks like this is only used by GitHooksService in
+# Gitaly note: JV: looks like this is only used by Gitlab::Git::HooksService in
 # app/services. We shouldn't bother migrating this until we know how
-# GitHooksService will be migrated.
+# Gitlab::Git::HooksService will be migrated.
 
 module Gitlab
   module Git
     class Hook
       GL_PROTOCOL = 'web'.freeze
-      attr_reader :name, :repo_path, :path
+      attr_reader :name, :path, :repository
 
-      def initialize(name, project)
+      def initialize(name, repository)
         @name = name
-        @project = project
-        @repo_path = project.repository.path
+        @repository = repository
         @path = File.join(repo_path.strip, 'hooks', name)
       end
 
+      def repo_path
+        repository.path
+      end
+
       def exists?
         File.exist?(path)
       end
@@ -44,7 +47,7 @@ module Gitlab
           'GL_ID' => gl_id,
           'PWD' => repo_path,
           'GL_PROTOCOL' => GL_PROTOCOL,
-          'GL_REPOSITORY' => Gitlab::GlRepository.gl_repository(@project, false)
+          'GL_REPOSITORY' => repository.gl_repository
         }
 
         options = {
diff --git a/lib/gitlab/git/hooks_service.rb b/lib/gitlab/git/hooks_service.rb
new file mode 100644
index 00000000000..ea8a87a1290
--- /dev/null
+++ b/lib/gitlab/git/hooks_service.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module Git
+    class HooksService
+      PreReceiveError = Class.new(StandardError)
+
+      attr_accessor :oldrev, :newrev, :ref
+
+      def execute(committer, repository, oldrev, newrev, ref)
+        @repository = repository
+        @gl_id      = committer.gl_id
+        @oldrev     = oldrev
+        @newrev     = newrev
+        @ref        = ref
+
+        %w(pre-receive update).each do |hook_name|
+          status, message = run_hook(hook_name)
+
+          unless status
+            raise PreReceiveError, message
+          end
+        end
+
+        yield(self).tap do
+          run_hook('post-receive')
+        end
+      end
+
+      private
+
+      def run_hook(name)
+        hook = Gitlab::Git::Hook.new(name, @repository)
+        hook.trigger(@gl_id, oldrev, newrev, ref)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index eb3731ba35a..b835dec24eb 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -49,13 +49,14 @@ module Gitlab
       # Rugged repo object
       attr_reader :rugged
 
-      attr_reader :storage
+      attr_reader :storage, :gl_repository, :relative_path
 
       # 'path' must be the path to a _bare_ git repository, e.g.
       # /path/to/my-repo.git
-      def initialize(storage, relative_path)
+      def initialize(storage, relative_path, gl_repository)
         @storage = storage
         @relative_path = relative_path
+        @gl_repository = gl_repository
 
         storage_path = Gitlab.config.repositories.storages[@storage]['path']
         @path = File.join(storage_path, @relative_path)
@@ -153,7 +154,7 @@ module Gitlab
           if is_enabled
             gitaly_ref_client.count_branch_names
           else
-            rugged.branches.count do |ref|
+            rugged.branches.each(:local).count do |ref|
               begin
                 ref.name && ref.target # ensures the branch is valid
 
@@ -201,6 +202,19 @@ module Gitlab
         end
       end
 
+      # Returns true if the given ref name exists
+      #
+      # Ref names must start with `refs/`.
+      def ref_exists?(ref_name)
+        gitaly_migrate(:ref_exists) do |is_enabled|
+          if is_enabled
+            gitaly_ref_exists?(ref_name)
+          else
+            rugged_ref_exists?(ref_name)
+          end
+        end
+      end
+
       # Returns true if the given tag exists
       #
       # name - The name of the tag as a String.
@@ -992,6 +1006,16 @@ module Gitlab
       # Returns true if the given ref name exists
       #
       # Ref names must start with `refs/`.
+      def rugged_ref_exists?(ref_name)
+        raise ArgumentError, 'invalid refname' unless ref_name.start_with?('refs/')
+        rugged.references.exist?(ref_name)
+      rescue Rugged::ReferenceError
+        false
+      end
+
+      # Returns true if the given ref name exists
+      #
+      # Ref names must start with `refs/`.
       def gitaly_ref_exists?(ref_name)
         gitaly_ref_client.ref_exists?(ref_name)
       end
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index 2d58fb0186e..57f42bd35ee 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -175,8 +175,8 @@ module Gitlab
       def raw_blame(revision, path)
         request = Gitaly::RawBlameRequest.new(
           repository: @gitaly_repo,
-          revision: revision,
-          path: path
+          revision: GitalyClient.encode(revision),
+          path: GitalyClient.encode(path)
         )
 
         response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request)
@@ -194,6 +194,16 @@ module Gitlab
         response.commit
       end
 
+      def patch(revision)
+        request = Gitaly::CommitPatchRequest.new(
+          repository: @gitaly_repo,
+          revision: GitalyClient.encode(revision)
+        )
+        response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request)
+
+        response.sum(&:data)
+      end
+
       private
 
       def commit_diff_request_params(commit, options = {})
diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb
index cdcfed36740..8c0008c6971 100644
--- a/lib/gitlab/gitaly_client/ref_service.rb
+++ b/lib/gitlab/gitaly_client/ref_service.rb
@@ -71,7 +71,7 @@ module Gitlab
       end
 
       def ref_exists?(ref_name)
-        request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: ref_name)
+        request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: GitalyClient.encode(ref_name))
         response = GitalyClient.call(@storage, :ref_service, :ref_exists, request)
         response.value
       rescue GRPC::InvalidArgument => e
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 894950e341f..e5d4bb686e7 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -134,5 +134,7 @@ methods:
     - :utf8_diff
   merge_requests:
     - :diff_head_sha
+    - :source_branch_sha
+    - :target_branch_sha
   project:
     - :description_html
diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb
index c20adc20bfd..81a213e8321 100644
--- a/lib/gitlab/import_export/merge_request_parser.rb
+++ b/lib/gitlab/import_export/merge_request_parser.rb
@@ -30,7 +30,7 @@ module Gitlab
       end
 
       def branch_exists?(branch_name)
-        @project.repository.branch_exists?(branch_name)
+        @project.repository.raw.branch_exists?(branch_name)
       end
 
       def fork_merge_request?
diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb
index 8867a91c244..cd7e4ca7b7e 100644
--- a/lib/gitlab/ldap/adapter.rb
+++ b/lib/gitlab/ldap/adapter.rb
@@ -73,7 +73,7 @@ module Gitlab
       private
 
       def user_options(field, value, limit)
-        options = { attributes: user_attributes }
+        options = { attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq }
         options[:size] = limit if limit
 
         if field.to_sym == :dn
@@ -99,10 +99,6 @@ module Gitlab
           filter
         end
       end
-
-      def user_attributes
-        %W(#{config.uid} cn dn) + config.attributes['username'] + config.attributes['email']
-      end
     end
   end
 end
diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb
index e138b466a34..4d6f8ac79de 100644
--- a/lib/gitlab/ldap/person.rb
+++ b/lib/gitlab/ldap/person.rb
@@ -21,6 +21,15 @@ module Gitlab
         adapter.dn_matches_filter?(dn, AD_USER_DISABLED)
       end
 
+      def self.ldap_attributes(config)
+        [
+          'dn', # Used in `dn`
+          config.uid, # Used in `uid`
+          *config.attributes['name'], # Used in `name`
+          *config.attributes['email'] # Used in `email`
+        ]
+      end
+
       def initialize(entry, provider)
         Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" }
         @entry = entry
diff --git a/lib/gitlab/logger.rb b/lib/gitlab/logger.rb
index 59b21149a9a..6bffd410ed0 100644
--- a/lib/gitlab/logger.rb
+++ b/lib/gitlab/logger.rb
@@ -14,13 +14,9 @@ module Gitlab
 
     def self.read_latest
       path = Rails.root.join("log", file_name)
-      self.build unless File.exist?(path)
-      tail_output, _ = Gitlab::Popen.popen(%W(tail -n 2000 #{path}))
-      tail_output.split("\n")
-    end
 
-    def self.read_latest_for(filename)
-      path = Rails.root.join("log", filename)
+      return [] unless File.readable?(path)
+
       tail_output, _ = Gitlab::Popen.popen(%W(tail -n 2000 #{path}))
       tail_output.split("\n")
     end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 1adc5ec952a..58f6245579a 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -53,7 +53,8 @@ module Gitlab
     end
 
     def kubernetes_namespace_regex_message
-      "can contain only letters, digits or '-', and cannot start or end with '-'"
+      "can contain only lowercase letters, digits, and '-'. " \
+      "Must start with a letter, and cannot end with '-'"
     end
 
     def environment_slug_regex
diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake
index 6e10ba374bf..d227a0c8bdb 100644
--- a/lib/tasks/gitlab/import.rake
+++ b/lib/tasks/gitlab/import.rake
@@ -9,6 +9,7 @@ namespace :gitlab do
     #  * The project owner will set to the first administator of the system
     #  * Existing projects will be skipped
     #
+    #
     desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance"
     task repos: :environment do
       if Project.current_application_settings.hashed_storage_enabled
@@ -17,69 +18,7 @@ namespace :gitlab do
         exit 1
       end
 
-      Gitlab.config.repositories.storages.each_value do |repository_storage|
-        git_base_path = repository_storage['path']
-        repos_to_import = Dir.glob(git_base_path + '/**/*.git')
-
-        repos_to_import.each do |repo_path|
-          # strip repo base path
-          repo_path[0..git_base_path.length] = ''
-
-          path = repo_path.sub(/\.git$/, '')
-          group_name, name = File.split(path)
-          group_name = nil if group_name == '.'
-
-          puts "Processing #{repo_path}".color(:yellow)
-
-          if path.end_with?('.wiki')
-            puts " * Skipping wiki repo"
-            next
-          end
-
-          project = Project.find_by_full_path(path)
-
-          if project
-            puts " * #{project.name} (#{repo_path}) exists"
-          else
-            user = User.admins.reorder("id").first
-
-            project_params = {
-              name: name,
-              path: name
-            }
-
-            # find group namespace
-            if group_name
-              group = Namespace.find_by(path: group_name)
-              # create group namespace
-              unless group
-                group = Group.new(name: group_name)
-                group.path = group_name
-                group.owner = user
-                if group.save
-                  puts " * Created Group #{group.name} (#{group.id})".color(:green)
-                else
-                  puts " * Failed trying to create group #{group.name}".color(:red)
-                end
-              end
-              # set project group
-              project_params[:namespace_id] = group.id
-            end
-
-            project = Projects::CreateService.new(user, project_params).execute
-
-            if project.persisted?
-              puts " * Created #{project.name} (#{repo_path})".color(:green)
-              ProjectCacheWorker.perform_async(project.id)
-            else
-              puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
-              puts "   Errors: #{project.errors.messages}".color(:red)
-            end
-          end
-        end
-      end
-
-      puts "Done!".color(:green)
+      Gitlab::BareRepositoryImporter.execute
     end
   end
 end
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
index 96b8f59242c..1206302cb76 100644
--- a/lib/tasks/import.rake
+++ b/lib/tasks/import.rake
@@ -72,23 +72,7 @@ class GithubImport
     return @current_user.namespace if names == @current_user.namespace_path
     return @current_user.namespace unless @current_user.can_create_group?
 
-    full_path_namespace = Namespace.find_by_full_path(names)
-
-    return full_path_namespace if full_path_namespace
-
-    names.split('/').inject(nil) do |parent, name|
-      begin
-        namespace = Group.create!(name: name,
-                                  path: name,
-                                  owner: @current_user,
-                                  parent: parent)
-        namespace.add_owner(@current_user)
-
-        namespace
-      rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid
-        Namespace.where(parent: parent).find_by_path_or_name(name)
-      end
-    end
+    Groups::NestedCreateService.new(@current_user, group_path: names).execute
   end
 
   def full_path_namespace(names)