Merge branch 'forked-repo-creation-async' into 'master'

Import forked repositories asynchronously to prevent large repositories from timing out

Preserve synchronous mode of adding a repository since some tasks (e.g. restore/check) require the former.

One side bug fix is that the `ProjectCacheWorker` is also run, updating the repository size and commit numbers properly. Previously all the values were set to 0 when a project were forked.

* Closes #2388
* Closes #2400

See merge request !1231

4 files changed, 57 insertions, 17 deletions

diff --git a/app/controllers/projects/forks_controller.rb b/app/controllers/projects/forks_controller.rb
index 9e72597ea87..8a785076bb7 100644
--- a/app/controllers/projects/forks_controller.rb
+++ b/app/controllers/projects/forks_controller.rb
@@ -13,10 +13,14 @@ class Projects::ForksController < Projects::ApplicationController
     @forked_project = ::Projects::ForkService.new(project, current_user, namespace: namespace).execute
 
     if @forked_project.saved? && @forked_project.forked?
-      redirect_to(
-        namespace_project_path(@forked_project.namespace, @forked_project),
-        notice: 'Project was successfully forked.'
-      )
+      if @forked_project.import_in_progress?
+        redirect_to namespace_project_import_path(@forked_project.namespace, @forked_project)
+      else
+        redirect_to(
+          namespace_project_path(@forked_project.namespace, @forked_project),
+          notice: 'Project was successfully forked.'
+        )
+      end
     else
       render :error
     end
diff --git a/app/models/project.rb b/app/models/project.rb
index 072d7d73f41..49525eb9227 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -144,7 +144,7 @@ class Project < ActiveRecord::Base
   validates_uniqueness_of :path, scope: :namespace_id
   validates :import_url,
     format: { with: /\A#{URI.regexp(%w(ssh git http https))}\z/, message: 'should be a valid url' },
-    if: :import?
+    if: :external_import?
   validates :star_count, numericality: { greater_than_or_equal_to: 0 }
   validate :check_limit, on: :create
   validate :avatar_type,
@@ -275,7 +275,13 @@ class Project < ActiveRecord::Base
   end
 
   def add_import_job
-    RepositoryImportWorker.perform_in(2.seconds, id)
+    if forked?
+      unless RepositoryForkWorker.perform_async(id, forked_from_project.path_with_namespace, self.namespace.path)
+        import_fail
+      end
+    else
+      RepositoryImportWorker.perform_in(2.seconds, id)
+    end
   end
 
   def clear_import_data
@@ -283,6 +289,10 @@ class Project < ActiveRecord::Base
   end
 
   def import?
+    external_import? || forked?
+  end
+
+  def external_import?
     import_url.present?
   end
 
@@ -702,14 +712,8 @@ class Project < ActiveRecord::Base
   end
 
   def create_repository
-    if forked?
-      if gitlab_shell.fork_repository(forked_from_project.path_with_namespace, self.namespace.path)
-        true
-      else
-        errors.add(:base, 'Failed to fork repository via gitlab-shell')
-        false
-      end
-    else
+    # Forked import is handled asynchronously
+    unless forked?
       if gitlab_shell.add_repository(path_with_namespace)
         true
       else
diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb
index 1bb2462565a..e54a13ed6c5 100644
--- a/app/services/projects/create_service.rb
+++ b/app/services/projects/create_service.rb
@@ -55,9 +55,7 @@ module Projects
         @project.save
 
         if @project.persisted? && !@project.import?
-          unless @project.create_repository
-            raise 'Failed to create repository'
-          end
+          raise 'Failed to create repository' unless @project.create_repository
         end
       end
 
diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb
new file mode 100644
index 00000000000..acd1c43f06b
--- /dev/null
+++ b/app/workers/repository_fork_worker.rb
@@ -0,0 +1,34 @@
+class RepositoryForkWorker
+  include Sidekiq::Worker
+  include Gitlab::ShellAdapter
+
+  sidekiq_options queue: :gitlab_shell
+
+  def perform(project_id, source_path, target_path)
+    project = Project.find_by_id(project_id)
+
+    unless project.present?
+      logger.error("Project #{project_id} no longer exists!")
+      return
+    end
+
+    result = gitlab_shell.fork_repository(source_path, target_path)
+
+    unless result
+      logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}")
+      project.import_fail
+      project.save
+      return
+    end
+
+    if project.valid_repo?
+      ProjectCacheWorker.perform_async(project.id)
+      project.import_finish
+    else
+      project.import_fail
+      logger.error("Project #{id} had an invalid repository after fork")
+    end
+
+    project.save
+  end
+end