Merge remote-tracking branch 'upstream/master' into pipeline-notifications

* upstream/master: (70 commits)
  Fix routing spec for group controller
  Add small improvements to constrainers and specs
  Faster search
  Fix broken commits search
  Changed helper method to check for none on params Moved if statements around in view
  API: Return 400 when creating a systemhook fails
  Update non-exist group spinach test to match routing
  Bump omniauth-gitlab to 1.0.2 to fix incompatibility with omniauth-oauth2
  Replace trigger with the new ID of the docs project
  Refactor method name
  17492 Update link color for more accessible contrast
  Fixed todos empty state when filtering
  Refactor namespace regex
  implements reset incoming email token on issues modal and account page, reactivates all tests and writes more tests for it
  Use separate email-friendly token for incoming email and let incoming email token be reset
  Use the Gitlab Workhorse HTTP header in the admin dashboard
  Refactor project routing
  Fix 404 when visit /projects page
  Rewritten spinach git_blame tests to rspec feature tests
  Add tests for project#index routing
  ...

2 files changed, 111 insertions, 3 deletions

diff --git a/app/workers/git_garbage_collect_worker.rb b/app/workers/git_garbage_collect_worker.rb
index 65f8093b5b0..d369b639ae9 100644
--- a/app/workers/git_garbage_collect_worker.rb
+++ b/app/workers/git_garbage_collect_worker.rb
@@ -1,17 +1,58 @@
 class GitGarbageCollectWorker
   include Sidekiq::Worker
-  include Gitlab::ShellAdapter
   include DedicatedSidekiqQueue
+  include Gitlab::CurrentSettings
 
   sidekiq_options retry: false
 
-  def perform(project_id)
+  def perform(project_id, task = :gc, lease_key = nil, lease_uuid = nil)
     project = Project.find(project_id)
+    task = task.to_sym
+
+    cmd = command(task)
+    repo_path = project.repository.path_to_repo
+    description = "'#{cmd.join(' ')}' in #{repo_path}"
+
+    Gitlab::GitLogger.info(description)
+
+    output, status = Gitlab::Popen.popen(cmd, repo_path)
+    Gitlab::GitLogger.error("#{description} failed:\n#{output}") unless status.zero?
 
-    gitlab_shell.gc(project.repository_storage_path, project.path_with_namespace)
     # Refresh the branch cache in case garbage collection caused a ref lookup to fail
+    flush_ref_caches(project) if task == :gc
+  ensure
+    Gitlab::ExclusiveLease.cancel(lease_key, lease_uuid) if lease_key.present? && lease_uuid.present?
+  end
+
+  private
+
+  def command(task)
+    case task
+    when :gc
+      git(write_bitmaps: bitmaps_enabled?) + %w[gc]
+    when :full_repack
+      git(write_bitmaps: bitmaps_enabled?) + %w[repack -A -d --pack-kept-objects]
+    when :incremental_repack
+      # Normal git repack fails when bitmaps are enabled. It is impossible to
+      # create a bitmap here anyway.
+      git(write_bitmaps: false) + %w[repack -d]
+    else
+      raise "Invalid gc task: #{task.inspect}"
+    end
+  end
+
+  def flush_ref_caches(project)
     project.repository.after_create_branch
     project.repository.branch_names
     project.repository.has_visible_content?
   end
+
+  def bitmaps_enabled?
+    current_application_settings.housekeeping_bitmaps_enabled
+  end
+
+  def git(write_bitmaps:)
+    config_value = write_bitmaps ? 'true' : 'false'
+    %W[git -c repack.writeBitmaps=#{config_value}]
+  end
 end
diff --git a/app/workers/process_commit_worker.rb b/app/workers/process_commit_worker.rb
new file mode 100644
index 00000000000..071741fbacd
--- /dev/null
+++ b/app/workers/process_commit_worker.rb
@@ -0,0 +1,67 @@
+# Worker for processing individiual commit messages pushed to a repository.
+#
+# Jobs for this worker are scheduled for every commit that is being pushed. As a
+# result of this the workload of this worker should be kept to a bare minimum.
+# Consider using an extra worker if you need to add any extra (and potentially
+# slow) processing of commits.
+class ProcessCommitWorker
+  include Sidekiq::Worker
+  include DedicatedSidekiqQueue
+
+  # project_id - The ID of the project this commit belongs to.
+  # user_id - The ID of the user that pushed the commit.
+  # commit_sha - The SHA1 of the commit to process.
+  # default - The data was pushed to the default branch.
+  def perform(project_id, user_id, commit_sha, default = false)
+    project = Project.find_by(id: project_id)
+
+    return unless project
+
+    user = User.find_by(id: user_id)
+
+    return unless user
+
+    commit = find_commit(project, commit_sha)
+
+    return unless commit
+
+    author = commit.author || user
+
+    process_commit_message(project, commit, user, author, default)
+
+    update_issue_metrics(commit, author)
+  end
+
+  def process_commit_message(project, commit, user, author, default = false)
+    closed_issues = default ? commit.closes_issues(user) : []
+
+    unless closed_issues.empty?
+      close_issues(project, user, author, commit, closed_issues)
+    end
+
+    commit.create_cross_references!(author, closed_issues)
+  end
+
+  def close_issues(project, user, author, commit, issues)
+    # We don't want to run permission related queries for every single issue,
+    # therefor we use IssueCollection here and skip the authorization check in
+    # Issues::CloseService#execute.
+    IssueCollection.new(issues).updatable_by_user(user).each do |issue|
+      Issues::CloseService.new(project, author).
+        close_issue(issue, commit: commit)
+    end
+  end
+
+  def update_issue_metrics(commit, author)
+    mentioned_issues = commit.all_references(author).issues
+
+    Issue::Metrics.where(issue_id: mentioned_issues.map(&:id), first_mentioned_in_commit_at: nil).
+      update_all(first_mentioned_in_commit_at: commit.committed_date)
+  end
+
+  private
+
+  def find_commit(project, sha)
+    project.commit(sha)
+  end
+end