Rebase repo check MR

3 files changed, 92 insertions, 0 deletions

diff --git a/app/workers/admin_email_worker.rb b/app/workers/admin_email_worker.rb
new file mode 100644
index 00000000000..fcccb9ea669
--- /dev/null
+++ b/app/workers/admin_email_worker.rb
@@ -0,0 +1,12 @@
+class AdminEmailWorker
+  include Sidekiq::Worker
+
+  sidekiq_options retry: false # this job auto-repeats via sidekiq-cron
+
+  def perform
+    repo_check_failed_count = Project.where(last_repo_check_failed: true).count
+    return if repo_check_failed_count.zero?
+
+    RepoCheckMailer.notify(repo_check_failed_count).deliver_now
+  end
+end
diff --git a/app/workers/repo_check_worker.rb b/app/workers/repo_check_worker.rb
new file mode 100644
index 00000000000..9be795309e0
--- /dev/null
+++ b/app/workers/repo_check_worker.rb
@@ -0,0 +1,46 @@
+class RepoCheckWorker
+  include Sidekiq::Worker
+
+  RUN_TIME = 3600
+
+  sidekiq_options retry: false
+
+  def perform
+    start = Time.now
+
+    # This loop will break after a little more than one hour ('a little
+    # more' because `git fsck` may take a few minutes), or if it runs out of
+    # projects to check. By default sidekiq-cron will start a new
+    # RepoCheckWorker each hour so that as long as there are repositories to
+    # check, only one (or two) will be checked at a time.
+    project_ids.each do |project_id|
+      break if Time.now - start >= RUN_TIME
+
+      next if !try_obtain_lease(project_id)
+
+      SingleRepoCheckWorker.new.perform(project_id)
+    end
+  end
+
+  private
+
+  # In an ideal world we would use Project.where(...).find_each.
+  # Unfortunately, calling 'find_each' drops the 'where', so we must build
+  # an array of IDs instead.
+  def project_ids
+    limit = 10_000
+    never_checked_projects = Project.where('last_repo_check_at IS NULL').limit(limit).
+      pluck(:id)
+    old_check_projects = Project.where('last_repo_check_at < ?', 1.week.ago).
+      reorder('last_repo_check_at ASC').limit(limit).pluck(:id)
+    never_checked_projects + old_check_projects
+  end
+
+  def try_obtain_lease(id)
+    lease = Gitlab::ExclusiveLease.new(
+      "project_repo_check:#{id}",
+      timeout: RUN_TIME
+    )
+    lease.try_obtain
+  end
+end
diff --git a/app/workers/single_repo_check_worker.rb b/app/workers/single_repo_check_worker.rb
new file mode 100644
index 00000000000..f8b245247c5
--- /dev/null
+++ b/app/workers/single_repo_check_worker.rb
@@ -0,0 +1,34 @@
+class SingleRepoCheckWorker
+  include Sidekiq::Worker
+
+  sidekiq_options retry: false
+
+  def perform(project_id)
+    project = Project.find(project_id)
+    update(project, success: check(project))
+  end
+
+  private
+
+  def check(project)
+    [project.repository.path_to_repo, project.wiki.wiki.path].all? do |path|
+      git_fsck(path)
+    end
+  end
+
+  def git_fsck(path)
+    cmd = %W(nice git --git-dir=#{path} fsck)
+    output, status = Gitlab::Popen.popen(cmd)
+    return true if status.zero?
+
+    Gitlab::RepoCheckLogger.error("command failed: #{cmd.join(' ')}\n#{output}")
+    false
+  end
+
+  def update(project, success:)
+    project.update_columns(
+      last_repo_check_failed: !success,
+      last_repo_check_at: Time.now,
+    )
+  end
+end