Merge branch '19711-many-git-gc-instances' into 'master'

Reset project pushes_since_gc when we enqueue the git gc call

## What does this MR do?

Reset counters just when we really enqueue the `git gc` command. So as this is part of the `#execute`method we throttle the enqueue and the reset (database statement) as we throttle the `#increment!` call. And we only call `#execute` if it's needed.

## What are the relevant issue numbers?

Closes #19711 

## Does this MR meet the acceptance criteria?

- [x] [CHANGELOG](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CHANGELOG) entry added
- ~~[ ] [Documentation created/updated](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/development/doc_styleguide.md)~~
- ~~[ ] API support added~~
- Tests
  - [x] Added for this feature/bug
  - [x] All builds are passing
- [x] Conform by the [style guides](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md#style-guides)
- [x] Branch has no merge conflicts with `master` (if you do - rebase it please)
- [x] [Squashed related commits together](https://git-scm.com/book/en/Git-Tools-Rewriting-History#Squashing-Commits)

See merge request !5201

3 files changed, 48 insertions, 19 deletions

diff --git a/CHANGELOG b/CHANGELOG
index ee3ee4c37d6..2527290f0fd 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -75,6 +75,7 @@ v 8.10.0 (unreleased)
   - Fix 404 redirect after validation fails importing a GitLab project
   - Added setting to set new users by default as external !4545 (Dravere)
   - Add min value for project limit field on user's form !3622 (jastkand)
+  - Reset project pushes_since_gc when we enqueue the git gc call
   - Add reminder to not paste private SSH keys !4399 (Ingo Blechschmidt)
   - Remove duplicate `description` field in `MergeRequest` entities (Ben Boeckel)
   - Style of import project buttons were fixed in the new project page. !5183 (rdemirbay)
diff --git a/app/services/projects/housekeeping_service.rb b/app/services/projects/housekeeping_service.rb
index c9ad710b7bf..29b3981f49f 100644
--- a/app/services/projects/housekeeping_service.rb
+++ b/app/services/projects/housekeeping_service.rb
@@ -22,11 +22,7 @@ module Projects
     def execute
       raise LeaseTaken unless try_obtain_lease
 
-      GitGarbageCollectWorker.perform_async(@project.id)
-    ensure
-      Gitlab::Metrics.measure(:reset_pushes_since_gc) do
-        update_pushes_since_gc(0)
-      end
+      execute_gitlab_shell_gc
     end
 
     def needed?
@@ -34,19 +30,27 @@ module Projects
     end
 
     def increment!
-      Gitlab::Metrics.measure(:increment_pushes_since_gc) do
-        update_pushes_since_gc(@project.pushes_since_gc + 1)
+      if Gitlab::ExclusiveLease.new("project_housekeeping:increment!:#{@project.id}", timeout: 60).try_obtain
+        Gitlab::Metrics.measure(:increment_pushes_since_gc) do
+          update_pushes_since_gc(@project.pushes_since_gc + 1)
+        end
       end
     end
 
     private
 
-    def update_pushes_since_gc(new_value)
-      if Gitlab::ExclusiveLease.new("project_housekeeping:update_pushes_since_gc:#{project.id}", timeout: 60).try_obtain
-        @project.update_column(:pushes_since_gc, new_value)
+    def execute_gitlab_shell_gc
+      GitGarbageCollectWorker.perform_async(@project.id)
+    ensure
+      Gitlab::Metrics.measure(:reset_pushes_since_gc) do
+        update_pushes_since_gc(0)
       end
     end
 
+    def update_pushes_since_gc(new_value)
+      @project.update_column(:pushes_since_gc, new_value)
+    end
+
     def try_obtain_lease
       Gitlab::Metrics.measure(:obtain_housekeeping_lease) do
         lease = ::Gitlab::ExclusiveLease.new("project_housekeeping:#{@project.id}", timeout: LEASE_TIMEOUT)
diff --git a/spec/services/projects/housekeeping_service_spec.rb b/spec/services/projects/housekeeping_service_spec.rb
index 7ab95e042ce..ad0d58672b3 100644
--- a/spec/services/projects/housekeeping_service_spec.rb
+++ b/spec/services/projects/housekeeping_service_spec.rb
@@ -15,15 +15,25 @@ describe Projects::HousekeepingService do
       expect(GitGarbageCollectWorker).to receive(:perform_async).with(project.id)
 
       subject.execute
-      expect(project.pushes_since_gc).to eq(0)
+      expect(project.reload.pushes_since_gc).to eq(0)
     end
 
-    it 'does not enqueue a job when no lease can be obtained' do
-      expect(subject).to receive(:try_obtain_lease).and_return(false)
-      expect(GitGarbageCollectWorker).not_to receive(:perform_async)
+    context 'when no lease can be obtained' do
+      before(:each) do
+        expect(subject).to receive(:try_obtain_lease).and_return(false)
+      end
 
-      expect { subject.execute }.to raise_error(Projects::HousekeepingService::LeaseTaken)
-      expect(project.pushes_since_gc).to eq(0)
+      it 'does not enqueue a job' do
+        expect(GitGarbageCollectWorker).not_to receive(:perform_async)
+
+        expect { subject.execute }.to raise_error(Projects::HousekeepingService::LeaseTaken)
+      end
+
+      it 'does not reset pushes_since_gc' do
+        expect do
+          expect { subject.execute }.to raise_error(Projects::HousekeepingService::LeaseTaken)
+        end.not_to change { project.pushes_since_gc }.from(3)
+      end
     end
   end
 
@@ -39,10 +49,24 @@ describe Projects::HousekeepingService do
   end
 
   describe 'increment!' do
+    let(:lease_key) { "project_housekeeping:increment!:#{project.id}" }
+
     it 'increments the pushes_since_gc counter' do
-      expect(project.pushes_since_gc).to eq(0)
-      subject.increment!
-      expect(project.pushes_since_gc).to eq(1)
+      lease = double(:lease, try_obtain: true)
+      expect(Gitlab::ExclusiveLease).to receive(:new).with(lease_key, anything).and_return(lease)
+
+      expect do
+        subject.increment!
+      end.to change { project.pushes_since_gc }.from(0).to(1)
+    end
+
+    it 'does not increment when no lease can be obtained' do
+      lease = double(:lease, try_obtain: false)
+      expect(Gitlab::ExclusiveLease).to receive(:new).with(lease_key, anything).and_return(lease)
+
+      expect do
+        subject.increment!
+      end.not_to change { project.pushes_since_gc }
     end
   end
 end