diff options
author | Hiroyuki Sato <sathiroyuki@gmail.com> | 2019-04-01 09:11:08 +0300 |
---|---|---|
committer | Hiroyuki Sato <sathiroyuki@gmail.com> | 2019-04-04 18:22:56 +0300 |
commit | 0adedbb4822a8daaa215b33c88ace136c31d042d (patch) | |
tree | 7adcd76e9bb9172c5c7d5200803957bb00ed7be5 | |
parent | c7f918aa691b1b9c6ee6a489ae28a094eacff72c (diff) |
Fix the bug that the project statistics is not updated
-rw-r--r-- | app/workers/all_queues.yml | 1 | ||||
-rw-r--r-- | app/workers/project_cache_worker.rb | 14 | ||||
-rw-r--r-- | app/workers/update_project_statistics_worker.rb | 22 | ||||
-rw-r--r-- | changelogs/unreleased/delay-update-statictics.yml | 5 | ||||
-rw-r--r-- | config/sidekiq_queues.yml | 3 | ||||
-rw-r--r-- | spec/workers/project_cache_worker_spec.rb | 36 | ||||
-rw-r--r-- | spec/workers/update_project_statistics_worker_spec.rb | 48 |
7 files changed, 107 insertions, 22 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 3e8c2a1209a..f9b2e698fc9 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -143,6 +143,7 @@ - repository_remove_remote - system_hook_push - update_merge_requests +- update_project_statistics - upload_checksum - web_hook - repository_update_remote_mirror diff --git a/app/workers/project_cache_worker.rb b/app/workers/project_cache_worker.rb index b31099bc670..73eb461768c 100644 --- a/app/workers/project_cache_worker.rb +++ b/app/workers/project_cache_worker.rb @@ -28,18 +28,20 @@ class ProjectCacheWorker def update_statistics(project, statistics = []) return if Gitlab::Database.read_only? - return unless try_obtain_lease_for(project.id, :update_statistics) + return unless try_obtain_lease_for(project.id, statistics) - Rails.logger.info("Updating statistics for project #{project.id}") - - project.statistics.refresh!(only: statistics) + UpdateProjectStatisticsWorker.perform_in(LEASE_TIMEOUT, project.id, statistics) end private - def try_obtain_lease_for(project_id, section) + def try_obtain_lease_for(project_id, statistics) Gitlab::ExclusiveLease - .new("project_cache_worker:#{project_id}:#{section}", timeout: LEASE_TIMEOUT) + .new(project_cache_worker_key(project_id, statistics), timeout: LEASE_TIMEOUT) .try_obtain end + + def project_cache_worker_key(project_id, statistics) + (["project_cache_worker", project_id] + statistics.sort).join(":") + end end diff --git a/app/workers/update_project_statistics_worker.rb b/app/workers/update_project_statistics_worker.rb new file mode 100644 index 00000000000..46673c4b07e --- /dev/null +++ b/app/workers/update_project_statistics_worker.rb @@ -0,0 +1,22 @@ + +# frozen_string_literal: true + +# Worker for updating project statistics. +class UpdateProjectStatisticsWorker + include ApplicationWorker + + # project_id - The ID of the project for which to flush the cache. + # statistics - An Array containing columns from ProjectStatistics to + # refresh, if empty all columns will be refreshed + # rubocop: disable CodeReuse/ActiveRecord + def perform(project_id, statistics = []) + project = Project.find_by(id: project_id) + + return unless project && project.repository.exists? + + Rails.logger.info("Updating statistics for project #{project.id}") + + project.statistics.refresh!(only: statistics.map(&:to_sym)) + end + # rubocop: enable CodeReuse/ActiveRecord +end diff --git a/changelogs/unreleased/delay-update-statictics.yml b/changelogs/unreleased/delay-update-statictics.yml new file mode 100644 index 00000000000..d0201fb6db8 --- /dev/null +++ b/changelogs/unreleased/delay-update-statictics.yml @@ -0,0 +1,5 @@ +--- +title: Fix the bug that the project statistics is not updated +merge_request: 26854 +author: Hiroyuki Sato +type: fixed diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 2dc0da00919..8bc2426ec4c 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -89,4 +89,5 @@ - [project_daily_statistics, 1] - [import_issues_csv, 2] - [chat_notification, 2] - - [migrate_external_diffs, 1] + - [migrate_external_diffs, 1] + - [update_project_statistics, 1] diff --git a/spec/workers/project_cache_worker_spec.rb b/spec/workers/project_cache_worker_spec.rb index a7353227043..724947228b7 100644 --- a/spec/workers/project_cache_worker_spec.rb +++ b/spec/workers/project_cache_worker_spec.rb @@ -7,9 +7,9 @@ describe ProjectCacheWorker do let(:worker) { described_class.new } let(:project) { create(:project, :repository) } - let(:statistics) { project.statistics } - let(:lease_key) { "project_cache_worker:#{project.id}:update_statistics" } + let(:lease_key) { (["project_cache_worker", project.id] + statistics.sort).join(":") } let(:lease_timeout) { ProjectCacheWorker::LEASE_TIMEOUT } + let(:statistics) { [] } describe '#perform' do before do @@ -35,14 +35,6 @@ describe ProjectCacheWorker do end context 'with an existing project' do - it 'updates the project statistics' do - expect(worker).to receive(:update_statistics) - .with(kind_of(Project), %i(repository_size)) - .and_call_original - - worker.perform(project.id, [], %w(repository_size)) - end - it 'refreshes the method caches' do expect_any_instance_of(Repository).to receive(:refresh_method_caches) .with(%i(readme)) @@ -51,6 +43,18 @@ describe ProjectCacheWorker do worker.perform(project.id, %w(readme)) end + context 'with statistics' do + let(:statistics) { %w(repository_size) } + + it 'updates the project statistics' do + expect(worker).to receive(:update_statistics) + .with(kind_of(Project), %i(repository_size)) + .and_call_original + + worker.perform(project.id, [], statistics) + end + end + context 'with plain readme' do it 'refreshes the method caches' do allow(MarkupHelper).to receive(:gitlab_markdown?).and_return(false) @@ -66,13 +70,15 @@ describe ProjectCacheWorker do end describe '#update_statistics' do + let(:statistics) { %w(repository_size) } + context 'when a lease could not be obtained' do it 'does not update the repository size' do stub_exclusive_lease_taken(lease_key, timeout: lease_timeout) - expect(statistics).not_to receive(:refresh!) + expect(UpdateProjectStatisticsWorker).not_to receive(:perform_in) - worker.update_statistics(project) + worker.update_statistics(project, statistics.map(&:to_sym)) end end @@ -80,11 +86,11 @@ describe ProjectCacheWorker do it 'updates the project statistics' do stub_exclusive_lease(lease_key, timeout: lease_timeout) - expect(statistics).to receive(:refresh!) - .with(only: %i(repository_size)) + expect(UpdateProjectStatisticsWorker).to receive(:perform_in) + .with(lease_timeout, project.id, statistics.map(&:to_sym)) .and_call_original - worker.update_statistics(project, %i(repository_size)) + worker.update_statistics(project, statistics.map(&:to_sym)) end end end diff --git a/spec/workers/update_project_statistics_worker_spec.rb b/spec/workers/update_project_statistics_worker_spec.rb new file mode 100644 index 00000000000..3411e10da7e --- /dev/null +++ b/spec/workers/update_project_statistics_worker_spec.rb @@ -0,0 +1,48 @@ +require 'spec_helper' + +describe UpdateProjectStatisticsWorker do + let(:worker) { described_class.new } + let(:project) { create(:project, :repository) } + + describe '#perform' do + context 'with a non-existing project' do + it 'does nothing' do + expect_any_instance_of(ProjectStatistics).not_to receive(:refresh!) + + worker.perform(-1) + end + end + + context 'with an existing project without a repository' do + it 'does nothing' do + allow_any_instance_of(Repository).to receive(:exists?).and_return(false) + + expect_any_instance_of(ProjectStatistics).not_to receive(:refresh!) + + worker.perform(project.id) + end + end + + context 'with an existing project' do + it 'refreshes the project statistics' do + expect_any_instance_of(ProjectStatistics).to receive(:refresh!) + .with(only: []) + .and_call_original + + worker.perform(project.id) + end + + context 'with a specific statistics target' do + it 'refreshes the project repository size' do + statistics_target = %w(repository_size) + + expect_any_instance_of(ProjectStatistics).to receive(:refresh!) + .with(only: statistics_target.map(&:to_sym)) + .and_call_original + + worker.perform(project.id, statistics_target) + end + end + end + end +end |