diff options
author | Stan Hu <stanhu@gmail.com> | 2016-10-28 17:41:24 +0300 |
---|---|---|
committer | Stan Hu <stanhu@gmail.com> | 2016-10-28 17:41:24 +0300 |
commit | d306b0d7c2c1f9384382c2a90a9d7c43bd20573c (patch) | |
tree | 307b59b8e83d752eba286ab883a4a2b723deee3e /app | |
parent | 144358e98ee1b25b61854a3471e21e100ace9db5 (diff) | |
parent | 2822526e7b996c90fb3bbd7c286c2777e5e37360 (diff) |
Merge branch 'use-optimistic-locking' into 'master'
Use optimistic locking
## What does this MR do?
Removes the usage of pessimistic locking in favor of optimistic which is way cheaper and doesn't block database operation.
Since this is very simple change it should be safe. If we receive `StaleObjectError` message we will reload object a retry operations in lock.
However, I still believe that we need this one: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7005 as this will reduce a load on Database and FS.
This changes a behavior from:
### Pesimistic locking (previous behavior)
#### For updating
1. SELECT * FOR UPDATE (other updates wait on this)
2. we update ci_pipeline
3. latest_build_status
4. enqueue: (use: transition :created -> :pending)
5. [state_machine] we are in state created, we can go to pending
6. [state_machine] ci_pipeline.status = created
7. [state_machine] ci_pipeline.save
8. [state_machine] after_transition: (if for success): PipelineSuccessWorker on Sidekiq
9. release DB lock
#### If no update is required
1. SELECT * FOR UPDATE (other updates wait on this)
2. we update ci_pipeline
3. latest_build_status
4. we are in pending, we can't transition to pending, because it's forbidden
5. release DB lock
### Optimistic locking (implemented by this MR)
#### For updating
1. latest_build_status
2. enqueue: (use `transition :created -> :pending`)
3. [state_machine] we are in state created, we can go to pending
4. [state_machine] ci_pipeline.status = created
5. [state_machine] ci_pipeline.save
6. [state_machine] [save] where(lock_version: ci_pipeline.lock_version).update_all(status: :created, updated_at: Time.now)
7. [state_machine] [save] unless we_updated_row then raise ObjectInconsistentError
#### If no update is required
1. we update ci_pipeline
2. latest_build_status
3. we are in pending, we can't transition to pending, because it's forbidden
## Why was this MR needed?
We have been seeing a number of problems when we migrated Pipeline/Build processing to Sidekiq. Especially we started seeing a lot of blocking queries.
We used a pessimistic locking which doesn't seem to be required. This effectively allows us to fix our issues with blocked queries by using more efficient method of operation.
## What are the relevant issue numbers?
Issues: https://gitlab.com/gitlab-com/infrastructure/issues/623 and https://gitlab.com/gitlab-com/infrastructure/issues/584, but also there's a bunch of Merge Requests that try to improve behavior of scheduled jobs.
cc @pcarranza @yorickpeterse @stanhu
See merge request !7040
Diffstat (limited to 'app')
-rw-r--r-- | app/models/ci/pipeline.rb | 12 | ||||
-rw-r--r-- | app/models/commit_status.rb | 12 | ||||
-rw-r--r-- | app/services/ci/process_pipeline_service.rb | 23 | ||||
-rw-r--r-- | app/services/ci/register_build_service.rb | 11 |
4 files changed, 27 insertions, 31 deletions
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index adda3b8f40c..d3432632899 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -30,23 +30,23 @@ module Ci end event :run do - transition any => :running + transition any - [:running] => :running end event :skip do - transition any => :skipped + transition any - [:skipped] => :skipped end event :drop do - transition any => :failed + transition any - [:failed] => :failed end event :succeed do - transition any => :success + transition any - [:success] => :success end event :cancel do - transition any => :canceled + transition any - [:canceled] => :canceled end # IMPORTANT @@ -260,7 +260,7 @@ module Ci end def update_status - with_lock do + Gitlab::OptimisticLocking.retry_lock(self) do case latest_builds_status when 'pending' then enqueue when 'running' then run diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 4cb3a69416e..d159fc6c5c7 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -73,16 +73,16 @@ class CommitStatus < ActiveRecord::Base transition [:created, :pending, :running] => :canceled end - after_transition created: [:pending, :running] do |commit_status| - commit_status.update_attributes queued_at: Time.now + before_transition created: [:pending, :running] do |commit_status| + commit_status.queued_at = Time.now end - after_transition [:created, :pending] => :running do |commit_status| - commit_status.update_attributes started_at: Time.now + before_transition [:created, :pending] => :running do |commit_status| + commit_status.started_at = Time.now end - after_transition any => [:success, :failed, :canceled] do |commit_status| - commit_status.update_attributes finished_at: Time.now + before_transition any => [:success, :failed, :canceled] do |commit_status| + commit_status.finished_at = Time.now end after_transition do |commit_status, transition| diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb index d3dd30b2588..8face432d97 100644 --- a/app/services/ci/process_pipeline_service.rb +++ b/app/services/ci/process_pipeline_service.rb @@ -10,17 +10,14 @@ module Ci create_builds! end - @pipeline.with_lock do - new_builds = - stage_indexes_of_created_builds.map do |index| - process_stage(index) - end + new_builds = + stage_indexes_of_created_builds.map do |index| + process_stage(index) + end - @pipeline.update_status + @pipeline.update_status - # Return a flag if a when builds got enqueued - new_builds.flatten.any? - end + new_builds.flatten.any? end private @@ -32,9 +29,11 @@ module Ci def process_stage(index) current_status = status_for_prior_stages(index) - created_builds_in_stage(index).select do |build| - if HasStatus::COMPLETED_STATUSES.include?(current_status) - process_build(build, current_status) + if HasStatus::COMPLETED_STATUSES.include?(current_status) + created_builds_in_stage(index).select do |build| + Gitlab::OptimisticLocking.retry_lock(build) do |subject| + process_build(subject, current_status) + end end end end diff --git a/app/services/ci/register_build_service.rb b/app/services/ci/register_build_service.rb index 6973191b203..74b5ebf372b 100644 --- a/app/services/ci/register_build_service.rb +++ b/app/services/ci/register_build_service.rb @@ -28,17 +28,14 @@ module Ci if build # In case when 2 runners try to assign the same build, second runner will be declined - # with StateMachines::InvalidTransition in run! method. - build.with_lock do - build.runner_id = current_runner.id - build.save! - build.run! - end + # with StateMachines::InvalidTransition or StaleObjectError when doing run! or save method. + build.runner_id = current_runner.id + build.run! end build - rescue StateMachines::InvalidTransition + rescue StateMachines::InvalidTransition, ActiveRecord::StaleObjectError nil end |