diff options
Diffstat (limited to 'app/services/issue_rebalancing_service.rb')
| -rw-r--r-- | app/services/issue_rebalancing_service.rb | 50 |
1 files changed, 47 insertions, 3 deletions
diff --git a/app/services/issue_rebalancing_service.rb b/app/services/issue_rebalancing_service.rb index f9c3388204f..6a8d45b92b2 100644 --- a/app/services/issue_rebalancing_service.rb +++ b/app/services/issue_rebalancing_service.rb @@ -3,8 +3,18 @@ class IssueRebalancingService MAX_ISSUE_COUNT = 10_000 BATCH_SIZE = 100 + SMALLEST_BATCH_SIZE = 5 + RETRIES_LIMIT = 3 TooManyIssues = Class.new(StandardError) + TIMING_CONFIGURATION = [ + [0.1.seconds, 0.05.seconds], # short timings, lock_timeout: 100ms, sleep after LockWaitTimeout: 50ms + [0.5.seconds, 0.05.seconds], + [1.second, 0.5.seconds], + [1.second, 0.5.seconds], + [5.seconds, 1.second] + ].freeze + def initialize(issue) @issue = issue @base = Issue.relative_positioning_query_base(issue) @@ -23,14 +33,23 @@ class IssueRebalancingService assign_positions(start, indexed_ids) .sort_by(&:first) .each_slice(BATCH_SIZE) do |pairs_with_position| - update_positions(pairs_with_position, 'rebalance issue positions in batches ordered by id') + if Feature.enabled?(:issue_rebalancing_with_retry) + update_positions_with_retry(pairs_with_position, 'rebalance issue positions in batches ordered by id') + else + update_positions(pairs_with_position, 'rebalance issue positions in batches ordered by id') + end end end else Issue.transaction do indexed_ids.each_slice(BATCH_SIZE) do |pairs| pairs_with_position = assign_positions(start, pairs) - update_positions(pairs_with_position, 'rebalance issue positions') + + if Feature.enabled?(:issue_rebalancing_with_retry) + update_positions_with_retry(pairs_with_position, 'rebalance issue positions') + else + update_positions(pairs_with_position, 'rebalance issue positions') + end end end end @@ -52,12 +71,37 @@ class IssueRebalancingService end end + def update_positions_with_retry(pairs_with_position, query_name) + retries = 0 + batch_size = pairs_with_position.size + + until pairs_with_position.empty? + begin + update_positions(pairs_with_position.first(batch_size), query_name) + pairs_with_position = pairs_with_position.drop(batch_size) + retries = 0 + rescue ActiveRecord::StatementTimeout, ActiveRecord::QueryCanceled => ex + raise ex if batch_size < SMALLEST_BATCH_SIZE + + if (retries += 1) == RETRIES_LIMIT + # shrink the batch size in half when RETRIES limit is reached and update still fails perhaps because batch size is still too big + batch_size = (batch_size / 2).to_i + retries = 0 + end + + retry + end + end + end + def update_positions(pairs_with_position, query_name) values = pairs_with_position.map do |id, index| "(#{id}, #{index})" end.join(', ') - run_update_query(values, query_name) + Gitlab::Database::WithLockRetries.new(timing_configuration: TIMING_CONFIGURATION, klass: self.class).run do + run_update_query(values, query_name) + end end def run_update_query(values, query_name) |