diff options
author | Kamil Trzciński <ayufan@ayufan.eu> | 2018-02-07 15:44:23 +0300 |
---|---|---|
committer | Kamil Trzciński <ayufan@ayufan.eu> | 2018-02-07 15:44:23 +0300 |
commit | 9bd044d9a74b6dd2fa252ed085562d56aef5a6ac (patch) | |
tree | 1aea80a68828ad50fcf0fab08e5280a63bd613ec /app | |
parent | d08bf247bc2a99120498e939ba573f44e5f27f07 (diff) | |
parent | e27ea805457a3b794d7a8b3b6b0355eddb1c1eca (diff) |
Merge branch '38265-stuckcijobsworker-wrongly-detects-cancels-stuck-builds-when-per-job-timeout-is-more-than-an-hour' into 'master'
Resolve "StuckCiJobsWorker wrongly detects, cancels 'stuck' builds when per-job timeout is more than an hour"
Closes #38265, #42196, and #42750
See merge request gitlab-org/gitlab-ce!16756
Diffstat (limited to 'app')
-rw-r--r-- | app/models/ci/runner.rb | 27 | ||||
-rw-r--r-- | app/models/concerns/redis_cacheable.rb | 41 |
2 files changed, 68 insertions, 0 deletions
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index dcbb397fb78..13c784bea0d 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -2,9 +2,11 @@ module Ci class Runner < ActiveRecord::Base extend Gitlab::Ci::Model include Gitlab::SQL::Pattern + include RedisCacheable RUNNER_QUEUE_EXPIRY_TIME = 60.minutes ONLINE_CONTACT_TIMEOUT = 1.hour + UPDATE_DB_RUNNER_INFO_EVERY = 40.minutes AVAILABLE_SCOPES = %w[specific shared active paused online].freeze FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level].freeze @@ -47,6 +49,8 @@ module Ci ref_protected: 1 } + cached_attr_reader :version, :revision, :platform, :architecture, :contacted_at + # Searches for runners matching the given query. # # This method uses ILIKE on PostgreSQL and LIKE on MySQL. @@ -152,6 +156,18 @@ module Ci ensure_runner_queue_value == value if value.present? end + def update_cached_info(values) + values = values&.slice(:version, :revision, :platform, :architecture) || {} + values[:contacted_at] = Time.now + + cache_attributes(values) + + if persist_cached_data? + self.assign_attributes(values) + self.save if self.changed? + end + end + private def cleanup_runner_queue @@ -164,6 +180,17 @@ module Ci "runner:build_queue:#{self.token}" end + def persist_cached_data? + # Use a random threshold to prevent beating DB updates. + # It generates a distribution between [40m, 80m]. + + contacted_at_max_age = UPDATE_DB_RUNNER_INFO_EVERY + Random.rand(UPDATE_DB_RUNNER_INFO_EVERY) + + real_contacted_at = read_attribute(:contacted_at) + real_contacted_at.nil? || + (Time.now - real_contacted_at) >= contacted_at_max_age + end + def tag_constraints unless has_tags? || run_untagged? errors.add(:tags_list, diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb new file mode 100644 index 00000000000..b889f4202dc --- /dev/null +++ b/app/models/concerns/redis_cacheable.rb @@ -0,0 +1,41 @@ +module RedisCacheable + extend ActiveSupport::Concern + include Gitlab::Utils::StrongMemoize + + CACHED_ATTRIBUTES_EXPIRY_TIME = 24.hours + + class_methods do + def cached_attr_reader(*attributes) + attributes.each do |attribute| + define_method("#{attribute}") do + cached_attribute(attribute) || read_attribute(attribute) + end + end + end + end + + def cached_attribute(attribute) + (cached_attributes || {})[attribute] + end + + def cache_attributes(values) + Gitlab::Redis::SharedState.with do |redis| + redis.set(cache_attribute_key, values.to_json, ex: CACHED_ATTRIBUTES_EXPIRY_TIME) + end + end + + private + + def cache_attribute_key + "cache:#{self.class.name}:#{self.id}:attributes" + end + + def cached_attributes + strong_memoize(:cached_attributes) do + Gitlab::Redis::SharedState.with do |redis| + data = redis.get(cache_attribute_key) + JSON.parse(data, symbolize_names: true) if data + end + end + end +end |