Merge branch '38265-stuckcijobsworker-wrongly-detects-cancels-stuck-builds-when-per-job-timeout-is-more-than-an-hour' into 'master'

Resolve "StuckCiJobsWorker wrongly detects, cancels 'stuck' builds when per-job timeout is more than an hour"

Closes #38265, #42196, and #42750

See merge request gitlab-org/gitlab-ce!16756

3 files changed, 137 insertions, 13 deletions

diff --git a/spec/models/ci/runner_spec.rb b/spec/models/ci/runner_spec.rb
index b2b64e6ff48..ab170e6351c 100644
--- a/spec/models/ci/runner_spec.rb
+++ b/spec/models/ci/runner_spec.rb
@@ -95,28 +95,68 @@ describe Ci::Runner do
 
     subject { runner.online? }
 
-    context 'never contacted' do
+    before do
+      allow_any_instance_of(described_class).to receive(:cached_attribute).and_call_original
+      allow_any_instance_of(described_class).to receive(:cached_attribute)
+        .with(:platform).and_return("darwin")
+    end
+
+    context 'no cache value' do
       before do
-        runner.contacted_at = nil
+        stub_redis_runner_contacted_at(nil)
       end
 
-      it { is_expected.to be_falsey }
-    end
+      context 'never contacted' do
+        before do
+          runner.contacted_at = nil
+        end
 
-    context 'contacted long time ago time' do
-      before do
-        runner.contacted_at = 1.year.ago
+        it { is_expected.to be_falsey }
+      end
+
+      context 'contacted long time ago time' do
+        before do
+          runner.contacted_at = 1.year.ago
+        end
+
+        it { is_expected.to be_falsey }
       end
 
-      it { is_expected.to be_falsey }
+      context 'contacted 1s ago' do
+        before do
+          runner.contacted_at = 1.second.ago
+        end
+
+        it { is_expected.to be_truthy }
+      end
     end
 
-    context 'contacted 1s ago' do
-      before do
-        runner.contacted_at = 1.second.ago
+    context 'with cache value' do
+      context 'contacted long time ago time' do
+        before do
+          runner.contacted_at = 1.year.ago
+          stub_redis_runner_contacted_at(1.year.ago.to_s)
+        end
+
+        it { is_expected.to be_falsey }
+      end
+
+      context 'contacted 1s ago' do
+        before do
+          runner.contacted_at = 50.minutes.ago
+          stub_redis_runner_contacted_at(1.second.ago.to_s)
+        end
+
+        it { is_expected.to be_truthy }
       end
+    end
 
-      it { is_expected.to be_truthy }
+    def stub_redis_runner_contacted_at(value)
+      Gitlab::Redis::SharedState.with do |redis|
+        cache_key = runner.send(:cache_attribute_key)
+        expect(redis).to receive(:get).with(cache_key)
+          .and_return({ contacted_at: value }.to_json).at_least(:once)
+      end
     end
   end
 
@@ -361,6 +401,50 @@ describe Ci::Runner do
     end
   end
 
+  describe '#update_cached_info' do
+    let(:runner) { create(:ci_runner) }
+
+    subject { runner.update_cached_info(architecture: '18-bit') }
+
+    context 'when database was updated recently' do
+      before do
+        runner.contacted_at = Time.now
+      end
+
+      it 'updates cache' do
+        expect_redis_update
+
+        subject
+      end
+    end
+
+    context 'when database was not updated recently' do
+      before do
+        runner.contacted_at = 2.hours.ago
+      end
+
+      it 'updates database' do
+        expect_redis_update
+
+        expect { subject }.to change { runner.reload.read_attribute(:contacted_at) }
+          .and change { runner.reload.read_attribute(:architecture) }
+      end
+
+      it 'updates cache' do
+        expect_redis_update
+
+        subject
+      end
+    end
+
+    def expect_redis_update
+      Gitlab::Redis::SharedState.with do |redis|
+        redis_key = runner.send(:cache_attribute_key)
+        expect(redis).to receive(:set).with(redis_key, anything, any_args)
+      end
+    end
+  end
+
   describe '#destroy' do
     let(:runner) { create(:ci_runner) }
 
diff --git a/spec/models/concerns/redis_cacheable_spec.rb b/spec/models/concerns/redis_cacheable_spec.rb
new file mode 100644
index 00000000000..3d7963120b6
--- /dev/null
+++ b/spec/models/concerns/redis_cacheable_spec.rb
@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+describe RedisCacheable do
+  let(:model) { double }
+
+  before do
+    model.extend(described_class)
+    allow(model).to receive(:cache_attribute_key).and_return('key')
+  end
+
+  describe '#cached_attribute' do
+    let(:payload) { { attribute: 'value' } }
+
+    subject { model.cached_attribute(payload.keys.first) }
+
+    it 'gets the cache attribute' do
+      Gitlab::Redis::SharedState.with do |redis|
+        expect(redis).to receive(:get).with('key')
+          .and_return(payload.to_json)
+      end
+
+      expect(subject).to eq(payload.values.first)
+    end
+  end
+
+  describe '#cache_attributes' do
+    let(:values) { { name: 'new_name' } }
+
+    subject { model.cache_attributes(values) }
+
+    it 'sets the cache attributes' do
+      Gitlab::Redis::SharedState.with do |redis|
+        expect(redis).to receive(:set).with('key', values.to_json, anything)
+      end
+
+      subject
+    end
+  end
+end
diff --git a/spec/requests/api/runner_spec.rb b/spec/requests/api/runner_spec.rb
index 0bd88748479..f10b6e43d09 100644
--- a/spec/requests/api/runner_spec.rb
+++ b/spec/requests/api/runner_spec.rb
@@ -8,6 +8,7 @@ describe API::Runner do
   before do
     stub_gitlab_calls
     stub_application_setting(runners_registration_token: registration_token)
+    allow_any_instance_of(Ci::Runner).to receive(:cache_attributes)
   end
 
   describe '/api/v4/runners' do
@@ -408,7 +409,7 @@ describe API::Runner do
             expect { request_job }.to change { runner.reload.contacted_at }
           end
 
-          %w(name version revision platform architecture).each do |param|
+          %w(version revision platform architecture).each do |param|
             context "when info parameter '#{param}' is present" do
               let(:value) { "#{param}_value" }