path: root/spec/lib/gitlab/exclusive_lease_spec.rb

# frozen_string_literal: true

require 'spec_helper'

RSpec.describe Gitlab::ExclusiveLease, :request_store, :clean_gitlab_redis_shared_state,
  :clean_gitlab_redis_cluster_shared_state, feature_category: :shared do
  let(:unique_key) { SecureRandom.hex(10) }

  describe '#try_obtain' do
    it 'cannot obtain twice before the lease has expired' do
      lease = described_class.new(unique_key, timeout: 3600)
      expect(lease.try_obtain).to be_present
      expect(lease.try_obtain).to eq(false)
    end

    it 'can obtain after the lease has expired' do
      timeout = 1
      lease = described_class.new(unique_key, timeout: timeout)
      lease.try_obtain # start the lease
      sleep(2 * timeout) # lease should have expired now
      expect(lease.try_obtain).to be_present
    end

    context 'when migrating across stores' do
      let(:lease) { described_class.new(unique_key, timeout: 3600) }

      before do
        stub_feature_flags(use_cluster_shared_state_for_exclusive_lease: false)
        allow(lease).to receive(:same_store).and_return(false)
      end

      it 'acquires 2 locks' do
        # stub first SETNX
        Gitlab::Redis::SharedState.with { |r| expect(r).to receive(:set).and_return(true) }
        Gitlab::Redis::ClusterSharedState.with { |r| expect(r).to receive(:set).and_call_original }

        expect(lease.try_obtain).to be_truthy
      end

      it 'rollback first lock if second lock is not acquired' do
        Gitlab::Redis::ClusterSharedState.with do |r|
          expect(r).to receive(:set).and_return(false)
          expect(r).to receive(:eval).and_call_original
        end

        Gitlab::Redis::SharedState.with do |r|
          expect(r).to receive(:set).and_call_original
          expect(r).to receive(:eval).and_call_original
        end

        expect(lease.try_obtain).to be_falsey
      end
    end

    context 'when cutting over to ClusterSharedState' do
      context 'when lock is not acquired' do
        it 'waits for existing holder to yield the lock' do
          Gitlab::Redis::ClusterSharedState.with { |r| expect(r).to receive(:set).and_call_original }
          Gitlab::Redis::SharedState.with { |r| expect(r).not_to receive(:set) }

          lease = described_class.new(unique_key, timeout: 3600)
          expect(lease.try_obtain).to be_truthy
        end
      end

      context 'when lock is still acquired' do
        let(:lease) { described_class.new(unique_key, timeout: 3600) }

        before do
          # simulates cutover where some application's feature-flag has not updated
          stub_feature_flags(use_cluster_shared_state_for_exclusive_lease: false)
          lease.try_obtain
          stub_feature_flags(use_cluster_shared_state_for_exclusive_lease: true)
        end

        it 'waits for existing holder to yield the lock' do
          Gitlab::Redis::ClusterSharedState.with { |r| expect(r).not_to receive(:set) }
          Gitlab::Redis::SharedState.with { |r| expect(r).not_to receive(:set) }

          expect(lease.try_obtain).to be_falsey
        end
      end
    end
  end

  describe '.redis_shared_state_key' do
    it 'provides a namespaced key' do
      expect(described_class.redis_shared_state_key(unique_key))
        .to start_with(described_class::PREFIX)
        .and include(unique_key)
    end
  end

  describe '.ensure_prefixed_key' do
    it 'does not double prefix a key' do
      prefixed = described_class.redis_shared_state_key(unique_key)

      expect(described_class.ensure_prefixed_key(unique_key))
        .to eq(described_class.ensure_prefixed_key(prefixed))
    end

    it 'raises errors when there is no key' do
      expect { described_class.ensure_prefixed_key(nil) }.to raise_error(described_class::NoKey)
    end
  end

  shared_examples 'write operations' do
    describe '#renew' do
      it 'returns true when we have the existing lease' do
        lease = described_class.new(unique_key, timeout: 3600)
        expect(lease.try_obtain).to be_present
        expect(lease.renew).to be_truthy
      end

      it 'returns false when we dont have a lease' do
        lease = described_class.new(unique_key, timeout: 3600)
        expect(lease.renew).to be_falsey
      end
    end

    describe 'cancellation' do
      def new_lease(key)
        described_class.new(key, timeout: 3600)
      end

      shared_examples 'cancelling a lease' do
        let(:lease) { new_lease(unique_key) }

        it 'releases the held lease' do
          uuid = lease.try_obtain
          expect(uuid).to be_present
          expect(new_lease(unique_key).try_obtain).to eq(false)

          cancel_lease(uuid)

          expect(new_lease(unique_key).try_obtain).to be_present
        end
      end

      describe '.cancel' do
        def cancel_lease(uuid)
          described_class.cancel(release_key, uuid)
        end

        context 'when called with the unprefixed key' do
          it_behaves_like 'cancelling a lease' do
            let(:release_key) { unique_key }
          end
        end

        context 'when called with the prefixed key' do
          it_behaves_like 'cancelling a lease' do
            let(:release_key) { described_class.redis_shared_state_key(unique_key) }
          end
        end

        it 'does not raise errors when given a nil key' do
          expect { described_class.cancel(nil, nil) }.not_to raise_error
        end
      end

      describe '#cancel' do
        def cancel_lease(_uuid)
          lease.cancel
        end

        it_behaves_like 'cancelling a lease'

        it 'is safe to call even if the lease was never obtained' do
          lease = new_lease(unique_key)

          lease.cancel

          expect(new_lease(unique_key).try_obtain).to be_present
        end
      end
    end

    describe '.reset_all!' do
      it 'removes all existing lease keys from redis' do
        uuid = described_class.new(unique_key, timeout: 3600).try_obtain

        expect(described_class.get_uuid(unique_key)).to eq(uuid)

        described_class.reset_all!

        expect(described_class.get_uuid(unique_key)).to be_falsey
      end
    end
  end

  shared_examples 'read operations' do
    describe '#exists?' do
      it 'returns true for an existing lease' do
        lease = described_class.new(unique_key, timeout: 3600)
        lease.try_obtain

        expect(lease.exists?).to eq(true)
      end

      it 'returns false for a lease that does not exist' do
        lease = described_class.new(unique_key, timeout: 3600)

        expect(lease.exists?).to eq(false)
      end
    end

    describe '.get_uuid' do
      it 'gets the uuid if lease with the key associated exists' do
        uuid = described_class.new(unique_key, timeout: 3600).try_obtain

        expect(described_class.get_uuid(unique_key)).to eq(uuid)
      end

      it 'returns false if the lease does not exist' do
        expect(described_class.get_uuid(unique_key)).to be false
      end
    end

    describe '#ttl' do
      it 'returns the TTL of the Redis key' do
        lease = described_class.new('kittens', timeout: 100)
        lease.try_obtain

        expect(lease.ttl <= 100).to eq(true)
      end

      it 'returns nil when the lease does not exist' do
        lease = described_class.new('kittens', timeout: 10)

        expect(lease.ttl).to be_nil
      end
    end
  end

  context 'when migrating across stores' do
    before do
      stub_feature_flags(use_cluster_shared_state_for_exclusive_lease: false)
    end

    it_behaves_like 'read operations'
    it_behaves_like 'write operations'
  end

  context 'when feature flags are all disabled' do
    before do
      stub_feature_flags(
        use_cluster_shared_state_for_exclusive_lease: false,
        enable_exclusive_lease_double_lock_rw: false
      )
    end

    it_behaves_like 'read operations'
    it_behaves_like 'write operations'
  end

  it_behaves_like 'read operations'
  it_behaves_like 'write operations'

  describe '.throttle' do
    it 'prevents repeated execution of the block' do
      number = 0

      action = -> { described_class.throttle(1) { number += 1 } }

      action.call
      action.call

      expect(number).to eq 1
    end

    it 'is distinct by block' do
      number = 0

      described_class.throttle(1) { number += 1 }
      described_class.throttle(1) { number += 1 }

      expect(number).to eq 2
    end

    it 'is distinct by key' do
      number = 0

      action = ->(k) { described_class.throttle(k) { number += 1 } }

      action.call(:a)
      action.call(:b)
      action.call(:a)

      expect(number).to eq 2
    end

    it 'allows a group to be passed' do
      number = 0

      described_class.throttle(1, group: :a) { number += 1 }
      described_class.throttle(1, group: :b) { number += 1 }
      described_class.throttle(1, group: :a) { number += 1 }
      described_class.throttle(1, group: :b) { number += 1 }

      expect(number).to eq 2
    end

    it 'defaults to a 60min timeout' do
      expect(described_class).to receive(:new).with(anything, hash_including(timeout: 1.hour.to_i)).and_call_original

      described_class.throttle(1) {}
    end

    it 'allows count to be specified' do
      expect(described_class)
        .to receive(:new)
        .with(anything, hash_including(timeout: 15.minutes.to_i))
        .and_call_original

      described_class.throttle(1, count: 4) {}
    end

    it 'allows period to be specified' do
      expect(described_class)
        .to receive(:new)
        .with(anything, hash_including(timeout: 1.day.to_i))
        .and_call_original

      described_class.throttle(1, period: 1.day) {}
    end

    it 'allows period and count to be specified' do
      expect(described_class)
        .to receive(:new)
        .with(anything, hash_including(timeout: 30.minutes.to_i))
        .and_call_original

      described_class.throttle(1, count: 48, period: 1.day) {}
    end
  end

  describe 'transitions between feature-flag toggles' do
    shared_examples 'retains behaviours across transitions' do |flag|
      it 'retains read behaviour' do
        lease = described_class.new(unique_key, timeout: 3600)
        uuid = lease.try_obtain

        expect(lease.ttl).not_to eq(nil)
        expect(lease.exists?).to be_truthy
        expect(described_class.get_uuid(unique_key)).to eq(uuid)

        # simulates transition
        stub_feature_flags({ flag => true })
        Gitlab::SafeRequestStore.clear!

        expect(lease.ttl).not_to eq(nil)
        expect(lease.exists?).to be_truthy
        expect(described_class.get_uuid(unique_key)).to eq(uuid)
      end

      it 'retains renew behaviour' do
        lease = described_class.new(unique_key, timeout: 3600)
        lease.try_obtain

        expect(lease.renew).to be_truthy

        # simulates transition
        stub_feature_flags({ flag => true })
        Gitlab::SafeRequestStore.clear!

        expect(lease.renew).to be_truthy
      end

      it 'retains renew behaviour' do
        lease = described_class.new(unique_key, timeout: 3600)
        uuid = lease.try_obtain
        lease.cancel

        # proves successful cancellation
        expect(lease.try_obtain).to eq(uuid)

        # simulates transition
        stub_feature_flags({ flag => true })
        Gitlab::SafeRequestStore.clear!

        expect(lease.try_obtain).to be_falsey
        lease.cancel
        expect(lease.try_obtain).to eq(uuid)
      end
    end

    context 'when enabling enable_exclusive_lease_double_lock_rw' do
      before do
        stub_feature_flags(
          enable_exclusive_lease_double_lock_rw: false,
          use_cluster_shared_state_for_exclusive_lease: false
        )
      end

      it_behaves_like 'retains behaviours across transitions', :enable_exclusive_lease_double_lock_rw
    end

    context 'when enabling use_cluster_shared_state_for_exclusive_lease' do
      before do
        stub_feature_flags(use_cluster_shared_state_for_exclusive_lease: false)
      end

      it_behaves_like 'retains behaviours across transitions', :use_cluster_shared_state_for_exclusive_lease
    end
  end
end