diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-09-02 15:09:50 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-09-02 15:09:50 +0300 |
commit | 5ed2199ea73700ec5a489b5d7d1c746c6b4f9d34 (patch) | |
tree | fb8503e320267658580d2e058287cc1b631b436a | |
parent | 28c25a9eb01f0bcaccec86438ff52056c78d360c (diff) |
Add latest changes from gitlab-org/gitlab@master
-rw-r--r-- | config/feature_flags/development/use_primary_and_secondary_stores_for_workhorse.yml | 8 | ||||
-rw-r--r-- | config/feature_flags/development/use_primary_store_as_default_for_workhorse.yml | 8 | ||||
-rw-r--r-- | lib/gitlab/redis.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/redis/workhorse.rb | 22 | ||||
-rw-r--r-- | lib/gitlab/setup_helper.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/workhorse.rb | 11 | ||||
-rw-r--r-- | spec/lib/gitlab/redis/workhorse_spec.rb | 44 | ||||
-rw-r--r-- | spec/lib/gitlab/setup_helper/workhorse_spec.rb | 10 | ||||
-rw-r--r-- | spec/lib/gitlab/workhorse_spec.rb | 37 |
9 files changed, 136 insertions, 11 deletions
diff --git a/config/feature_flags/development/use_primary_and_secondary_stores_for_workhorse.yml b/config/feature_flags/development/use_primary_and_secondary_stores_for_workhorse.yml new file mode 100644 index 00000000000..a5577c5d5d0 --- /dev/null +++ b/config/feature_flags/development/use_primary_and_secondary_stores_for_workhorse.yml @@ -0,0 +1,8 @@ +--- +name: use_primary_and_secondary_stores_for_workhorse +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/127577 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/423420 +milestone: '16.4' +type: development +group: group::scalability +default_enabled: false diff --git a/config/feature_flags/development/use_primary_store_as_default_for_workhorse.yml b/config/feature_flags/development/use_primary_store_as_default_for_workhorse.yml new file mode 100644 index 00000000000..e6d21627035 --- /dev/null +++ b/config/feature_flags/development/use_primary_store_as_default_for_workhorse.yml @@ -0,0 +1,8 @@ +--- +name: use_primary_store_as_default_for_workhorse +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/127577 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/423420 +milestone: '16.4' +type: development +group: group::scalability +default_enabled: false diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 6bed6937d70..5159a1fef86 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -19,7 +19,8 @@ module Gitlab Gitlab::Redis::Sessions, Gitlab::Redis::SharedState, Gitlab::Redis::TraceChunks, - Gitlab::Redis::Chat + Gitlab::Redis::Chat, + Gitlab::Redis::Workhorse ].freeze end end diff --git a/lib/gitlab/redis/workhorse.rb b/lib/gitlab/redis/workhorse.rb new file mode 100644 index 00000000000..ea0fca515fe --- /dev/null +++ b/lib/gitlab/redis/workhorse.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class Workhorse < ::Gitlab::Redis::Wrapper + class << self + def config_fallback + SharedState + end + + private + + def redis + primary_store = ::Redis.new(params) + secondary_store = ::Redis.new(config_fallback.params) + + MultiStore.new(primary_store, secondary_store, store_name) + end + end + end + end +end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 2e09a4fce12..8f737977e80 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -38,11 +38,11 @@ module Gitlab end def redis_url - Gitlab::Redis::SharedState.url + Gitlab::Redis::Workhorse.url end def redis_db - Gitlab::Redis::SharedState.params.fetch(:db, 0) + Gitlab::Redis::Workhorse.params.fetch(:db, 0) end def get_config_path(dir, _) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 79131404465..f8948c4959f 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -228,7 +228,7 @@ module Gitlab end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Redis::SharedState.with do |redis| + with_redis do |redis| result = redis.set(key, value, ex: expire, nx: !overwrite) if result redis.publish(NOTIFICATION_PREFIX + key, value) @@ -249,6 +249,15 @@ module Gitlab protected + def with_redis(&blk) + if Feature.enabled?(:use_primary_and_secondary_stores_for_workhorse) || + Feature.enabled?(:use_primary_store_as_default_for_workhorse) + Gitlab::Redis::Workhorse.with(&blk) # rubocop:disable CodeReuse/ActiveRecord + else + Gitlab::Redis::SharedState.with(&blk) # rubocop:disable CodeReuse/ActiveRecord + end + end + # This is the outermost encoding of a senddata: header. It is safe for # inclusion in HTTP response headers def encode(hash) diff --git a/spec/lib/gitlab/redis/workhorse_spec.rb b/spec/lib/gitlab/redis/workhorse_spec.rb new file mode 100644 index 00000000000..46931a6afcb --- /dev/null +++ b/spec/lib/gitlab/redis/workhorse_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Redis::Workhorse, feature_category: :scalability do + include_examples "redis_new_instance_shared_examples", 'workhorse', Gitlab::Redis::SharedState + include_examples "redis_shared_examples" + + describe '#pool' do + let(:config_new_format_host) { "spec/fixtures/config/redis_new_format_host.yml" } + let(:config_new_format_socket) { "spec/fixtures/config/redis_new_format_socket.yml" } + + subject { described_class.pool } + + before do + allow(described_class).to receive(:config_file_name).and_return(config_new_format_host) + + # Override rails root to avoid having our fixtures overwritten by `redis.yml` if it exists + allow(Gitlab::Redis::SharedState).to receive(:rails_root).and_return(mktmpdir) + allow(Gitlab::Redis::SharedState).to receive(:config_file_name).and_return(config_new_format_socket) + end + + around do |example| + clear_pool + example.run + ensure + clear_pool + end + + it 'instantiates an instance of MultiStore' do + subject.with do |redis_instance| + expect(redis_instance).to be_instance_of(::Gitlab::Redis::MultiStore) + + expect(redis_instance.primary_store.connection[:id]).to eq("redis://test-host:6379/99") + expect(redis_instance.secondary_store.connection[:id]).to eq("unix:///path/to/redis.sock/0") + + expect(redis_instance.instance_name).to eq('Workhorse') + end + end + + it_behaves_like 'multi store feature flags', :use_primary_and_secondary_stores_for_workhorse, + :use_primary_store_as_default_for_workhorse + end +end diff --git a/spec/lib/gitlab/setup_helper/workhorse_spec.rb b/spec/lib/gitlab/setup_helper/workhorse_spec.rb index 726b73a9dfe..e5a44abc731 100644 --- a/spec/lib/gitlab/setup_helper/workhorse_spec.rb +++ b/spec/lib/gitlab/setup_helper/workhorse_spec.rb @@ -24,8 +24,8 @@ RSpec.describe Gitlab::SetupHelper::Workhorse do end describe '.redis_url' do - it 'matches the SharedState URL' do - expect(Gitlab::Redis::SharedState).to receive(:url).and_return('foo') + it 'matches the Workhorse URL' do + expect(Gitlab::Redis::Workhorse).to receive(:url).and_return('foo') expect(described_class.redis_url).to eq('foo') end @@ -34,14 +34,14 @@ RSpec.describe Gitlab::SetupHelper::Workhorse do describe '.redis_db' do subject { described_class.redis_db } - it 'matches the SharedState DB' do - expect(Gitlab::Redis::SharedState).to receive(:params).and_return(db: 1) + it 'matches the Workhorse DB' do + expect(Gitlab::Redis::Workhorse).to receive(:params).and_return(db: 1) is_expected.to eq(1) end it 'defaults to 0 if unspecified' do - expect(Gitlab::Redis::SharedState).to receive(:params).and_return({}) + expect(Gitlab::Redis::Workhorse).to receive(:params).and_return({}) is_expected.to eq(0) end diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb index a1c2f7d667f..f726e0ff998 100644 --- a/spec/lib/gitlab/workhorse_spec.rb +++ b/spec/lib/gitlab/workhorse_spec.rb @@ -371,13 +371,13 @@ RSpec.describe Gitlab::Workhorse do subject { described_class.set_key_and_notify(key, value, overwrite: overwrite) } - shared_examples 'set and notify' do + shared_examples 'set and notify' do |redis = Gitlab::Redis::Workhorse| it 'set and return the same value' do is_expected.to eq(value) end it 'set and notify' do - expect(Gitlab::Redis::SharedState).to receive(:with).and_call_original + expect(redis).to receive(:with).and_call_original expect_any_instance_of(::Redis).to receive(:publish) .with(described_class::NOTIFICATION_PREFIX + 'test-key', "test-value") @@ -389,6 +389,39 @@ RSpec.describe Gitlab::Workhorse do let(:overwrite) { true } it_behaves_like 'set and notify' + + context 'when workhorse migration feature flags are disabled' do + before do + stub_feature_flags( + use_primary_and_secondary_stores_for_workhorse: false, + use_primary_store_as_default_for_workhorse: false + ) + end + + it_behaves_like 'set and notify', Gitlab::Redis::SharedState + end + + context 'when either workhorse migration feature flags are enabled' do + context 'when use_primary_and_secondary_stores_for_workhorse is enabled' do + before do + stub_feature_flags( + use_primary_store_as_default_for_workhorse: false + ) + end + + it_behaves_like 'set and notify' + end + + context 'when use_primary_store_as_default_for_workhorse is enabled' do + before do + stub_feature_flags( + use_primary_and_secondary_stores_for_workhorse: false + ) + end + + it_behaves_like 'set and notify' + end + end end context 'when we set an existing key' do |