Add latest changes from gitlab-org/gitlab@master

7 files changed, 366 insertions, 11 deletions

diff --git a/spec/lib/gitlab/anonymous_session_spec.rb b/spec/lib/gitlab/anonymous_session_spec.rb
index 245ca02e91a..64186e9003a 100644
--- a/spec/lib/gitlab/anonymous_session_spec.rb
+++ b/spec/lib/gitlab/anonymous_session_spec.rb
@@ -2,7 +2,7 @@
 
 require 'spec_helper'
 
-RSpec.describe Gitlab::AnonymousSession, :clean_gitlab_redis_shared_state do
+RSpec.describe Gitlab::AnonymousSession, :clean_gitlab_redis_sessions do
   let(:default_session_id) { '6919a6f1bb119dd7396fadc38fd18d0d' }
   let(:additional_session_id) { '7919a6f1bb119dd7396fadc38fd18d0d' }
 
@@ -16,7 +16,7 @@ RSpec.describe Gitlab::AnonymousSession, :clean_gitlab_redis_shared_state do
     it 'adds session id to proper key' do
       subject.count_session_ip
 
-      Gitlab::Redis::SharedState.with do |redis|
+      Gitlab::Redis::Sessions.with do |redis|
         expect(redis.get("session:lookup:ip:gitlab2:127.0.0.1").to_i).to eq 1
       end
     end
@@ -25,7 +25,7 @@ RSpec.describe Gitlab::AnonymousSession, :clean_gitlab_redis_shared_state do
       freeze_time do
         subject.count_session_ip
 
-        Gitlab::Redis::SharedState.with do |redis|
+        Gitlab::Redis::Sessions.with do |redis|
           expect(redis.ttl("session:lookup:ip:gitlab2:127.0.0.1")).to eq(24.hours.to_i)
         end
       end
@@ -36,7 +36,7 @@ RSpec.describe Gitlab::AnonymousSession, :clean_gitlab_redis_shared_state do
         subject.count_session_ip
         new_anonymous_session.count_session_ip
 
-        Gitlab::Redis::SharedState.with do |redis|
+        Gitlab::Redis::Sessions.with do |redis|
           expect(redis.get("session:lookup:ip:gitlab2:127.0.0.1").to_i).to eq(2)
         end
       end
@@ -45,7 +45,7 @@ RSpec.describe Gitlab::AnonymousSession, :clean_gitlab_redis_shared_state do
 
   describe '#stored_sessions' do
     it 'returns all anonymous sessions per ip' do
-      Gitlab::Redis::SharedState.with do |redis|
+      Gitlab::Redis::Sessions.with do |redis|
         redis.set("session:lookup:ip:gitlab2:127.0.0.1", 2)
       end
 
@@ -54,13 +54,13 @@ RSpec.describe Gitlab::AnonymousSession, :clean_gitlab_redis_shared_state do
   end
 
   it 'removes obsolete lookup through ip entries' do
-    Gitlab::Redis::SharedState.with do |redis|
+    Gitlab::Redis::Sessions.with do |redis|
       redis.set("session:lookup:ip:gitlab2:127.0.0.1", 2)
     end
 
     subject.cleanup_session_per_ip_count
 
-    Gitlab::Redis::SharedState.with do |redis|
+    Gitlab::Redis::Sessions.with do |redis|
       expect(redis.exists("session:lookup:ip:gitlab2:127.0.0.1")).to eq(false)
     end
   end
diff --git a/spec/lib/gitlab/application_context_spec.rb b/spec/lib/gitlab/application_context_spec.rb
index ecd68caba79..5ecec978017 100644
--- a/spec/lib/gitlab/application_context_spec.rb
+++ b/spec/lib/gitlab/application_context_spec.rb
@@ -152,6 +152,38 @@ RSpec.describe Gitlab::ApplicationContext do
         end
       end
     end
+
+    context 'when using a runner project' do
+      let_it_be_with_reload(:runner) { create(:ci_runner, :project) }
+
+      it 'sets project path from runner project' do
+        context = described_class.new(runner: runner)
+
+        expect(result(context)).to include(project: runner.runner_projects.first.project.full_path)
+      end
+
+      context 'when the runner serves multiple projects' do
+        before do
+          create(:ci_runner_project, runner: runner, project: create(:project))
+        end
+
+        it 'does not set project path' do
+          context = described_class.new(runner: runner)
+
+          expect(result(context)).to include(project: nil)
+        end
+      end
+    end
+
+    context 'when using an instance runner' do
+      let_it_be(:runner) { create(:ci_runner, :instance) }
+
+      it 'does not sets project path' do
+        context = described_class.new(runner: runner)
+
+        expect(result(context)).to include(project: nil)
+      end
+    end
   end
 
   describe '#use' do
diff --git a/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb b/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb
index 1c6f5c5c694..09f50c6507a 100644
--- a/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb
+++ b/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb
@@ -16,7 +16,7 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do
     subject(:sync_partitions) { described_class.new(model).sync_partitions }
 
     let(:model) { double(partitioning_strategy: partitioning_strategy, table_name: table, connection: connection) }
-    let(:partitioning_strategy) { double(missing_partitions: partitions, extra_partitions: []) }
+    let(:partitioning_strategy) { double(missing_partitions: partitions, extra_partitions: [], after_adding_partitions: nil) }
     let(:connection) { ActiveRecord::Base.connection }
     let(:table) { "some_table" }
 
@@ -83,7 +83,7 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do
 
     let(:manager) { described_class.new(model) }
     let(:model) { double(partitioning_strategy: partitioning_strategy, table_name: table, connection: connection) }
-    let(:partitioning_strategy) { double(extra_partitions: extra_partitions, missing_partitions: []) }
+    let(:partitioning_strategy) { double(extra_partitions: extra_partitions, missing_partitions: [], after_adding_partitions: nil) }
     let(:connection) { ActiveRecord::Base.connection }
     let(:table) { "foo" }
 
diff --git a/spec/lib/gitlab/database/partitioning/single_numeric_list_partition_spec.rb b/spec/lib/gitlab/database/partitioning/single_numeric_list_partition_spec.rb
new file mode 100644
index 00000000000..9941241e846
--- /dev/null
+++ b/spec/lib/gitlab/database/partitioning/single_numeric_list_partition_spec.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Database::Partitioning::SingleNumericListPartition do
+  describe '.from_sql' do
+    subject(:parsed_partition) { described_class.from_sql(table, partition_name, definition) }
+
+    let(:table) { 'partitioned_table' }
+    let(:partition_value) { 0 }
+    let(:partition_name) { "partitioned_table_#{partition_value}" }
+    let(:definition) { "FOR VALUES IN ('#{partition_value}')" }
+
+    it 'uses specified table name' do
+      expect(parsed_partition.table).to eq(table)
+    end
+
+    it 'uses specified partition name' do
+      expect(parsed_partition.partition_name).to eq(partition_name)
+    end
+
+    it 'parses the definition' do
+      expect(parsed_partition.value).to eq(partition_value)
+    end
+  end
+
+  describe '#partition_name' do
+    it 'is the explicit name if provided' do
+      expect(described_class.new('table', 1, partition_name: 'some_other_name').partition_name).to eq('some_other_name')
+    end
+
+    it 'defaults to the table name followed by the partition value' do
+      expect(described_class.new('table', 1).partition_name).to eq('table_1')
+    end
+  end
+
+  context 'sorting' do
+    it 'is incomparable if the tables do not match' do
+      expect(described_class.new('table1', 1) <=> described_class.new('table2', 2)).to be_nil
+    end
+
+    it 'sorts by the value when the tables match' do
+      expect(described_class.new('table1', 1) <=> described_class.new('table1', 2)).to eq(1 <=> 2)
+    end
+
+    it 'sorts by numeric value rather than text value' do
+      expect(described_class.new('table', 10)).to be > described_class.new('table', 9)
+    end
+  end
+end
diff --git a/spec/lib/gitlab/database/partitioning/sliding_list_strategy_spec.rb b/spec/lib/gitlab/database/partitioning/sliding_list_strategy_spec.rb
new file mode 100644
index 00000000000..73474e8b38a
--- /dev/null
+++ b/spec/lib/gitlab/database/partitioning/sliding_list_strategy_spec.rb
@@ -0,0 +1,214 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Database::Partitioning::SlidingListStrategy do
+  let(:connection) { ActiveRecord::Base.connection }
+  let(:table_name) { :_test_partitioned_test }
+  let(:model) { double('model', table_name: table_name, ignored_columns: %w[partition]) }
+  let(:next_partition_if) { double('next_partition_if') }
+  let(:detach_partition_if) { double('detach_partition_if') }
+
+  subject(:strategy) do
+    described_class.new(model, :partition,
+                        next_partition_if: next_partition_if,
+                        detach_partition_if: detach_partition_if)
+  end
+
+  before do
+    connection.execute(<<~SQL)
+      create table #{table_name}
+        (
+          id serial not null,
+          partition bigint not null default 2,
+          created_at timestamptz not null,
+          primary key (id, partition)
+        )
+        partition by list(partition);
+
+      create table #{table_name}_1
+      partition of #{table_name} for values in (1);
+
+      create table #{table_name}_2
+      partition of #{table_name} for values in (2);
+    SQL
+  end
+
+  describe '#current_partitions' do
+    it 'detects both partitions' do
+      expect(strategy.current_partitions).to eq([
+                                         Gitlab::Database::Partitioning::SingleNumericListPartition.new(table_name, 1, partition_name: '_test_partitioned_test_1'),
+                                         Gitlab::Database::Partitioning::SingleNumericListPartition.new(table_name, 2, partition_name: '_test_partitioned_test_2')
+      ])
+    end
+  end
+
+  describe '#active_partition' do
+    it 'is the partition with the largest value' do
+      expect(strategy.active_partition.value).to eq(2)
+    end
+  end
+
+  describe '#missing_partitions' do
+    context 'when next_partition_if returns true' do
+      let(:next_partition_if) { proc { true } }
+
+      it 'is a partition definition for the next partition in the series' do
+        extra = strategy.missing_partitions
+
+        expect(extra.length).to eq(1)
+        expect(extra.first.value).to eq(3)
+      end
+    end
+
+    context 'when next_partition_if returns false' do
+      let(:next_partition_if) { proc { false } }
+
+      it 'is empty' do
+        expect(strategy.missing_partitions).to be_empty
+      end
+    end
+
+    context 'when there are no partitions for the table' do
+      it 'returns a partition for value 1' do
+        connection.execute("drop table #{table_name}_1; drop table #{table_name}_2;")
+
+        missing_partitions = strategy.missing_partitions
+
+        expect(missing_partitions.size).to eq(1)
+        missing_partition = missing_partitions.first
+
+        expect(missing_partition.value).to eq(1)
+      end
+    end
+  end
+
+  describe '#extra_partitions' do
+    before do
+      (3..10).each do |i|
+        connection.execute("CREATE TABLE #{table_name}_#{i} PARTITION OF #{table_name} FOR VALUES IN (#{i})")
+      end
+    end
+
+    context 'when some partitions are true for detach_partition_if' do
+      let(:detach_partition_if) { ->(p) { p != 5 } }
+
+      it 'is the leading set of partitions before that value' do
+        expect(strategy.extra_partitions.map(&:value)).to contain_exactly(1, 2, 3, 4)
+      end
+    end
+
+    context 'when all partitions are true for detach_partition_if' do
+      let(:detach_partition_if) { proc { true } }
+
+      it 'is all but the most recent partition', :aggregate_failures do
+        expect(strategy.extra_partitions.map(&:value)).to contain_exactly(1, 2, 3, 4, 5, 6, 7, 8, 9)
+
+        expect(strategy.current_partitions.map(&:value).max).to eq(10)
+      end
+    end
+  end
+
+  describe '#initial_partition' do
+    it 'starts with the value 1', :aggregate_failures do
+      initial_partition = strategy.initial_partition
+      expect(initial_partition.value).to eq(1)
+      expect(initial_partition.table).to eq(strategy.table_name)
+      expect(initial_partition.partition_name).to eq("#{strategy.table_name}_1")
+    end
+  end
+
+  describe '#next_partition' do
+    it 'is one after the active partition', :aggregate_failures do
+      expect(strategy).to receive(:active_partition).and_return(double(value: 5))
+      next_partition = strategy.next_partition
+
+      expect(next_partition.value).to eq(6)
+      expect(next_partition.table).to eq(strategy.table_name)
+      expect(next_partition.partition_name).to eq("#{strategy.table_name}_6")
+    end
+  end
+
+  describe '#ensure_partitioning_column_ignored!' do
+    it 'raises when the column is not ignored' do
+      expect do
+        Class.new(ApplicationRecord) do
+          include PartitionedTable
+
+          partitioned_by :partition, strategy: :sliding_list,
+                       next_partition_if: proc { false },
+                       detach_partition_if: proc { false }
+        end
+      end.to raise_error(/ignored_columns/)
+    end
+
+    it 'does not raise when the column is ignored' do
+      expect do
+        Class.new(ApplicationRecord) do
+          include PartitionedTable
+
+          self.ignored_columns = [:partition]
+
+          partitioned_by :partition, strategy: :sliding_list,
+                         next_partition_if: proc { false },
+                         detach_partition_if: proc { false }
+        end
+      end.not_to raise_error
+    end
+  end
+  context 'redirecting inserts as the active partition changes' do
+    let(:model) do
+      Class.new(ApplicationRecord) do
+        include PartitionedTable
+
+        self.table_name = '_test_partitioned_test'
+        self.primary_key = :id
+
+        self.ignored_columns = %w[partition]
+
+        # method().call cannot be detected by rspec, so we add a layer of indirection here
+        def self.next_partition_if_wrapper(...)
+          next_partition?(...)
+        end
+
+        def self.detach_partition_if_wrapper(...)
+          detach_partition?(...)
+        end
+        partitioned_by :partition, strategy: :sliding_list,
+                       next_partition_if: method(:next_partition_if_wrapper),
+                       detach_partition_if: method(:detach_partition_if_wrapper)
+
+        def self.next_partition?(current_partition)
+        end
+
+        def self.detach_partition?(partition)
+        end
+      end
+    end
+
+    it 'redirects to the new partition', :aggregate_failures do
+      partition_2_model = model.create! # Goes in partition 2
+
+      allow(model).to receive(:next_partition?) do
+        model.partitioning_strategy.active_partition.value < 3
+      end
+
+      allow(model).to receive(:detach_partition?).and_return(false)
+
+      Gitlab::Database::Partitioning::PartitionManager.new(model).sync_partitions
+
+      partition_3_model = model.create!
+
+      # Rails doesn't pick up on database default changes, so we need to reload
+      # We also want to grab the partition column to verify what it was set to.
+      # In normal operation we make rails ignore it so that we can use a changing default
+      # So we force select * to load it
+      all_columns = model.select(model.arel_table[Arel.star])
+      partition_2_model = all_columns.find(partition_2_model.id)
+      partition_3_model = all_columns.find(partition_3_model.id)
+
+      expect(partition_2_model.partition).to eq(2)
+      expect(partition_3_model.partition).to eq(3)
+    end
+  end
+end
diff --git a/spec/lib/gitlab/redis/multi_store_spec.rb b/spec/lib/gitlab/redis/multi_store_spec.rb
index bf1bf65bb9b..1bb622663e8 100644
--- a/spec/lib/gitlab/redis/multi_store_spec.rb
+++ b/spec/lib/gitlab/redis/multi_store_spec.rb
@@ -114,6 +114,12 @@ RSpec.describe Gitlab::Redis::MultiStore do
     end
 
     RSpec.shared_examples_for 'fallback read from the secondary store' do
+      let(:counter) { Gitlab::Metrics::NullMetric.instance }
+
+      before do
+        allow(Gitlab::Metrics).to receive(:counter).and_return(counter)
+      end
+
       it 'fallback and execute on secondary instance' do
         expect(secondary_store).to receive(name).with(*args).and_call_original
 
@@ -128,7 +134,7 @@ RSpec.describe Gitlab::Redis::MultiStore do
       end
 
       it 'increment read fallback count metrics' do
-        expect(multi_store).to receive(:increment_read_fallback_count).with(name)
+        expect(counter).to receive(:increment).with(command: name, instance_name: instance_name)
 
         subject
       end
@@ -401,9 +407,12 @@ RSpec.describe Gitlab::Redis::MultiStore do
   end
 
   context 'with unsupported command' do
+    let(:counter) { Gitlab::Metrics::NullMetric.instance }
+
     before do
       primary_store.flushdb
       secondary_store.flushdb
+      allow(Gitlab::Metrics).to receive(:counter).and_return(counter)
     end
 
     let_it_be(:key) { "redis:counter" }
@@ -426,7 +435,7 @@ RSpec.describe Gitlab::Redis::MultiStore do
     end
 
     it 'increments method missing counter' do
-      expect(multi_store).to receive(:increment_method_missing_count).with(:incr)
+      expect(counter).to receive(:increment).with(command: :incr, instance_name: instance_name)
 
       subject
     end
diff --git a/spec/lib/gitlab/redis/sessions_spec.rb b/spec/lib/gitlab/redis/sessions_spec.rb
index 7e239c08e9f..c62d10eb566 100644
--- a/spec/lib/gitlab/redis/sessions_spec.rb
+++ b/spec/lib/gitlab/redis/sessions_spec.rb
@@ -4,4 +4,54 @@ require 'spec_helper'
 
 RSpec.describe Gitlab::Redis::Sessions do
   include_examples "redis_new_instance_shared_examples", 'sessions', Gitlab::Redis::SharedState
+
+  describe 'redis instance used in connection pool' do
+    before do
+      clear_pool
+    end
+
+    context 'when redis.sessions configuration is not provided' do
+      it 'uses ::Redis instance' do
+        expect(described_class).to receive(:config_fallback?).and_return(true)
+
+        described_class.pool.with do |redis_instance|
+          expect(redis_instance).to be_instance_of(::Redis)
+        end
+      end
+    end
+
+    context 'when redis.sessions configuration is provided' do
+      it 'instantiates an instance of MultiStore' do
+        expect(described_class).to receive(:config_fallback?).and_return(false)
+
+        described_class.pool.with do |redis_instance|
+          expect(redis_instance).to be_instance_of(::Gitlab::Redis::MultiStore)
+        end
+      end
+    end
+
+    def clear_pool
+      described_class.remove_instance_variable(:@pool)
+    rescue NameError
+      # raised if @pool was not set; ignore
+    end
+  end
+
+  describe '#store' do
+    subject { described_class.store(namespace: described_class::SESSION_NAMESPACE) }
+
+    context 'when redis.sessions configuration is provided' do
+      it 'instantiates ::Redis instance' do
+        expect(described_class).to receive(:config_fallback?).and_return(true)
+        expect(subject).to be_instance_of(::Redis::Store)
+      end
+    end
+
+    context 'when redis.sessions configuration is not provided' do
+      it 'instantiates an instance of MultiStore' do
+        expect(described_class).to receive(:config_fallback?).and_return(false)
+        expect(subject).to be_instance_of(::Gitlab::Redis::MultiStore)
+      end
+    end
+  end
 end