Add latest changes from gitlab-org/gitlab@master

4 files changed, 74 insertions, 105 deletions

diff --git a/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb
index a70ff0bd82d..c1b0f4df29a 100644
--- a/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb
+++ b/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb
@@ -104,8 +104,13 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsImporter do
         .and_yield(pull_request)
 
       expect(Gitlab::GithubImport::ImportPullRequestWorker)
-        .to receive(:perform_async)
-        .with(project.id, an_instance_of(Hash), an_instance_of(String))
+        .to receive(:bulk_perform_in)
+        .with(
+          1.second,
+          [[project.id, an_instance_of(Hash), an_instance_of(String)]],
+          batch_delay: 1.minute,
+          batch_size: 200
+        )
 
       waiter = importer.parallel_import
 
diff --git a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb
index f375e84e0fd..6a19afbc60d 100644
--- a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb
+++ b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb
@@ -22,6 +22,10 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do
       def collection_method
         :issues
       end
+
+      def parallel_import_batch
+        { size: 10, delay: 1.minute }
+      end
     end
   end
 
@@ -254,35 +258,61 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do
 
   describe '#parallel_import' do
     let(:importer) { importer_class.new(project, client) }
+    let(:repr_class) { double(:representation) }
+    let(:worker_class) { double(:worker) }
+    let(:object) { double(:object) }
+    let(:batch_size) { 200 }
+    let(:batch_delay) { 1.minute }
 
-    it 'imports data in parallel' do
-      repr_class = double(:representation)
-      worker_class = double(:worker)
-      object = double(:object)
-
-      expect(importer)
-        .to receive(:each_object_to_import)
-        .and_yield(object)
-
-      expect(importer)
+    before do
+      allow(importer)
         .to receive(:representation_class)
         .and_return(repr_class)
 
-      expect(importer)
+      allow(importer)
         .to receive(:sidekiq_worker_class)
         .and_return(worker_class)
 
-      expect(repr_class)
+      allow(repr_class)
         .to receive(:from_api_response)
         .with(object)
         .and_return({ title: 'Foo' })
+    end
+
+    context 'with multiple objects' do
+      before do
+        allow(importer).to receive(:parallel_import_batch) { { size: batch_size, delay: batch_delay } }
+        expect(importer).to receive(:each_object_to_import).and_yield(object).and_yield(object).and_yield(object)
+      end
 
-      expect(worker_class)
-        .to receive(:perform_async)
-        .with(project.id, { title: 'Foo' }, an_instance_of(String))
+      it 'imports data in parallel batches with delays' do
+        expect(worker_class).to receive(:bulk_perform_in).with(1.second, [
+          [project.id, { title: 'Foo' }, an_instance_of(String)],
+          [project.id, { title: 'Foo' }, an_instance_of(String)],
+          [project.id, { title: 'Foo' }, an_instance_of(String)]
+        ], batch_size: batch_size, batch_delay: batch_delay)
+
+        importer.parallel_import
+      end
+    end
 
-      expect(importer.parallel_import)
-        .to be_an_instance_of(Gitlab::JobWaiter)
+    context 'when FF is disabled' do
+      before do
+        stub_feature_flags(spread_parallel_import: false)
+      end
+
+      it 'imports data in parallel' do
+        expect(importer)
+          .to receive(:each_object_to_import)
+          .and_yield(object)
+
+        expect(worker_class)
+          .to receive(:perform_async)
+          .with(project.id, { title: 'Foo' }, an_instance_of(String))
+
+        expect(importer.parallel_import)
+          .to be_an_instance_of(Gitlab::JobWaiter)
+      end
     end
   end
 
diff --git a/spec/lib/gitlab/seeder_spec.rb b/spec/lib/gitlab/seeder_spec.rb
index 877461a7064..71d0a41ef98 100644
--- a/spec/lib/gitlab/seeder_spec.rb
+++ b/spec/lib/gitlab/seeder_spec.rb
@@ -4,6 +4,26 @@ require 'spec_helper'
 
 RSpec.describe Gitlab::Seeder do
   describe '.quiet' do
+    let(:database_base_models) do
+      {
+        main: ApplicationRecord,
+        ci: Ci::ApplicationRecord
+      }
+    end
+
+    it 'disables database logging' do
+      allow(Gitlab::Database).to receive(:database_base_models)
+        .and_return(database_base_models.with_indifferent_access)
+
+      described_class.quiet do
+        expect(ApplicationRecord.logger).to be_nil
+        expect(Ci::ApplicationRecord.logger).to be_nil
+      end
+
+      expect(ApplicationRecord.logger).not_to be_nil
+      expect(Ci::ApplicationRecord.logger).not_to be_nil
+    end
+
     it 'disables mail deliveries' do
       expect(ActionMailer::Base.perform_deliveries).to eq(true)
 
diff --git a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb
index 833de6ae624..8d46845548a 100644
--- a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb
+++ b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb
@@ -122,20 +122,6 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gi
         it_behaves_like 'sets Redis keys with correct TTL'
       end
 
-      context 'when preserve_latest_wal_locations_for_idempotent_jobs feature flag is disabled' do
-        before do
-          stub_feature_flags(preserve_latest_wal_locations_for_idempotent_jobs: false)
-        end
-
-        it "does not change the existing wal locations key's TTL" do
-          expect { duplicate_job.check! }
-            .to not_change { read_idempotency_key_with_ttl(existing_wal_location_key(idempotency_key, :main)) }
-                  .from([nil, -2])
-            .and not_change { read_idempotency_key_with_ttl(existing_wal_location_key(idempotency_key, :ci)) }
-                  .from([nil, -2])
-        end
-      end
-
       it "adds the idempotency key to the jobs payload" do
         expect { duplicate_job.check! }.to change { job['idempotency_key'] }.from(nil).to(idempotency_key)
       end
@@ -186,28 +172,6 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gi
       duplicate_job.check!
     end
 
-    context 'when preserve_latest_wal_locations_for_idempotent_jobs feature flag is disabled' do
-      let(:existing_wal) { {} }
-
-      before do
-        stub_feature_flags(preserve_latest_wal_locations_for_idempotent_jobs: false)
-      end
-
-      it "doesn't call Sidekiq.redis" do
-        expect(Sidekiq).not_to receive(:redis)
-
-        duplicate_job.update_latest_wal_location!
-      end
-
-      it "doesn't update a wal location to redis with an offset" do
-        expect { duplicate_job.update_latest_wal_location! }
-          .to not_change { read_range_from_redis(wal_location_key(idempotency_key, :main)) }
-                .from([])
-         .and not_change { read_range_from_redis(wal_location_key(idempotency_key, :ci)) }
-                .from([])
-      end
-    end
-
     context "when the key doesn't exists in redis" do
       let(:existing_wal) do
         {
@@ -328,20 +292,6 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gi
     context 'when job is not deduplication and wal locations were not persisted' do
       it { expect(duplicate_job.latest_wal_locations).to be_empty }
     end
-
-    context 'when preserve_latest_wal_locations_for_idempotent_jobs feature flag is disabled' do
-      before do
-        stub_feature_flags(preserve_latest_wal_locations_for_idempotent_jobs: false)
-      end
-
-      it "doesn't call Sidekiq.redis" do
-        expect(Sidekiq).not_to receive(:redis)
-
-        duplicate_job.latest_wal_locations
-      end
-
-      it { expect(duplicate_job.latest_wal_locations).to eq({}) }
-    end
   end
 
   describe '#delete!' do
@@ -406,32 +356,6 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gi
           let(:key) { wal_location_key(idempotency_key, :ci) }
           let(:from_value) { wal_locations[:ci] }
         end
-
-        context 'when preserve_latest_wal_locations_for_idempotent_jobs feature flag is disabled' do
-          before do
-            stub_feature_flags(preserve_latest_wal_locations_for_idempotent_jobs: false)
-          end
-
-          it_behaves_like 'does not delete key from redis', 'latest wal location keys for main database' do
-            let(:key) { existing_wal_location_key(idempotency_key, :main) }
-            let(:from_value) { wal_locations[:main] }
-          end
-
-          it_behaves_like 'does not delete key from redis', 'latest wal location keys for ci database' do
-            let(:key) { existing_wal_location_key(idempotency_key, :ci) }
-            let(:from_value) { wal_locations[:ci] }
-          end
-
-          it_behaves_like 'does not delete key from redis', 'latest wal location keys for main database' do
-            let(:key) { wal_location_key(idempotency_key, :main) }
-            let(:from_value) { wal_locations[:main] }
-          end
-
-          it_behaves_like 'does not delete key from redis', 'latest wal location keys for ci database' do
-            let(:key) { wal_location_key(idempotency_key, :ci) }
-            let(:from_value) { wal_locations[:ci] }
-          end
-        end
       end
 
       context 'when the idempotency key is not part of the job' do
@@ -666,16 +590,6 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gi
       it 'returns true' do
         expect(duplicate_job).to be_idempotent
       end
-
-      context 'when preserve_latest_wal_locations_for_idempotent_jobs feature flag is disabled' do
-        before do
-          stub_feature_flags(preserve_latest_wal_locations_for_idempotent_jobs: false)
-        end
-
-        it 'returns false' do
-          expect(duplicate_job).not_to be_idempotent
-        end
-      end
     end
   end