Add latest changes from gitlab-org/gitlab@master

9 files changed, 222 insertions, 71 deletions

diff --git a/spec/frontend/pipeline_editor/components/drawer/cards/first_pipeline_card_spec.js b/spec/frontend/pipeline_editor/components/drawer/cards/first_pipeline_card_spec.js
index 8a4f07c4d88..e435c0dcc08 100644
--- a/spec/frontend/pipeline_editor/components/drawer/cards/first_pipeline_card_spec.js
+++ b/spec/frontend/pipeline_editor/components/drawer/cards/first_pipeline_card_spec.js
@@ -1,13 +1,11 @@
 import { getByRole } from '@testing-library/dom';
 import { mount } from '@vue/test-utils';
 import FirstPipelineCard from '~/pipeline_editor/components/drawer/cards/first_pipeline_card.vue';
-import PipelineVisualReference from '~/pipeline_editor/components/drawer/ui/pipeline_visual_reference.vue';
 
 describe('First pipeline card', () => {
   let wrapper;
 
   const defaultProvide = {
-    ciExamplesHelpPagePath: '/pipelines/examples',
     runnerHelpPagePath: '/help/runners',
   };
 
@@ -20,9 +18,9 @@ describe('First pipeline card', () => {
   };
 
   const getLinkByName = (name) => getByRole(wrapper.element, 'link', { name }).href;
-  const findPipelinesLink = () => getLinkByName(/examples and templates/i);
   const findRunnersLink = () => getLinkByName(/make sure your instance has runners available/i);
-  const findVisualReference = () => wrapper.findComponent(PipelineVisualReference);
+  const findInstructionsList = () => wrapper.find('ol');
+  const findAllInstructions = () => findInstructionsList().findAll('li');
 
   beforeEach(() => {
     createComponent();
@@ -37,11 +35,11 @@ describe('First pipeline card', () => {
   });
 
   it('renders the content', () => {
-    expect(findVisualReference().exists()).toBe(true);
+    expect(findInstructionsList().exists()).toBe(true);
+    expect(findAllInstructions()).toHaveLength(3);
   });
 
-  it('renders the links', () => {
+  it('renders the link', () => {
     expect(findRunnersLink()).toContain(defaultProvide.runnerHelpPagePath);
-    expect(findPipelinesLink()).toContain(defaultProvide.ciExamplesHelpPagePath);
   });
 });
diff --git a/spec/frontend/pipeline_editor/components/drawer/ui/pipeline_visual_reference_spec.js b/spec/frontend/pipeline_editor/components/drawer/ui/pipeline_visual_reference_spec.js
deleted file mode 100644
index e4834544484..00000000000
--- a/spec/frontend/pipeline_editor/components/drawer/ui/pipeline_visual_reference_spec.js
+++ /dev/null
@@ -1,31 +0,0 @@
-import { shallowMount } from '@vue/test-utils';
-import DemoJobPill from '~/pipeline_editor/components/drawer/ui/demo_job_pill.vue';
-import PipelineVisualReference from '~/pipeline_editor/components/drawer/ui/pipeline_visual_reference.vue';
-
-describe('Demo job pill', () => {
-  let wrapper;
-
-  const createComponent = () => {
-    wrapper = shallowMount(PipelineVisualReference);
-  };
-
-  const findAllDemoJobPills = () => wrapper.findAllComponents(DemoJobPill);
-
-  beforeEach(() => {
-    createComponent();
-  });
-
-  afterEach(() => {
-    wrapper.destroy();
-  });
-
-  it('renders all stage names', () => {
-    expect(wrapper.text()).toContain(wrapper.vm.$options.i18n.stageNames.build);
-    expect(wrapper.text()).toContain(wrapper.vm.$options.i18n.stageNames.test);
-    expect(wrapper.text()).toContain(wrapper.vm.$options.i18n.stageNames.deploy);
-  });
-
-  it('renders all job pills', () => {
-    expect(findAllDemoJobPills()).toHaveLength(4);
-  });
-});
diff --git a/spec/lib/gitlab/cache/import/caching_spec.rb b/spec/lib/gitlab/cache/import/caching_spec.rb
index d6911dad9d4..8ce12f5d32e 100644
--- a/spec/lib/gitlab/cache/import/caching_spec.rb
+++ b/spec/lib/gitlab/cache/import/caching_spec.rb
@@ -88,6 +88,18 @@ RSpec.describe Gitlab::Cache::Import::Caching, :clean_gitlab_redis_cache do
     end
   end
 
+  describe '.values_from_set' do
+    it 'returns empty list when the set is empty' do
+      expect(described_class.values_from_set('foo')).to eq([])
+    end
+
+    it 'returns the set list of values' do
+      described_class.set_add('foo', 10)
+
+      expect(described_class.values_from_set('foo')).to eq(['10'])
+    end
+  end
+
   describe '.write_multiple' do
     it 'sets multiple keys when key_prefix not set' do
       mapping = { 'foo' => 10, 'bar' => 20 }
diff --git a/spec/lib/gitlab/database/dynamic_model_helpers_spec.rb b/spec/lib/gitlab/database/dynamic_model_helpers_spec.rb
index 23ad621d0ee..0844616ee1c 100644
--- a/spec/lib/gitlab/database/dynamic_model_helpers_spec.rb
+++ b/spec/lib/gitlab/database/dynamic_model_helpers_spec.rb
@@ -3,12 +3,12 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::Database::DynamicModelHelpers do
+  let(:including_class) { Class.new.include(described_class) }
+  let(:table_name) { 'projects' }
+
   describe '#define_batchable_model' do
     subject { including_class.new.define_batchable_model(table_name) }
 
-    let(:including_class) { Class.new.include(described_class) }
-    let(:table_name) { 'projects' }
-
     it 'is an ActiveRecord model' do
       expect(subject.ancestors).to include(ActiveRecord::Base)
     end
@@ -25,4 +25,86 @@ RSpec.describe Gitlab::Database::DynamicModelHelpers do
       expect(subject.inheritance_column).to eq('_type_disabled')
     end
   end
+
+  describe '#each_batch' do
+    subject { including_class.new }
+
+    before do
+      create_list(:project, 2)
+    end
+
+    context 'when no transaction is open' do
+      before do
+        allow(subject).to receive(:transaction_open?).and_return(false)
+      end
+
+      it 'iterates table in batches' do
+        each_batch_size = ->(&block) do
+          subject.each_batch(table_name, of: 1) do |batch|
+            block.call(batch.size)
+          end
+        end
+
+        expect { |b| each_batch_size.call(&b) }
+          .to yield_successive_args(1, 1)
+      end
+    end
+
+    context 'when transaction is open' do
+      before do
+        allow(subject).to receive(:transaction_open?).and_return(true)
+      end
+
+      it 'raises an error' do
+        expect { subject.each_batch(table_name, of: 1) { |batch| batch.size } }
+          .to raise_error(RuntimeError, /each_batch should not run inside a transaction/)
+      end
+    end
+  end
+
+  describe '#each_batch_range' do
+    subject { including_class.new }
+
+    let(:first_project) { create(:project) }
+    let(:second_project) { create(:project) }
+
+    context 'when no transaction is open' do
+      before do
+        allow(subject).to receive(:transaction_open?).and_return(false)
+      end
+
+      it 'iterates table in batch ranges' do
+        expect { |b| subject.each_batch_range(table_name, of: 1, &b) }
+          .to yield_successive_args(
+            [first_project.id, first_project.id],
+            [second_project.id, second_project.id]
+          )
+      end
+
+      it 'yields only one batch if bigger than the table size' do
+        expect { |b| subject.each_batch_range(table_name, of: 2, &b) }
+          .to yield_successive_args([first_project.id, second_project.id])
+      end
+
+      it 'makes it possible to apply a scope' do
+        each_batch_limited = ->(&b) do
+          subject.each_batch_range(table_name, scope: ->(table) { table.limit(1) }, of: 1, &b)
+        end
+
+        expect { |b| each_batch_limited.call(&b) }
+          .to yield_successive_args([first_project.id, first_project.id])
+      end
+    end
+
+    context 'when transaction is open' do
+      before do
+        allow(subject).to receive(:transaction_open?).and_return(true)
+      end
+
+      it 'raises an error' do
+        expect { subject.each_batch_range(table_name, of: 1) { 1 } }
+          .to raise_error(RuntimeError, /each_batch should not run inside a transaction/)
+      end
+    end
+  end
 end
diff --git a/spec/lib/gitlab/github_import/importer/pull_requests_reviews_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_requests_reviews_importer_spec.rb
index f18064f10aa..08be350f0f9 100644
--- a/spec/lib/gitlab/github_import/importer/pull_requests_reviews_importer_spec.rb
+++ b/spec/lib/gitlab/github_import/importer/pull_requests_reviews_importer_spec.rb
@@ -27,30 +27,100 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsReviewsImporter do
   end
 
   describe '#each_object_to_import', :clean_gitlab_redis_cache do
-    it 'fetchs the merged pull requests data' do
-      merge_request = create(
-        :merged_merge_request,
-        iid: 999,
-        source_project: project,
-        target_project: project
-      )
-
-      review = double
-
-      expect(review)
-        .to receive(:merge_request_id=)
-        .with(merge_request.id)
-
-      allow(client)
-        .to receive(:pull_request_reviews)
-        .exactly(:once) # ensure to be cached on the second call
-        .with('github/repo', merge_request.iid)
-        .and_return([review])
-
-      expect { |b| subject.each_object_to_import(&b) }
-        .to yield_with_args(review)
-
-      subject.each_object_to_import {}
+    context 'when github_review_importer_query_only_unimported_merge_requests is enabled' do
+      before do
+        stub_feature_flags(github_review_importer_query_only_unimported_merge_requests: true)
+      end
+
+      let(:merge_request) do
+        create(
+          :merged_merge_request,
+          iid: 999,
+          source_project: project,
+          target_project: project
+        )
+      end
+
+      let(:review) { double(id: 1) }
+
+      it 'fetches the pull requests reviews data' do
+        page = double(objects: [review], number: 1)
+
+        expect(review)
+          .to receive(:merge_request_id=)
+          .with(merge_request.id)
+
+        expect(client)
+          .to receive(:each_page)
+          .exactly(:once) # ensure to be cached on the second call
+          .with(:pull_request_reviews, 'github/repo', merge_request.iid, page: 1)
+          .and_yield(page)
+
+        expect { |b| subject.each_object_to_import(&b) }
+          .to yield_with_args(review)
+
+        subject.each_object_to_import {}
+      end
+
+      it 'skips cached pages' do
+        Gitlab::GithubImport::PageCounter
+          .new(project, "merge_request/#{merge_request.id}/pull_request_reviews")
+          .set(2)
+
+        expect(review).not_to receive(:merge_request_id=)
+
+        expect(client)
+          .to receive(:each_page)
+          .exactly(:once) # ensure to be cached on the second call
+          .with(:pull_request_reviews, 'github/repo', merge_request.iid, page: 2)
+
+        subject.each_object_to_import {}
+      end
+
+      it 'skips cached merge requests' do
+        Gitlab::Cache::Import::Caching.set_add(
+          "github-importer/merge_request/already-imported/#{project.id}",
+          merge_request.id
+        )
+
+        expect(review).not_to receive(:merge_request_id=)
+
+        expect(client).not_to receive(:each_page)
+
+        subject.each_object_to_import {}
+      end
+    end
+
+    context 'when github_review_importer_query_only_unimported_merge_requests is disabled' do
+      before do
+        stub_feature_flags(github_review_importer_query_only_unimported_merge_requests: false)
+      end
+
+      it 'fetchs the merged pull requests data' do
+        merge_request = create(
+          :merged_merge_request,
+          iid: 999,
+          source_project: project,
+          target_project: project
+        )
+
+        review = double
+
+        expect(review)
+          .to receive(:merge_request_id=)
+          .with(merge_request.id)
+
+        allow(client)
+          .to receive(:pull_request_reviews)
+          .exactly(:once) # ensure to be cached on the second call
+          .with('github/repo', merge_request.iid)
+          .and_return([review])
+
+        expect { |b| subject.each_object_to_import(&b) }
+          .to yield_with_args(review)
+
+        subject.each_object_to_import {}
+      end
     end
   end
 end
diff --git a/spec/lib/gitlab/github_import/page_counter_spec.rb b/spec/lib/gitlab/github_import/page_counter_spec.rb
index a1305b714b5..568bc8cbbef 100644
--- a/spec/lib/gitlab/github_import/page_counter_spec.rb
+++ b/spec/lib/gitlab/github_import/page_counter_spec.rb
@@ -31,4 +31,15 @@ RSpec.describe Gitlab::GithubImport::PageCounter, :clean_gitlab_redis_cache do
       expect(counter.current).to eq(2)
     end
   end
+
+  describe '#expire!' do
+    it 'expires the current page counter' do
+      counter.set(2)
+
+      counter.expire!
+
+      expect(Gitlab::Cache::Import::Caching.read_integer(counter.cache_key)).to be_nil
+      expect(counter.current).to eq(1)
+    end
+  end
 end
diff --git a/spec/models/commit_spec.rb b/spec/models/commit_spec.rb
index 7c00f367844..693e754c53d 100644
--- a/spec/models/commit_spec.rb
+++ b/spec/models/commit_spec.rb
@@ -471,16 +471,25 @@ eos
   end
 
   it_behaves_like 'a mentionable' do
-    subject { create(:project, :repository).commit }
+    subject(:commit) { create(:project, :repository).commit }
 
     let(:author) { create(:user, email: subject.author_email) }
     let(:backref_text) { "commit #{subject.id}" }
     let(:set_mentionable_text) do
-      ->(txt) { allow(subject).to receive(:safe_message).and_return(txt) }
+      ->(txt) { allow(commit).to receive(:safe_message).and_return(txt) }
     end
 
     # Include the subject in the repository stub.
-    let(:extra_commits) { [subject] }
+    let(:extra_commits) { [commit] }
+
+    it 'uses the CachedMarkdownField cache instead of the Mentionable cache', :use_clean_rails_redis_caching do
+      expect(commit.title_html).not_to be_present
+
+      commit.all_references(project.owner).all
+
+      expect(commit.title_html).to be_present
+      expect(Rails.cache.read("banzai/commit:#{commit.id}/safe_message/single_line")).to be_nil
+    end
   end
 
   describe '#hook_attrs' do
diff --git a/spec/support/shared_examples/models/mentionable_shared_examples.rb b/spec/support/shared_examples/models/mentionable_shared_examples.rb
index 2392658e584..04630484964 100644
--- a/spec/support/shared_examples/models/mentionable_shared_examples.rb
+++ b/spec/support/shared_examples/models/mentionable_shared_examples.rb
@@ -66,7 +66,7 @@ RSpec.shared_examples 'a mentionable' do
     expect(subject.gfm_reference).to eq(backref_text)
   end
 
-  it "extracts references from its reference property" do
+  it "extracts references from its reference property", :clean_gitlab_redis_cache do
     # De-duplicate and omit itself
     refs = subject.referenced_mentionables
     expect(refs.size).to eq(6)
@@ -98,7 +98,7 @@ RSpec.shared_examples 'a mentionable' do
     end
   end
 
-  it 'creates cross-reference notes' do
+  it 'creates cross-reference notes', :clean_gitlab_redis_cache do
     mentioned_objects = [mentioned_issue, mentioned_mr, mentioned_commit,
                          ext_issue, ext_mr, ext_commit]
 
diff --git a/spec/workers/process_commit_worker_spec.rb b/spec/workers/process_commit_worker_spec.rb
index 294a05c652b..3df26c774ba 100644
--- a/spec/workers/process_commit_worker_spec.rb
+++ b/spec/workers/process_commit_worker_spec.rb
@@ -138,7 +138,7 @@ RSpec.describe ProcessCommitWorker do
     end
   end
 
-  describe '#update_issue_metrics' do
+  describe '#update_issue_metrics', :clean_gitlab_redis_cache do
     context 'when commit has issue reference' do
       subject(:update_metrics_and_reload) do
         -> {