Add latest changes from gitlab-org/gitlab@master

17 files changed, 549 insertions, 325 deletions

diff --git a/spec/features/issues/filtered_search/filter_issues_spec.rb b/spec/features/issues/filtered_search/filter_issues_spec.rb
index 756699fb854..5e2b5921e06 100644
--- a/spec/features/issues/filtered_search/filter_issues_spec.rb
+++ b/spec/features/issues/filtered_search/filter_issues_spec.rb
@@ -224,14 +224,6 @@ describe 'Filter issues', :js do
         expect_no_issues_list
         expect_filtered_search_input_empty
       end
-
-      it 'does show issues for bug label' do
-        input_filtered_search("label:!=~#{bug_label.title}")
-
-        expect_tokens([label_token(bug_label.title)])
-        expect_issues_list_count(6)
-        expect_filtered_search_input_empty
-      end
     end
 
     context 'label with multiple words' do
diff --git a/spec/features/projects/import_export/export_file_spec.rb b/spec/features/projects/import_export/export_file_spec.rb
index 54a6ac1551b..1e13afc6033 100644
--- a/spec/features/projects/import_export/export_file_spec.rb
+++ b/spec/features/projects/import_export/export_file_spec.rb
@@ -38,51 +38,109 @@ describe 'Import/Export - project export integration test', :js do
       sign_in(user)
     end
 
-    it 'exports a project successfully', :sidekiq_might_not_need_inline do
-      visit edit_project_path(project)
+    shared_examples 'export file without sensitive words' do
+      it 'exports a project successfully', :sidekiq_inline do
+        export_project_and_download_file(page, project)
 
-      expect(page).to have_content('Export project')
+        in_directory_with_expanded_export(project) do |exit_status, tmpdir|
+          expect(exit_status).to eq(0)
 
-      find(:link, 'Export project').send_keys(:return)
+          project_json_path = File.join(tmpdir, 'project.json')
+          expect(File).to exist(project_json_path)
 
-      visit edit_project_path(project)
+          project_hash = JSON.parse(IO.read(project_json_path))
 
-      expect(page).to have_content('Download export')
+          sensitive_words.each do |sensitive_word|
+            found = find_sensitive_attributes(sensitive_word, project_hash)
 
-      expect(project.export_status).to eq(:finished)
-      expect(project.export_file.path).to include('tar.gz')
+            expect(found).to be_nil, failure_message(found.try(:key_found), found.try(:parent), sensitive_word)
+          end
+        end
+      end
+    end
+
+    context "with legacy export" do
+      before do
+        stub_feature_flags(streaming_serializer: false)
+        stub_feature_flags(project_export_as_ndjson: false)
+      end
+
+      it_behaves_like "export file without sensitive words"
+    end
+
+    context "with streaming serializer" do
+      before do
+        stub_feature_flags(streaming_serializer: true)
+        stub_feature_flags(project_export_as_ndjson: false)
+      end
+
+      it_behaves_like "export file without sensitive words"
+    end
 
-      in_directory_with_expanded_export(project) do |exit_status, tmpdir|
-        expect(exit_status).to eq(0)
+    context "with ndjson" do
+      before do
+        stub_feature_flags(streaming_serializer: true)
+        stub_feature_flags(project_export_as_ndjson: true)
+      end
+
+      it 'exports a project successfully', :sidekiq_inline do
+        export_project_and_download_file(page, project)
+
+        in_directory_with_expanded_export(project) do |exit_status, tmpdir|
+          expect(exit_status).to eq(0)
 
-        project_json_path = File.join(tmpdir, 'project.json')
-        expect(File).to exist(project_json_path)
+          project_json_path = File.join(tmpdir, 'tree', 'project.json')
+          expect(File).to exist(project_json_path)
 
-        project_hash = JSON.parse(IO.read(project_json_path))
+          relations = []
+          relations << JSON.parse(IO.read(project_json_path))
+          Dir.glob(File.join(tmpdir, 'tree/project', '*.ndjson')) do |rb_filename|
+            File.foreach(rb_filename) do |line|
+              json = ActiveSupport::JSON.decode(line)
+              relations << json
+            end
+          end
 
-        sensitive_words.each do |sensitive_word|
-          found = find_sensitive_attributes(sensitive_word, project_hash)
+          relations.each do |relation_hash|
+            sensitive_words.each do |sensitive_word|
+              found = find_sensitive_attributes(sensitive_word, relation_hash)
 
-          expect(found).to be_nil, failure_message(found.try(:key_found), found.try(:parent), sensitive_word)
+              expect(found).to be_nil, failure_message(found.try(:key_found), found.try(:parent), sensitive_word)
+            end
+          end
         end
       end
     end
+  end
 
-    def failure_message(key_found, parent, sensitive_word)
-      <<-MSG
-        Found a new sensitive word <#{key_found}>, which is part of the hash #{parent.inspect}
+  def export_project_and_download_file(page, project)
+    visit edit_project_path(project)
 
-        If you think this information shouldn't get exported, please exclude the model or attribute in IMPORT_EXPORT_CONFIG.
+    expect(page).to have_content('Export project')
 
-        Otherwise, please add the exception to +safe_list+ in CURRENT_SPEC using #{sensitive_word} as the key and the
-        correspondent hash or model as the value.
+    find(:link, 'Export project').send_keys(:return)
 
-        Also, if the attribute is a generated unique token, please add it to RelationFactory::TOKEN_RESET_MODELS if it needs to be
-        reset (to prevent duplicate column problems while importing to the same instance).
+    visit edit_project_path(project)
 
-        IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file}
-        CURRENT_SPEC: #{__FILE__}
-      MSG
-    end
+    expect(page).to have_content('Download export')
+    expect(project.export_status).to eq(:finished)
+    expect(project.export_file.path).to include('tar.gz')
+  end
+
+  def failure_message(key_found, parent, sensitive_word)
+    <<-MSG
+      Found a new sensitive word <#{key_found}>, which is part of the hash #{parent.inspect}
+
+      If you think this information shouldn't get exported, please exclude the model or attribute in IMPORT_EXPORT_CONFIG.
+
+      Otherwise, please add the exception to +safe_list+ in CURRENT_SPEC using #{sensitive_word} as the key and the
+      correspondent hash or model as the value.
+
+      Also, if the attribute is a generated unique token, please add it to RelationFactory::TOKEN_RESET_MODELS if it needs to be
+      reset (to prevent duplicate column problems while importing to the same instance).
+
+      IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file}
+      CURRENT_SPEC: #{__FILE__}
+    MSG
   end
 end
diff --git a/spec/finders/environments_finder_spec.rb b/spec/finders/environments_finder_spec.rb
index 7100376478a..850d160506c 100644
--- a/spec/finders/environments_finder_spec.rb
+++ b/spec/finders/environments_finder_spec.rb
@@ -37,11 +37,6 @@ describe EnvironmentsFinder do
           .to be_empty
       end
 
-      it 'returns environment when with_tags is set' do
-        expect(described_class.new(project, user, ref: 'master', commit: commit, with_tags: true).execute)
-          .to contain_exactly(environment, environment_two)
-      end
-
       # We expect two Gitaly calls: FindCommit, CommitIsAncestor
       # This tests to ensure we don't call one CommitIsAncestor per environment
       it 'only calls Gitaly twice when multiple environments are present', :request_store do
diff --git a/spec/helpers/users_helper_spec.rb b/spec/helpers/users_helper_spec.rb
index 893d5cde24a..e3e599007a4 100644
--- a/spec/helpers/users_helper_spec.rb
+++ b/spec/helpers/users_helper_spec.rb
@@ -95,9 +95,9 @@ describe UsersHelper do
     end
 
     it 'includes the settings tab if the user can update themself' do
-      expect(helper).to receive(:can?).with(user, :read_user, user) { true }
+      expect(helper).to receive(:can?).with(user, :update_user, user) { true }
 
-      expect(items).to include(:profile)
+      expect(items).to include(:settings)
     end
 
     context 'when terms are enforced' do
diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb
index 0b34e887716..62adba4319e 100644
--- a/spec/lib/gitlab/ci/yaml_processor_spec.rb
+++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb
@@ -1647,48 +1647,6 @@ module Gitlab
 
           it { expect { subject }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, /is not defined in prior stages/) }
         end
-
-        context 'when trigger job includes artifact generated by a dependency' do
-          context 'when dependency is defined in previous stages' do
-            let(:config) do
-              {
-                build1: { stage: 'build', script: 'test' },
-                test1: { stage: 'test', trigger: {
-                  include: [{ job: 'build1', artifact: 'generated.yml' }]
-                } }
-              }
-            end
-
-            it { expect { subject }.not_to raise_error }
-          end
-
-          context 'when dependency is defined in later stages' do
-            let(:config) do
-              {
-                build1: { stage: 'build', script: 'test' },
-                test1: { stage: 'test', trigger: {
-                  include: [{ job: 'deploy1', artifact: 'generated.yml' }]
-                } },
-                deploy1: { stage: 'deploy', script: 'test' }
-              }
-            end
-
-            it { expect { subject }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, /is not defined in prior stages/) }
-          end
-
-          context 'when dependency is not defined' do
-            let(:config) do
-              {
-                build1: { stage: 'build', script: 'test' },
-                test1: { stage: 'test', trigger: {
-                  include: [{ job: 'non-existent', artifact: 'generated.yml' }]
-                } }
-              }
-            end
-
-            it { expect { subject }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, /undefined dependency: non-existent/) }
-          end
-        end
       end
 
       describe "Job Needs" do
@@ -2094,34 +2052,6 @@ module Gitlab
         end
       end
 
-      describe 'with trigger:include' do
-        context 'when artifact and job are specified' do
-          let(:config) do
-            YAML.dump({
-              build1: { stage: 'build', script: 'test' },
-              test1: { stage: 'test', trigger: {
-                include: [{ artifact: 'generated.yml', job: 'build1' }]
-              } }
-            })
-          end
-
-          it { expect { subject }.not_to raise_error }
-        end
-
-        context 'when artifact is specified without job' do
-          let(:config) do
-            YAML.dump({
-              build1: { stage: 'build', script: 'test' },
-              test1: { stage: 'test', trigger: {
-                include: [{ artifact: 'generated.yml' }]
-              } }
-            })
-          end
-
-          it { expect { subject }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, /must specify the job where to fetch the artifact from/) }
-        end
-      end
-
       describe "Error handling" do
         it "fails to parse YAML" do
           expect do
diff --git a/spec/lib/gitlab/import_export/fork_spec.rb b/spec/lib/gitlab/import_export/fork_spec.rb
index 8aa28353c04..bb79331efac 100644
--- a/spec/lib/gitlab/import_export/fork_spec.rb
+++ b/spec/lib/gitlab/import_export/fork_spec.rb
@@ -32,6 +32,8 @@ describe 'forked project import' do
   end
 
   before do
+    stub_feature_flags(project_export_as_ndjson: false)
+
     allow_next_instance_of(Gitlab::ImportExport) do |instance|
       allow(instance).to receive(:storage_path).and_return(export_path)
     end
diff --git a/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb b/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb
index 50b26637cb1..707975f20b6 100644
--- a/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb
+++ b/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb
@@ -20,6 +20,10 @@ describe Gitlab::ImportExport do
 
   let(:json_fixture) { 'complex' }
 
+  before do
+    stub_feature_flags(project_export_as_ndjson: false)
+  end
+
   it 'yields the initial tree when importing and exporting it again' do
     project = create(:project, creator: create(:user, :admin))
 
diff --git a/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb b/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb
new file mode 100644
index 00000000000..bae3672474c
--- /dev/null
+++ b/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe Gitlab::ImportExport::JSON::NdjsonWriter do
+  include ImportExport::CommonUtil
+
+  let(:path) { "#{Dir.tmpdir}/ndjson_writer_spec/tree" }
+  let(:exportable_path) { 'projects' }
+
+  subject { described_class.new(path) }
+
+  after do
+    FileUtils.rm_rf(path)
+  end
+
+  describe "#write_attributes" do
+    it "writes correct json to root" do
+      expected_hash = { "key" => "value_1", "key_1" => "value_2" }
+      subject.write_attributes(exportable_path, expected_hash)
+
+      expect(consume_attributes(path, exportable_path)).to eq(expected_hash)
+    end
+  end
+
+  describe "#write_relation" do
+    context "when single relation is serialized" do
+      it "appends json in correct file " do
+        relation = "relation"
+        value =  { "key" => "value_1", "key_1" => "value_1" }
+        subject.write_relation(exportable_path, relation, value)
+
+        expect(consume_relations(path, exportable_path, relation)).to eq([value])
+      end
+    end
+
+    context "when single relation is already serialized" do
+      it "raise exception" do
+        values = [{ "key" => "value_1", "key_1" => "value_1" }, { "key" => "value_2", "key_1" => "value_2" }]
+        relation = "relation"
+        file_path = File.join(path, exportable_path, "#{relation}.ndjson")
+        subject.write_relation(exportable_path, relation, values[0])
+
+        expect {subject.write_relation(exportable_path, relation, values[1])}.to raise_exception("The #{file_path} already exist")
+      end
+    end
+  end
+
+  describe "#write_relation_array" do
+    it "writes json in correct files" do
+      values =  [{ "key" => "value_1", "key_1" => "value_1" }, { "key" => "value_2", "key_1" => "value_2" }]
+      relations = %w(relation1 relation2)
+      relations.each do |relation|
+        subject.write_relation_array(exportable_path, relation, values.to_enum)
+      end
+      subject.close
+
+      relations.each do |relation|
+        expect(consume_relations(path, exportable_path, relation)).to eq(values)
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/import_export/project/relation_factory_spec.rb b/spec/lib/gitlab/import_export/project/relation_factory_spec.rb
index 73ae6810706..175da623c1b 100644
--- a/spec/lib/gitlab/import_export/project/relation_factory_spec.rb
+++ b/spec/lib/gitlab/import_export/project/relation_factory_spec.rb
@@ -160,7 +160,7 @@ describe Gitlab::ImportExport::Project::RelationFactory do
     end
 
     it 'has preloaded target project' do
-      expect(created_object.source_project).to equal(project)
+      expect(created_object.target_project).to equal(project)
     end
   end
 
diff --git a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb
index c910ee5430b..d859af5df02 100644
--- a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb
+++ b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb
@@ -3,314 +3,397 @@
 require 'spec_helper'
 
 describe Gitlab::ImportExport::Project::TreeSaver do
-  describe 'saves the project tree into a json object' do
-    let(:shared) { project.import_export_shared }
-    let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared) }
-    let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
-    let(:user) { create(:user) }
-    let!(:project) { setup_project }
-
-    before do
-      project.add_maintainer(user)
-      allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
-      allow_any_instance_of(MergeRequest).to receive(:source_branch_sha).and_return('ABCD')
-      allow_any_instance_of(MergeRequest).to receive(:target_branch_sha).and_return('DCBA')
-    end
+  let_it_be(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
+  let_it_be(:exportable_path) { 'project' }
 
-    after do
-      FileUtils.rm_rf(export_path)
-    end
+  shared_examples 'saves project tree successfully' do |ndjson_enabled|
+    include ImportExport::CommonUtil
 
-    it 'saves project successfully' do
-      expect(project_tree_saver.save).to be true
-    end
+    subject { get_json(full_path, exportable_path, relation_name, ndjson_enabled) }
 
-    context 'JSON' do
-      let(:saved_project_json) do
-        project_tree_saver.save
-        project_json(project_tree_saver.full_path)
-      end
+    describe 'saves project tree attributes' do
+      let_it_be(:user) { create(:user) }
+      let_it_be(:group) { create(:group) }
+      let_it_be(:project) { setup_project }
+      let_it_be(:shared) { project.import_export_shared }
+      let_it_be(:project_tree_saver ) { described_class.new(project: project, current_user: user, shared: shared) }
 
-      # It is not duplicated in
-      # `spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb`
-      context 'with description override' do
-        let(:params) { { description: 'Foo Bar' } }
-        let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared, params: params) }
+      let(:relation_name) { :projects }
 
-        it 'overrides the project description' do
-          expect(saved_project_json).to include({ 'description' => params[:description] })
+      let_it_be(:full_path) do
+        if ndjson_enabled
+          File.join(shared.export_path, 'tree')
+        else
+          File.join(shared.export_path, Gitlab::ImportExport.project_filename)
         end
       end
 
-      it 'saves the correct json' do
-        expect(saved_project_json).to include({ 'description' => 'description', 'visibility_level' => 20 })
+      before_all do
+        Feature.enable(:project_export_as_ndjson) if ndjson_enabled
+        project.add_maintainer(user)
+        project_tree_saver.save
       end
 
-      it 'has approvals_before_merge set' do
-        expect(saved_project_json['approvals_before_merge']).to eq(1)
+      after :all do
+        FileUtils.rm_rf(export_path)
       end
 
-      it 'has milestones' do
-        expect(saved_project_json['milestones']).not_to be_empty
-      end
+      context 'with project root' do
+        it { is_expected.to include({ 'description' => 'description', 'visibility_level' => 20 }) }
 
-      it 'has merge requests' do
-        expect(saved_project_json['merge_requests']).not_to be_empty
-      end
+        it { is_expected.not_to include("runners_token" => 'token') }
 
-      it 'has merge request\'s milestones' do
-        expect(saved_project_json['merge_requests'].first['milestone']).not_to be_empty
+        it 'has approvals_before_merge set' do
+          expect(subject['approvals_before_merge']).to eq(1)
+        end
       end
 
-      it 'has merge request\'s source branch SHA' do
-        expect(saved_project_json['merge_requests'].first['source_branch_sha']).to eq('ABCD')
-      end
+      context 'with milestones' do
+        let(:relation_name) { :milestones }
 
-      it 'has merge request\'s target branch SHA' do
-        expect(saved_project_json['merge_requests'].first['target_branch_sha']).to eq('DCBA')
+        it { is_expected.not_to be_empty }
       end
 
-      it 'has events' do
-        expect(saved_project_json['merge_requests'].first['milestone']['events']).not_to be_empty
-      end
+      context 'with merge_requests' do
+        let(:relation_name) { :merge_requests }
 
-      it 'has snippets' do
-        expect(saved_project_json['snippets']).not_to be_empty
-      end
+        it { is_expected.not_to be_empty }
 
-      it 'has snippet notes' do
-        expect(saved_project_json['snippets'].first['notes']).not_to be_empty
-      end
+        it 'has merge request\'s milestones' do
+          expect(subject.first['milestone']).not_to be_empty
+        end
+        it 'has merge request\'s source branch SHA' do
+          expect(subject.first['source_branch_sha']).to eq('b83d6e391c22777fca1ed3012fce84f633d7fed0')
+        end
 
-      it 'has releases' do
-        expect(saved_project_json['releases']).not_to be_empty
-      end
+        it 'has merge request\'s target branch SHA' do
+          expect(subject.first['target_branch_sha']).to eq('0b4bc9a49b562e85de7cc9e834518ea6828729b9')
+        end
 
-      it 'has no author on releases' do
-        expect(saved_project_json['releases'].first['author']).to be_nil
-      end
+        it 'has events' do
+          expect(subject.first['milestone']['events']).not_to be_empty
+        end
 
-      it 'has the author ID on releases' do
-        expect(saved_project_json['releases'].first['author_id']).not_to be_nil
-      end
+        it 'has merge requests diffs' do
+          expect(subject.first['merge_request_diff']).not_to be_empty
+        end
 
-      it 'has issues' do
-        expect(saved_project_json['issues']).not_to be_empty
-      end
+        it 'has merge request diff files' do
+          expect(subject.first['merge_request_diff']['merge_request_diff_files']).not_to be_empty
+        end
 
-      it 'has issue comments' do
-        notes = saved_project_json['issues'].first['notes']
+        it 'has merge request diff commits' do
+          expect(subject.first['merge_request_diff']['merge_request_diff_commits']).not_to be_empty
+        end
 
-        expect(notes).not_to be_empty
-        expect(notes.first['type']).to eq('DiscussionNote')
-      end
+        it 'has merge requests comments' do
+          expect(subject.first['notes']).not_to be_empty
+        end
 
-      it 'has issue assignees' do
-        expect(saved_project_json['issues'].first['issue_assignees']).not_to be_empty
-      end
+        it 'has author on merge requests comments' do
+          expect(subject.first['notes'].first['author']).not_to be_empty
+        end
 
-      it 'has author on issue comments' do
-        expect(saved_project_json['issues'].first['notes'].first['author']).not_to be_empty
+        it 'has merge request resource label events' do
+          expect(subject.first['resource_label_events']).not_to be_empty
+        end
       end
 
-      it 'has system note metadata on issue comments' do
-        metadata = saved_project_json['issues'].first['notes'].first['system_note_metadata']
+      context 'with snippets' do
+        let(:relation_name) { :snippets }
 
-        expect(metadata['action']).to eq('description')
-      end
+        it { is_expected.not_to be_empty }
 
-      it 'has project members' do
-        expect(saved_project_json['project_members']).not_to be_empty
+        it 'has snippet notes' do
+          expect(subject.first['notes']).not_to be_empty
+        end
       end
 
-      it 'has merge requests diffs' do
-        expect(saved_project_json['merge_requests'].first['merge_request_diff']).not_to be_empty
-      end
+      context 'with releases' do
+        let(:relation_name) { :releases }
 
-      it 'has merge request diff files' do
-        expect(saved_project_json['merge_requests'].first['merge_request_diff']['merge_request_diff_files']).not_to be_empty
-      end
+        it { is_expected.not_to be_empty }
 
-      it 'has merge request diff commits' do
-        expect(saved_project_json['merge_requests'].first['merge_request_diff']['merge_request_diff_commits']).not_to be_empty
-      end
+        it 'has no author on releases' do
+          expect(subject.first['author']).to be_nil
+        end
 
-      it 'has merge requests comments' do
-        expect(saved_project_json['merge_requests'].first['notes']).not_to be_empty
+        it 'has the author ID on releases' do
+          expect(subject.first['author_id']).not_to be_nil
+        end
       end
 
-      it 'has author on merge requests comments' do
-        expect(saved_project_json['merge_requests'].first['notes'].first['author']).not_to be_empty
-      end
+      context 'with issues' do
+        let(:relation_name) { :issues }
 
-      it 'has system note metadata on merge requests comments' do
-        metadata = saved_project_json['merge_requests'].first['notes'].first['system_note_metadata']
+        it { is_expected.not_to be_empty }
 
-        expect(metadata['commit_count']).to eq(1)
-        expect(metadata['action']).to eq('commit')
-      end
+        it 'has issue comments' do
+          notes = subject.first['notes']
 
-      it 'has pipeline stages' do
-        expect(saved_project_json.dig('ci_pipelines', 0, 'stages')).not_to be_empty
-      end
+          expect(notes).not_to be_empty
+          expect(notes.first['type']).to eq('DiscussionNote')
+        end
 
-      it 'has pipeline statuses' do
-        expect(saved_project_json.dig('ci_pipelines', 0, 'stages', 0, 'statuses')).not_to be_empty
-      end
+        it 'has issue assignees' do
+          expect(subject.first['issue_assignees']).not_to be_empty
+        end
 
-      it 'has pipeline builds' do
-        builds_count = saved_project_json
-          .dig('ci_pipelines', 0, 'stages', 0, 'statuses')
-          .count { |hash| hash['type'] == 'Ci::Build' }
+        it 'has author on issue comments' do
+          expect(subject.first['notes'].first['author']).not_to be_empty
+        end
 
-        expect(builds_count).to eq(1)
-      end
+        it 'has labels associated to records' do
+          expect(subject.first['label_links'].first['label']).not_to be_empty
+        end
 
-      it 'has no when YML attributes but only the DB column' do
-        expect_any_instance_of(Gitlab::Ci::YamlProcessor).not_to receive(:build_attributes)
+        it 'has project and group labels' do
+          label_types = subject.first['label_links'].map { |link| link['label']['type'] }
 
-        saved_project_json
-      end
+          expect(label_types).to match_array(%w(ProjectLabel GroupLabel))
+        end
 
-      it 'has pipeline commits' do
-        expect(saved_project_json['ci_pipelines']).not_to be_empty
-      end
+        it 'has priorities associated to labels' do
+          priorities = subject.first['label_links'].flat_map { |link| link['label']['priorities'] }
 
-      it 'has ci pipeline notes' do
-        expect(saved_project_json['ci_pipelines'].first['notes']).not_to be_empty
-      end
+          expect(priorities).not_to be_empty
+        end
 
-      it 'has labels with no associations' do
-        expect(saved_project_json['labels']).not_to be_empty
+        it 'has issue resource label events' do
+          expect(subject.first['resource_label_events']).not_to be_empty
+        end
       end
 
-      it 'has labels associated to records' do
-        expect(saved_project_json['issues'].first['label_links'].first['label']).not_to be_empty
-      end
+      context 'with ci_pipelines' do
+        let(:relation_name) { :ci_pipelines }
+
+        it { is_expected.not_to be_empty }
+
+        it 'has pipeline stages' do
+          expect(subject.dig(0, 'stages')).not_to be_empty
+        end
+
+        it 'has pipeline statuses' do
+          expect(subject.dig(0, 'stages', 0, 'statuses')).not_to be_empty
+        end
 
-      it 'has project and group labels' do
-        label_types = saved_project_json['issues'].first['label_links'].map { |link| link['label']['type'] }
+        it 'has pipeline builds' do
+          builds_count = subject.dig(0, 'stages', 0, 'statuses')
+                           .count { |hash| hash['type'] == 'Ci::Build' }
 
-        expect(label_types).to match_array(%w(ProjectLabel GroupLabel))
+          expect(builds_count).to eq(1)
+        end
+
+        it 'has ci pipeline notes' do
+          expect(subject.first['notes']).not_to be_empty
+        end
       end
 
-      it 'has priorities associated to labels' do
-        priorities = saved_project_json['issues'].first['label_links'].flat_map { |link| link['label']['priorities'] }
+      context 'with labels' do
+        let(:relation_name) { :labels }
 
-        expect(priorities).not_to be_empty
+        it { is_expected.not_to be_empty }
       end
 
-      it 'has issue resource label events' do
-        expect(saved_project_json['issues'].first['resource_label_events']).not_to be_empty
+      context 'with services' do
+        let(:relation_name) { :services }
+
+        it 'saves the correct service type' do
+          expect(subject.first['type']).to eq('CustomIssueTrackerService')
+        end
+
+        it 'saves the properties for a service' do
+          expect(subject.first['properties']).to eq('one' => 'value')
+        end
       end
 
-      it 'has merge request resource label events' do
-        expect(saved_project_json['merge_requests'].first['resource_label_events']).not_to be_empty
+      context 'with project_feature' do
+        let(:relation_name) { :project_feature }
+
+        it { is_expected.not_to be_empty }
+
+        it 'has project feature' do
+          expect(subject["issues_access_level"]).to eq(ProjectFeature::DISABLED)
+          expect(subject["wiki_access_level"]).to eq(ProjectFeature::ENABLED)
+          expect(subject["builds_access_level"]).to eq(ProjectFeature::PRIVATE)
+        end
       end
 
-      it 'saves the correct service type' do
-        expect(saved_project_json['services'].first['type']).to eq('CustomIssueTrackerService')
+      context 'with custom_attributes' do
+        let(:relation_name) { :custom_attributes }
+
+        it 'has custom attributes' do
+          expect(subject.count).to eq(2)
+        end
       end
 
-      it 'saves the properties for a service' do
-        expect(saved_project_json['services'].first['properties']).to eq('one' => 'value')
+      context 'with badges' do
+        let(:relation_name) { :custom_attributes }
+
+        it 'has badges' do
+          expect(subject.count).to eq(2)
+        end
       end
 
-      it 'has project feature' do
-        project_feature = saved_project_json['project_feature']
-        expect(project_feature).not_to be_empty
-        expect(project_feature["issues_access_level"]).to eq(ProjectFeature::DISABLED)
-        expect(project_feature["wiki_access_level"]).to eq(ProjectFeature::ENABLED)
-        expect(project_feature["builds_access_level"]).to eq(ProjectFeature::PRIVATE)
+      context 'with project_members' do
+        let(:relation_name) { :project_members }
+
+        it { is_expected.not_to be_empty }
       end
 
-      it 'has custom attributes' do
-        expect(saved_project_json['custom_attributes'].count).to eq(2)
+      context 'with boards' do
+        let(:relation_name) { :boards }
+
+        it { is_expected.not_to be_empty }
       end
+    end
 
-      it 'has badges' do
-        expect(saved_project_json['project_badges'].count).to eq(2)
+    describe '#saves project tree' do
+      let_it_be(:user) { create(:user) }
+      let_it_be(:group) { create(:group) }
+      let(:project) { setup_project }
+      let(:full_path) do
+        if ndjson_enabled
+          File.join(shared.export_path, 'tree')
+        else
+          File.join(shared.export_path, Gitlab::ImportExport.project_filename)
+        end
       end
+      let(:shared) { project.import_export_shared }
+      let(:params) { {} }
 
-      it 'does not complain about non UTF-8 characters in MR diff files' do
-        ActiveRecord::Base.connection.execute("UPDATE merge_request_diff_files SET diff = '---\n- :diff: !binary |-\n    LS0tIC9kZXYvbnVsbAorKysgYi9pbWFnZXMvbnVjb3IucGRmCkBAIC0wLDAg\n    KzEsMTY3OSBAQAorJVBERi0xLjUNJeLjz9MNCisxIDAgb2JqDTw8L01ldGFk\n    YXR'")
+      let(:project_tree_saver ) { described_class.new(project: project, current_user: user, shared: shared, params: params) }
 
-        expect(project_tree_saver.save).to be true
+      before do
+        stub_feature_flags(project_export_as_ndjson: ndjson_enabled)
+        project.add_maintainer(user)
+
+        FileUtils.rm_rf(export_path)
       end
 
-      context 'group members' do
+      after do
+        FileUtils.rm_rf(export_path)
+      end
+
+      context 'overrides group members' do
         let(:user2) { create(:user, email: 'group@member.com') }
+        let(:relation_name) { :project_members }
+
         let(:member_emails) do
-          saved_project_json['project_members'].map do |pm|
+          emails = subject.map do |pm|
             pm['user']['email']
           end
+          emails
         end
 
         before do
-          Group.first.add_developer(user2)
+          group.add_developer(user2)
         end
 
-        it 'does not export group members if it has no permission' do
-          Group.first.add_developer(user)
+        context 'when has no permission' do
+          before do
+            group.add_developer(user)
+            project_tree_saver.save
+          end
 
-          expect(member_emails).not_to include('group@member.com')
+          it 'does not export group members' do
+            expect(member_emails).not_to include('group@member.com')
+          end
         end
 
-        it 'does not export group members as maintainer' do
-          Group.first.add_maintainer(user)
+        context 'when has permission as maintainer' do
+          before do
+            group.add_maintainer(user)
 
-          expect(member_emails).not_to include('group@member.com')
+            project_tree_saver.save
+          end
+
+          it 'does not export group members' do
+            expect(member_emails).not_to include('group@member.com')
+          end
         end
 
-        it 'exports group members as group owner' do
-          Group.first.add_owner(user)
+        context 'when has permission as group owner' do
+          before do
+            group.add_owner(user)
 
-          expect(member_emails).to include('group@member.com')
+            project_tree_saver.save
+          end
+
+          it 'exports group members as group owner' do
+            expect(member_emails).to include('group@member.com')
+          end
         end
 
         context 'as admin' do
           let(:user) { create(:admin) }
 
+          before do
+            project_tree_saver.save
+          end
+
           it 'exports group members as admin' do
             expect(member_emails).to include('group@member.com')
           end
 
           it 'exports group members as project members' do
-            member_types = saved_project_json['project_members'].map { |pm| pm['source_type'] }
+            member_types = subject.map { |pm| pm['source_type'] }
 
             expect(member_types).to all(eq('Project'))
           end
         end
       end
 
-      context 'project attributes' do
-        it 'does not contain the runners token' do
-          expect(saved_project_json).not_to include("runners_token" => 'token')
+      context 'with description override' do
+        let(:params) { { description: 'Foo Bar' } }
+        let(:relation_name) { :projects }
+
+        before do
+          project_tree_saver.save
         end
+
+        it { is_expected.to include({ 'description' => params[:description] }) }
+      end
+
+      it 'saves project successfully' do
+        expect(project_tree_saver.save).to be true
+      end
+
+      it 'does not complain about non UTF-8 characters in MR diff files' do
+        ActiveRecord::Base.connection.execute("UPDATE merge_request_diff_files SET diff = '---\n- :diff: !binary |-\n    LS0tIC9kZXYvbnVsbAorKysgYi9pbWFnZXMvbnVjb3IucGRmCkBAIC0wLDAg\n    KzEsMTY3OSBAQAorJVBERi0xLjUNJeLjz9MNCisxIDAgb2JqDTw8L01ldGFk\n    YXR'")
+
+        expect(project_tree_saver.save).to be true
       end
 
-      it 'has a board and a list' do
-        expect(saved_project_json['boards'].first['lists']).not_to be_empty
+      it 'has no when YML attributes but only the DB column' do
+        expect_any_instance_of(Gitlab::Ci::YamlProcessor).not_to receive(:build_attributes)
+
+        project_tree_saver.save
       end
     end
   end
 
+  context 'with JSON' do
+    it_behaves_like "saves project tree successfully", false
+  end
+
+  context 'with NDJSON' do
+    it_behaves_like "saves project tree successfully", true
+  end
+
   def setup_project
     release = create(:release)
-    group = create(:group)
 
     project = create(:project,
-                     :public,
-                     :repository,
-                     :issues_disabled,
-                     :wiki_enabled,
-                     :builds_private,
-                     description: 'description',
-                     releases: [release],
-                     group: group,
-                     approvals_before_merge: 1
-                    )
-    allow(project).to receive(:commit).and_return(Commit.new(RepoHelpers.sample_commit, project))
+      :public,
+      :repository,
+      :issues_disabled,
+      :wiki_enabled,
+      :builds_private,
+      description: 'description',
+      releases: [release],
+      group: group,
+      approvals_before_merge: 1)
 
     issue = create(:issue, assignees: [user], project: project)
     snippet = create(:project_snippet, project: project)
@@ -331,9 +414,9 @@ describe Gitlab::ImportExport::Project::TreeSaver do
     mr_note = create(:note, noteable: merge_request, project: project)
     create(:note, noteable: snippet, project: project)
     create(:note_on_commit,
-           author: user,
-           project: project,
-           commit_id: ci_build.pipeline.sha)
+      author: user,
+      project: project,
+      commit_id: ci_build.pipeline.sha)
 
     create(:system_note_metadata, action: 'description', note: discussion_note)
     create(:system_note_metadata, commit_count: 1, action: 'commit', note: mr_note)
@@ -355,8 +438,4 @@ describe Gitlab::ImportExport::Project::TreeSaver do
 
     project
   end
-
-  def project_json(filename)
-    ::JSON.parse(IO.read(filename))
-  end
 end
diff --git a/spec/lib/gitlab/sidekiq_middleware_spec.rb b/spec/lib/gitlab/sidekiq_middleware_spec.rb
index 88f83ebc2ac..32c1807ba6e 100644
--- a/spec/lib/gitlab/sidekiq_middleware_spec.rb
+++ b/spec/lib/gitlab/sidekiq_middleware_spec.rb
@@ -134,12 +134,12 @@ describe Gitlab::SidekiqMiddleware do
     let(:middleware_expected_args) { [worker_class_arg, job, queue, redis_pool] }
     let(:expected_middlewares) do
       [
-        Gitlab::SidekiqStatus::ClientMiddleware,
-        Gitlab::SidekiqMiddleware::ClientMetrics,
-        Gitlab::SidekiqMiddleware::WorkerContext::Client,
-        Labkit::Middleware::Sidekiq::Client,
-        Gitlab::SidekiqMiddleware::AdminMode::Client,
-        Gitlab::SidekiqMiddleware::DuplicateJobs::Client
+         ::Gitlab::SidekiqMiddleware::WorkerContext::Client,
+         ::Labkit::Middleware::Sidekiq::Client,
+         ::Gitlab::SidekiqMiddleware::DuplicateJobs::Client,
+         ::Gitlab::SidekiqStatus::ClientMiddleware,
+         ::Gitlab::SidekiqMiddleware::AdminMode::Client,
+         ::Gitlab::SidekiqMiddleware::ClientMetrics
       ]
     end
 
diff --git a/spec/models/merge_request_spec.rb b/spec/models/merge_request_spec.rb
index 137795dcbc3..bfad1da1e21 100644
--- a/spec/models/merge_request_spec.rb
+++ b/spec/models/merge_request_spec.rb
@@ -3701,4 +3701,18 @@ describe MergeRequest do
       end
     end
   end
+
+  describe '#predefined_variables' do
+    let(:merge_request) { create(:merge_request) }
+
+    it 'caches all SQL-sourced data on the first call' do
+      control = ActiveRecord::QueryRecorder.new { merge_request.predefined_variables }.count
+
+      expect(control).to be > 0
+
+      count = ActiveRecord::QueryRecorder.new { merge_request.predefined_variables }.count
+
+      expect(count).to eq(0)
+    end
+  end
 end
diff --git a/spec/services/projects/hashed_storage/migration_service_spec.rb b/spec/services/projects/hashed_storage/migration_service_spec.rb
index f3ac26e7761..0a7975305dc 100644
--- a/spec/services/projects/hashed_storage/migration_service_spec.rb
+++ b/spec/services/projects/hashed_storage/migration_service_spec.rb
@@ -5,6 +5,11 @@ require 'spec_helper'
 describe Projects::HashedStorage::MigrationService do
   let(:project) { create(:project, :empty_repo, :wiki_repo, :legacy_storage) }
   let(:logger) { double }
+  let!(:project_attachment) { build(:file_uploader, project: project) }
+  let(:project_hashed_path) { Storage::Hashed.new(project).disk_path }
+  let(:project_legacy_path) { Storage::LegacyProject.new(project).disk_path }
+  let(:wiki_hashed_path) { "#{project_hashed_path}.wiki" }
+  let(:wiki_legacy_path) { "#{project_legacy_path}.wiki" }
 
   subject(:service) { described_class.new(project, project.full_path, logger: logger) }
 
@@ -29,9 +34,24 @@ describe Projects::HashedStorage::MigrationService do
 
         service.execute
       end
+
+      it 'migrates legacy repositories to hashed storage' do
+        legacy_attachments_path = FileUploader.absolute_base_dir(project)
+        hashed_project = project.dup.tap { |p| p.id = project.id }
+        hashed_project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments]
+        hashed_attachments_path = FileUploader.absolute_base_dir(hashed_project)
+
+        expect(logger).to receive(:info).with(/Repository moved from '#{project_legacy_path}' to '#{project_hashed_path}'/)
+        expect(logger).to receive(:info).with(/Repository moved from '#{wiki_legacy_path}' to '#{wiki_hashed_path}'/)
+        expect(logger).to receive(:info).with(/Project attachments moved from '#{legacy_attachments_path}' to '#{hashed_attachments_path}'/)
+
+        expect { service.execute }.to change { project.storage_version }.from(nil).to(2)
+      end
     end
 
     context 'attachments migration' do
+      let(:project) { create(:project, :empty_repo, :wiki_repo, storage_version: ::Project::HASHED_STORAGE_FEATURES[:repository]) }
+
       let(:attachments_service) do
         Projects::HashedStorage::MigrateAttachmentsService.new(project: project,
                                                                old_disk_path: project.full_path,
@@ -51,6 +71,17 @@ describe Projects::HashedStorage::MigrationService do
 
         service.execute
       end
+
+      it 'migrates legacy attachments to hashed storage' do
+        legacy_attachments_path = FileUploader.absolute_base_dir(project)
+        hashed_project = project.dup.tap { |p| p.id = project.id }
+        hashed_project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments]
+        hashed_attachments_path = FileUploader.absolute_base_dir(hashed_project)
+
+        expect(logger).to receive(:info).with(/Project attachments moved from '#{legacy_attachments_path}' to '#{hashed_attachments_path}'/)
+
+        expect { service.execute }.to change { project.storage_version }.from(1).to(2)
+      end
     end
   end
 end
diff --git a/spec/services/projects/hashed_storage/rollback_service_spec.rb b/spec/services/projects/hashed_storage/rollback_service_spec.rb
index 48d4eac9eb7..e6b7daba99e 100644
--- a/spec/services/projects/hashed_storage/rollback_service_spec.rb
+++ b/spec/services/projects/hashed_storage/rollback_service_spec.rb
@@ -5,6 +5,11 @@ require 'spec_helper'
 describe Projects::HashedStorage::RollbackService do
   let(:project) { create(:project, :empty_repo, :wiki_repo) }
   let(:logger) { double }
+  let!(:project_attachment) { build(:file_uploader, project: project) }
+  let(:project_hashed_path) { Storage::Hashed.new(project).disk_path }
+  let(:project_legacy_path) { Storage::LegacyProject.new(project).disk_path }
+  let(:wiki_hashed_path) { "#{project_hashed_path}.wiki" }
+  let(:wiki_legacy_path) { "#{project_legacy_path}.wiki" }
 
   subject(:service) { described_class.new(project, project.disk_path, logger: logger) }
 
@@ -26,6 +31,20 @@ describe Projects::HashedStorage::RollbackService do
 
         service.execute
       end
+
+      it 'rollbacks to legacy storage' do
+        hashed_attachments_path = FileUploader.absolute_base_dir(project)
+        legacy_project = project.dup
+        legacy_project.storage_version = nil
+        legacy_attachments_path = FileUploader.absolute_base_dir(legacy_project)
+
+        expect(logger).to receive(:info).with(/Project attachments moved from '#{hashed_attachments_path}' to '#{legacy_attachments_path}'/)
+
+        expect(logger).to receive(:info).with(/Repository moved from '#{project_hashed_path}' to '#{project_legacy_path}'/)
+        expect(logger).to receive(:info).with(/Repository moved from '#{wiki_hashed_path}' to '#{wiki_legacy_path}'/)
+
+        expect { service.execute }.to change { project.storage_version }.from(2).to(nil)
+      end
     end
 
     context 'repository rollback' do
@@ -47,6 +66,13 @@ describe Projects::HashedStorage::RollbackService do
 
         service.execute
       end
+
+      it 'rollbacks to legacy storage' do
+        expect(logger).to receive(:info).with(/Repository moved from '#{project_hashed_path}' to '#{project_legacy_path}'/)
+        expect(logger).to receive(:info).with(/Repository moved from '#{wiki_hashed_path}' to '#{wiki_legacy_path}'/)
+
+        expect { service.execute }.to change { project.storage_version }.from(1).to(nil)
+      end
     end
   end
 end
diff --git a/spec/services/web_hook_service_spec.rb b/spec/services/web_hook_service_spec.rb
index d8f13bc2e61..4a917ecdbb5 100644
--- a/spec/services/web_hook_service_spec.rb
+++ b/spec/services/web_hook_service_spec.rb
@@ -71,16 +71,6 @@ describe WebHookService do
       end
     end
 
-    it 'POSTs to the webhook URL' do
-      stub_full_request(project_hook.url, method: :post)
-
-      service_instance.execute
-
-      expect(WebMock).to have_requested(:post, stubbed_hostname(project_hook.url)).with(
-        headers: headers
-      ).once
-    end
-
     it 'POSTs the data as JSON' do
       stub_full_request(project_hook.url, method: :post)
 
diff --git a/spec/support/import_export/common_util.rb b/spec/support/import_export/common_util.rb
index 9281937e4ba..efe14b7244c 100644
--- a/spec/support/import_export/common_util.rb
+++ b/spec/support/import_export/common_util.rb
@@ -26,6 +26,21 @@ module ImportExport
       "tmp/tests/gitlab-test/import_export"
     end
 
+    def get_json(path, exportable_path, key, ndjson_enabled)
+      if ndjson_enabled
+        json = if key == :projects
+                 consume_attributes(path, exportable_path)
+               else
+                 consume_relations(path, exportable_path, key)
+               end
+      else
+        json = project_json(path)
+        json = json[key.to_s] unless key == :projects
+      end
+
+      json
+    end
+
     def restore_then_save_project(project, import_path:, export_path:)
       project_restorer = get_project_restorer(project, import_path)
       project_saver = get_project_saver(project, export_path)
@@ -50,5 +65,30 @@ module ImportExport
         allow(shared).to receive(:export_path).and_return(path)
       end
     end
+
+    def consume_attributes(dir_path, exportable_path)
+      path = File.join(dir_path, "#{exportable_path}.json")
+      return unless File.exist?(path)
+
+      ActiveSupport::JSON.decode(IO.read(path))
+    end
+
+    def consume_relations(dir_path, exportable_path, key)
+      path = File.join(dir_path, exportable_path, "#{key}.ndjson")
+      return unless File.exist?(path)
+
+      relations = []
+
+      File.foreach(path) do |line|
+        json = ActiveSupport::JSON.decode(line)
+        relations << json
+      end
+
+      key == :project_feature ? relations.first : relations.flatten
+    end
+
+    def project_json(filename)
+      ActiveSupport::JSON.decode(IO.read(filename))
+    end
   end
 end
diff --git a/spec/support/shared_examples/lib/gitlab/import_export/project_tree_restorer_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/import_export/project/tree_restorer_shared_examples.rb
index 5f9316c1cde..5f9316c1cde 100644
--- a/spec/support/shared_examples/lib/gitlab/import_export/project_tree_restorer_shared_examples.rb
+++ b/spec/support/shared_examples/lib/gitlab/import_export/project/tree_restorer_shared_examples.rb