Add latest changes from gitlab-org/gitlab@master

7 files changed, 227 insertions, 41 deletions

diff --git a/spec/lib/error_tracking/collector/payload_validator_spec.rb b/spec/lib/error_tracking/collector/payload_validator_spec.rb
index 94708f63bf4..96ad66e9b58 100644
--- a/spec/lib/error_tracking/collector/payload_validator_spec.rb
+++ b/spec/lib/error_tracking/collector/payload_validator_spec.rb
@@ -24,7 +24,7 @@ RSpec.describe ErrorTracking::Collector::PayloadValidator do
       end
 
       with_them do
-        let(:payload) { Gitlab::Json.parse(fixture_file(event_fixture)) }
+        let(:payload) { Gitlab::Json.parse(File.read(event_fixture)) }
 
         it_behaves_like 'valid payload'
       end
diff --git a/spec/lib/gitlab/ci/config/external/context_spec.rb b/spec/lib/gitlab/ci/config/external/context_spec.rb
index f1640822c6b..fc68d3d62a2 100644
--- a/spec/lib/gitlab/ci/config/external/context_spec.rb
+++ b/spec/lib/gitlab/ci/config/external/context_spec.rb
@@ -7,7 +7,16 @@ RSpec.describe Gitlab::Ci::Config::External::Context, feature_category: :pipelin
   let(:user) { double('User') }
   let(:sha) { '12345' }
   let(:variables) { Gitlab::Ci::Variables::Collection.new([{ 'key' => 'a', 'value' => 'b' }]) }
-  let(:attributes) { { project: project, user: user, sha: sha, variables: variables } }
+  let(:pipeline_config) { instance_double(Gitlab::Ci::ProjectConfig) }
+  let(:attributes) do
+    {
+      project: project,
+      user: user,
+      sha: sha,
+      variables: variables,
+      pipeline_config: pipeline_config
+    }
+  end
 
   subject(:subject) { described_class.new(**attributes) }
 
@@ -20,6 +29,7 @@ RSpec.describe Gitlab::Ci::Config::External::Context, feature_category: :pipelin
       it { expect(subject.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) }
       it { expect(subject.variables_hash).to be_instance_of(ActiveSupport::HashWithIndifferentAccess) }
       it { expect(subject.variables_hash).to include('a' => 'b') }
+      it { expect(subject.pipeline_config).to eq(pipeline_config) }
     end
 
     context 'without values' do
@@ -31,6 +41,7 @@ RSpec.describe Gitlab::Ci::Config::External::Context, feature_category: :pipelin
       it { expect(subject.execution_deadline).to eq(0) }
       it { expect(subject.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) }
       it { expect(subject.variables_hash).to be_instance_of(ActiveSupport::HashWithIndifferentAccess) }
+      it { expect(subject.pipeline_config).to be_nil }
     end
   end
 
@@ -144,27 +155,24 @@ RSpec.describe Gitlab::Ci::Config::External::Context, feature_category: :pipelin
     it { expect(subject.sentry_payload).to match(a_hash_including(:project, :user)) }
   end
 
-  describe '#contains_internal_include?' do
+  describe '#internal_include?' do
     context 'when pipeline_config is provided' do
-      let(:pipeline_config) { instance_double(Gitlab::Ci::ProjectConfig) }
-      let(:attributes) do
-        { project: project, user: user, sha: sha, variables: variables, pipeline_config: pipeline_config }
-      end
-
       where(:value) { [true, false] }
 
       with_them do
-        it 'returns the value of .contains_internal_include?' do
-          allow(pipeline_config).to receive(:contains_internal_include?).and_return(value)
+        it 'returns the value of .internal_include_prepended?' do
+          allow(pipeline_config).to receive(:internal_include_prepended?).and_return(value)
 
-          expect(subject.contains_internal_include?).to eq(value)
+          expect(subject.internal_include?).to eq(value)
         end
       end
     end
 
     context 'when pipeline_config is not provided' do
+      let(:pipeline_config) { nil }
+
       it 'returns false' do
-        expect(subject.contains_internal_include?).to eq(false)
+        expect(subject.internal_include?).to eq(false)
       end
     end
   end
diff --git a/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb
index 478b26e97cd..056a06337af 100644
--- a/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb
+++ b/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb
@@ -209,7 +209,30 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category:
       end
     end
 
-    context 'when total file count exceeds max_includes' do
+    describe 'max includes detection' do
+      shared_examples 'verifies max includes' do
+        context 'when total file count is equal to max_includes' do
+          before do
+            allow(context).to receive(:max_includes).and_return(expected_total_file_count)
+          end
+
+          it 'adds the expected number of files to expandset' do
+            expect { process }.not_to raise_error
+            expect(context.expandset.count).to eq(expected_total_file_count)
+          end
+        end
+
+        context 'when total file count exceeds max_includes' do
+          before do
+            allow(context).to receive(:max_includes).and_return(expected_total_file_count - 1)
+          end
+
+          it 'raises error' do
+            expect { process }.to raise_error(expected_error_class)
+          end
+        end
+      end
+
       context 'when files are nested' do
         let(:files) do
           [
@@ -217,9 +240,21 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category:
           ]
         end
 
-        it 'raises Processor::IncludeError' do
-          allow(context).to receive(:max_includes).and_return(1)
-          expect { process }.to raise_error(Gitlab::Ci::Config::External::Processor::IncludeError)
+        let(:expected_total_file_count) { 4 } # Includes nested_configs.yml + 3 nested files
+        let(:expected_error_class) { Gitlab::Ci::Config::External::Processor::IncludeError }
+
+        it_behaves_like 'verifies max includes'
+
+        context 'when duplicate files are included' do
+          let(:expected_total_file_count) { 8 } # 2 x (Includes nested_configs.yml + 3 nested files)
+          let(:files) do
+            [
+              Gitlab::Ci::Config::External::File::Local.new({ local: 'nested_configs.yml' }, context),
+              Gitlab::Ci::Config::External::File::Local.new({ local: 'nested_configs.yml' }, context)
+            ]
+          end
+
+          it_behaves_like 'verifies max includes'
         end
       end
 
@@ -231,24 +266,163 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category:
           ]
         end
 
-        it 'raises Mapper::TooManyIncludesError' do
-          allow(context).to receive(:max_includes).and_return(1)
-          expect { process }.to raise_error(Gitlab::Ci::Config::External::Mapper::TooManyIncludesError)
+        let(:expected_total_file_count) { files.count }
+        let(:expected_error_class) { Gitlab::Ci::Config::External::Mapper::TooManyIncludesError }
+
+        it_behaves_like 'verifies max includes'
+
+        context 'when duplicate files are included' do
+          let(:files) do
+            [
+              Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context),
+              Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file2.yml' }, context),
+              Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file2.yml' }, context)
+            ]
+          end
+
+          let(:expected_total_file_count) { files.count }
+
+          it_behaves_like 'verifies max includes'
         end
       end
 
-      context 'when files are duplicates' do
+      context 'when there is a circular include' do
+        let(:project_files) do
+          {
+            'myfolder/file1.yml' => <<~YAML
+              include: myfolder/file1.yml
+            YAML
+          }
+        end
+
         let(:files) do
           [
-            Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context),
-            Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context),
             Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context)
           ]
         end
 
+        before do
+          allow(context).to receive(:max_includes).and_return(10)
+        end
+
         it 'raises error' do
-          allow(context).to receive(:max_includes).and_return(2)
-          expect { process }.to raise_error(Gitlab::Ci::Config::External::Mapper::TooManyIncludesError)
+          expect { process }.to raise_error(Gitlab::Ci::Config::External::Processor::IncludeError)
+        end
+      end
+
+      context 'when a file is an internal include' do
+        let(:project_files) do
+          {
+            'myfolder/file1.yml' => <<~YAML,
+              my_build:
+                script: echo Hello World
+            YAML
+            '.internal-include.yml' => <<~YAML
+              include:
+                - local: myfolder/file1.yml
+            YAML
+          }
+        end
+
+        let(:files) do
+          [
+            Gitlab::Ci::Config::External::File::Local.new({ local: '.internal-include.yml' }, context)
+          ]
+        end
+
+        let(:total_file_count) { 2 } # Includes .internal-include.yml + myfolder/file1.yml
+        let(:pipeline_config) { instance_double(Gitlab::Ci::ProjectConfig) }
+
+        let(:context) do
+          Gitlab::Ci::Config::External::Context.new(
+            project: project,
+            user: user,
+            sha: project.commit.id,
+            pipeline_config: pipeline_config
+          )
+        end
+
+        before do
+          allow(pipeline_config).to receive(:internal_include_prepended?).and_return(true)
+          allow(context).to receive(:max_includes).and_return(1)
+        end
+
+        context 'when total file count excluding internal include is equal to max_includes' do
+          it 'does not add the internal include to expandset' do
+            expect { process }.not_to raise_error
+            expect(context.expandset.count).to eq(total_file_count - 1)
+            expect(context.expandset.first.location).to eq('myfolder/file1.yml')
+          end
+        end
+
+        context 'when total file count excluding internal include exceeds max_includes' do
+          let(:project_files) do
+            {
+              'myfolder/file1.yml' => <<~YAML,
+                my_build:
+                  script: echo Hello World
+              YAML
+              '.internal-include.yml' => <<~YAML
+                include:
+                  - local: myfolder/file1.yml
+                  - local: myfolder/file1.yml
+              YAML
+            }
+          end
+
+          it 'raises error' do
+            expect { process }.to raise_error(Gitlab::Ci::Config::External::Processor::IncludeError)
+          end
+        end
+      end
+    end
+
+    context 'when FF ci_fix_max_includes is disabled' do
+      before do
+        stub_feature_flags(ci_fix_max_includes: false)
+      end
+
+      context 'when total file count exceeds max_includes' do
+        context 'when files are nested' do
+          let(:files) do
+            [
+              Gitlab::Ci::Config::External::File::Local.new({ local: 'nested_configs.yml' }, context)
+            ]
+          end
+
+          it 'raises Processor::IncludeError' do
+            allow(context).to receive(:max_includes).and_return(1)
+            expect { process }.to raise_error(Gitlab::Ci::Config::External::Processor::IncludeError)
+          end
+        end
+
+        context 'when files are not nested' do
+          let(:files) do
+            [
+              Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context),
+              Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file2.yml' }, context)
+            ]
+          end
+
+          it 'raises Mapper::TooManyIncludesError' do
+            allow(context).to receive(:max_includes).and_return(1)
+            expect { process }.to raise_error(Gitlab::Ci::Config::External::Mapper::TooManyIncludesError)
+          end
+        end
+
+        context 'when files are duplicates' do
+          let(:files) do
+            [
+              Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context),
+              Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context),
+              Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context)
+            ]
+          end
+
+          it 'raises error' do
+            allow(context).to receive(:max_includes).and_return(2)
+            expect { process }.to raise_error(Gitlab::Ci::Config::External::Mapper::TooManyIncludesError)
+          end
         end
       end
     end
diff --git a/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb
index 434acfb5274..a9a52972294 100644
--- a/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb
@@ -26,7 +26,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content, feature_category: :
 
         expect(pipeline.config_source).to eq 'bridge_source'
         expect(command.config_content).to eq 'the-yaml'
-        expect(command.pipeline_config.contains_internal_include?).to eq(false)
+        expect(command.pipeline_config.internal_include_prepended?).to eq(false)
       end
     end
 
@@ -53,7 +53,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content, feature_category: :
         expect(pipeline.config_source).to eq 'repository_source'
         expect(pipeline.pipeline_config.content).to eq(config_content_result)
         expect(command.config_content).to eq(config_content_result)
-        expect(command.pipeline_config.contains_internal_include?).to eq(true)
+        expect(command.pipeline_config.internal_include_prepended?).to eq(true)
       end
     end
 
@@ -73,7 +73,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content, feature_category: :
         expect(pipeline.config_source).to eq 'remote_source'
         expect(pipeline.pipeline_config.content).to eq(config_content_result)
         expect(command.config_content).to eq(config_content_result)
-        expect(command.pipeline_config.contains_internal_include?).to eq(true)
+        expect(command.pipeline_config.internal_include_prepended?).to eq(true)
       end
     end
 
@@ -94,7 +94,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content, feature_category: :
         expect(pipeline.config_source).to eq 'external_project_source'
         expect(pipeline.pipeline_config.content).to eq(config_content_result)
         expect(command.config_content).to eq(config_content_result)
-        expect(command.pipeline_config.contains_internal_include?).to eq(true)
+        expect(command.pipeline_config.internal_include_prepended?).to eq(true)
       end
 
       context 'when path specifies a refname' do
@@ -115,7 +115,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content, feature_category: :
           expect(pipeline.config_source).to eq 'external_project_source'
           expect(pipeline.pipeline_config.content).to eq(config_content_result)
           expect(command.config_content).to eq(config_content_result)
-          expect(command.pipeline_config.contains_internal_include?).to eq(true)
+          expect(command.pipeline_config.internal_include_prepended?).to eq(true)
         end
       end
     end
@@ -143,7 +143,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content, feature_category: :
         expect(pipeline.config_source).to eq 'repository_source'
         expect(pipeline.pipeline_config.content).to eq(config_content_result)
         expect(command.config_content).to eq(config_content_result)
-        expect(command.pipeline_config.contains_internal_include?).to eq(true)
+        expect(command.pipeline_config.internal_include_prepended?).to eq(true)
       end
     end
 
@@ -167,7 +167,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content, feature_category: :
         expect(pipeline.config_source).to eq 'auto_devops_source'
         expect(pipeline.pipeline_config.content).to eq(config_content_result)
         expect(command.config_content).to eq(config_content_result)
-        expect(command.pipeline_config.contains_internal_include?).to eq(true)
+        expect(command.pipeline_config.internal_include_prepended?).to eq(true)
       end
     end
 
@@ -188,7 +188,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content, feature_category: :
         expect(pipeline.config_source).to eq 'parameter_source'
         expect(pipeline.pipeline_config.content).to eq(content)
         expect(command.config_content).to eq(content)
-        expect(command.pipeline_config.contains_internal_include?).to eq(false)
+        expect(command.pipeline_config.internal_include_prepended?).to eq(false)
       end
     end
 
diff --git a/spec/lib/gitlab/ci/project_config/repository_spec.rb b/spec/lib/gitlab/ci/project_config/repository_spec.rb
index b31a9099348..e8a997a7e43 100644
--- a/spec/lib/gitlab/ci/project_config/repository_spec.rb
+++ b/spec/lib/gitlab/ci/project_config/repository_spec.rb
@@ -45,8 +45,8 @@ RSpec.describe Gitlab::Ci::ProjectConfig::Repository, feature_category: :continu
     it { is_expected.to eq(:repository_source) }
   end
 
-  describe '#contains_internal_include?' do
-    subject { config.contains_internal_include? }
+  describe '#internal_include_prepended?' do
+    subject { config.internal_include_prepended? }
 
     it { is_expected.to eq(true) }
   end
diff --git a/spec/lib/gitlab/ci/project_config/source_spec.rb b/spec/lib/gitlab/ci/project_config/source_spec.rb
index 5248cf080e8..eefabe1babb 100644
--- a/spec/lib/gitlab/ci/project_config/source_spec.rb
+++ b/spec/lib/gitlab/ci/project_config/source_spec.rb
@@ -21,9 +21,9 @@ RSpec.describe Gitlab::Ci::ProjectConfig::Source, feature_category: :continuous_
     it { expect { source }.to raise_error(NotImplementedError) }
   end
 
-  describe '#contains_internal_include?' do
-    subject(:contains_internal_include) { custom_config.contains_internal_include? }
+  describe '#internal_include_prepended?' do
+    subject(:internal_include_prepended) { custom_config.internal_include_prepended? }
 
-    it { expect(contains_internal_include).to eq(false) }
+    it { expect(internal_include_prepended).to eq(false) }
   end
 end
diff --git a/spec/lib/gitlab/email/html_to_markdown_parser_spec.rb b/spec/lib/gitlab/email/html_to_markdown_parser_spec.rb
index fe585d47d59..59c488739dc 100644
--- a/spec/lib/gitlab/email/html_to_markdown_parser_spec.rb
+++ b/spec/lib/gitlab/email/html_to_markdown_parser_spec.rb
@@ -1,17 +1,21 @@
 # frozen_string_literal: true
 
-require 'spec_helper'
+require 'kramdown'
+require 'html2text'
+require 'fast_spec_helper'
+require 'support/helpers/fixture_helpers'
 
 RSpec.describe Gitlab::Email::HtmlToMarkdownParser, feature_category: :service_desk do
+  include FixtureHelpers
+
   subject { described_class.convert(html) }
 
   describe '.convert' do
     let(:html) { fixture_file("lib/gitlab/email/basic.html") }
 
     it 'parses html correctly' do
-      expect(subject)
-      .to eq(
-        <<-BODY.strip_heredoc.chomp
+      expect(subject).to eq(
+        <<~BODY.chomp
           Hello, World!
           This is some e-mail content. Even though it has whitespace and newlines, the e-mail converter will handle it correctly.
           *Even* mismatched tags.