Add latest changes from gitlab-org/gitlab@master

6 files changed, 93 insertions, 65 deletions

diff --git a/lib/gitlab/ci/config/entry/include.rb b/lib/gitlab/ci/config/entry/include.rb
index b2586714636..cd09d83b728 100644
--- a/lib/gitlab/ci/config/entry/include.rb
+++ b/lib/gitlab/ci/config/entry/include.rb
@@ -15,11 +15,6 @@ module Gitlab
           validations do
             validates :config, hash_or_string: true
             validates :config, allowed_keys: ALLOWED_KEYS
-            validate do
-              if config[:artifact] && config[:job].blank?
-                errors.add(:config, "must specify the job where to fetch the artifact from")
-              end
-            end
           end
         end
       end
diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
index 020d1f323ee..10ef33e71d5 100644
--- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
@@ -1,7 +1,7 @@
 # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/
 
 # Configure the scanning tool through the environment variables.
-# List of the variables: https://gitlab.com/gitlab-org/security-products/dast#settings
+# List of the variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables
 # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
 
 stages:
@@ -19,26 +19,10 @@ dast:
     name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION"
   variables:
     GIT_STRATEGY: none
-  # URL to scan:
-  # DAST_WEBSITE: https://example.com/
-  #
-  # Time limit for target availability (scan is attempted even when timeout):
-  # DAST_TARGET_AVAILABILITY_TIMEOUT: 60
-  #
-  # Set these variables to scan with an authenticated user:
-  # DAST_AUTH_URL: https://example.com/sign-in
-  # DAST_USERNAME: john.doe@example.com
-  # DAST_PASSWORD: john-doe-password
-  # DAST_USERNAME_FIELD: session[user] # the name of username field at the sign-in HTML form
-  # DAST_PASSWORD_FIELD: session[password] # the name of password field at the sign-in HTML form
-  # DAST_AUTH_EXCLUDE_URLS: http://example.com/sign-out,http://example.com/sign-out-2 # optional: URLs to skip during the authenticated scan; comma-separated, no spaces in between
-  #
-  # Perform ZAP Full Scan, which includes both passive and active scanning:
-  # DAST_FULL_SCAN_ENABLED: "true"
   allow_failure: true
   script:
     - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
-    - /analyze -t $DAST_WEBSITE
+    - /analyze
   artifacts:
     reports:
       dast: gl-dast-report.json
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 4b0062549f0..764047dae6d 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -142,7 +142,6 @@ module Gitlab
           validate_job_stage!(name, job)
           validate_job_dependencies!(name, job)
           validate_job_needs!(name, job)
-          validate_dynamic_child_pipeline_dependencies!(name, job)
           validate_job_environment!(name, job)
         end
       end
@@ -164,50 +163,35 @@ module Gitlab
       def validate_job_dependencies!(name, job)
         return unless job[:dependencies]
 
-        job[:dependencies].each do |dependency|
-          validate_job_dependency!(name, dependency)
-        end
-      end
+        stage_index = @stages.index(job[:stage])
 
-      def validate_dynamic_child_pipeline_dependencies!(name, job)
-        return unless includes = job.dig(:trigger, :include)
+        job[:dependencies].each do |dependency|
+          raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym]
 
-        includes.each do |included|
-          next unless dependency = included[:job]
+          dependency_stage_index = @stages.index(@jobs[dependency.to_sym][:stage])
 
-          validate_job_dependency!(name, dependency)
+          unless dependency_stage_index.present? && dependency_stage_index < stage_index
+            raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages"
+          end
         end
       end
 
       def validate_job_needs!(name, job)
-        return unless needs = job.dig(:needs, :job)
+        return unless job.dig(:needs, :job)
 
-        needs.each do |need|
-          dependency = need[:name]
-          validate_job_dependency!(name, dependency, 'need')
-        end
-      end
+        stage_index = @stages.index(job[:stage])
 
-      def validate_job_dependency!(name, dependency, dependency_type = 'dependency')
-        unless @jobs[dependency.to_sym]
-          raise ValidationError, "#{name} job: undefined #{dependency_type}: #{dependency}"
-        end
+        job.dig(:needs, :job).each do |need|
+          need_job_name = need[:name]
 
-        job_stage_index = stage_index(name)
-        dependency_stage_index = stage_index(dependency)
+          raise ValidationError, "#{name} job: undefined need: #{need_job_name}" unless @jobs[need_job_name.to_sym]
 
-        # A dependency might be defined later in the configuration
-        # with a stage that does not exist
-        unless dependency_stage_index.present? && dependency_stage_index < job_stage_index
-          raise ValidationError, "#{name} job: #{dependency_type} #{dependency} is not defined in prior stages"
-        end
-      end
+          needs_stage_index = @stages.index(@jobs[need_job_name.to_sym][:stage])
 
-      def stage_index(name)
-        job = @jobs[name.to_sym]
-        return unless job
-
-        @stages.index(job[:stage])
+          unless needs_stage_index.present? && needs_stage_index < stage_index
+            raise ValidationError, "#{name} job: need #{need_job_name} is not defined in prior stages"
+          end
+        end
       end
 
       def validate_job_environment!(name, job)
diff --git a/lib/gitlab/import_export/json/ndjson_writer.rb b/lib/gitlab/import_export/json/ndjson_writer.rb
new file mode 100644
index 00000000000..e74fdd74049
--- /dev/null
+++ b/lib/gitlab/import_export/json/ndjson_writer.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module JSON
+      class NdjsonWriter
+        include Gitlab::ImportExport::CommandLineUtil
+
+        def initialize(dir_path)
+          @dir_path = dir_path
+        end
+
+        def close
+        end
+
+        def write_attributes(exportable_path, hash)
+          # It will create:
+          # tree/project.json
+          with_file("#{exportable_path}.json") do |file|
+            file.write(hash.to_json)
+          end
+        end
+
+        def write_relation(exportable_path, relation, value)
+          # It will create:
+          # tree/project/ci_cd_setting.ndjson
+          with_file(exportable_path, "#{relation}.ndjson") do |file|
+            file.write(value.to_json)
+          end
+        end
+
+        def write_relation_array(exportable_path, relation, items)
+          # It will create:
+          # tree/project/merge_requests.ndjson
+          with_file(exportable_path, "#{relation}.ndjson") do |file|
+            items.each do |item|
+              file.write(item.to_json)
+              file.write("\n")
+            end
+          end
+        end
+
+        private
+
+        def with_file(*path)
+          file_path = File.join(@dir_path, *path)
+          raise ArgumentError, "The #{file_path} already exist" if File.exist?(file_path)
+
+          # ensure that path is created
+          mkdir_p(File.dirname(file_path))
+
+          File.open(file_path, "wb") do |file|
+            yield(file)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb
index 988776fe600..0017aa523c1 100644
--- a/lib/gitlab/import_export/project/tree_saver.rb
+++ b/lib/gitlab/import_export/project/tree_saver.rb
@@ -11,15 +11,9 @@ module Gitlab
           @project      = project
           @current_user = current_user
           @shared       = shared
-          @full_path    = File.join(@shared.export_path, ImportExport.project_filename)
         end
 
         def save
-          json_writer = ImportExport::JSON::LegacyWriter.new(
-            @full_path,
-            allowed_path: "project"
-          )
-
           ImportExport::JSON::StreamingSerializer.new(
             exportable,
             reader.project_tree,
@@ -57,6 +51,18 @@ module Gitlab
         def presenter_class
           Projects::ImportExport::ProjectExportPresenter
         end
+
+        def json_writer
+          @json_writer ||= begin
+            if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace)
+              full_path = File.join(@shared.export_path, 'tree')
+              Gitlab::ImportExport::JSON::NdjsonWriter.new(full_path)
+            else
+              full_path = File.join(@shared.export_path, ImportExport.project_filename)
+              Gitlab::ImportExport::JSON::LegacyWriter.new(full_path, allowed_path: 'project')
+            end
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb
index 37165d787c7..1b155570f18 100644
--- a/lib/gitlab/sidekiq_middleware.rb
+++ b/lib/gitlab/sidekiq_middleware.rb
@@ -29,12 +29,12 @@ module Gitlab
     # eg: `config.client_middleware(&Gitlab::SidekiqMiddleware.client_configurator)`
     def self.client_configurator
       lambda do |chain|
-        chain.add ::Gitlab::SidekiqStatus::ClientMiddleware
-        chain.add ::Gitlab::SidekiqMiddleware::ClientMetrics
         chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Client # needs to be before the Labkit middleware
         chain.add ::Labkit::Middleware::Sidekiq::Client
-        chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Client
         chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Client
+        chain.add ::Gitlab::SidekiqStatus::ClientMiddleware
+        chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Client
+        chain.add ::Gitlab::SidekiqMiddleware::ClientMetrics
       end
     end
   end