diff options
Diffstat (limited to 'lib/gitlab/ci')
72 files changed, 1214 insertions, 208 deletions
diff --git a/lib/gitlab/ci/build/artifacts/expire_in_parser.rb b/lib/gitlab/ci/build/artifacts/expire_in_parser.rb new file mode 100644 index 00000000000..3e8a1fb86fc --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/expire_in_parser.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Artifacts + class ExpireInParser + def self.validate_duration(value) + new(value).validate_duration + end + + def initialize(value) + @value = value + end + + def validate_duration + return true if never? + + parse + rescue ChronicDuration::DurationParseError + false + end + + def seconds_from_now + parse&.seconds&.from_now + end + + private + + attr_reader :value + + def parse + return if never? + + ChronicDuration.parse(value) + end + + def never? + value.to_s.casecmp('never') == 0 + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index ef354832e8e..355fffbf9c6 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -16,7 +16,7 @@ module Gitlab # class Entry attr_reader :entries - attr_accessor :name + attr_writer :name def initialize(path, entries) @entries = entries diff --git a/lib/gitlab/ci/build/auto_retry.rb b/lib/gitlab/ci/build/auto_retry.rb new file mode 100644 index 00000000000..e6ef12975c2 --- /dev/null +++ b/lib/gitlab/ci/build/auto_retry.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +class Gitlab::Ci::Build::AutoRetry + include Gitlab::Utils::StrongMemoize + + DEFAULT_RETRIES = { + scheduler_failure: 2 + }.freeze + + def initialize(build) + @build = build + end + + def allowed? + return false unless @build.retryable? + + within_max_retry_limit? + end + + private + + def within_max_retry_limit? + max_allowed_retries > 0 && max_allowed_retries > @build.retries_count + end + + def max_allowed_retries + strong_memoize(:max_allowed_retries) do + options_retry_max || DEFAULT_RETRIES.fetch(@build.failure_reason.to_sym, 0) + end + end + + def options_retry_max + Integer(options_retry[:max], exception: false) if retry_on_reason_or_always? + end + + def options_retry_when + options_retry.fetch(:when, ['always']) + end + + def retry_on_reason_or_always? + options_retry_when.include?(@build.failure_reason.to_s) || + options_retry_when.include?('always') + end + + # The format of the retry option changed in GitLab 11.5: Before it was + # integer only, after it is a hash. New builds are created with the new + # format, but builds created before GitLab 11.5 and saved in database still + # have the old integer only format. This method returns the retry option + # normalized as a hash in 11.5+ format. + def options_retry + strong_memoize(:options_retry) do + value = @build.options&.dig(:retry) + value = value.is_a?(Integer) ? { max: value } : value.to_h + value.with_indifferent_access + end + end +end diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb index f8550b50905..3f0ccefa9e5 100644 --- a/lib/gitlab/ci/build/step.rb +++ b/lib/gitlab/ci/build/step.rb @@ -21,8 +21,6 @@ module Gitlab end def from_release(job) - return unless Gitlab::Ci::Features.release_generation_enabled? - release = job.options[:release] return unless release diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index a9a9636637f..206dbaea272 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -42,7 +42,7 @@ module Gitlab inclusion: { in: %w[on_success on_failure always], message: 'should be on_success, on_failure ' \ 'or always' } - validates :expire_in, duration: true + validates :expire_in, duration: { parser: ::Gitlab::Ci::Build::Artifacts::ExpireInParser } end end diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb index f4362d3b0ce..a8b67a1db4f 100644 --- a/lib/gitlab/ci/config/entry/bridge.rb +++ b/lib/gitlab/ci/config/entry/bridge.rb @@ -11,7 +11,7 @@ module Gitlab class Bridge < ::Gitlab::Config::Entry::Node include ::Gitlab::Ci::Config::Entry::Processable - ALLOWED_KEYS = %i[trigger allow_failure when needs].freeze + ALLOWED_KEYS = %i[trigger].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index a615cab1a80..f960cec1f26 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -11,9 +11,8 @@ module Gitlab include ::Gitlab::Ci::Config::Entry::Processable ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze - ALLOWED_KEYS = %i[tags script type image services - allow_failure type when start_in artifacts cache - dependencies before_script needs after_script + ALLOWED_KEYS = %i[tags script type image services start_in artifacts + cache dependencies before_script after_script environment coverage retry parallel interruptible timeout resource_group release secrets].freeze @@ -23,18 +22,9 @@ module Gitlab validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS validates :config, required_keys: REQUIRED_BY_NEEDS, if: :has_needs? validates :script, presence: true - validates :config, - disallowed_keys: { - in: %i[release], - message: 'release features are not enabled' - }, - unless: -> { Gitlab::Ci::Features.release_generation_enabled? } with_options allow_nil: true do validates :allow_failure, boolean: true - validates :parallel, numericality: { only_integer: true, - greater_than_or_equal_to: 2, - less_than_or_equal_to: 50 } validates :when, inclusion: { in: ALLOWED_WHEN, message: "should be one of: #{ALLOWED_WHEN.join(', ')}" @@ -124,13 +114,47 @@ module Gitlab description: 'This job will produce a release.', inherit: false + entry :parallel, Entry::Product::Parallel, + description: 'Parallel configuration for this job.', + inherit: false + attributes :script, :tags, :allow_failure, :when, :dependencies, :needs, :retry, :parallel, :start_in, :interruptible, :timeout, :resource_group, :release + Matcher = Struct.new(:name, :config) do + def applies? + job_is_not_hidden? && + config_is_a_hash? && + has_job_keys? + end + + private + + def job_is_not_hidden? + !name.to_s.start_with?('.') + end + + def config_is_a_hash? + config.is_a?(Hash) + end + + def has_job_keys? + if name == :default + config.key?(:script) + else + (ALLOWED_KEYS & config.keys).any? + end + end + end + def self.matching?(name, config) - !name.to_s.start_with?('.') && - config.is_a?(Hash) && config.key?(:script) + if Gitlab::Ci::Features.job_entry_matches_all_keys? + Matcher.new(name, config).applies? + else + !name.to_s.start_with?('.') && + config.is_a?(Hash) && config.key?(:script) + end end def self.visible? @@ -174,7 +198,7 @@ module Gitlab environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, retry: retry_defined? ? retry_value : nil, - parallel: has_parallel? ? parallel.to_i : nil, + parallel: has_parallel? ? parallel_value : nil, interruptible: interruptible_defined? ? interruptible_value : nil, timeout: has_timeout? ? ChronicDuration.parse(timeout.to_s) : nil, artifacts: artifacts_value, diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index b4539475d88..f10c509d0cc 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -14,7 +14,8 @@ module Gitlab include ::Gitlab::Config::Entry::Attributable include ::Gitlab::Config::Entry::Inheritable - PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables inherit].freeze + PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables + inherit allow_failure when needs].freeze included do validations do @@ -82,8 +83,8 @@ module Gitlab @entries.delete(:except) unless except_defined? # rubocop:disable Gitlab/ModuleWithInstanceVariables end - if has_rules? && !has_workflow_rules && Gitlab::Ci::Features.raise_job_rules_without_workflow_rules_warning? - add_warning('uses `rules` without defining `workflow:rules`') + unless has_workflow_rules + validate_against_warnings end # inherit root variables @@ -93,6 +94,19 @@ module Gitlab end end + def validate_against_warnings + # If rules are valid format and workflow rules are not specified + return unless rules_value + return unless Gitlab::Ci::Features.raise_job_rules_without_workflow_rules_warning? + + last_rule = rules_value.last + + if last_rule&.keys == [:when] && last_rule[:when] != 'never' + docs_url = 'read more: https://docs.gitlab.com/ee/ci/troubleshooting.html#pipeline-warnings' + add_warning("may allow multiple pipelines to run for a single action due to `rules:when` clause with no `workflow:rules` - #{docs_url}") + end + end + def name metadata[:name] end diff --git a/lib/gitlab/ci/config/entry/product/matrix.rb b/lib/gitlab/ci/config/entry/product/matrix.rb new file mode 100644 index 00000000000..6af809d46c1 --- /dev/null +++ b/lib/gitlab/ci/config/entry/product/matrix.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents matrix style parallel builds. + # + module Product + class Matrix < ::Gitlab::Config::Entry::Node + include ::Gitlab::Utils::StrongMemoize + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + validations do + validates :config, array_of_hashes: true + + validate on: :composed do + limit = Entry::Product::Parallel::PARALLEL_LIMIT + + if number_of_generated_jobs > limit + errors.add(:config, "generates too many jobs (maximum is #{limit})") + end + end + end + + def compose!(deps = nil) + super(deps) do + @config.each_with_index do |variables, index| + @entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Product::Variables) + .value(variables) + .with(parent: self, description: 'matrix variables definition.') # rubocop:disable CodeReuse/ActiveRecord + .create! + end + + @entries.each_value do |entry| + entry.compose!(deps) + end + end + end + + def value + strong_memoize(:value) do + @entries.values.map(&:value) + end + end + + # rubocop:disable CodeReuse/ActiveRecord + def number_of_generated_jobs + value.sum do |config| + config.values.reduce(1) { |acc, values| acc * values.size } + end + end + # rubocop:enable CodeReuse/ActiveRecord + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/product/parallel.rb b/lib/gitlab/ci/config/entry/product/parallel.rb new file mode 100644 index 00000000000..cd9eabbbc66 --- /dev/null +++ b/lib/gitlab/ci/config/entry/product/parallel.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a parallel job config. + # + module Product + class Parallel < ::Gitlab::Config::Entry::Simplifiable + strategy :ParallelBuilds, if: -> (config) { config.is_a?(Numeric) } + strategy :MatrixBuilds, if: -> (config) { config.is_a?(Hash) } + + PARALLEL_LIMIT = 50 + + class ParallelBuilds < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, numericality: { only_integer: true, + greater_than_or_equal_to: 2, + less_than_or_equal_to: Entry::Product::Parallel::PARALLEL_LIMIT }, + allow_nil: true + end + + def value + { number: super.to_i } + end + end + + class MatrixBuilds < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Configurable + + PERMITTED_KEYS = %i[matrix].freeze + + validations do + validates :config, allowed_keys: PERMITTED_KEYS + validates :config, required_keys: PERMITTED_KEYS + end + + entry :matrix, Entry::Product::Matrix, + description: 'Variables definition for matrix builds' + end + + class UnknownStrategy < ::Gitlab::Config::Entry::Node + def errors + ["#{location} should be an integer or a hash"] + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb new file mode 100644 index 00000000000..ac4f70fb69e --- /dev/null +++ b/lib/gitlab/ci/config/entry/product/variables.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents variables for parallel matrix builds. + # + module Product + class Variables < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, variables: { array_values: true } + validates :config, length: { + minimum: 2, + too_short: 'requires at least %{count} items' + } + end + + def self.default(**) + {} + end + + def value + @config + .map { |key, value| [key.to_s, Array(value).map(&:to_s)] } + .to_h + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index 814dcc66362..cf6c2961ee7 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -54,7 +54,7 @@ module Gitlab end def execution_expired? - return false if execution_deadline.zero? + return false if execution_deadline == 0 current_monotonic_time > execution_deadline end diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb index 1139efee9e8..451ba14bb89 100644 --- a/lib/gitlab/ci/config/normalizer.rb +++ b/lib/gitlab/ci/config/normalizer.rb @@ -32,7 +32,7 @@ module Gitlab return unless job_names job_names.flat_map do |job_name| - parallelized_jobs[job_name.to_sym] || job_name + parallelized_jobs[job_name.to_sym]&.map(&:name) || job_name end end @@ -42,10 +42,8 @@ module Gitlab job_needs.flat_map do |job_need| job_need_name = job_need[:name].to_sym - if all_job_names = parallelized_jobs[job_need_name] - all_job_names.map do |job_name| - job_need.merge(name: job_name) - end + if all_jobs = parallelized_jobs[job_need_name] + all_jobs.map { |job| job_need.merge(name: job.name) } else job_need end @@ -57,7 +55,7 @@ module Gitlab @jobs_config.each_with_object({}) do |(job_name, config), hash| next unless config[:parallel] - hash[job_name] = self.class.parallelize_job_names(job_name, config[:parallel]) + hash[job_name] = parallelize_job_config(job_name, config[:parallel]) end end end @@ -65,9 +63,9 @@ module Gitlab def expand_parallelize_jobs @jobs_config.each_with_object({}) do |(job_name, config), hash| if parallelized_jobs.key?(job_name) - parallelized_jobs[job_name].each_with_index do |name, index| - hash[name.to_sym] = - yield(name, config.merge(name: name, instance: index + 1)) + parallelized_jobs[job_name].each do |job| + hash[job.name.to_sym] = + yield(job.name, config.deep_merge(job.attributes)) end else hash[job_name] = yield(job_name, config) @@ -75,8 +73,8 @@ module Gitlab end end - def self.parallelize_job_names(name, total) - Array.new(total) { |index| "#{name} #{index + 1}/#{total}" } + def parallelize_job_config(name, config) + Normalizer::Factory.new(name, config).create end end end diff --git a/lib/gitlab/ci/config/normalizer/factory.rb b/lib/gitlab/ci/config/normalizer/factory.rb new file mode 100644 index 00000000000..bf813f8e878 --- /dev/null +++ b/lib/gitlab/ci/config/normalizer/factory.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Normalizer + class Factory + include Gitlab::Utils::StrongMemoize + + def initialize(name, config) + @name = name + @config = config + end + + def create + return [] unless strategy + + strategy.build_from(@name, @config) + end + + private + + def strategy + strong_memoize(:strategy) do + strategies.find do |strategy| + strategy.applies_to?(@config) + end + end + end + + def strategies + [NumberStrategy, MatrixStrategy] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb new file mode 100644 index 00000000000..db21274a9ed --- /dev/null +++ b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Normalizer + class MatrixStrategy + class << self + def applies_to?(config) + config.is_a?(Hash) && config.key?(:matrix) + end + + def build_from(job_name, initial_config) + config = expand(initial_config[:matrix]) + total = config.size + + config.map.with_index do |vars, index| + new(job_name, index.next, vars, total) + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def expand(config) + config.flat_map do |config| + values = config.values + + values[0] + .product(*values.from(1)) + .map { |vals| config.keys.zip(vals).to_h } + end + end + # rubocop: enable CodeReuse/ActiveRecord + end + + def initialize(job_name, instance, variables, total) + @job_name = job_name + @instance = instance + @variables = variables.to_h + @total = total + end + + def attributes + { + name: name, + instance: instance, + variables: variables, + parallel: { total: total } + } + end + + def name_with_details + vars = variables.map { |key, value| "#{key}=#{value}"}.join('; ') + + "#{job_name} (#{vars})" + end + + def name + "#{job_name} #{instance}/#{total}" + end + + private + + attr_reader :job_name, :instance, :variables, :total + end + end + end + end +end diff --git a/lib/gitlab/ci/config/normalizer/number_strategy.rb b/lib/gitlab/ci/config/normalizer/number_strategy.rb new file mode 100644 index 00000000000..4754e7b46d4 --- /dev/null +++ b/lib/gitlab/ci/config/normalizer/number_strategy.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Normalizer + class NumberStrategy + class << self + def applies_to?(config) + config.is_a?(Integer) || config.is_a?(Hash) && config.key?(:number) + end + + def build_from(job_name, config) + total = config.is_a?(Hash) ? config[:number] : config + + Array.new(total) do |index| + new(job_name, index.next, total) + end + end + end + + def initialize(job_name, instance, total) + @job_name = job_name + @instance = instance + @total = total + end + + def attributes + { + name: name, + instance: instance, + parallel: { total: total } + } + end + + def name + "#{job_name} #{instance}/#{total}" + end + + private + + attr_reader :job_name, :instance, :total + end + end + end + end +end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index 6130baeb9d5..2f6667d3600 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -14,32 +14,16 @@ module Gitlab ::Feature.enabled?(:ci_job_heartbeats_runner, project, default_enabled: true) end - def self.pipeline_fixed_notifications? - ::Feature.enabled?(:ci_pipeline_fixed_notifications, default_enabled: true) - end - def self.instance_variables_ui_enabled? ::Feature.enabled?(:ci_instance_variables_ui, default_enabled: true) end - def self.composite_status?(project) - ::Feature.enabled?(:ci_composite_status, project, default_enabled: true) - end - - def self.atomic_processing?(project) - ::Feature.enabled?(:ci_atomic_processing, project, default_enabled: true) - end - def self.pipeline_latest? ::Feature.enabled?(:ci_pipeline_latest, default_enabled: true) end def self.pipeline_status_omit_commit_sha_in_cache_key?(project) - Feature.enabled?(:ci_pipeline_status_omit_commit_sha_in_cache_key, project) - end - - def self.release_generation_enabled? - ::Feature.enabled?(:ci_release_generation, default_enabled: true) + Feature.enabled?(:ci_pipeline_status_omit_commit_sha_in_cache_key, project, default_enabled: true) end # Remove in https://gitlab.com/gitlab-org/gitlab/-/issues/224199 @@ -49,13 +33,11 @@ module Gitlab # Remove in https://gitlab.com/gitlab-org/gitlab/-/issues/227052 def self.variables_api_filter_environment_scope? - ::Feature.enabled?(:ci_variables_api_filter_environment_scope, default_enabled: false) + ::Feature.enabled?(:ci_variables_api_filter_environment_scope, default_enabled: true) end - # This FF is only used for development purpose to test that warnings can be - # raised and propagated to the UI. def self.raise_job_rules_without_workflow_rules_warning? - ::Feature.enabled?(:ci_raise_job_rules_without_workflow_rules_warning) + ::Feature.enabled?(:ci_raise_job_rules_without_workflow_rules_warning, default_enabled: true) end def self.keep_latest_artifacts_for_ref_enabled?(project) @@ -70,8 +52,32 @@ module Gitlab ::Feature.enabled?(:ci_bulk_insert_on_create, project, default_enabled: true) end + def self.ci_if_parenthesis_enabled? + ::Feature.enabled?(:ci_if_parenthesis_enabled, default_enabled: true) + end + def self.allow_to_create_merge_request_pipelines_in_target_project?(target_project) - ::Feature.enabled?(:ci_allow_to_create_merge_request_pipelines_in_target_project, target_project) + ::Feature.enabled?(:ci_allow_to_create_merge_request_pipelines_in_target_project, target_project, default_enabled: true) + end + + def self.ci_plan_needs_size_limit?(project) + ::Feature.enabled?(:ci_plan_needs_size_limit, project, default_enabled: true) + end + + def self.job_entry_matches_all_keys? + ::Feature.enabled?(:ci_job_entry_matches_all_keys) + end + + def self.lint_creates_pipeline_with_dry_run?(project) + ::Feature.enabled?(:ci_lint_creates_pipeline_with_dry_run, project, default_enabled: true) + end + + def self.reset_ci_minutes_for_all_namespaces? + ::Feature.enabled?(:reset_ci_minutes_for_all_namespaces, default_enabled: false) + end + + def self.expand_names_for_cross_pipeline_artifacts?(project) + ::Feature.enabled?(:ci_expand_names_for_cross_pipeline_artifacts, project) end end end diff --git a/lib/gitlab/ci/parsers/coverage/cobertura.rb b/lib/gitlab/ci/parsers/coverage/cobertura.rb index 006d5097148..934c797580c 100644 --- a/lib/gitlab/ci/parsers/coverage/cobertura.rb +++ b/lib/gitlab/ci/parsers/coverage/cobertura.rb @@ -28,6 +28,8 @@ module Gitlab end def parse_node(key, value, coverage_report) + return if key == 'sources' + if key == 'class' Array.wrap(value).each do |item| parse_class(item, coverage_report) diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb new file mode 100644 index 00000000000..468f3bc4689 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class CancelPendingPipelines < Chain::Base + include Chain::Helpers + + def perform! + return unless project.auto_cancel_pending_pipelines? + + Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines) do |cancelables| + cancelables.find_each do |cancelable| + cancelable.auto_cancel_running(pipeline) + end + end + end + + def break? + false + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def auto_cancelable_pipelines + project.ci_pipelines + .where(ref: pipeline.ref) + .where.not(id: pipeline.same_family_pipeline_ids) + .where.not(sha: project.commit(pipeline.ref).try(:id)) + .alive_or_scheduled + .with_only_interruptible_builds + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 74b28b181bc..dbaa6951e64 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -10,7 +10,7 @@ module Gitlab :trigger_request, :schedule, :merge_request, :external_pull_request, :ignore_skip_ci, :save_incompleted, :seeds_block, :variables_attributes, :push_options, - :chat_data, :allow_mirror_update, :bridge, :content, + :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, # These attributes are set by Chains during processing: :config_content, :config_processor, :stage_seeds ) do @@ -22,6 +22,8 @@ module Gitlab end end + alias_method :dry_run?, :dry_run + def branch_exists? strong_memoize(:is_branch) do project.repository.branch_exists?(ref) diff --git a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb b/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb index 3dd216b33d1..9954aedc4b7 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb @@ -12,7 +12,6 @@ module Gitlab def content strong_memoize(:content) do next unless command.content.present? - raise UnsupportedSourceError, "#{command.source} not a dangling build" unless command.dangling_build? command.content end diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index aba7dab508d..d7271df1694 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -6,13 +6,13 @@ module Gitlab module Chain module Helpers def error(message, config_error: false, drop_reason: nil) - if config_error && command.save_incompleted + if config_error drop_reason = :config_error pipeline.yaml_errors = message end pipeline.add_error_message(message) - pipeline.drop!(drop_reason) if drop_reason + pipeline.drop!(drop_reason) if drop_reason && persist_pipeline? # TODO: consider not to rely on AR errors directly as they can be # polluted with other unrelated errors (e.g. state machine) @@ -23,6 +23,10 @@ module Gitlab def warning(message) pipeline.add_warning_message(message) end + + def persist_pipeline? + command.save_incompleted && !pipeline.readonly? + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/metrics.rb b/lib/gitlab/ci/pipeline/chain/metrics.rb new file mode 100644 index 00000000000..0d7449813b4 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/metrics.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class Metrics < Chain::Base + def perform! + counter.increment(source: @pipeline.source) + end + + def break? + false + end + + def counter + ::Gitlab::Ci::Pipeline::Metrics.new.pipelines_created_counter + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/pipeline/process.rb b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb new file mode 100644 index 00000000000..1eb7474e915 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Pipeline + # After pipeline has been successfully created we can start processing it. + class Process < Chain::Base + def perform! + ::Ci::ProcessPipelineService + .new(@pipeline) + .execute + end + + def break? + false + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index 204c7725214..dc648568129 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -9,30 +9,21 @@ module Gitlab @pipeline = pipeline @command = command @sequence = sequence - @completed = [] @start = Time.now end def build! - @sequence.each do |chain| - step = chain.new(@pipeline, @command) + @sequence.each do |step_class| + step = step_class.new(@pipeline, @command) step.perform! break if step.break? - - @completed.push(step) end - @pipeline.tap do - yield @pipeline, self if block_given? - - @command.observe_creation_duration(Time.now - @start) - @command.observe_pipeline_size(@pipeline) - end - end + @command.observe_creation_duration(Time.now - @start) + @command.observe_pipeline_size(@pipeline) - def complete? - @completed.size == @sequence.size + @pipeline end end end diff --git a/lib/gitlab/ci/pipeline/chain/stop_dry_run.rb b/lib/gitlab/ci/pipeline/chain/stop_dry_run.rb new file mode 100644 index 00000000000..0e9add4ee74 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/stop_dry_run.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + # During the dry run we don't want to persist the pipeline and skip + # all the other steps that operate on a persisted context. + # This causes the chain to break at this point. + class StopDryRun < Chain::Base + def perform! + # no-op + end + + def break? + @command.dry_run? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index 769d0dffd0b..8f1e690c081 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -34,7 +34,7 @@ module Gitlab end def allowed_to_write_ref? - access = Gitlab::UserAccess.new(current_user, project: project) + access = Gitlab::UserAccess.new(current_user, container: project) if @command.branch_exists? access.can_update_branch?(@command.ref) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/and.rb b/lib/gitlab/ci/pipeline/expression/lexeme/and.rb index 54a0e2ad9dd..422735bd104 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/and.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/and.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Expression module Lexeme - class And < Lexeme::Operator + class And < Lexeme::LogicalOperator PATTERN = /&&/.freeze def evaluate(variables = {}) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb index 7ebd2e25398..676857183cf 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb @@ -10,6 +10,10 @@ module Gitlab raise NotImplementedError end + def name + self.class.name.demodulize.underscore + end + def self.build(token) raise NotImplementedError end @@ -23,6 +27,10 @@ module Gitlab def self.pattern self::PATTERN end + + def self.consume?(lexeme) + lexeme && precedence >= lexeme.precedence + end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb index 62f4c14f597..d35be12c996 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Expression module Lexeme - class Equals < Lexeme::Operator + class Equals < Lexeme::LogicalOperator PATTERN = /==/.freeze def evaluate(variables = {}) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/logical_operator.rb b/lib/gitlab/ci/pipeline/expression/lexeme/logical_operator.rb new file mode 100644 index 00000000000..05d5043c06e --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/logical_operator.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class LogicalOperator < Lexeme::Operator + # This operator class is design to handle single operators that take two + # arguments. Expression::Parser was originally designed to read infix operators, + # and so the two operands are called "left" and "right" here. If we wish to + # implement an Operator that takes a greater or lesser number of arguments, a + # structural change or additional Operator superclass will likely be needed. + + def initialize(left, right) + raise OperatorError, 'Invalid left operand' unless left.respond_to? :evaluate + raise OperatorError, 'Invalid right operand' unless right.respond_to? :evaluate + + @left = left + @right = right + end + + def inspect + "#{name}(#{@left.inspect}, #{@right.inspect})" + end + + def self.type + :logical_operator + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb index f7b0720d4a9..4d65b914d8d 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Expression module Lexeme - class Matches < Lexeme::Operator + class Matches < Lexeme::LogicalOperator PATTERN = /=~/.freeze def evaluate(variables = {}) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb index 8166bcd5730..64485a7e6b3 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Expression module Lexeme - class NotEquals < Lexeme::Operator + class NotEquals < Lexeme::LogicalOperator PATTERN = /!=/.freeze def evaluate(variables = {}) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb index 02479ed28a4..29c5aa5d753 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Expression module Lexeme - class NotMatches < Lexeme::Operator + class NotMatches < Lexeme::LogicalOperator PATTERN = /\!~/.freeze def evaluate(variables = {}) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb index be7258c201a..e7f7945532b 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb @@ -9,13 +9,17 @@ module Gitlab PATTERN = /null/.freeze def initialize(value = nil) - @value = nil + super end def evaluate(variables = {}) nil end + def inspect + 'null' + end + def self.build(_value) self.new end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb index 3ddab7800c8..a740c50c900 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb @@ -6,24 +6,10 @@ module Gitlab module Expression module Lexeme class Operator < Lexeme::Base - # This operator class is design to handle single operators that take two - # arguments. Expression::Parser was originally designed to read infix operators, - # and so the two operands are called "left" and "right" here. If we wish to - # implement an Operator that takes a greater or lesser number of arguments, a - # structural change or additional Operator superclass will likely be needed. - OperatorError = Class.new(Expression::ExpressionError) - def initialize(left, right) - raise OperatorError, 'Invalid left operand' unless left.respond_to? :evaluate - raise OperatorError, 'Invalid right operand' unless right.respond_to? :evaluate - - @left = left - @right = right - end - def self.type - :operator + raise NotImplementedError end def self.precedence diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/or.rb b/lib/gitlab/ci/pipeline/expression/lexeme/or.rb index 807876f905a..c7d653ac859 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/or.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/or.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Expression module Lexeme - class Or < Lexeme::Operator + class Or < Lexeme::LogicalOperator PATTERN = /\|\|/.freeze def evaluate(variables = {}) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb new file mode 100644 index 00000000000..b0ca26c9f5d --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class ParenthesisClose < Lexeme::Operator + PATTERN = /\)/.freeze + + def self.type + :parenthesis_close + end + + def self.precedence + 900 + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb new file mode 100644 index 00000000000..924fe0663ab --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class ParenthesisOpen < Lexeme::Operator + PATTERN = /\(/.freeze + + def self.type + :parenthesis_open + end + + def self.precedence + # Needs to be higher than `ParenthesisClose` and all other Lexemes + 901 + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index 0212fa9d661..514241e8ae2 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -11,7 +11,7 @@ module Gitlab PATTERN = %r{^\/([^\/]|\\/)+[^\\]\/[ismU]*}.freeze def initialize(regexp) - @value = regexp.gsub(/\\\//, '/') + super(regexp.gsub(/\\\//, '/')) unless Gitlab::UntrustedRegexp::RubySyntax.valid?(@value) raise Lexer::SyntaxError, 'Invalid regular expression!' @@ -24,6 +24,10 @@ module Gitlab raise Expression::RuntimeError, 'Invalid regular expression!' end + def inspect + "/#{value}/" + end + def self.pattern PATTERN end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb index 2db2bf011f1..e90e764bcd9 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb @@ -9,13 +9,17 @@ module Gitlab PATTERN = /("(?<string>.*?)")|('(?<string>.*?)')/.freeze def initialize(value) - @value = value + super(value) end def evaluate(variables = {}) @value.to_s end + def inspect + @value.inspect + end + def self.build(string) new(string.match(PATTERN)[:string]) end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb index ef9ddb6cae9..6d872fee39d 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb @@ -9,6 +9,10 @@ module Gitlab def self.type :value end + + def initialize(value) + @value = value + end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb index 85c0899e4f6..11d2010909f 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb @@ -8,12 +8,12 @@ module Gitlab class Variable < Lexeme::Value PATTERN = /\$(?<name>\w+)/.freeze - def initialize(name) - @name = name + def evaluate(variables = {}) + variables.with_indifferent_access.fetch(@value, nil) end - def evaluate(variables = {}) - variables.with_indifferent_access.fetch(@name, nil) + def inspect + "$#{@value}" end def self.build(string) diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb index 7d7582612f9..5b7365cb33b 100644 --- a/lib/gitlab/ci/pipeline/expression/lexer.rb +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -10,6 +10,8 @@ module Gitlab SyntaxError = Class.new(Expression::ExpressionError) LEXEMES = [ + Expression::Lexeme::ParenthesisOpen, + Expression::Lexeme::ParenthesisClose, Expression::Lexeme::Variable, Expression::Lexeme::String, Expression::Lexeme::Pattern, @@ -22,6 +24,28 @@ module Gitlab Expression::Lexeme::Or ].freeze + # To be removed with `ci_if_parenthesis_enabled` + LEGACY_LEXEMES = [ + Expression::Lexeme::Variable, + Expression::Lexeme::String, + Expression::Lexeme::Pattern, + Expression::Lexeme::Null, + Expression::Lexeme::Equals, + Expression::Lexeme::Matches, + Expression::Lexeme::NotEquals, + Expression::Lexeme::NotMatches, + Expression::Lexeme::And, + Expression::Lexeme::Or + ].freeze + + def self.lexemes + if ::Gitlab::Ci::Features.ci_if_parenthesis_enabled? + LEXEMES + else + LEGACY_LEXEMES + end + end + MAX_TOKENS = 100 def initialize(statement, max_tokens: MAX_TOKENS) @@ -47,7 +71,7 @@ module Gitlab return tokens if @scanner.eos? - lexeme = LEXEMES.find do |type| + lexeme = self.class.lexemes.find do |type| type.scan(@scanner).tap do |token| tokens.push(token) if token.present? end diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb index edb55edf356..27d7aa2f37e 100644 --- a/lib/gitlab/ci/pipeline/expression/parser.rb +++ b/lib/gitlab/ci/pipeline/expression/parser.rb @@ -15,11 +15,18 @@ module Gitlab def tree results = [] - tokens_rpn.each do |token| + tokens = + if ::Gitlab::Ci::Features.ci_if_parenthesis_enabled? + tokens_rpn + else + legacy_tokens_rpn + end + + tokens.each do |token| case token.type when :value results.push(token.build) - when :operator + when :logical_operator right_operand = results.pop left_operand = results.pop @@ -27,7 +34,7 @@ module Gitlab results.push(res) end else - raise ParseError, 'Unprocessable token found in parse tree' + raise ParseError, "Unprocessable token found in parse tree: #{token.type}" end end @@ -45,6 +52,7 @@ module Gitlab # Parse the expression into Reverse Polish Notation # (See: Shunting-yard algorithm) + # Taken from: https://en.wikipedia.org/wiki/Shunting-yard_algorithm#The_algorithm_in_detail def tokens_rpn output = [] operators = [] @@ -53,7 +61,34 @@ module Gitlab case token.type when :value output.push(token) - when :operator + when :logical_operator + output.push(operators.pop) while token.lexeme.consume?(operators.last&.lexeme) + + operators.push(token) + when :parenthesis_open + operators.push(token) + when :parenthesis_close + output.push(operators.pop) while token.lexeme.consume?(operators.last&.lexeme) + + raise ParseError, 'Unmatched parenthesis' unless operators.last + + operators.pop if operators.last.lexeme.type == :parenthesis_open + end + end + + output.concat(operators.reverse) + end + + # To be removed with `ci_if_parenthesis_enabled` + def legacy_tokens_rpn + output = [] + operators = [] + + @tokens.each do |token| + case token.type + when :value + output.push(token) + when :logical_operator if operators.any? && token.lexeme.precedence >= operators.last.lexeme.precedence output.push(operators.pop) end diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 649da745eea..db6cca27f1c 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -36,6 +36,15 @@ module Gitlab Gitlab::Metrics.counter(name, comment) end end + + def pipelines_created_counter + strong_memoize(:pipelines_created_count) do + name = :pipelines_created_total + comment = 'Counter of pipelines created' + + Gitlab::Metrics.counter(name, comment) + end + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 114a46ca9f6..3be3fa63b92 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -11,9 +11,7 @@ module Gitlab delegate :dig, to: :@seed_attributes - # When the `ci_dag_limit_needs` is enabled it uses the lower limit - LOW_NEEDS_LIMIT = 10 - HARD_NEEDS_LIMIT = 50 + DEFAULT_NEEDS_LIMIT = 10 def initialize(pipeline, attributes, previous_stages) @pipeline = pipeline @@ -142,10 +140,10 @@ module Gitlab end def max_needs_allowed - if Feature.enabled?(:ci_dag_limit_needs, @project, default_enabled: true) - LOW_NEEDS_LIMIT + if ::Gitlab::Ci::Features.ci_plan_needs_size_limit?(@pipeline.project) + @pipeline.project.actual_limits.ci_needs_size_limit else - HARD_NEEDS_LIMIT + DEFAULT_NEEDS_LIMIT end end diff --git a/lib/gitlab/ci/reports/accessibility_reports_comparer.rb b/lib/gitlab/ci/reports/accessibility_reports_comparer.rb index fa6337166d5..210eb17f2d3 100644 --- a/lib/gitlab/ci/reports/accessibility_reports_comparer.rb +++ b/lib/gitlab/ci/reports/accessibility_reports_comparer.rb @@ -17,7 +17,7 @@ module Gitlab end def status - head_reports.errors_count.positive? ? STATUS_FAILED : STATUS_SUCCESS + head_reports.errors_count > 0 ? STATUS_FAILED : STATUS_SUCCESS end def existing_errors diff --git a/lib/gitlab/ci/reports/test_report_summary.rb b/lib/gitlab/ci/reports/test_report_summary.rb index 85b83b790e7..3e7227b7223 100644 --- a/lib/gitlab/ci/reports/test_report_summary.rb +++ b/lib/gitlab/ci/reports/test_report_summary.rb @@ -4,42 +4,17 @@ module Gitlab module Ci module Reports class TestReportSummary - attr_reader :all_results - - def initialize(all_results) - @all_results = all_results + def initialize(build_report_results) + @build_report_results = build_report_results + @suite_summary = TestSuiteSummary.new(@build_report_results) end def total - TestSuiteSummary.new(all_results) - end - - def total_time - total.total_time - end - - def total_count - total.total_count - end - - def success_count - total.success_count - end - - def failed_count - total.failed_count - end - - def skipped_count - total.skipped_count - end - - def error_count - total.error_count + @suite_summary.to_h end def test_suites - all_results + @build_report_results .group_by(&:tests_name) .transform_values { |results| TestSuiteSummary.new(results) } end diff --git a/lib/gitlab/ci/reports/test_reports.rb b/lib/gitlab/ci/reports/test_reports.rb index 86ba725c71e..a5a630642e5 100644 --- a/lib/gitlab/ci/reports/test_reports.rb +++ b/lib/gitlab/ci/reports/test_reports.rb @@ -43,9 +43,7 @@ module Gitlab end def suite_errors - test_suites.each_with_object({}) do |(name, suite), errors| - errors[suite.name] = suite.suite_error if suite.suite_error - end + test_suites.transform_values(&:suite_error).compact end TestCase::STATUS_TYPES.each do |status_type| diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb index 28b81e7a471..5ee779227ec 100644 --- a/lib/gitlab/ci/reports/test_suite.rb +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -28,7 +28,7 @@ module Gitlab def total_count return 0 if suite_error - test_cases.values.sum(&:count) + [success_count, failed_count, skipped_count, error_count].sum end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/reports/test_suite_summary.rb b/lib/gitlab/ci/reports/test_suite_summary.rb index f9b0bedb712..32b06d0ad49 100644 --- a/lib/gitlab/ci/reports/test_suite_summary.rb +++ b/lib/gitlab/ci/reports/test_suite_summary.rb @@ -4,45 +4,54 @@ module Gitlab module Ci module Reports class TestSuiteSummary - attr_reader :results - - def initialize(results) - @results = results + def initialize(build_report_results) + @build_report_results = build_report_results end def name - @name ||= results.first.tests_name + @name ||= @build_report_results.first.tests_name end # rubocop: disable CodeReuse/ActiveRecord def build_ids - results.pluck(:build_id) + @build_report_results.pluck(:build_id) end def total_time - @total_time ||= results.sum(&:tests_duration) + @total_time ||= @build_report_results.sum(&:tests_duration) end def success_count - @success_count ||= results.sum(&:tests_success) + @success_count ||= @build_report_results.sum(&:tests_success) end def failed_count - @failed_count ||= results.sum(&:tests_failed) + @failed_count ||= @build_report_results.sum(&:tests_failed) end def skipped_count - @skipped_count ||= results.sum(&:tests_skipped) + @skipped_count ||= @build_report_results.sum(&:tests_skipped) end def error_count - @error_count ||= results.sum(&:tests_errored) + @error_count ||= @build_report_results.sum(&:tests_errored) end def total_count @total_count ||= [success_count, failed_count, skipped_count, error_count].sum end # rubocop: disable CodeReuse/ActiveRecord + + def to_h + { + time: total_time, + count: total_count, + success: success_count, + failed: failed_count, + skipped: skipped_count, + error: error_count + } + end end end end diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb new file mode 100644 index 00000000000..2171637687f --- /dev/null +++ b/lib/gitlab/ci/runner_instructions.rb @@ -0,0 +1,137 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class RunnerInstructions + class ArgumentError < ::ArgumentError; end + + include Gitlab::Allowable + + OS = { + linux: { + human_readable_name: "Linux", + download_locations: { + amd64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-amd64", + '386': "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-386", + arm: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-arm", + arm64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-arm64" + }, + install_script_template_path: "lib/gitlab/ci/runner_instructions/templates/linux/install.sh", + runner_executable: "sudo gitlab-runner" + }, + osx: { + human_readable_name: "macOS", + download_locations: { + amd64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-darwin-amd64" + }, + install_script_template_path: "lib/gitlab/ci/runner_instructions/templates/osx/install.sh", + runner_executable: "sudo gitlab-runner" + }, + windows: { + human_readable_name: "Windows", + download_locations: { + amd64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-windows-amd64.exe", + '386': "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-windows-386.exe" + }, + install_script_template_path: "lib/gitlab/ci/runner_instructions/templates/windows/install.ps1", + runner_executable: "./gitlab-runner.exe" + } + }.freeze + + OTHER_ENVIRONMENTS = { + docker: { + human_readable_name: "Docker", + installation_instructions_url: "https://docs.gitlab.com/runner/install/docker.html" + }, + kubernetes: { + human_readable_name: "Kubernetes", + installation_instructions_url: "https://docs.gitlab.com/runner/install/kubernetes.html" + } + }.freeze + + attr_reader :errors + + def initialize(current_user:, group: nil, project: nil, os:, arch:) + @current_user = current_user + @group = group + @project = project + @os = os + @arch = arch + @errors = [] + + validate_params + end + + def install_script + with_error_handling [Gitlab::Ci::RunnerInstructions::ArgumentError] do + raise Gitlab::Ci::RunnerInstructions::ArgumentError, s_('Architecture not found for OS') unless environment[:download_locations].key?(@arch.to_sym) + + replace_variables(get_file(environment[:install_script_template_path])) + end + end + + def register_command + with_error_handling [Gitlab::Ci::RunnerInstructions::ArgumentError, Gitlab::Access::AccessDeniedError] do + raise Gitlab::Ci::RunnerInstructions::ArgumentError, s_('No runner executable') unless environment[:runner_executable] + + server_url = Gitlab::Routing.url_helpers.root_url(only_path: false) + runner_executable = environment[:runner_executable] + + "#{runner_executable} register --url #{server_url} --registration-token #{registration_token}" + end + end + + private + + def with_error_handling(exceptions) + return if errors.present? + + yield + rescue *exceptions => e + @errors << e.message + nil + end + + def environment + @environment ||= OS[@os.to_sym] || ( raise Gitlab::Ci::RunnerInstructions::ArgumentError, s_('Invalid OS') ) + end + + def validate_params + @errors << s_('Missing OS') unless @os.present? + @errors << s_('Missing arch') unless @arch.present? + end + + def replace_variables(expression) + expression.sub('${GITLAB_CI_RUNNER_DOWNLOAD_LOCATION}', "#{environment[:download_locations][@arch.to_sym]}") + end + + def get_file(path) + File.read(path) + end + + def registration_token + project_token || group_token || instance_token + end + + def project_token + return unless @project + raise Gitlab::Access::AccessDeniedError unless can?(@current_user, :admin_pipeline, @project) + + @project.runners_token + end + + def group_token + return unless @group + raise Gitlab::Access::AccessDeniedError unless can?(@current_user, :admin_group, @group) + + @group.runners_token + end + + def instance_token + raise Gitlab::Access::AccessDeniedError unless @current_user&.admin? + + Gitlab::CurrentSettings.runners_registration_token + end + end + end +end diff --git a/lib/gitlab/ci/runner_instructions/templates/linux/install.sh b/lib/gitlab/ci/runner_instructions/templates/linux/install.sh new file mode 100644 index 00000000000..6c8a0796d23 --- /dev/null +++ b/lib/gitlab/ci/runner_instructions/templates/linux/install.sh @@ -0,0 +1,12 @@ +# Download the binary for your system +sudo curl -L --output /usr/local/bin/gitlab-runner ${GITLAB_CI_RUNNER_DOWNLOAD_LOCATION} + +# Give it permissions to execute +sudo chmod +x /usr/local/bin/gitlab-runner + +# Create a GitLab CI user +sudo useradd --comment 'GitLab Runner' --create-home gitlab-runner --shell /bin/bash + +# Install and run as service +sudo gitlab-runner install --user=gitlab-runner --working-directory=/home/gitlab-runner +sudo gitlab-runner start diff --git a/lib/gitlab/ci/runner_instructions/templates/osx/install.sh b/lib/gitlab/ci/runner_instructions/templates/osx/install.sh new file mode 100644 index 00000000000..de4ee3e52fc --- /dev/null +++ b/lib/gitlab/ci/runner_instructions/templates/osx/install.sh @@ -0,0 +1,11 @@ +# Download the binary for your system +sudo curl --output /usr/local/bin/gitlab-runner ${GITLAB_CI_RUNNER_DOWNLOAD_LOCATION} + +# Give it permissions to execute +sudo chmod +x /usr/local/bin/gitlab-runner + +# The rest of commands execute as the user who will run the Runner +# Register the Runner (steps below), then run +cd ~ +gitlab-runner install +gitlab-runner start diff --git a/lib/gitlab/ci/runner_instructions/templates/windows/install.ps1 b/lib/gitlab/ci/runner_instructions/templates/windows/install.ps1 new file mode 100644 index 00000000000..dc37f88543c --- /dev/null +++ b/lib/gitlab/ci/runner_instructions/templates/windows/install.ps1 @@ -0,0 +1,13 @@ +# Run PowerShell: https://docs.microsoft.com/en-us/powershell/scripting/windows-powershell/starting-windows-powershell?view=powershell-7#with-administrative-privileges-run-as-administrator +# Create a folder somewhere in your system ex.: C:\GitLab-Runner +New-Item -Path 'C:\GitLab-Runner' -ItemType Directory + +# Enter the folder +cd 'C:\GitLab-Runner' + +# Dowload binary +Invoke-WebRequest -Uri "${GITLAB_CI_RUNNER_DOWNLOAD_LOCATION}" -OutFile "gitlab-runner.exe" + +# Register the Runner (steps below), then run +.\gitlab-runner.exe install +.\gitlab-runner.exe start diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 76ad113aad9..88846f724e7 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -24,7 +24,8 @@ module Gitlab downstream_bridge_project_not_found: 'downstream project could not be found', insufficient_bridge_permissions: 'no permissions to trigger downstream pipeline', bridge_pipeline_is_child_pipeline: 'creation of child pipeline not allowed from another child pipeline', - downstream_pipeline_creation_failed: 'downstream pipeline can not be created' + downstream_pipeline_creation_failed: 'downstream pipeline can not be created', + secrets_provider_not_found: 'secrets provider can not be found' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index c10d87a537b..968ff0fce89 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -162,4 +162,4 @@ include: - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml - template: Security/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml - - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml + - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml index 5f4bd631db6..c1815baf7e6 100644 --- a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml @@ -1,4 +1,4 @@ -# This template is deprecated and will be removed as part of GitLab 13.2! +# This template is deprecated. # # If you have referenced this template in your CI pipeline, please # update your CI configuration by replacing the following occurrence(s): @@ -20,12 +20,8 @@ stages: - deploy - production -before_script: - - printf '\nWARNING!\nThis job includes "Deploy-ECS.gitlab-ci.yml". Please rename this to "AWS/Deploy-ECS.gitlab-ci.yml".\n' - -variables: - AUTO_DEVOPS_PLATFORM_TARGET: ECS - -include: - - template: Jobs/Build.gitlab-ci.yml - - template: Jobs/Deploy/ECS.gitlab-ci.yml +"error: Template has moved": + stage: deploy + script: + - echo "Deploy-ECS.gitlab-ci.yml has been moved to AWS/Deploy-ECS.gitlab-ci.yml, see https://docs.gitlab.com/ee/ci/cloud_deployment/#deploy-your-application-to-the-aws-elastic-container-service-ecs for more details." + - exit 1 diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index dbe870953ae..0c3598a61a7 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,6 +1,6 @@ build: stage: build - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.3.1" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.4.0" variables: DOCKER_TLS_CERTDIR: "" services: diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 6b76d7e0c9b..cf851c875ee 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.10" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.10-gitlab.1" needs: [] script: - | diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index d7d927ac8ee..f234008dad4 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.17.2" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.0" dast_environment_deploy: extends: .dast-auto-deploy @@ -23,7 +23,7 @@ dast_environment_deploy: when: never - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH when: never - - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given + - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given when: never - if: $CI_COMMIT_BRANCH && $CI_KUBERNETES_ACTIVE && @@ -46,7 +46,7 @@ stop_dast_environment: when: never - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH when: never - - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given + - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given when: never - if: $CI_COMMIT_BRANCH && $CI_KUBERNETES_ACTIVE && diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 66c60e85892..76fb2948144 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.17.2" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.0" dependencies: [] include: diff --git a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml index b437ddbd734..4a9849c85c9 100644 --- a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml @@ -5,7 +5,7 @@ load_performance: variables: DOCKER_TLS_CERTDIR: "" K6_IMAGE: loadimpact/k6 - K6_VERSION: 0.26.2 + K6_VERSION: 0.27.0 K6_TEST_FILE: github.com/loadimpact/k6/samples/http_get.js K6_OPTIONS: '' services: diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml new file mode 100644 index 00000000000..e87f0f28d01 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -0,0 +1,146 @@ +stages: + - build + - test + - deploy + - fuzz + +variables: + FUZZAPI_PROFILE: Quick + FUZZAPI_VERSION: latest + FUZZAPI_CONFIG: "/app/.gitlab-api-fuzzing.yml" + FUZZAPI_TIMEOUT: 30 + FUZZAPI_REPORT: gl-api-fuzzing-report.xml + # + FUZZAPI_D_NETWORK: testing-net + # + # Wait up to 5 minutes for API Fuzzer and target url to become + # available (non 500 response to HTTP(s)) + FUZZAPI_SERVICE_START_TIMEOUT: "300" + # + +apifuzzer_fuzz: + stage: fuzz + image: docker:19.03.12 + variables: + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + FUZZAPI_PROJECT: $CI_PROJECT_PATH + FUZZAPI_API: http://apifuzzer:80 + allow_failure: true + rules: + - if: $API_FUZZING_DISABLED + when: never + - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $FUZZAPI_HAR == null && + $FUZZAPI_OPENAPI == null && + $FUZZAPI_D_WORKER_IMAGE == null + when: never + - if: $FUZZAPI_D_WORKER_IMAGE == null && + $FUZZAPI_TARGET_URL == null + when: never + - if: $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ + services: + - docker:19.03.12-dind + script: + # + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + # + - docker network create --driver bridge $FUZZAPI_D_NETWORK + # + # Run user provided pre-script + - sh -c "$FUZZAPI_PRE_SCRIPT" + # + # Start peach testing engine container + - | + docker run -d \ + --name apifuzzer \ + --network $FUZZAPI_D_NETWORK \ + -e Proxy:Port=8000 \ + -e TZ=America/Los_Angeles \ + -e FUZZAPI_API=http://127.0.0.1:80 \ + -e FUZZAPI_PROJECT \ + -e FUZZAPI_PROFILE \ + -e FUZZAPI_CONFIG \ + -e FUZZAPI_REPORT \ + -e FUZZAPI_HAR \ + -e FUZZAPI_OPENAPI \ + -e FUZZAPI_TARGET_URL \ + -e FUZZAPI_OVERRIDES_FILE \ + -e FUZZAPI_OVERRIDES_ENV \ + -e FUZZAPI_OVERRIDES_CMD \ + -e FUZZAPI_OVERRIDES_INTERVAL \ + -e FUZZAPI_TIMEOUT \ + -e FUZZAPI_VERBOSE \ + -e FUZZAPI_SERVICE_START_TIMEOUT \ + -e GITLAB_FEATURES \ + -v $CI_PROJECT_DIR:/app \ + -p 80:80 \ + -p 8000:8000 \ + -p 514:514 \ + --restart=no \ + registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing-src:${FUZZAPI_VERSION}-engine + # + # Start target container + - | + if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \ + docker run -d \ + --name target \ + --network $FUZZAPI_D_NETWORK \ + $FUZZAPI_D_TARGET_ENV \ + $FUZZAPI_D_TARGET_PORTS \ + $FUZZAPI_D_TARGET_VOLUME \ + --restart=no \ + $FUZZAPI_D_TARGET_IMAGE \ + ; fi + # + # Start worker container + - | + if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \ + echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE" \ + docker run \ + --name worker \ + --network $FUZZAPI_D_NETWORK \ + -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_PROJECT \ + -e FUZZAPI_PROFILE \ + -e FUZZAPI_AUTOMATION_CMD \ + -e FUZZAPI_CONFIG \ + -e FUZZAPI_REPORT \ + -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \ + $FUZZAPI_D_WORKER_ENV \ + $FUZZAPI_D_WORKER_PORTS \ + $FUZZAPI_D_WORKER_VOLUME \ + --restart=no \ + $FUZZAPI_D_WORKER_IMAGE \ + ; fi + # + # Wait for testing to complete if api fuzzer is scanning + - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI" != "" ]; then echo "Waiting for API Fuzzer to exit"; docker wait apifuzzer; fi + # + # Run user provided pre-script + - sh -c "$FUZZAPI_POST_SCRIPT" + # + after_script: + # + # Shutdown all containers + - echo "Stopping all containers" + - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi + - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then docker stop worker; fi + - docker stop apifuzzer + # + # Save docker logs + - docker logs apifuzzer &> gl-api_fuzzing-logs.log + - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi + - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then docker logs worker &> gl-api_fuzzing-worker-logs.log; fi + # + artifacts: + when: always + paths: + - ./gl-api_fuzzing*.log + - ./gl-api_fuzzing*.zip + reports: + junit: $FUZZAPI_REPORT + +# end diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index 2fab8b95a3d..3f47e575afd 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -3,22 +3,26 @@ variables: # Which branch we want to run full fledged long running fuzzing jobs. # All others will run fuzzing regression - COVERAGE_FUZZING_BRANCH: "$CI_DEFAULT_BRANCH" - # This is using semantic version and will always download latest v1 gitlab-cov-fuzz release - COVERAGE_FUZZING_VERSION: v1 + COVFUZZ_BRANCH: "$CI_DEFAULT_BRANCH" + # This is using semantic version and will always download latest v2 gitlab-cov-fuzz release + COVFUZZ_VERSION: v2 # This is for users who have an offline environment and will have to replicate gitlab-cov-fuzz release binaries # to their own servers - COVERAGE_FUZZING_URL_PREFIX: "https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw" + COVFUZZ_URL_PREFIX: "https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw" + .fuzz_base: stage: fuzz allow_failure: true before_script: + - export COVFUZZ_JOB_TOKEN=$CI_JOB_TOKEN + - export COVFUZZ_PRIVATE_TOKEN=$CI_PRIVATE_TOKEN + - export COVFUZZ_PROJECT_ID=$CI_PROJECT_ID - if [ -x "$(command -v apt-get)" ] ; then apt-get update && apt-get install -y wget; fi - - wget -O gitlab-cov-fuzz "${COVERAGE_FUZZING_URL_PREFIX}"/"${COVERAGE_FUZZING_VERSION}"/binaries/gitlab-cov-fuzz_Linux_x86_64 + - wget -O gitlab-cov-fuzz "${COVFUZZ_URL_PREFIX}"/"${COVFUZZ_VERSION}"/binaries/gitlab-cov-fuzz_Linux_x86_64 - chmod a+x gitlab-cov-fuzz - export REGRESSION=true - - if [[ $CI_COMMIT_BRANCH = $COVERAGE_FUZZING_BRANCH ]]; then REGRESSION=false; fi; + - if [[ $CI_COMMIT_BRANCH = $COVFUZZ_BRANCH ]]; then REGRESSION=false; fi; artifacts: paths: - corpus @@ -28,7 +32,7 @@ variables: coverage_fuzzing: gl-coverage-fuzzing-report.json when: always rules: - - if: $COVERAGE_FUZZING_DISABLED + - if: $COVFUZZ_DISABLED when: never - if: $GITLAB_FEATURES =~ /\bcoverage_fuzzing\b/ - if: $CI_RUNNER_EXECUTABLE_ARCH == "linux" diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 37f6cd216ca..d5275c57ef8 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -111,6 +111,7 @@ gemnasium-dependency_scanning: - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}' - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' + - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}' gemnasium-maven-dependency_scanning: extends: .ds-analyzer @@ -144,8 +145,8 @@ gemnasium-python-dependency_scanning: - '{Pipfile,*/Pipfile,*/*/Pipfile}' - '{requires.txt,*/requires.txt,*/*/requires.txt}' - '{setup.py,*/setup.py,*/*/setup.py}' - # Support passing of $PIP_REQUIREMENTS_FILE - # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning + # Support passing of $PIP_REQUIREMENTS_FILE + # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ && diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index f0e2f48dd5c..6eb17341472 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -52,8 +52,7 @@ sast: rules: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ + - if: $CI_COMMIT_BRANCH script: - /analyzer run @@ -65,7 +64,6 @@ bandit-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /bandit/ exists: - '**/*.py' @@ -106,7 +104,6 @@ flawfinder-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ exists: - '**/*.c' @@ -120,7 +117,6 @@ kubesec-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /kubesec/ && $SCAN_KUBERNETES_MANIFESTS == 'true' @@ -132,7 +128,6 @@ gosec-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /gosec/ exists: - '**/*.go' @@ -145,7 +140,6 @@ nodejs-scan-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ exists: - 'package.json' @@ -158,7 +152,6 @@ phpcs-security-audit-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/ exists: - '**/*.php' @@ -171,7 +164,6 @@ pmd-apex-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/ exists: - '**/*.cls' @@ -184,7 +176,6 @@ secrets-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /secrets/ security-code-scan-sast: @@ -195,7 +186,6 @@ security-code-scan-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/ exists: - '**/*.csproj' @@ -209,7 +199,6 @@ sobelow-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /sobelow/ exists: - 'mix.exs' @@ -222,7 +211,6 @@ spotbugs-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /spotbugs/ exists: - '**/*.groovy' diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index 441a57048e1..b897c7b482f 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -7,6 +7,8 @@ variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" SECRETS_ANALYZER_VERSION: "3" + SECRET_DETECTION_EXCLUDED_PATHS: "" + .secret-analyzer: stage: test @@ -21,8 +23,7 @@ secret_detection_default_branch: rules: - if: $SECRET_DETECTION_DISABLED when: never - - if: $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsecret_detection\b/ + - if: $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH script: - /analyzer run @@ -31,8 +32,7 @@ secret_detection: rules: - if: $SECRET_DETECTION_DISABLED when: never - - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH && - $GITLAB_FEATURES =~ /\bsecret_detection\b/ + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH script: - git fetch origin $CI_DEFAULT_BRANCH $CI_BUILD_REF_NAME - export SECRET_DETECTION_COMMIT_TO=$(git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_BUILD_REF_NAME | tail -n 1) diff --git a/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml index 77a1b57d92f..584e6966180 100644 --- a/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml @@ -1,4 +1,5 @@ rspec-rails-modified-path-specs: + image: ruby:2.6 stage: .pre rules: - if: $CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "merge_train" diff --git a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml index d39bd234020..f964b3b2caf 100644 --- a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml @@ -11,7 +11,7 @@ load_performance: image: docker:git variables: K6_IMAGE: loadimpact/k6 - K6_VERSION: 0.26.2 + K6_VERSION: 0.27.0 K6_TEST_FILE: github.com/loadimpact/k6/samples/http_get.js K6_OPTIONS: '' services: diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 8cf355bbfc1..b7046064f44 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -115,7 +115,7 @@ module Gitlab end def release(job) - job[:release] if Gitlab::Ci::Features.release_generation_enabled? + job[:release] end def stage_builds_attributes(stage) |