diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-09-19 04:45:44 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-09-19 04:45:44 +0300 |
| commit | 85dc423f7090da0a52c73eb66faf22ddb20efff9 (patch) | |
| tree | 9160f299afd8c80c038f08e1545be119f5e3f1e1 /lib/gitlab/ci | |
| parent | 15c2c8c66dbe422588e5411eee7e68f1fa440bb8 (diff) | |
Add latest changes from gitlab-org/gitlab@13-4-stable-ee
Diffstat (limited to 'lib/gitlab/ci')
50 files changed, 1070 insertions, 561 deletions
diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index e145bd2e9df..1fac00337a3 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -31,105 +31,205 @@ module Gitlab end class Converter - def on_0(_) reset end + def on_0(_) + reset + end - def on_1(_) enable(STYLE_SWITCHES[:bold]) end + def on_1(_) + enable(STYLE_SWITCHES[:bold]) + end - def on_3(_) enable(STYLE_SWITCHES[:italic]) end + def on_3(_) + enable(STYLE_SWITCHES[:italic]) + end - def on_4(_) enable(STYLE_SWITCHES[:underline]) end + def on_4(_) + enable(STYLE_SWITCHES[:underline]) + end - def on_8(_) enable(STYLE_SWITCHES[:conceal]) end + def on_8(_) + enable(STYLE_SWITCHES[:conceal]) + end - def on_9(_) enable(STYLE_SWITCHES[:cross]) end + def on_9(_) + enable(STYLE_SWITCHES[:cross]) + end - def on_21(_) disable(STYLE_SWITCHES[:bold]) end + def on_21(_) + disable(STYLE_SWITCHES[:bold]) + end - def on_22(_) disable(STYLE_SWITCHES[:bold]) end + def on_22(_) + disable(STYLE_SWITCHES[:bold]) + end - def on_23(_) disable(STYLE_SWITCHES[:italic]) end + def on_23(_) + disable(STYLE_SWITCHES[:italic]) + end - def on_24(_) disable(STYLE_SWITCHES[:underline]) end + def on_24(_) + disable(STYLE_SWITCHES[:underline]) + end - def on_28(_) disable(STYLE_SWITCHES[:conceal]) end + def on_28(_) + disable(STYLE_SWITCHES[:conceal]) + end - def on_29(_) disable(STYLE_SWITCHES[:cross]) end + def on_29(_) + disable(STYLE_SWITCHES[:cross]) + end - def on_30(_) set_fg_color(0) end + def on_30(_) + set_fg_color(0) + end - def on_31(_) set_fg_color(1) end + def on_31(_) + set_fg_color(1) + end - def on_32(_) set_fg_color(2) end + def on_32(_) + set_fg_color(2) + end - def on_33(_) set_fg_color(3) end + def on_33(_) + set_fg_color(3) + end - def on_34(_) set_fg_color(4) end + def on_34(_) + set_fg_color(4) + end - def on_35(_) set_fg_color(5) end + def on_35(_) + set_fg_color(5) + end - def on_36(_) set_fg_color(6) end + def on_36(_) + set_fg_color(6) + end - def on_37(_) set_fg_color(7) end + def on_37(_) + set_fg_color(7) + end - def on_38(stack) set_fg_color_256(stack) end + def on_38(stack) + set_fg_color_256(stack) + end - def on_39(_) set_fg_color(9) end + def on_39(_) + set_fg_color(9) + end - def on_40(_) set_bg_color(0) end + def on_40(_) + set_bg_color(0) + end - def on_41(_) set_bg_color(1) end + def on_41(_) + set_bg_color(1) + end - def on_42(_) set_bg_color(2) end + def on_42(_) + set_bg_color(2) + end - def on_43(_) set_bg_color(3) end + def on_43(_) + set_bg_color(3) + end - def on_44(_) set_bg_color(4) end + def on_44(_) + set_bg_color(4) + end - def on_45(_) set_bg_color(5) end + def on_45(_) + set_bg_color(5) + end - def on_46(_) set_bg_color(6) end + def on_46(_) + set_bg_color(6) + end - def on_47(_) set_bg_color(7) end + def on_47(_) + set_bg_color(7) + end - def on_48(stack) set_bg_color_256(stack) end + def on_48(stack) + set_bg_color_256(stack) + end - def on_49(_) set_bg_color(9) end + def on_49(_) + set_bg_color(9) + end - def on_90(_) set_fg_color(0, 'l') end + def on_90(_) + set_fg_color(0, 'l') + end - def on_91(_) set_fg_color(1, 'l') end + def on_91(_) + set_fg_color(1, 'l') + end - def on_92(_) set_fg_color(2, 'l') end + def on_92(_) + set_fg_color(2, 'l') + end - def on_93(_) set_fg_color(3, 'l') end + def on_93(_) + set_fg_color(3, 'l') + end - def on_94(_) set_fg_color(4, 'l') end + def on_94(_) + set_fg_color(4, 'l') + end - def on_95(_) set_fg_color(5, 'l') end + def on_95(_) + set_fg_color(5, 'l') + end - def on_96(_) set_fg_color(6, 'l') end + def on_96(_) + set_fg_color(6, 'l') + end - def on_97(_) set_fg_color(7, 'l') end + def on_97(_) + set_fg_color(7, 'l') + end - def on_99(_) set_fg_color(9, 'l') end + def on_99(_) + set_fg_color(9, 'l') + end - def on_100(_) set_bg_color(0, 'l') end + def on_100(_) + set_bg_color(0, 'l') + end - def on_101(_) set_bg_color(1, 'l') end + def on_101(_) + set_bg_color(1, 'l') + end - def on_102(_) set_bg_color(2, 'l') end + def on_102(_) + set_bg_color(2, 'l') + end - def on_103(_) set_bg_color(3, 'l') end + def on_103(_) + set_bg_color(3, 'l') + end - def on_104(_) set_bg_color(4, 'l') end + def on_104(_) + set_bg_color(4, 'l') + end - def on_105(_) set_bg_color(5, 'l') end + def on_105(_) + set_bg_color(5, 'l') + end - def on_106(_) set_bg_color(6, 'l') end + def on_106(_) + set_bg_color(6, 'l') + end - def on_107(_) set_bg_color(7, 'l') end + def on_107(_) + set_bg_color(7, 'l') + end - def on_109(_) set_bg_color(9, 'l') end + def on_109(_) + set_bg_color(9, 'l') + end attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask, :sections, :lineno_in_section diff --git a/lib/gitlab/ci/artifact_file_reader.rb b/lib/gitlab/ci/artifact_file_reader.rb index c2d17cc176e..6395a20ca99 100644 --- a/lib/gitlab/ci/artifact_file_reader.rb +++ b/lib/gitlab/ci/artifact_file_reader.rb @@ -45,6 +45,31 @@ module Gitlab end def read_zip_file!(file_path) + if ::Gitlab::Ci::Features.new_artifact_file_reader_enabled?(job.project) + read_with_new_artifact_file_reader(file_path) + else + read_with_legacy_artifact_file_reader(file_path) + end + end + + def read_with_new_artifact_file_reader(file_path) + job.artifacts_file.use_open_file do |file| + zip_file = Zip::File.new(file, false, true) + entry = zip_file.find_entry(file_path) + + unless entry + raise Error, "Path `#{file_path}` does not exist inside the `#{job.name}` artifacts archive!" + end + + if entry.name_is_directory? + raise Error, "Path `#{file_path}` was expected to be a file but it was a directory!" + end + + zip_file.read(entry) + end + end + + def read_with_legacy_artifact_file_reader(file_path) job.artifacts_file.use_file do |archive_path| Zip::File.open(archive_path) do |zip_file| entry = zip_file.find_entry(file_path) diff --git a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb index b64990d6a7a..72ef0a8d067 100644 --- a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb +++ b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb @@ -33,7 +33,7 @@ module Gitlab def kubernetes_namespace strong_memoize(:kubernetes_namespace) do - Clusters::KubernetesNamespaceFinder.new( + ::Clusters::KubernetesNamespaceFinder.new( deployment_cluster, project: environment.project, environment_name: environment.name, @@ -47,7 +47,7 @@ module Gitlab return if conflicting_ci_namespace_requested?(namespace) - Clusters::Kubernetes::CreateOrUpdateNamespaceService.new( + ::Clusters::Kubernetes::CreateOrUpdateNamespaceService.new( cluster: deployment_cluster, kubernetes_namespace: namespace ).execute @@ -71,7 +71,7 @@ module Gitlab end def build_namespace_record - Clusters::BuildKubernetesNamespaceService.new( + ::Clusters::BuildKubernetesNamespaceService.new( deployment_cluster, environment: environment ).execute diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index d81a3fef1f5..9d269831679 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -62,6 +62,10 @@ module Gitlab root.jobs_value end + def normalized_jobs + @normalized_jobs ||= Ci::Config::Normalizer.new(jobs).normalize_jobs + end + private def expand_config(config) diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index f960cec1f26..ecc2c5cb729 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -122,39 +122,9 @@ module Gitlab :needs, :retry, :parallel, :start_in, :interruptible, :timeout, :resource_group, :release - Matcher = Struct.new(:name, :config) do - def applies? - job_is_not_hidden? && - config_is_a_hash? && - has_job_keys? - end - - private - - def job_is_not_hidden? - !name.to_s.start_with?('.') - end - - def config_is_a_hash? - config.is_a?(Hash) - end - - def has_job_keys? - if name == :default - config.key?(:script) - else - (ALLOWED_KEYS & config.keys).any? - end - end - end - def self.matching?(name, config) - if Gitlab::Ci::Features.job_entry_matches_all_keys? - Matcher.new(name, config).applies? - else - !name.to_s.start_with?('.') && - config.is_a?(Hash) && config.key?(:script) - end + !name.to_s.start_with?('.') && + config.is_a?(Hash) && config.key?(:script) end def self.visible? diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb index 1d3036189b0..b5ce42969a5 100644 --- a/lib/gitlab/ci/config/entry/jobs.rb +++ b/lib/gitlab/ci/config/entry/jobs.rb @@ -14,8 +14,8 @@ module Gitlab validates :config, type: Hash validate do - unless has_valid_jobs? - errors.add(:config, 'should contain valid jobs') + each_unmatched_job do |name| + errors.add(name, 'config should implement a script: or a trigger: keyword') end unless has_visible_job? @@ -23,9 +23,9 @@ module Gitlab end end - def has_valid_jobs? - config.all? do |name, value| - Jobs.find_type(name, value) + def each_unmatched_job + config.each do |name, value| + yield(name) unless Jobs.find_type(name, value) end end diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index 19d6a470941..2d93f1ab06e 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -134,7 +134,7 @@ module Gitlab @jobs_config = @config .except(*self.class.reserved_nodes_names) .select do |name, config| - Entry::Jobs.find_type(name, config).present? + Entry::Jobs.find_type(name, config).present? || ALLOWED_KEYS.exclude?(name) end @config = @config.except(*@jobs_config.keys) diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb index 451ba14bb89..22fcd84c968 100644 --- a/lib/gitlab/ci/config/normalizer.rb +++ b/lib/gitlab/ci/config/normalizer.rb @@ -11,6 +11,7 @@ module Gitlab end def normalize_jobs + return {} unless @jobs_config return @jobs_config if parallelized_jobs.empty? expand_parallelize_jobs do |job_name, config| diff --git a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb index db21274a9ed..5a23836d8a0 100644 --- a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb +++ b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb @@ -48,14 +48,13 @@ module Gitlab } end - def name_with_details - vars = variables.map { |key, value| "#{key}=#{value}"}.join('; ') - - "#{job_name} (#{vars})" - end - def name - "#{job_name} #{instance}/#{total}" + vars = variables + .values + .compact + .join(', ') + + "#{job_name}: [#{vars}]" end private diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index 2f6667d3600..e770187b124 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -31,56 +31,49 @@ module Gitlab ::Feature.enabled?(:ci_store_pipeline_messages, project, default_enabled: true) end - # Remove in https://gitlab.com/gitlab-org/gitlab/-/issues/227052 - def self.variables_api_filter_environment_scope? - ::Feature.enabled?(:ci_variables_api_filter_environment_scope, default_enabled: true) - end - def self.raise_job_rules_without_workflow_rules_warning? ::Feature.enabled?(:ci_raise_job_rules_without_workflow_rules_warning, default_enabled: true) end - def self.keep_latest_artifacts_for_ref_enabled?(project) - ::Feature.enabled?(:keep_latest_artifacts_for_ref, project, default_enabled: false) - end - - def self.destroy_only_unlocked_expired_artifacts_enabled? - ::Feature.enabled?(:destroy_only_unlocked_expired_artifacts, default_enabled: false) - end - def self.bulk_insert_on_create?(project) ::Feature.enabled?(:ci_bulk_insert_on_create, project, default_enabled: true) end - def self.ci_if_parenthesis_enabled? - ::Feature.enabled?(:ci_if_parenthesis_enabled, default_enabled: true) + # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` + # is a safe switch to disable the feature for a parituclar project when something went wrong, + # therefore it's not supposed to be enabled by default. + def self.disallow_to_create_merge_request_pipelines_in_target_project?(target_project) + ::Feature.enabled?(:ci_disallow_to_create_merge_request_pipelines_in_target_project, target_project) end - def self.allow_to_create_merge_request_pipelines_in_target_project?(target_project) - ::Feature.enabled?(:ci_allow_to_create_merge_request_pipelines_in_target_project, target_project, default_enabled: true) + def self.lint_creates_pipeline_with_dry_run?(project) + ::Feature.enabled?(:ci_lint_creates_pipeline_with_dry_run, project, default_enabled: true) end - def self.ci_plan_needs_size_limit?(project) - ::Feature.enabled?(:ci_plan_needs_size_limit, project, default_enabled: true) + def self.project_transactionless_destroy?(project) + Feature.enabled?(:project_transactionless_destroy, project, default_enabled: false) end - def self.job_entry_matches_all_keys? - ::Feature.enabled?(:ci_job_entry_matches_all_keys) + def self.coverage_report_view?(project) + ::Feature.enabled?(:coverage_report_view, project, default_enabled: true) end - def self.lint_creates_pipeline_with_dry_run?(project) - ::Feature.enabled?(:ci_lint_creates_pipeline_with_dry_run, project, default_enabled: true) + def self.child_of_child_pipeline_enabled?(project) + ::Feature.enabled?(:ci_child_of_child_pipeline, project, default_enabled: true) end - def self.reset_ci_minutes_for_all_namespaces? - ::Feature.enabled?(:reset_ci_minutes_for_all_namespaces, default_enabled: false) + def self.trace_overwrite? + ::Feature.enabled?(:ci_trace_overwrite, type: :ops, default_enabled: false) end - def self.expand_names_for_cross_pipeline_artifacts?(project) - ::Feature.enabled?(:ci_expand_names_for_cross_pipeline_artifacts, project) + def self.accept_trace?(project) + ::Feature.enabled?(:ci_enable_live_trace, project) && + ::Feature.enabled?(:ci_accept_trace, project, type: :ops, default_enabled: false) + end + + def self.new_artifact_file_reader_enabled?(project) + ::Feature.enabled?(:ci_new_artifact_file_reader, project, default_enabled: false) end end end end - -::Gitlab::Ci::Features.prepend_if_ee('::EE::Gitlab::Ci::Features') diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb new file mode 100644 index 00000000000..86a9ebfa451 --- /dev/null +++ b/lib/gitlab/ci/lint.rb @@ -0,0 +1,108 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Lint + class Result + attr_reader :jobs, :errors, :warnings + + def initialize(jobs:, errors:, warnings:) + @jobs = jobs + @errors = errors + @warnings = warnings + end + + def valid? + @errors.empty? + end + end + + def initialize(project:, current_user:) + @project = project + @current_user = current_user + end + + def validate(content, dry_run: false) + if dry_run && Gitlab::Ci::Features.lint_creates_pipeline_with_dry_run?(@project) + simulate_pipeline_creation(content) + else + static_validation(content) + end + end + + private + + def simulate_pipeline_creation(content) + pipeline = ::Ci::CreatePipelineService + .new(@project, @current_user, ref: @project.default_branch) + .execute(:push, dry_run: true, content: content) + + Result.new( + jobs: dry_run_convert_to_jobs(pipeline.stages), + errors: pipeline.error_messages.map(&:content), + warnings: pipeline.warning_messages(limit: ::Gitlab::Ci::Warnings::MAX_LIMIT).map(&:content) + ) + end + + def static_validation(content) + result = Gitlab::Ci::YamlProcessor.new( + content, + project: @project, + user: @current_user, + sha: @project.repository.commit.sha + ).execute + + Result.new( + jobs: static_validation_convert_to_jobs(result), + errors: result.errors, + warnings: result.warnings.take(::Gitlab::Ci::Warnings::MAX_LIMIT) # rubocop: disable CodeReuse/ActiveRecord + ) + end + + def dry_run_convert_to_jobs(stages) + stages.reduce([]) do |jobs, stage| + jobs + stage.statuses.map do |job| + { + name: job.name, + stage: stage.name, + before_script: job.options[:before_script].to_a, + script: job.options[:script].to_a, + after_script: job.options[:after_script].to_a, + tag_list: (job.tag_list if job.is_a?(::Ci::Build)).to_a, + environment: job.options.dig(:environment, :name), + when: job.when, + allow_failure: job.allow_failure + } + end + end + end + + def static_validation_convert_to_jobs(result) + jobs = [] + return jobs unless result.valid? + + result.stages.each do |stage_name| + result.builds.each do |job| + next unless job[:stage] == stage_name + + jobs << { + name: job[:name], + stage: stage_name, + before_script: job.dig(:options, :before_script).to_a, + script: job.dig(:options, :script).to_a, + after_script: job.dig(:options, :after_script).to_a, + tag_list: job[:tag_list].to_a, + only: job[:only], + except: job[:except], + environment: job[:environment], + when: job[:when], + allow_failure: job[:allow_failure] + } + end + end + + jobs + end + end + end +end diff --git a/lib/gitlab/ci/mask_secret.rb b/lib/gitlab/ci/mask_secret.rb index 58d55b1bd6f..e5a7151b823 100644 --- a/lib/gitlab/ci/mask_secret.rb +++ b/lib/gitlab/ci/mask_secret.rb @@ -8,6 +8,11 @@ module Gitlab # We assume 'value' must be mutable, given # that frozen string is enabled. + + ## + # TODO We need to remove this because it is going to change checksum of + # a trace. + # value.gsub!(token, 'x' * token.length) value end diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index 4190c40eb66..9662209f88e 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -20,11 +20,7 @@ module Gitlab pipeline_schedule: @command.schedule, merge_request: @command.merge_request, external_pull_request: @command.external_pull_request, - variables_attributes: Array(@command.variables_attributes), - # This should be removed and set on the database column default - # level when the keep_latest_artifacts_for_ref feature flag is - # removed. - locked: ::Gitlab::Ci::Features.keep_latest_artifacts_for_ref_enabled?(@command.project) ? :artifacts_locked : :unlocked + variables_attributes: Array(@command.variables_attributes) ) end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index dbaa6951e64..d1882059dd8 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -12,7 +12,7 @@ module Gitlab :seeds_block, :variables_attributes, :push_options, :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, # These attributes are set by Chains during processing: - :config_content, :config_processor, :stage_seeds + :config_content, :yaml_processor_result, :stage_seeds ) do include Gitlab::Utils::StrongMemoize diff --git a/lib/gitlab/ci/pipeline/chain/config/content/remote.rb b/lib/gitlab/ci/pipeline/chain/config/content/remote.rb index dcc336b8929..4990a5a6eb5 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content/remote.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content/remote.rb @@ -9,7 +9,7 @@ module Gitlab class Remote < Source def content strong_memoize(:content) do - next unless ci_config_path =~ URI.regexp(%w[http https]) + next unless ci_config_path =~ URI::DEFAULT_PARSER.make_regexp(%w[http https]) YAML.dump('include' => [{ 'remote' => ci_config_path }]) end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 2cfcb295407..8ccb33ffd34 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -11,20 +11,23 @@ module Gitlab def perform! raise ArgumentError, 'missing config content' unless @command.config_content - @command.config_processor = ::Gitlab::Ci::YamlProcessor.new( + result = ::Gitlab::Ci::YamlProcessor.new( @command.config_content, { project: project, sha: @pipeline.sha, user: current_user, parent_pipeline: parent_pipeline } - ) + ).execute + + add_warnings_to_pipeline(result.warnings) - add_warnings_to_pipeline(@command.config_processor.warnings) - rescue Gitlab::Ci::YamlProcessor::ValidationError => ex - add_warnings_to_pipeline(ex.warnings) + if result.valid? + @command.yaml_processor_result = result + else + error(result.errors.first, config_error: true) + end - error(ex.message, config_error: true) rescue => ex Gitlab::ErrorTracking.track_exception(ex, project_id: project.id, diff --git a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb index a793ae9cc24..3c910963a2a 100644 --- a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb +++ b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb @@ -39,7 +39,7 @@ module Gitlab end def workflow_config - @command.config_processor.workflow_attributes || {} + @command.yaml_processor_result.workflow_attributes || {} end end end diff --git a/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb b/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb index 9267c72efa4..71f22c52869 100644 --- a/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb +++ b/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb @@ -6,13 +6,13 @@ module Gitlab module Chain class RemoveUnwantedChatJobs < Chain::Base def perform! - raise ArgumentError, 'missing config processor' unless @command.config_processor + raise ArgumentError, 'missing YAML processor result' unless @command.yaml_processor_result return unless pipeline.chat? # When scheduling a chat pipeline we only want to run the build # that matches the chat command. - @command.config_processor.jobs.select! do |name, _| + @command.yaml_processor_result.jobs.select! do |name, _| name.to_s == command.chat_data[:command].to_s end end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index e48e79d561b..e10a0bc3718 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -9,7 +9,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize def perform! - raise ArgumentError, 'missing config processor' unless @command.config_processor + raise ArgumentError, 'missing YAML processor result' unless @command.yaml_processor_result # Allocate next IID. This operation must be outside of transactions of pipeline creations. pipeline.ensure_project_iid! @@ -56,7 +56,7 @@ module Gitlab end def stages_attributes - @command.config_processor.stages_attributes + @command.yaml_processor_result.stages_attributes end end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index 24628338dd2..d056501a6d3 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -51,7 +51,7 @@ module Gitlab def validate_service_request Gitlab::HTTP.post( validation_service_url, timeout: VALIDATION_REQUEST_TIMEOUT, - body: validation_service_payload(@pipeline, @command.config_processor.stages_attributes) + body: validation_service_payload(@pipeline, @command.yaml_processor_result.stages_attributes) ) end diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb index 5b7365cb33b..ac03ef79ccb 100644 --- a/lib/gitlab/ci/pipeline/expression/lexer.rb +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -24,26 +24,8 @@ module Gitlab Expression::Lexeme::Or ].freeze - # To be removed with `ci_if_parenthesis_enabled` - LEGACY_LEXEMES = [ - Expression::Lexeme::Variable, - Expression::Lexeme::String, - Expression::Lexeme::Pattern, - Expression::Lexeme::Null, - Expression::Lexeme::Equals, - Expression::Lexeme::Matches, - Expression::Lexeme::NotEquals, - Expression::Lexeme::NotMatches, - Expression::Lexeme::And, - Expression::Lexeme::Or - ].freeze - def self.lexemes - if ::Gitlab::Ci::Features.ci_if_parenthesis_enabled? - LEXEMES - else - LEGACY_LEXEMES - end + LEXEMES end MAX_TOKENS = 100 diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb index 27d7aa2f37e..a20b0015e05 100644 --- a/lib/gitlab/ci/pipeline/expression/parser.rb +++ b/lib/gitlab/ci/pipeline/expression/parser.rb @@ -15,12 +15,7 @@ module Gitlab def tree results = [] - tokens = - if ::Gitlab::Ci::Features.ci_if_parenthesis_enabled? - tokens_rpn - else - legacy_tokens_rpn - end + tokens = tokens_rpn tokens.each do |token| case token.type @@ -78,27 +73,6 @@ module Gitlab output.concat(operators.reverse) end - - # To be removed with `ci_if_parenthesis_enabled` - def legacy_tokens_rpn - output = [] - operators = [] - - @tokens.each do |token| - case token.type - when :value - output.push(token) - when :logical_operator - if operators.any? && token.lexeme.precedence >= operators.last.lexeme.precedence - output.push(operators.pop) - end - - operators.push(token) - end - end - - output.concat(operators.reverse) - end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 3be3fa63b92..91dbcc616ea 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -11,8 +11,6 @@ module Gitlab delegate :dig, to: :@seed_attributes - DEFAULT_NEEDS_LIMIT = 10 - def initialize(pipeline, attributes, previous_stages) @pipeline = pipeline @seed_attributes = attributes @@ -140,11 +138,7 @@ module Gitlab end def max_needs_allowed - if ::Gitlab::Ci::Features.ci_plan_needs_size_limit?(@pipeline.project) - @pipeline.project.actual_limits.ci_needs_size_limit - else - DEFAULT_NEEDS_LIMIT - end + @pipeline.project.actual_limits.ci_needs_size_limit end def pipeline_attributes diff --git a/lib/gitlab/ci/pipeline_object_hierarchy.rb b/lib/gitlab/ci/pipeline_object_hierarchy.rb new file mode 100644 index 00000000000..de3262b10e0 --- /dev/null +++ b/lib/gitlab/ci/pipeline_object_hierarchy.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class PipelineObjectHierarchy < ::Gitlab::ObjectHierarchy + private + + def middle_table + ::Ci::Sources::Pipeline.arel_table + end + + def from_tables(cte) + [objects_table, cte.table, middle_table] + end + + def parent_id_column(_cte) + middle_table[:source_pipeline_id] + end + + def ancestor_conditions(cte) + middle_table[:source_pipeline_id].eq(objects_table[:id]).and( + middle_table[:pipeline_id].eq(cte.table[:id]) + ).and( + same_project_condition + ) + end + + def descendant_conditions(cte) + middle_table[:pipeline_id].eq(objects_table[:id]).and( + middle_table[:source_pipeline_id].eq(cte.table[:id]) + ).and( + same_project_condition + ) + end + + def same_project_condition + if options[:same_project] + middle_table[:source_project_id].eq(middle_table[:project_id]) + else + Arel.sql('TRUE') + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_case.rb b/lib/gitlab/ci/reports/test_case.rb index 75898745366..15a3c862c9e 100644 --- a/lib/gitlab/ci/reports/test_case.rb +++ b/lib/gitlab/ci/reports/test_case.rb @@ -8,7 +8,7 @@ module Gitlab STATUS_FAILED = 'failed' STATUS_SKIPPED = 'skipped' STATUS_ERROR = 'error' - STATUS_TYPES = [STATUS_SUCCESS, STATUS_FAILED, STATUS_SKIPPED, STATUS_ERROR].freeze + STATUS_TYPES = [STATUS_ERROR, STATUS_FAILED, STATUS_SUCCESS, STATUS_SKIPPED].freeze attr_reader :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key, :attachment, :job diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb index 5ee779227ec..e9b78b841e4 100644 --- a/lib/gitlab/ci/reports/test_suite.rb +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -78,11 +78,27 @@ module Gitlab end end + def sorted + sort_by_status + sort_by_execution_time_desc + self + end + private def existing_key?(test_case) @test_cases[test_case.status]&.key?(test_case.key) end + + def sort_by_status + @test_cases = @test_cases.sort_by { |status, _| Gitlab::Ci::Reports::TestCase::STATUS_TYPES.index(status) }.to_h + end + + def sort_by_execution_time_desc + @test_cases = @test_cases.keys.each_with_object({}) do |key, hash| + hash[key] = @test_cases[key].sort_by { |_key, test_case| -test_case.execution_time }.to_h + end + end end end end diff --git a/lib/gitlab/ci/status/bridge/common.rb b/lib/gitlab/ci/status/bridge/common.rb index 4746195c618..b95565b5e09 100644 --- a/lib/gitlab/ci/status/bridge/common.rb +++ b/lib/gitlab/ci/status/bridge/common.rb @@ -10,14 +10,28 @@ module Gitlab end def has_details? - false + !!details_path + end + + def details_path + return unless Feature.enabled?(:ci_bridge_pipeline_details, subject.project, default_enabled: true) + return unless can?(user, :read_pipeline, downstream_pipeline) + + project_pipeline_path(downstream_project, downstream_pipeline) end def has_action? false end - def details_path + private + + def downstream_pipeline + subject.downstream_pipeline + end + + def downstream_project + downstream_pipeline&.project end end end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 88846f724e7..f6562737838 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -25,7 +25,8 @@ module Gitlab insufficient_bridge_permissions: 'no permissions to trigger downstream pipeline', bridge_pipeline_is_child_pipeline: 'creation of child pipeline not allowed from another child pipeline', downstream_pipeline_creation_failed: 'downstream pipeline can not be created', - secrets_provider_not_found: 'secrets provider can not be found' + secrets_provider_not_found: 'secrets provider can not be found', + reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb index 04a9fc29802..9a4f5644f7d 100644 --- a/lib/gitlab/ci/status/composite.rb +++ b/lib/gitlab/ci/status/composite.rb @@ -7,7 +7,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize # This class accepts an array of arrays/hashes/or objects - def initialize(all_statuses, with_allow_failure: true) + def initialize(all_statuses, with_allow_failure: true, dag: false) unless all_statuses.respond_to?(:pluck) raise ArgumentError, "all_statuses needs to respond to `.pluck`" end @@ -15,6 +15,7 @@ module Gitlab @status_set = Set.new @status_key = 0 @allow_failure_key = 1 if with_allow_failure + @dag = dag consume_all_statuses(all_statuses) end @@ -31,7 +32,13 @@ module Gitlab return if none? strong_memoize(:status) do - if only_of?(:skipped, :ignored) + if @dag && any_of?(:skipped) + # The DAG job is skipped if one of the needs does not run at all. + 'skipped' + elsif @dag && !only_of?(:success, :failed, :canceled, :skipped, :success_with_warnings) + # DAG is blocked from executing if a dependent is not "complete" + 'pending' + elsif only_of?(:skipped, :ignored) 'skipped' elsif only_of?(:success, :skipped, :success_with_warnings, :ignored) 'success' diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 968ff0fce89..6966ce88b30 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -150,16 +150,22 @@ workflow: - exists: - .static +# NOTE: These links point to the latest templates for development in GitLab canonical project, +# therefore the actual templates that were included for Auto DevOps pipelines +# could be different from the contents in the links. +# To view the actual templates, please replace `master` to the specific GitLab version when +# the Auto DevOps pipeline started running e.g. `v13.0.2-ee`. include: - - template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml - - template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml - - template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml - - template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml - - template: Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml - - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml - - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml - - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml - - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml - - template: Security/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml - - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml - - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml + - template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml + - template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml + - template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml + - template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml + - template: Jobs/Deploy/ECS.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml + - template: Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml + - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml + - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml + - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml + - template: Security/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml + - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index 8553a940bd7..5edb26a0b56 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -7,7 +7,7 @@ performance: variables: DOCKER_TLS_CERTDIR: "" SITESPEED_IMAGE: sitespeedio/sitespeed.io - SITESPEED_VERSION: 13.3.0 + SITESPEED_VERSION: 14.1.0 SITESPEED_OPTIONS: '' services: - docker:19.03.12-dind @@ -20,15 +20,15 @@ performance: fi - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) - mkdir gitlab-exporter - - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.0.1/index.js + - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - | if [ -f .gitlab-urls.txt ] then sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS else - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS fi - mv sitespeed-results/data/performance.json browser-performance.json artifacts: diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index cf851c875ee..568ceceeaa2 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -9,6 +9,8 @@ code_quality: DOCKER_TLS_CERTDIR: "" CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.10-gitlab.1" needs: [] + before_script: + - export SOURCE_CODE=$PWD script: - | if ! docker info &>/dev/null; then @@ -16,11 +18,27 @@ code_quality: export DOCKER_HOST='tcp://localhost:2375' fi fi + - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } - docker pull --quiet "$CODE_QUALITY_IMAGE" - - docker run - --env SOURCE_CODE="$PWD" - --volume "$PWD":/code - --volume /var/run/docker.sock:/var/run/docker.sock + - | + docker run \ + $(propagate_env_vars \ + SOURCE_CODE \ + TIMEOUT_SECONDS \ + CODECLIMATE_DEBUG \ + CODECLIMATE_DEV \ + REPORT_STDOUT \ + ENGINE_MEMORY_LIMIT_BYTES \ + ) \ + --volume "$PWD":/code \ + --volume /var/run/docker.sock:/var/run/docker.sock \ "$CODE_QUALITY_IMAGE" /code artifacts: reports: diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 2922e1c6e88..829fd7a722f 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -2,9 +2,6 @@ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3" dependencies: [] -include: - - template: Jobs/Deploy/ECS.gitlab-ci.yml - review: extends: .auto-deploy stage: review diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml new file mode 100644 index 00000000000..829fd7a722f --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -0,0 +1,249 @@ +.auto-deploy: + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3" + dependencies: [] + +review: + extends: .auto-deploy + stage: review + script: + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy + - auto-deploy persist_environment_url + environment: + name: review/$CI_COMMIT_REF_NAME + url: http://$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN + on_stop: stop_review + artifacts: + paths: [environment_url.txt, tiller.log] + when: always + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + when: never + - if: '$REVIEW_DISABLED' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + +stop_review: + extends: .auto-deploy + stage: cleanup + variables: + GIT_STRATEGY: none + script: + - auto-deploy initialize_tiller + - auto-deploy delete + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop + allow_failure: true + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + when: never + - if: '$REVIEW_DISABLED' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + when: manual + +# Staging deploys are disabled by default since +# continuous deployment to production is enabled by default +# If you prefer to automatically deploy to staging and +# only manually promote to production, enable this job by setting +# STAGING_ENABLED. + +staging: + extends: .auto-deploy + stage: staging + script: + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy + environment: + name: staging + url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + - if: '$STAGING_ENABLED' + +# Canaries are disabled by default, but if you want them, +# and know what the downsides are, you can enable this by setting +# CANARY_ENABLED. + +canary: + extends: .auto-deploy + stage: canary + allow_failure: true + script: + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy canary + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + - if: '$CANARY_ENABLED' + when: manual + +.production: &production_template + extends: .auto-deploy + stage: production + script: + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy + - auto-deploy delete canary + - auto-deploy delete rollout + - auto-deploy persist_environment_url + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN + artifacts: + paths: [environment_url.txt, tiller.log] + when: always + +production: + <<: *production_template + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$STAGING_ENABLED' + when: never + - if: '$CANARY_ENABLED' + when: never + - if: '$INCREMENTAL_ROLLOUT_ENABLED' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + +production_manual: + <<: *production_template + allow_failure: false + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$INCREMENTAL_ROLLOUT_ENABLED' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE' + when: never + - if: '$CI_COMMIT_BRANCH == "master" && $STAGING_ENABLED' + when: manual + - if: '$CI_COMMIT_BRANCH == "master" && $CANARY_ENABLED' + when: manual + +# This job implements incremental rollout on for every push to `master`. + +.rollout: &rollout_template + extends: .auto-deploy + script: + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy rollout $ROLLOUT_PERCENTAGE + - auto-deploy scale stable $((100-ROLLOUT_PERCENTAGE)) + - auto-deploy delete canary + - auto-deploy persist_environment_url + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN + artifacts: + paths: [environment_url.txt, tiller.log] + when: always + +.manual_rollout_template: &manual_rollout_template + <<: *rollout_template + stage: production + resource_group: production + allow_failure: true + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + # $INCREMENTAL_ROLLOUT_ENABLED is for compamtibilty with pre-GitLab 11.4 syntax + - if: '$INCREMENTAL_ROLLOUT_MODE == "manual" || $INCREMENTAL_ROLLOUT_ENABLED' + when: manual + +.timed_rollout_template: &timed_rollout_template + <<: *rollout_template + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE == "manual"' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' + when: delayed + start_in: 5 minutes + +timed rollout 10%: + <<: *timed_rollout_template + stage: incremental rollout 10% + variables: + ROLLOUT_PERCENTAGE: 10 + +timed rollout 25%: + <<: *timed_rollout_template + stage: incremental rollout 25% + variables: + ROLLOUT_PERCENTAGE: 25 + +timed rollout 50%: + <<: *timed_rollout_template + stage: incremental rollout 50% + variables: + ROLLOUT_PERCENTAGE: 50 + +timed rollout 100%: + <<: *timed_rollout_template + <<: *production_template + stage: incremental rollout 100% + variables: + ROLLOUT_PERCENTAGE: 100 + +rollout 10%: + <<: *manual_rollout_template + variables: + ROLLOUT_PERCENTAGE: 10 + +rollout 25%: + <<: *manual_rollout_template + variables: + ROLLOUT_PERCENTAGE: 25 + +rollout 50%: + <<: *manual_rollout_template + variables: + ROLLOUT_PERCENTAGE: 50 + +rollout 100%: + <<: *manual_rollout_template + <<: *production_template + allow_failure: false diff --git a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml index 4a9849c85c9..9a7c513c25f 100644 --- a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml @@ -8,6 +8,7 @@ load_performance: K6_VERSION: 0.27.0 K6_TEST_FILE: github.com/loadimpact/k6/samples/http_get.js K6_OPTIONS: '' + K6_DOCKER_OPTIONS: '' services: - docker:19.03.11-dind script: @@ -17,7 +18,7 @@ load_performance: export DOCKER_HOST='tcp://localhost:2375' fi fi - - docker run --rm -v "$(pwd)":/k6 -w /k6 $K6_IMAGE:$K6_VERSION run $K6_TEST_FILE --summary-export=load-performance.json $K6_OPTIONS + - docker run --rm -v "$(pwd)":/k6 -w /k6 $K6_DOCKER_OPTIONS $K6_IMAGE:$K6_VERSION run $K6_TEST_FILE --summary-export=load-performance.json $K6_OPTIONS artifacts: reports: load_performance: load-performance.json diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 3d0bacda853..7050b41e045 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,27 +1,11 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.24.2" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.29.0" environment: name: production variables: TILLER_NAMESPACE: gitlab-managed-apps GITLAB_MANAGED_APPS_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/config.yaml - INGRESS_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/ingress/values.yaml - CERT_MANAGER_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/cert-manager/values.yaml - SENTRY_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/sentry/values.yaml - GITLAB_RUNNER_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/gitlab-runner/values.yaml - CILIUM_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/cilium/values.yaml - CILIUM_HUBBLE_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/cilium/hubble-values.yaml - JUPYTERHUB_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/jupyterhub/values.yaml - PROMETHEUS_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/prometheus/values.yaml - ELASTIC_STACK_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/elastic-stack/values.yaml - VAULT_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/vault/values.yaml - CROSSPLANE_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/crossplane/values.yaml - FLUENTD_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/fluentd/values.yaml - KNATIVE_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/knative/values.yaml - POSTHOG_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/posthog/values.yaml - FALCO_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/falco/values.yaml - APPARMOR_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/apparmor/values.yaml script: - gitlab-managed-apps /usr/local/share/gitlab-managed-apps/helmfile.yaml only: diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index e87f0f28d01..c3a92b67a8b 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -37,9 +37,6 @@ apifuzzer_fuzz: $FUZZAPI_OPENAPI == null && $FUZZAPI_D_WORKER_IMAGE == null when: never - - if: $FUZZAPI_D_WORKER_IMAGE == null && - $FUZZAPI_TARGET_URL == null - when: never - if: $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ services: - docker:19.03.12-dind @@ -74,13 +71,15 @@ apifuzzer_fuzz: -e FUZZAPI_TIMEOUT \ -e FUZZAPI_VERBOSE \ -e FUZZAPI_SERVICE_START_TIMEOUT \ + -e FUZZAPI_HTTP_USERNAME \ + -e FUZZAPI_HTTP_PASSWORD \ -e GITLAB_FEATURES \ -v $CI_PROJECT_DIR:/app \ -p 80:80 \ -p 8000:8000 \ -p 514:514 \ --restart=no \ - registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing-src:${FUZZAPI_VERSION}-engine + registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine # # Start target container - | @@ -119,6 +118,9 @@ apifuzzer_fuzz: # Wait for testing to complete if api fuzzer is scanning - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI" != "" ]; then echo "Waiting for API Fuzzer to exit"; docker wait apifuzzer; fi # + # Propagate exit code from api fuzzer (if any) + - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing exited with an error. Logs are available as job artifacts."; docker logs apifuzzer; exit 1; fi + # # Run user provided pre-script - sh -c "$FUZZAPI_POST_SCRIPT" # diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index 3f47e575afd..4b957a8f771 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -34,5 +34,5 @@ variables: rules: - if: $COVFUZZ_DISABLED when: never - - if: $GITLAB_FEATURES =~ /\bcoverage_fuzzing\b/ + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bcoverage_fuzzing\b/ - if: $CI_RUNNER_EXECUTABLE_ARCH == "linux" diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index d5275c57ef8..3789f0edc1c 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -12,81 +12,24 @@ variables: DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python" DS_EXCLUDED_PATHS: "spec, test, tests, tmp" DS_MAJOR_VERSION: 2 - DS_DISABLE_DIND: "true" dependency_scanning: stage: test - image: docker:stable - variables: - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - allow_failure: true - services: - - docker:stable-dind script: - - | - if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then - export DOCKER_HOST='tcp://localhost:2375' - fi - fi - - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage - function propagate_env_vars() { - CURRENT_ENV=$(printenv) - - for VAR_NAME; do - echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " - done - } - - | - docker run \ - $(propagate_env_vars \ - DS_ANALYZER_IMAGES \ - SECURE_ANALYZERS_PREFIX \ - DS_ANALYZER_IMAGE_TAG \ - DS_DEFAULT_ANALYZERS \ - DS_EXCLUDED_PATHS \ - DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ - DS_PULL_ANALYZER_IMAGE_TIMEOUT \ - DS_RUN_ANALYZER_TIMEOUT \ - DS_PYTHON_VERSION \ - DS_PIP_VERSION \ - DS_PIP_DEPENDENCY_PATH \ - DS_JAVA_VERSION \ - GEMNASIUM_DB_LOCAL_PATH \ - GEMNASIUM_DB_REMOTE_URL \ - GEMNASIUM_DB_REF_NAME \ - PIP_INDEX_URL \ - PIP_EXTRA_INDEX_URL \ - PIP_REQUIREMENTS_FILE \ - MAVEN_CLI_OPTS \ - GRADLE_CLI_OPTS \ - SBT_CLI_OPTS \ - BUNDLER_AUDIT_UPDATE_DISABLED \ - BUNDLER_AUDIT_ADVISORY_DB_URL \ - BUNDLER_AUDIT_ADVISORY_DB_REF_NAME \ - RETIREJS_JS_ADVISORY_DB \ - RETIREJS_NODE_ADVISORY_DB \ - DS_REMEDIATE \ - ) \ - --volume "$PWD:/code" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$DS_MAJOR_VERSION" /code + - echo "$CI_JOB_NAME is used for configuration only, and its script should not be executed" + - exit 1 artifacts: reports: dependency_scanning: gl-dependency-scanning-report.json dependencies: [] rules: - - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'true' - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + - when: never .ds-analyzer: extends: dependency_scanning - services: [] + allow_failure: true rules: - - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + - if: $DEPENDENCY_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ @@ -96,9 +39,11 @@ dependency_scanning: gemnasium-dependency_scanning: extends: .ds-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/gemnasium:$DS_MAJOR_VERSION" + name: "$DS_ANALYZER_IMAGE" + variables: + DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium:$DS_MAJOR_VERSION" rules: - - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + - if: $DEPENDENCY_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && @@ -112,13 +57,16 @@ gemnasium-dependency_scanning: - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}' + - '{conan.lock,*/conan.lock,*/*/conan.lock}' gemnasium-maven-dependency_scanning: extends: .ds-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/gemnasium-maven:$DS_MAJOR_VERSION" + name: "$DS_ANALYZER_IMAGE" + variables: + DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-maven:$DS_MAJOR_VERSION" rules: - - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + - if: $DEPENDENCY_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && @@ -132,9 +80,11 @@ gemnasium-maven-dependency_scanning: gemnasium-python-dependency_scanning: extends: .ds-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/gemnasium-python:$DS_MAJOR_VERSION" + name: "$DS_ANALYZER_IMAGE" + variables: + DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-python:$DS_MAJOR_VERSION" rules: - - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + - if: $DEPENDENCY_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && @@ -155,9 +105,11 @@ gemnasium-python-dependency_scanning: bundler-audit-dependency_scanning: extends: .ds-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/bundler-audit:$DS_MAJOR_VERSION" + name: "$DS_ANALYZER_IMAGE" + variables: + DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bundler-audit:$DS_MAJOR_VERSION" rules: - - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + - if: $DEPENDENCY_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && @@ -168,9 +120,11 @@ bundler-audit-dependency_scanning: retire-js-dependency_scanning: extends: .ds-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/retire.js:$DS_MAJOR_VERSION" + name: "$DS_ANALYZER_IMAGE" + variables: + DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/retire.js:$DS_MAJOR_VERSION" rules: - - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + - if: $DEPENDENCY_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 6eb17341472..77ea11d01d1 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -9,48 +9,29 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec" + SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" SAST_ANALYZER_IMAGE_TAG: 2 - SAST_DISABLE_DIND: "true" SCAN_KUBERNETES_MANIFESTS: "false" sast: stage: test - allow_failure: true artifacts: reports: sast: gl-sast-report.json rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'true' - when: never - - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bsast\b/ - image: docker:stable + - when: never variables: SEARCH_MAX_DEPTH: 4 - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - services: - - docker:stable-dind script: - - | - if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then - export DOCKER_HOST='tcp://localhost:2375' - fi - fi - - | - docker run \ - $(awk 'BEGIN{for(v in ENVIRON) print v}' | grep -v -E '^(DOCKER_|CI|GITLAB_|FF_|HOME|PWD|OLDPWD|PATH|SHLVL|HOSTNAME)' | awk '{printf " -e %s", $0}') \ - --volume "$PWD:/code" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_ANALYZER_IMAGE_TAG" /app/bin/run /code + - echo "$CI_JOB_NAME is used for configuration only, and its script should not be executed" + - exit 1 .sast-analyzer: extends: sast - services: [] + allow_failure: true rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH script: @@ -59,9 +40,11 @@ sast: bandit-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /bandit/ @@ -71,9 +54,11 @@ bandit-sast: brakeman-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /brakeman/ @@ -83,9 +68,11 @@ brakeman-sast: eslint-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /eslint/ @@ -99,9 +86,11 @@ eslint-sast: flawfinder-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ @@ -112,9 +101,11 @@ flawfinder-sast: kubesec-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /kubesec/ && @@ -123,9 +114,11 @@ kubesec-sast: gosec-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /gosec/ @@ -135,9 +128,11 @@ gosec-sast: nodejs-scan-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ @@ -147,9 +142,11 @@ nodejs-scan-sast: phpcs-security-audit-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/ @@ -159,31 +156,25 @@ phpcs-security-audit-sast: pmd-apex-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/ exists: - '**/*.cls' -secrets-sast: - extends: .sast-analyzer - image: - name: "$SECURE_ANALYZERS_PREFIX/secrets:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /secrets/ - security-code-scan-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/ @@ -194,9 +185,11 @@ security-code-scan-sast: sobelow-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /sobelow/ @@ -206,9 +199,11 @@ sobelow-sast: spotbugs-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /spotbugs/ diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index b897c7b482f..bde6a0fbebb 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -35,6 +35,7 @@ secret_detection: - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH script: - git fetch origin $CI_DEFAULT_BRANCH $CI_BUILD_REF_NAME - - export SECRET_DETECTION_COMMIT_TO=$(git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_BUILD_REF_NAME | tail -n 1) - - export SECRET_DETECTION_COMMIT_FROM=$CI_COMMIT_SHA + - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_BUILD_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt + - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt - /analyzer run + - rm "$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index 9dbd9b679a8..e591e3cc1e2 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -12,15 +12,15 @@ performance: variables: URL: '' SITESPEED_IMAGE: sitespeedio/sitespeed.io - SITESPEED_VERSION: 13.3.0 + SITESPEED_VERSION: 14.1.0 SITESPEED_OPTIONS: '' services: - docker:stable-dind script: - mkdir gitlab-exporter - - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js + - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS + - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS - mv sitespeed-results/data/performance.json browser-performance.json artifacts: paths: diff --git a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml index f964b3b2caf..cd23af562e5 100644 --- a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml @@ -14,10 +14,11 @@ load_performance: K6_VERSION: 0.27.0 K6_TEST_FILE: github.com/loadimpact/k6/samples/http_get.js K6_OPTIONS: '' + K6_DOCKER_OPTIONS: '' services: - docker:stable-dind script: - - docker run --rm -v "$(pwd)":/k6 -w /k6 $K6_IMAGE:$K6_VERSION run $K6_TEST_FILE --summary-export=load-performance.json $K6_OPTIONS + - docker run --rm -v "$(pwd)":/k6 -w /k6 $K6_DOCKER_OPTIONS $K6_IMAGE:$K6_VERSION run $K6_TEST_FILE --summary-export=load-performance.json $K6_OPTIONS artifacts: reports: load_performance: load-performance.json diff --git a/lib/gitlab/ci/templates/npm.gitlab-ci.yml b/lib/gitlab/ci/templates/npm.gitlab-ci.yml index 035ba52da84..0a739cf122d 100644 --- a/lib/gitlab/ci/templates/npm.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/npm.gitlab-ci.yml @@ -55,5 +55,5 @@ publish_package: npm publish && echo "Successfully published version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} to GitLab's NPM registry: ${CI_PROJECT_URL}/-/packages" } || { - echo "No new version of ${NPM_PACKAGE_NAME} published. This is most likely because version ${NPM_PACKAGE_VERSION} already exists in GitLab's NPM registry." + echo "No new version of ${NPM_PACKAGE_NAME} published. This is most likely because version ${NPM_PACKAGE_VERSION} already exists in GitLab's NPM registry."; exit 1 } diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index f76aacc2d19..348e5472cb4 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -79,22 +79,13 @@ module Gitlab job.trace_chunks.any? || current_path.present? || old_trace.present? end - def read - stream = Gitlab::Ci::Trace::Stream.new do - if trace_artifact - trace_artifact.open - elsif job.trace_chunks.any? - Gitlab::Ci::Trace::ChunkedIO.new(job) - elsif current_path - File.open(current_path, "rb") - elsif old_trace - StringIO.new(old_trace) - end - end + def read(should_retry: true, &block) + read_stream(&block) + rescue Errno::ENOENT + raise unless should_retry - yield stream - ensure - stream&.close + job.reset + read_stream(&block) end def write(mode, &blk) @@ -141,6 +132,24 @@ module Gitlab private + def read_stream + stream = Gitlab::Ci::Trace::Stream.new do + if trace_artifact + trace_artifact.open + elsif job.trace_chunks.any? + Gitlab::Ci::Trace::ChunkedIO.new(job) + elsif current_path + File.open(current_path, "rb") + elsif old_trace + StringIO.new(old_trace) + end + end + + yield stream + ensure + stream&.close + end + def unsafe_write!(mode, &blk) stream = Gitlab::Ci::Trace::Stream.new do if trace_artifact diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb new file mode 100644 index 00000000000..82a7d5fb83c --- /dev/null +++ b/lib/gitlab/ci/trace/metrics.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Trace + class Metrics + extend Gitlab::Utils::StrongMemoize + + OPERATIONS = [:appended, :streamed, :chunked, :mutated, :overwrite, + :accepted, :finalized, :discarded, :conflict].freeze + + def increment_trace_operation(operation: :unknown) + unless OPERATIONS.include?(operation) + raise ArgumentError, "unknown trace operation: #{operation}" + end + + self.class.trace_operations.increment(operation: operation) + end + + def increment_trace_bytes(size) + self.class.trace_bytes.increment(by: size.to_i) + end + + def self.trace_operations + strong_memoize(:trace_operations) do + name = :gitlab_ci_trace_operations_total + comment = 'Total amount of different operations on a build trace' + + Gitlab::Metrics.counter(name, comment) + end + end + + def self.trace_bytes + strong_memoize(:trace_bytes) do + name = :gitlab_ci_trace_bytes_total + comment = 'Total amount of build trace bytes transferred' + + Gitlab::Metrics.counter(name, comment) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 20f5620dd64..618438c8887 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -8,7 +8,7 @@ module Gitlab BUFFER_SIZE = 4096 LIMIT_SIZE = 500.kilobytes - attr_reader :stream + attr_reader :stream, :metrics delegate :close, :tell, :seek, :size, :url, :truncate, to: :stream, allow_nil: true @@ -16,9 +16,10 @@ module Gitlab alias_method :present?, :valid? - def initialize + def initialize(metrics = Trace::Metrics.new) @stream = yield @stream&.binmode + @metrics = metrics end def valid? @@ -43,6 +44,9 @@ module Gitlab def append(data, offset) data = data.force_encoding(Encoding::BINARY) + metrics.increment_trace_operation(operation: :streamed) + metrics.increment_trace_bytes(data.bytesize) + stream.seek(offset, IO::SEEK_SET) stream.write(data) stream.truncate(offset + data.bytesize) diff --git a/lib/gitlab/ci/warnings.rb b/lib/gitlab/ci/warnings.rb new file mode 100644 index 00000000000..7138fd21b72 --- /dev/null +++ b/lib/gitlab/ci/warnings.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module Gitlab::Ci::Warnings + MAX_LIMIT = 25 +end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index b7046064f44..ee55eb8b22a 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -1,183 +1,65 @@ # frozen_string_literal: true +# This is the CI Linter component that runs the syntax validations +# while parsing the YAML config into a data structure that is +# then presented to the caller as result object. +# After syntax validations (done by Ci::Config), this component also +# runs logical validation on the built data structure. module Gitlab module Ci class YamlProcessor - # ValidationError is treated like a result object in the form of an exception. - # We can return any warnings, raised during the config validation, along with - # the error object until we support multiple messages to be returned. - class ValidationError < StandardError - attr_reader :warnings - - def initialize(message, warnings: []) - @warnings = warnings - super(message) - end - end - - include Gitlab::Config::Entry::LegacyValidationHelpers + ValidationError = Class.new(StandardError) - attr_reader :stages, :jobs + def self.validation_message(content, opts = {}) + result = new(content, opts).execute - class Result - attr_reader :config, :errors, :warnings + result.errors.first + end - def initialize(config: nil, errors: [], warnings: []) - @config = config - @errors = errors - @warnings = warnings - end + def initialize(config_content, opts = {}) + @config_content = config_content + @opts = opts + end - def valid? - config.present? && errors.empty? + def execute + if @config_content.blank? + return Result.new(errors: ['Please provide content of .gitlab-ci.yml']) end - end - def initialize(config, opts = {}) - @ci_config = Gitlab::Ci::Config.new(config, **opts) - @config = @ci_config.to_hash + @ci_config = Gitlab::Ci::Config.new(@config_content, **@opts) unless @ci_config.valid? - error!(@ci_config.errors.first) + return Result.new(ci_config: @ci_config, errors: @ci_config.errors, warnings: @ci_config.warnings) end - initial_parsing - rescue Gitlab::Ci::Config::ConfigError => e - error!(e.message) - end - - def self.new_with_validation_errors(content, opts = {}) - return Result.new(errors: ['Please provide content of .gitlab-ci.yml']) if content.blank? + run_logical_validations! - config = Gitlab::Ci::Config.new(content, **opts) - return Result.new(errors: config.errors, warnings: config.warnings) unless config.valid? - - config = Gitlab::Ci::YamlProcessor.new(content, opts) - Result.new(config: config, warnings: config.warnings) - - rescue ValidationError => e - Result.new(errors: [e.message], warnings: e.warnings) + Result.new(ci_config: @ci_config, warnings: @ci_config&.warnings) rescue Gitlab::Ci::Config::ConfigError => e - Result.new(errors: [e.message]) - end - - def warnings - @ci_config&.warnings || [] - end - - def builds - @jobs.map do |name, _| - build_attributes(name) - end - end - - def build_attributes(name) - job = @jobs.fetch(name.to_sym, {}) - - { stage_idx: @stages.index(job[:stage]), - stage: job[:stage], - tag_list: job[:tags], - name: job[:name].to_s, - allow_failure: job[:ignore], - when: job[:when] || 'on_success', - environment: job[:environment_name], - coverage_regex: job[:coverage], - yaml_variables: transform_to_yaml_variables(job[:variables]), - needs_attributes: job.dig(:needs, :job), - interruptible: job[:interruptible], - only: job[:only], - except: job[:except], - rules: job[:rules], - cache: job[:cache], - resource_group_key: job[:resource_group], - scheduling_type: job[:scheduling_type], - secrets: job[:secrets], - options: { - image: job[:image], - services: job[:services], - artifacts: job[:artifacts], - dependencies: job[:dependencies], - cross_dependencies: job.dig(:needs, :cross_dependency), - job_timeout: job[:timeout], - before_script: job[:before_script], - script: job[:script], - after_script: job[:after_script], - environment: job[:environment], - retry: job[:retry], - parallel: job[:parallel], - instance: job[:instance], - start_in: job[:start_in], - trigger: job[:trigger], - bridge_needs: job.dig(:needs, :bridge)&.first, - release: release(job) - }.compact }.compact - end + Result.new(ci_config: @ci_config, errors: [e.message], warnings: @ci_config&.warnings) - def release(job) - job[:release] - end - - def stage_builds_attributes(stage) - @jobs.values - .select { |job| job[:stage] == stage } - .map { |job| build_attributes(job[:name]) } - end - - def stages_attributes - @stages.uniq.map do |stage| - seeds = stage_builds_attributes(stage) - - { name: stage, index: @stages.index(stage), builds: seeds } - end - end - - def workflow_attributes - { - rules: @config.dig(:workflow, :rules), - yaml_variables: transform_to_yaml_variables(@variables) - } - end - - def self.validation_message(content, opts = {}) - return 'Please provide content of .gitlab-ci.yml' if content.blank? - - begin - Gitlab::Ci::YamlProcessor.new(content, opts) - nil - rescue ValidationError => e - e.message - end + rescue ValidationError => e + Result.new(ci_config: @ci_config, errors: [e.message], warnings: @ci_config&.warnings) end private - def initial_parsing - ## - # Global config - # - @variables = @ci_config.variables + def run_logical_validations! @stages = @ci_config.stages - - ## - # Jobs - # - @jobs = Ci::Config::Normalizer.new(@ci_config.jobs).normalize_jobs + @jobs = @ci_config.normalized_jobs @jobs.each do |name, job| - # logical validation for job - validate_job_stage!(name, job) - validate_job_dependencies!(name, job) - validate_job_needs!(name, job) - validate_dynamic_child_pipeline_dependencies!(name, job) - validate_job_environment!(name, job) + validate_job!(name, job) end end - def transform_to_yaml_variables(variables) - variables.to_h.map do |key, value| - { key: key.to_s, value: value, public: true } - end + def validate_job!(name, job) + validate_job_stage!(name, job) + validate_job_dependencies!(name, job) + validate_job_needs!(name, job) + validate_dynamic_child_pipeline_dependencies!(name, job) + validate_job_environment!(name, job) end def validate_job_stage!(name, job) @@ -188,10 +70,6 @@ module Gitlab end end - def error!(message) - raise ValidationError.new(message, warnings: warnings) - end - def validate_job_dependencies!(name, job) return unless job[:dependencies] @@ -267,6 +145,10 @@ module Gitlab error!("#{name} job: on_stop job #{on_stop} needs to have action stop defined") end end + + def error!(message) + raise ValidationError.new(message) + end end end end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb new file mode 100644 index 00000000000..68f61e52df7 --- /dev/null +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +# A data object that wraps `Ci::Config` and any messages +# (errors, warnings) generated by the YamlProcessor. +module Gitlab + module Ci + class YamlProcessor + class Result + attr_reader :errors, :warnings + + def initialize(ci_config: nil, errors: [], warnings: []) + @ci_config = ci_config + @errors = errors || [] + @warnings = warnings || [] + end + + def valid? + errors.empty? + end + + def stages_attributes + stages.uniq.map do |stage| + seeds = stage_builds_attributes(stage) + + { name: stage, index: stages.index(stage), builds: seeds } + end + end + + def builds + jobs.map do |name, _| + build_attributes(name) + end + end + + def stage_builds_attributes(stage) + jobs.values + .select { |job| job[:stage] == stage } + .map { |job| build_attributes(job[:name]) } + end + + def workflow_attributes + { + rules: hash_config.dig(:workflow, :rules), + yaml_variables: transform_to_yaml_variables(variables) + } + end + + def jobs + @jobs ||= @ci_config.normalized_jobs + end + + def stages + @stages ||= @ci_config.stages + end + + def build_attributes(name) + job = jobs.fetch(name.to_sym, {}) + + { stage_idx: stages.index(job[:stage]), + stage: job[:stage], + tag_list: job[:tags], + name: job[:name].to_s, + allow_failure: job[:ignore], + when: job[:when] || 'on_success', + environment: job[:environment_name], + coverage_regex: job[:coverage], + yaml_variables: transform_to_yaml_variables(job[:variables]), + needs_attributes: job.dig(:needs, :job), + interruptible: job[:interruptible], + only: job[:only], + except: job[:except], + rules: job[:rules], + cache: job[:cache], + resource_group_key: job[:resource_group], + scheduling_type: job[:scheduling_type], + secrets: job[:secrets], + options: { + image: job[:image], + services: job[:services], + artifacts: job[:artifacts], + dependencies: job[:dependencies], + cross_dependencies: job.dig(:needs, :cross_dependency), + job_timeout: job[:timeout], + before_script: job[:before_script], + script: job[:script], + after_script: job[:after_script], + environment: job[:environment], + retry: job[:retry], + parallel: job[:parallel], + instance: job[:instance], + start_in: job[:start_in], + trigger: job[:trigger], + bridge_needs: job.dig(:needs, :bridge)&.first, + release: release(job) + }.compact }.compact + end + + private + + def variables + @variables ||= @ci_config.variables + end + + def hash_config + @hash_config ||= @ci_config.to_hash + end + + def release(job) + job[:release] + end + + def transform_to_yaml_variables(variables) + variables.to_h.map do |key, value| + { key: key.to_s, value: value, public: true } + end + end + end + end + end +end |