diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-12-20 16:37:47 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-12-20 16:37:47 +0300 |
commit | aee0a117a889461ce8ced6fcf73207fe017f1d99 (patch) | |
tree | 891d9ef189227a8445d83f35c1b0fc99573f4380 /lib/gitlab/ci | |
parent | 8d46af3258650d305f53b819eabf7ab18d22f59e (diff) |
Add latest changes from gitlab-org/gitlab@14-6-stable-eev14.6.0-rc42
Diffstat (limited to 'lib/gitlab/ci')
47 files changed, 648 insertions, 137 deletions
diff --git a/lib/gitlab/ci/badge/coverage/report.rb b/lib/gitlab/ci/badge/coverage/report.rb index 78b51dbdaf0..4d1193176ad 100644 --- a/lib/gitlab/ci/badge/coverage/report.rb +++ b/lib/gitlab/ci/badge/coverage/report.rb @@ -30,7 +30,7 @@ module Gitlab::Ci @coverage ||= raw_coverage return unless @coverage - @coverage.to_f.round(2) + @coverage.round(2) end def metadata diff --git a/lib/gitlab/ci/badge/metadata.rb b/lib/gitlab/ci/badge/metadata.rb index eec9fedfaa9..244e3aff851 100644 --- a/lib/gitlab/ci/badge/metadata.rb +++ b/lib/gitlab/ci/badge/metadata.rb @@ -8,14 +8,13 @@ module Gitlab::Ci class Metadata include Gitlab::Routing include ActionView::Helpers::AssetTagHelper - include ActionView::Helpers::UrlHelper def initialize(badge) @badge = badge end def to_html - link_to(image_tag(image_url, alt: title), link_url) + ApplicationController.helpers.link_to(image_tag(image_url, alt: title), link_url) end def to_markdown diff --git a/lib/gitlab/ci/build/context/base.rb b/lib/gitlab/ci/build/context/base.rb index c7ea7c78e2f..81f96e822f4 100644 --- a/lib/gitlab/ci/build/context/base.rb +++ b/lib/gitlab/ci/build/context/base.rb @@ -17,6 +17,12 @@ module Gitlab raise NotImplementedError end + def variables_hash + strong_memoize(:variables_hash) do + variables.to_hash + end + end + def project pipeline.project end diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb index 7b1ce6330f0..810523052ae 100644 --- a/lib/gitlab/ci/build/policy/variables.rb +++ b/lib/gitlab/ci/build/policy/variables.rb @@ -10,7 +10,7 @@ module Gitlab end def satisfied_by?(pipeline, context) - variables = context.variables + variables = context.variables_hash statements = @expressions.map do |statement| ::Gitlab::Ci::Pipeline::Expression::Statement diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb index 9c2f6eea1dd..82a59fdb4e1 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/changes.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -23,7 +23,7 @@ module Gitlab return @globs unless context @globs.map do |glob| - ExpandVariables.expand_existing(glob, context.variables) + ExpandVariables.expand_existing(glob, -> { context.variables_hash }) end end end diff --git a/lib/gitlab/ci/build/rules/rule/clause/if.rb b/lib/gitlab/ci/build/rules/rule/clause/if.rb index 6143a736ca6..499a265a1e2 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/if.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/if.rb @@ -10,7 +10,7 @@ module Gitlab def satisfied_by?(pipeline, context) ::Gitlab::Ci::Pipeline::Expression::Statement.new( - @expression, context.variables).truthful? + @expression, context.variables_hash).truthful? end end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 6f149385969..42b487fdf81 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -17,21 +17,27 @@ module Gitlab Config::Yaml::Tags::TagError ].freeze - attr_reader :root, :context, :source_ref_path, :source + attr_reader :root, :context, :source_ref_path, :source, :logger - def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil) + def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil, logger: nil) + @logger = logger || ::Gitlab::Ci::Pipeline::Logger.new(project: project) @source_ref_path = pipeline&.source_ref_path - @context = build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline) + @context = self.logger.instrument(:config_build_context) do + build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline) + end + @context.set_deadline(TIMEOUT_SECONDS) @source = source - @config = expand_config(config) - - @root = Entry::Root.new(@config) - @root.compose! + @config = self.logger.instrument(:config_expand) do + expand_config(config) + end + @root = self.logger.instrument(:config_compose) do + Entry::Root.new(@config).tap(&:compose!) + end rescue *rescue_errors => e raise Config::ConfigError, e.message end @@ -94,11 +100,25 @@ module Gitlab end def build_config(config) - initial_config = Config::Yaml.load!(config) - initial_config = Config::External::Processor.new(initial_config, @context).perform - initial_config = Config::Extendable.new(initial_config).to_hash - initial_config = Config::Yaml::Tags::Resolver.new(initial_config).to_hash - Config::EdgeStagesInjector.new(initial_config).to_hash + initial_config = logger.instrument(:config_yaml_load) do + Config::Yaml.load!(config) + end + + initial_config = logger.instrument(:config_external_process) do + Config::External::Processor.new(initial_config, @context).perform + end + + initial_config = logger.instrument(:config_yaml_extend) do + Config::Extendable.new(initial_config).to_hash + end + + initial_config = logger.instrument(:config_tags_resolve) do + Config::Yaml::Tags::Resolver.new(initial_config).to_hash + end + + logger.instrument(:config_stages_inject) do + Config::EdgeStagesInjector.new(initial_config).to_hash + end end def find_sha(project) @@ -115,10 +135,20 @@ module Gitlab sha: sha || find_sha(project), user: user, parent_pipeline: parent_pipeline, - variables: build_variables(project: project, pipeline: pipeline)) + variables: build_variables(project: project, pipeline: pipeline), + logger: logger) end def build_variables(project:, pipeline:) + logger.instrument(:config_build_variables) do + build_variables_without_instrumentation( + project: project, + pipeline: pipeline + ) + end + end + + def build_variables_without_instrumentation(project:, pipeline:) Gitlab::Ci::Variables::Collection.new.tap do |variables| break variables unless project diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 520b1ce6119..43475742214 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -26,7 +26,7 @@ module Gitlab validates :name, length: { maximum: 255 }, if: -> { ::Feature.enabled?(:ci_validate_job_length, default_enabled: :yaml) } validates :config, disallowed_keys: { - in: %i[only except when start_in], + in: %i[only except start_in], message: 'key may not be used with `rules`' }, if: :has_rules? diff --git a/lib/gitlab/ci/config/entry/tags.rb b/lib/gitlab/ci/config/entry/tags.rb index ca3b48372e2..6044cfddbdc 100644 --- a/lib/gitlab/ci/config/entry/tags.rb +++ b/lib/gitlab/ci/config/entry/tags.rb @@ -16,8 +16,6 @@ module Gitlab validates :config, array_of_strings: true validate do - next unless ::Feature.enabled?(:ci_build_tags_limit, default_enabled: :yaml) - if config.is_a?(Array) && config.size >= TAGS_LIMIT errors.add(:config, _("must be less than the limit of %{tag_limit} tags") % { tag_limit: TAGS_LIMIT }) end diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index 51624dc30ea..308414af47d 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -9,17 +9,22 @@ module Gitlab TimeoutError = Class.new(StandardError) + include ::Gitlab::Utils::StrongMemoize + attr_reader :project, :sha, :user, :parent_pipeline, :variables - attr_reader :expandset, :execution_deadline + attr_reader :expandset, :execution_deadline, :logger + + delegate :instrument, to: :logger - def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: []) + def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil, logger: nil) @project = project @sha = sha @user = user @parent_pipeline = parent_pipeline - @variables = variables + @variables = variables || Ci::Variables::Collection.new @expandset = Set.new @execution_deadline = 0 + @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project) yield self if block_given? end @@ -36,10 +41,17 @@ module Gitlab end end + def variables_hash + strong_memoize(:variables_hash) do + variables.to_hash + end + end + def mutate(attrs = {}) self.class.new(**attrs) do |ctx| ctx.expandset = expandset ctx.execution_deadline = execution_deadline + ctx.logger = logger end end @@ -60,7 +72,7 @@ module Gitlab protected - attr_writer :expandset, :execution_deadline + attr_writer :expandset, :execution_deadline, :logger private diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 95f1a842c50..a5bf066c81f 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -30,6 +30,18 @@ module Gitlab def process return [] if locations.empty? + logger.instrument(:config_mapper_process) do + process_without_instrumentation + end + end + + private + + attr_reader :locations, :context + + delegate :expandset, :logger, to: :context + + def process_without_instrumentation locations .compact .map(&method(:normalize_location)) @@ -41,14 +53,14 @@ module Gitlab .map(&method(:select_first_matching)) end - private - - attr_reader :locations, :context - - delegate :expandset, to: :context + def normalize_location(location) + logger.instrument(:config_mapper_normalize) do + normalize_location_without_instrumentation(location) + end + end # convert location if String to canonical form - def normalize_location(location) + def normalize_location_without_instrumentation(location) if location.is_a?(String) expanded_location = expand_variables(location) normalize_location_string(expanded_location) @@ -58,6 +70,12 @@ module Gitlab end def verify_rules(location) + logger.instrument(:config_mapper_rules) do + verify_rules_without_instrumentation(location) + end + end + + def verify_rules_without_instrumentation(location) return unless Rules.new(location[:rules]).evaluate(context).pass? location @@ -72,6 +90,12 @@ module Gitlab end def expand_wildcard_paths(location) + logger.instrument(:config_mapper_wildcards) do + expand_wildcard_paths_without_instrumentation(location) + end + end + + def expand_wildcard_paths_without_instrumentation(location) # We only support local files for wildcard paths return location unless location[:local] && location[:local].include?('*') @@ -89,6 +113,12 @@ module Gitlab end def verify_duplicates!(location) + logger.instrument(:config_mapper_verify) do + verify_duplicates_without_instrumentation!(location) + end + end + + def verify_duplicates_without_instrumentation!(location) if expandset.count >= MAX_INCLUDES raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!" end @@ -106,6 +136,12 @@ module Gitlab end def select_first_matching(location) + logger.instrument(:config_mapper_select) do + select_first_matching_without_instrumentation(location) + end + end + + def select_first_matching_without_instrumentation(location) matching = FILE_CLASSES.map do |file_class| file_class.new(location, context) end.select(&:matching?) @@ -116,6 +152,12 @@ module Gitlab end def expand_variables(data) + logger.instrument(:config_mapper_variables) do + expand_variables_without_instrumentation(data) + end + end + + def expand_variables_without_instrumentation(data) if data.is_a?(String) expand(data) else @@ -137,7 +179,7 @@ module Gitlab end def expand(data) - ExpandVariables.expand(data, context.variables) + ExpandVariables.expand(data, -> { context.variables_hash }) end end end diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb index de69a1b1e8f..6a4aee26d80 100644 --- a/lib/gitlab/ci/config/external/processor.rb +++ b/lib/gitlab/ci/config/external/processor.rb @@ -7,10 +7,13 @@ module Gitlab class Processor IncludeError = Class.new(StandardError) + attr_reader :context, :logger + def initialize(values, context) @values = values @external_files = External::Mapper.new(values, context).process @content = {} + @logger = context.logger rescue External::Mapper::Error, OpenSSL::SSL::SSLError => e raise IncludeError, e.message @@ -29,13 +32,17 @@ module Gitlab def validate_external_files! @external_files.each do |file| - raise IncludeError, file.error_message unless file.valid? + logger.instrument(:config_external_verify) do + raise IncludeError, file.error_message unless file.valid? + end end end def merge_external_files! @external_files.each do |file| - @content.deep_merge!(file.to_hash) + logger.instrument(:config_external_merge) do + @content.deep_merge!(file.to_hash) + end end end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb deleted file mode 100644 index 51051b0490f..00000000000 --- a/lib/gitlab/ci/features.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - ## - # Deprecated: Ci::Features is a class that aggregates all CI/CD feature flags in one place. - # - module Features - # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` - # is a safe switch to disable the feature for a particular project when something went wrong, - # therefore it's not supposed to be enabled by default. - def self.disallow_to_create_merge_request_pipelines_in_target_project?(target_project) - ::Feature.enabled?(:ci_disallow_to_create_merge_request_pipelines_in_target_project, target_project) - end - - def self.accept_trace?(project) - ::Feature.enabled?(:ci_enable_live_trace, project) && - ::Feature.enabled?(:ci_accept_trace, project, type: :ops, default_enabled: true) - end - - def self.log_invalid_trace_chunks?(project) - ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) - end - - def self.gldropdown_tags_enabled? - ::Feature.enabled?(:gldropdown_tags, default_enabled: :yaml) - end - end - end -end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 73cfa02ce4b..651ed23eb25 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -34,7 +34,7 @@ module Gitlab end def file_name - "#{report_type.to_s.dasherize}-report-format.json" + report_type == :api_fuzzing ? "dast-report-format.json" : "#{report_type.to_s.dasherize}-report-format.json" end end diff --git a/lib/gitlab/ci/parsers/terraform/tfplan.rb b/lib/gitlab/ci/parsers/terraform/tfplan.rb index f9afa58f915..041d73cd914 100644 --- a/lib/gitlab/ci/parsers/terraform/tfplan.rb +++ b/lib/gitlab/ci/parsers/terraform/tfplan.rb @@ -34,7 +34,7 @@ module Gitlab def job_details(job) { 'job_id' => job.id.to_s, - 'job_name' => job.options.dig(:artifacts, :name).to_s, + 'job_name' => job.name, 'job_path' => Gitlab::Routing.url_helpers.project_job_path(job.project, job) } end diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb index 9b494f3a7ec..28567437719 100644 --- a/lib/gitlab/ci/pipeline/chain/base.rb +++ b/lib/gitlab/ci/pipeline/chain/base.rb @@ -7,7 +7,7 @@ module Gitlab class Base attr_reader :pipeline, :command, :config - delegate :project, :current_user, :parent_pipeline, to: :command + delegate :project, :current_user, :parent_pipeline, :logger, to: :command def initialize(pipeline, command) @pipeline = pipeline diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index 6feb693221b..bbdc6b65b96 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -21,6 +21,10 @@ module Gitlab merge_request: @command.merge_request, external_pull_request: @command.external_pull_request, locked: @command.project.default_pipeline_lock) + + # Initialize the feature flag at the beginning of the pipeline creation process + # so that the flag references in the latter chains return the same value. + @pipeline.create_deployment_in_separate_transaction? end def break? diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index beb8801096b..c466b8b36d0 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -11,7 +11,7 @@ module Gitlab :trigger_request, :schedule, :merge_request, :external_pull_request, :ignore_skip_ci, :save_incompleted, :seeds_block, :variables_attributes, :push_options, - :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, + :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, :logger, # These attributes are set by Chains during processing: :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed ) do @@ -88,7 +88,14 @@ module Gitlab @metrics ||= ::Gitlab::Ci::Pipeline::Metrics end + def logger + self[:logger] ||= ::Gitlab::Ci::Pipeline::Logger.new(project: project) + end + def observe_step_duration(step_class, duration) + step = step_class.name.underscore.parameterize(separator: '_') + logger.observe("pipeline_step_#{step}_duration_s", duration) + if Feature.enabled?(:ci_pipeline_creation_step_duration_tracking, type: :ops, default_enabled: :yaml) metrics.pipeline_creation_step_duration_histogram .observe({ step: step_class.name }, duration.seconds) @@ -96,11 +103,15 @@ module Gitlab end def observe_creation_duration(duration) + logger.observe(:pipeline_creation_duration_s, duration) + metrics.pipeline_creation_duration_histogram .observe({}, duration.seconds) end def observe_pipeline_size(pipeline) + logger.observe(:pipeline_size_count, pipeline.total_size) + metrics.pipeline_size_histogram .observe({ source: pipeline.source.to_s }, pipeline.total_size) end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index f3c937ddd28..64d1b001e3c 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -11,16 +11,21 @@ module Gitlab def perform! raise ArgumentError, 'missing config content' unless @command.config_content - result = ::Gitlab::Ci::YamlProcessor.new( - @command.config_content, { - project: project, - pipeline: @pipeline, - sha: @pipeline.sha, - source: @pipeline.source, - user: current_user, - parent_pipeline: parent_pipeline - } - ).execute + result = logger.instrument(:pipeline_config_process) do + processor = ::Gitlab::Ci::YamlProcessor.new( + @command.config_content, { + project: project, + pipeline: @pipeline, + sha: @pipeline.sha, + source: @pipeline.source, + user: current_user, + parent_pipeline: parent_pipeline, + logger: logger + } + ) + + processor.execute + end add_warnings_to_pipeline(result.warnings) diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 81ef3bb074d..15b0ff3c04d 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -6,10 +6,18 @@ module Gitlab module Chain class Create < Chain::Base include Chain::Helpers + include Gitlab::Utils::StrongMemoize def perform! - BulkInsertableAssociations.with_bulk_insert do - pipeline.save! + logger.instrument(:pipeline_save) do + BulkInsertableAssociations.with_bulk_insert do + tags = extract_tag_list_by_status + + pipeline.transaction do + pipeline.save! + CommitStatus.bulk_insert_tags!(statuses, tags) if bulk_insert_tags? + end + end end rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") @@ -18,6 +26,37 @@ module Gitlab def break? !pipeline.persisted? end + + private + + def statuses + strong_memoize(:statuses) do + pipeline.stages.flat_map(&:statuses) + end + end + + # We call `job.tag_list=` to assign tags to the jobs from the + # Chain::Seed step which uses the `@tag_list` instance variable to + # store them on the record. We remove them here because we want to + # bulk insert them, otherwise they would be inserted and assigned one + # by one with callbacks. We must use `remove_instance_variable` + # because having the instance variable defined would still run the callbacks + def extract_tag_list_by_status + return {} unless bulk_insert_tags? + + statuses.each.with_object({}) do |job, acc| + tag_list = job.clear_memoization(:tag_list) + next unless tag_list + + acc[job.name] = tag_list + end + end + + def bulk_insert_tags? + strong_memoize(:bulk_insert_tags) do + ::Feature.enabled?(:ci_bulk_insert_tags, project, default_enabled: :yaml) + end + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/create_deployments.rb b/lib/gitlab/ci/pipeline/chain/create_deployments.rb new file mode 100644 index 00000000000..b92aa89d62d --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/create_deployments.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class CreateDeployments < Chain::Base + DeploymentCreationError = Class.new(StandardError) + + def perform! + return unless pipeline.create_deployment_in_separate_transaction? + + create_deployments! + end + + def break? + false + end + + private + + def create_deployments! + pipeline.stages.map(&:statuses).flatten.map(&method(:create_deployment)) + end + + def create_deployment(build) + return unless build.instance_of?(::Ci::Build) && build.persisted_environment.present? + + deployment = ::Gitlab::Ci::Pipeline::Seed::Deployment + .new(build, build.persisted_environment).to_resource + + return unless deployment + + deployment.deployable = build + deployment.save! + rescue ActiveRecord::RecordInvalid => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception( + DeploymentCreationError.new(e.message), build_id: build.id) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb new file mode 100644 index 00000000000..424e1d87fb4 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class EnsureEnvironments < Chain::Base + def perform! + return unless pipeline.create_deployment_in_separate_transaction? + + pipeline.stages.map(&:statuses).flatten.each(&method(:ensure_environment)) + end + + def break? + false + end + + private + + def ensure_environment(build) + return unless build.instance_of?(::Ci::Build) && build.has_environment? + + environment = ::Gitlab::Ci::Pipeline::Seed::Environment.new(build).to_resource + + if environment.persisted? + build.persisted_environment = environment + build.assign_attributes(metadata_attributes: { expanded_environment_name: environment.name }) + else + build.assign_attributes(status: :failed, failure_reason: :environment_creation_failure) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/ensure_resource_groups.rb b/lib/gitlab/ci/pipeline/chain/ensure_resource_groups.rb new file mode 100644 index 00000000000..f4e5e6e467a --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/ensure_resource_groups.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class EnsureResourceGroups < Chain::Base + def perform! + return unless pipeline.create_deployment_in_separate_transaction? + + pipeline.stages.map(&:statuses).flatten.each(&method(:ensure_resource_group)) + end + + def break? + false + end + + private + + def ensure_resource_group(processable) + return unless processable.is_a?(::Ci::Processable) + + key = processable.options.delete(:resource_group_key) + + resource_group = ::Gitlab::Ci::Pipeline::Seed::Processable::ResourceGroup + .new(processable, key).to_resource + + processable.resource_group = resource_group + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index ef7447fa83d..356eeb76908 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -13,8 +13,10 @@ module Gitlab raise ArgumentError, 'missing workflow rules result' unless @command.workflow_rules_result # Allocate next IID. This operation must be outside of transactions of pipeline creations. - pipeline.ensure_project_iid! - pipeline.ensure_ci_ref! + logger.instrument(:pipeline_allocate_seed_attributes) do + pipeline.ensure_project_iid! + pipeline.ensure_ci_ref! + end # Protect the pipeline. This is assigned in Populate instead of # Build to prevent erroring out on ambiguous refs. @@ -23,8 +25,12 @@ module Gitlab ## # Gather all runtime build/stage errors # - if pipeline_seed.errors - return error(pipeline_seed.errors.join("\n"), config_error: true) + seed_errors = logger.instrument(:pipeline_seed_evaluation) do + pipeline_seed.errors + end + + if seed_errors + return error(seed_errors.join("\n"), config_error: true) end @command.pipeline_seed = pipeline_seed @@ -38,8 +44,11 @@ module Gitlab def pipeline_seed strong_memoize(:pipeline_seed) do - stages_attributes = @command.yaml_processor_result.stages_attributes - Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes) + logger.instrument(:pipeline_seed_initialization) do + stages_attributes = @command.yaml_processor_result.stages_attributes + + Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes) + end end end @@ -48,9 +57,11 @@ module Gitlab end def root_variables - ::Gitlab::Ci::Variables::Helpers.merge_variables( - @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables - ) + logger.instrument(:pipeline_seed_merge_variables) do + ::Gitlab::Ci::Variables::Helpers.merge_variables( + @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables + ) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index 845eb6c7a42..de147914850 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -9,30 +9,36 @@ module Gitlab @pipeline = pipeline @command = command @sequence = sequence - @start = Time.now + @start = current_monotonic_time end def build! @sequence.each do |step_class| - step_start = ::Gitlab::Metrics::System.monotonic_time + step_start = current_monotonic_time step = step_class.new(@pipeline, @command) step.perform! @command.observe_step_duration( step_class, - ::Gitlab::Metrics::System.monotonic_time - step_start + current_monotonic_time - step_start ) break if step.break? end - @command.observe_creation_duration(Time.now - @start) + @command.observe_creation_duration(current_monotonic_time - @start) @command.observe_pipeline_size(@pipeline) @command.observe_jobs_count_in_alive_pipelines @pipeline end + + private + + def current_monotonic_time + ::Gitlab::Metrics::System.monotonic_time + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index 28ba1cd4d47..85bd5f0a7c1 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -113,7 +113,7 @@ module Gitlab name: build[:name], stage: build[:stage], image: build.dig(:options, :image, :name), - services: build.dig(:options, :services)&.map { |service| service[:name] }, + services: service_names(build), script: [ build.dig(:options, :before_script), build.dig(:options, :script), @@ -122,6 +122,14 @@ module Gitlab } end + def service_names(build) + services = build.dig(:options, :services) + + return unless services + + services.compact.map { |service| service[:name] } + end + def stages_attributes command.yaml_processor_result.stages_attributes end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb index 11d2010909f..6da88fd287e 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb @@ -9,7 +9,11 @@ module Gitlab PATTERN = /\$(?<name>\w+)/.freeze def evaluate(variables = {}) - variables.with_indifferent_access.fetch(@value, nil) + unless variables.is_a?(ActiveSupport::HashWithIndifferentAccess) + variables = variables.with_indifferent_access + end + + variables.fetch(@value, nil) end def inspect diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index 5f3310dd668..4b13cae792e 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -9,7 +9,7 @@ module Gitlab def initialize(statement, variables = nil) @lexer = Expression::Lexer.new(statement) - @variables = variables&.to_hash + @variables = variables || {} end def parse_tree @@ -19,7 +19,7 @@ module Gitlab end def evaluate - parse_tree.evaluate(@variables.to_h) + parse_tree.evaluate(@variables) end def truthful? diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb new file mode 100644 index 00000000000..97f7dddd09a --- /dev/null +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + class Logger + include ::Gitlab::Utils::StrongMemoize + + def self.current_monotonic_time + ::Gitlab::Metrics::System.monotonic_time + end + + def initialize(project:, destination: Gitlab::AppJsonLogger) + @started_at = current_monotonic_time + @project = project + @destination = destination + @log_conditions = [] + + yield(self) if block_given? + end + + def log_when(&block) + log_conditions.push(block) + end + + def instrument(operation) + return yield unless enabled? + + raise ArgumentError, 'block not given' unless block_given? + + op_started_at = current_monotonic_time + + result = yield + + observe("#{operation}_duration_s", current_monotonic_time - op_started_at) + + result + end + + def observe(operation, value) + return unless enabled? + + observations[operation.to_s].push(value) + end + + def commit(pipeline:, caller:) + return unless log? + + attributes = { + class: self.class.name.to_s, + pipeline_creation_caller: caller, + project_id: project.id, + pipeline_id: pipeline.id, + pipeline_persisted: pipeline.persisted?, + pipeline_source: pipeline.source, + pipeline_creation_service_duration_s: age + }.stringify_keys.merge(observations_hash) + + destination.info(attributes) + end + + def observations_hash + observations.transform_values do |values| + next if values.empty? + + { + 'count' => values.size, + 'min' => values.min, + 'max' => values.max, + 'avg' => values.sum / values.size + } + end.compact + end + + private + + attr_reader :project, :destination, :started_at, :log_conditions + delegate :current_monotonic_time, to: :class + + def age + current_monotonic_time - started_at + end + + def log? + return false unless enabled? + return true if log_conditions.empty? + + log_conditions.any? { |cond| cond.call(observations) } + end + + def enabled? + strong_memoize(:enabled) do + ::Feature.enabled?(:ci_pipeline_creation_logger, project, type: :ops, default_enabled: :yaml) + end + end + + def observations + @observations ||= Hash.new { |hash, key| hash[key] = [] } + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 72837b8ec22..762292f0fa3 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -7,8 +7,6 @@ module Gitlab class Build < Seed::Base include Gitlab::Utils::StrongMemoize - EnvironmentCreationFailure = Class.new(StandardError) - delegate :dig, to: :@seed_attributes def initialize(context, attributes, stages_for_needs_lookup = []) @@ -30,7 +28,7 @@ module Gitlab @except = Gitlab::Ci::Build::Policy .fabricate(attributes.delete(:except)) @rules = Gitlab::Ci::Build::Rules - .new(attributes.delete(:rules), default_when: 'on_success') + .new(attributes.delete(:rules), default_when: attributes[:when]) @cache = Gitlab::Ci::Build::Cache .new(attributes.delete(:cache), @pipeline) @@ -80,7 +78,7 @@ module Gitlab def to_resource strong_memoize(:resource) do processable = initialize_processable - assign_resource_group(processable) + assign_resource_group(processable) unless @pipeline.create_deployment_in_separate_transaction? processable end end @@ -90,7 +88,9 @@ module Gitlab ::Ci::Bridge.new(attributes) else ::Ci::Build.new(attributes).tap do |build| - build.assign_attributes(self.class.deployment_attributes_for(build)) + unless @pipeline.create_deployment_in_separate_transaction? + build.assign_attributes(self.class.deployment_attributes_for(build)) + end end end end @@ -107,20 +107,7 @@ module Gitlab environment = Seed::Environment.new(build).to_resource if environment.nil? unless environment.persisted? - if Feature.enabled?(:surface_environment_creation_failure, build.project, default_enabled: :yaml) && - Feature.disabled?(:surface_environment_creation_failure_override, build.project) - return { status: :failed, failure_reason: :environment_creation_failure } - end - - # If there is a validation error on environment creation, such as - # the name contains invalid character, the build falls back to a - # non-environment job. - Gitlab::ErrorTracking.track_exception( - EnvironmentCreationFailure.new, - project_id: build.project_id, - reason: environment.errors.full_messages.to_sentence) - - return { environment: nil } + return { status: :failed, failure_reason: :environment_creation_failure } end build.persisted_environment = environment @@ -215,12 +202,14 @@ module Gitlab end def runner_tags - { tag_list: evaluate_runner_tags }.compact + strong_memoize(:runner_tags) do + { tag_list: evaluate_runner_tags }.compact + end end def evaluate_runner_tags - @seed_attributes[:tag_list]&.map do |tag| - ExpandVariables.expand_existing(tag, evaluate_context.variables) + @seed_attributes.delete(:tag_list)&.map do |tag| + ExpandVariables.expand_existing(tag, -> { evaluate_context.variables_hash }) end end diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb index 417319cb5be..3e4a44a2e70 100644 --- a/lib/gitlab/ci/reports/security/report.rb +++ b/lib/gitlab/ci/reports/security/report.rb @@ -51,7 +51,7 @@ module Gitlab def replace_with!(other) instance_variables.each do |ivar| - instance_variable_set(ivar, other.public_send(ivar.to_s[1..-1])) # rubocop:disable GitlabSecurity/PublicSend + instance_variable_set(ivar, other.public_send(ivar.to_s[1..])) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/gitlab/ci/status/bridge/common.rb b/lib/gitlab/ci/status/bridge/common.rb index d66d4b20bba..eaa87157716 100644 --- a/lib/gitlab/ci/status/bridge/common.rb +++ b/lib/gitlab/ci/status/bridge/common.rb @@ -16,7 +16,11 @@ module Gitlab def details_path return unless can?(user, :read_pipeline, downstream_pipeline) - project_pipeline_path(downstream_project, downstream_pipeline) + if Feature.enabled?(:ci_retry_downstream_pipeline, subject.project, default_enabled: :yaml) + project_job_path(subject.project, subject) + else + project_pipeline_path(downstream_project, downstream_pipeline) + end end def has_action? diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index b0f12ff7517..5dd28157b1f 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -34,7 +34,8 @@ module Gitlab no_matching_runner: 'no matching runner available', trace_size_exceeded: 'log size limit exceeded', builds_disabled: 'project builds are disabled', - environment_creation_failure: 'environment creation failure' + environment_creation_failure: 'environment creation failure', + deployment_rejected: 'deployment rejected' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/tags/bulk_insert.rb b/lib/gitlab/ci/tags/bulk_insert.rb new file mode 100644 index 00000000000..a299df7e2d9 --- /dev/null +++ b/lib/gitlab/ci/tags/bulk_insert.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Tags + class BulkInsert + TAGGINGS_BATCH_SIZE = 1000 + TAGS_BATCH_SIZE = 500 + + def initialize(statuses, tag_list_by_status) + @statuses = statuses + @tag_list_by_status = tag_list_by_status + end + + def insert! + return false if tag_list_by_status.empty? + + persist_build_tags! + end + + private + + attr_reader :statuses, :tag_list_by_status + + def persist_build_tags! + all_tags = tag_list_by_status.values.flatten.uniq.reject(&:blank?) + tag_records_by_name = create_tags(all_tags).index_by(&:name) + taggings = build_taggings_attributes(tag_records_by_name) + + return false if taggings.empty? + + taggings.each_slice(TAGGINGS_BATCH_SIZE) do |taggings_slice| + ActsAsTaggableOn::Tagging.insert_all!(taggings) + end + + true + end + + # rubocop: disable CodeReuse/ActiveRecord + def create_tags(tags) + existing_tag_records = ActsAsTaggableOn::Tag.where(name: tags).to_a + missing_tags = detect_missing_tags(tags, existing_tag_records) + return existing_tag_records if missing_tags.empty? + + missing_tags + .map { |tag| { name: tag } } + .each_slice(TAGS_BATCH_SIZE) do |tags_attributes| + ActsAsTaggableOn::Tag.insert_all!(tags_attributes) + end + + ActsAsTaggableOn::Tag.where(name: tags).to_a + end + # rubocop: enable CodeReuse/ActiveRecord + + def build_taggings_attributes(tag_records_by_name) + taggings = statuses.flat_map do |status| + tag_list = tag_list_by_status[status.name] + next unless tag_list + + tags = tag_records_by_name.values_at(*tag_list) + taggings_for(tags, status) + end + + taggings.compact! + taggings + end + + def taggings_for(tags, status) + tags.map do |tag| + { + tag_id: tag.id, + taggable_type: CommitStatus.name, + taggable_id: status.id, + created_at: Time.current, + context: 'tags' + } + end + end + + def detect_missing_tags(tags, tag_records) + if tags.size != tag_records.size + tags - tag_records.map(&:name) + else + [] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 89fd59d98f4..fddcc1492a8 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -53,6 +53,9 @@ variables: # KUBE_INGRESS_BASE_DOMAIN is the application deployment domain and should be set as a variable at the group or project level. # KUBE_INGRESS_BASE_DOMAIN: domain.example.com + # Allows Container-Scanning to correctly correlate image names when using Jobs/Build.gitlab-ci.yml + CS_DEFAULT_BRANCH_IMAGE: $CI_REGISTRY_IMAGE/$CI_DEFAULT_BRANCH:$CI_COMMIT_SHA + POSTGRES_USER: user POSTGRES_PASSWORD: testing-password POSTGRES_ENABLED: "true" diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml index b763705857e..fa7f6ffa2b7 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml @@ -24,7 +24,7 @@ kics-iac-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 0 + SAST_ANALYZER_IMAGE_TAG: 1 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml index 17ed1d2e87f..d32444833fb 100644 --- a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -9,6 +9,7 @@ pages: script: - mkdir .public - cp -r * .public + - rm -rf public - mv .public public artifacts: paths: diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index aec41c137a4..4917abf6ae9 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -23,7 +23,7 @@ cache: - venv/ before_script: - - python -V # Print out python version for debugging + - python --version # For debugging - pip install virtualenv - virtualenv venv - source venv/bin/activate diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml index ff8f9601189..de54d64dc42 100644 --- a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml @@ -13,8 +13,10 @@ before_script: - apt-get update -yqq - apt-get install apt-transport-https -yqq # Add keyserver for SBT - - echo "deb http://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list - - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 + - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list + - mkdir -p /root/.gnupg + - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823 + - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg # Install SBT - apt-get update -yqq - apt-get install sbt -yqq diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index 544774d3b06..01041f4f056 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -11,11 +11,11 @@ variables: FUZZAPI_VERSION: "1" SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION} + FUZZAPI_IMAGE: api-fuzzing apifuzzer_fuzz: stage: fuzz - image: $FUZZAPI_IMAGE + image: $SECURE_ANALYZERS_PREFIX/$FUZZAPI_IMAGE:$FUZZAPI_VERSION allow_failure: true rules: - if: $API_FUZZING_DISABLED diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 89e6743b0e4..65a2b20d5c0 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -38,7 +38,8 @@ container_scanning: artifacts: reports: container_scanning: gl-container-scanning-report.json - paths: [gl-container-scanning-report.json] + dependency_scanning: gl-dependency-scanning-report.json + paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json] dependencies: [] script: - gtcs scan diff --git a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml new file mode 100644 index 00000000000..57f1993921d --- /dev/null +++ b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml @@ -0,0 +1,52 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Dast-API.gitlab-ci.yml + +# To use this template, add the following to your .gitlab-ci.yml file: +# +# include: +# template: DAST-API.latest.gitlab-ci.yml +# +# You also need to add a `dast` stage to your `stages:` configuration. A sample configuration for DAST API: +# +# stages: +# - build +# - test +# - deploy +# - dast + +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html + +# Configure DAST API scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html#available-cicd-variables + +variables: + # Setting this variable affects all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + # + DAST_API_VERSION: "1" + DAST_API_IMAGE: api-fuzzing + +dast_api: + stage: dast + image: $SECURE_ANALYZERS_PREFIX/$DAST_API_IMAGE:$DAST_API_VERSION + allow_failure: true + rules: + - if: $DAST_API_DISABLED + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_COMMIT_BRANCH + script: + - /peach/analyzer-dast-api + artifacts: + when: always + paths: + - gl-assets + - gl-dast-api-report.json + - gl-*.log + reports: + dast: gl-dast-api-report.json diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml index 4f63ff93d4d..8f4a836441d 100644 --- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml @@ -13,7 +13,7 @@ stages: a11y: stage: accessibility - image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.0.1 + image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.1.1 script: /gitlab-accessibility.sh $a11y_urls allow_failure: true artifacts: diff --git a/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml index edd0fb0ba07..09fce67db2d 100644 --- a/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml @@ -1,4 +1,4 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: +# To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/dotNET-Core.yml @@ -14,10 +14,10 @@ # The 'latest' tag targets the latest available version of .NET Core SDK image. # If preferred, you can explicitly specify version of .NET Core (e.g. using '2.2-sdk' tag). # -# See other available tags for .NET Core: https://hub.docker.com/r/microsoft/dotnet +# See other available tags for .NET Core: https://hub.docker.com/_/microsoft-dotnet # Learn more about Docker tags: https://docs.docker.com/glossary/?term=tag # and the Docker itself: https://opensource.com/resources/what-docker -image: microsoft/dotnet:latest +image: mcr.microsoft.com/dotnet/sdk:latest # ### Define variables # diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index f4c5a06af97..3e2c2c7fc1a 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -12,7 +12,7 @@ module Gitlab def scoped_variables(job, environment:, dependencies:) Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.concat(predefined_variables(job)) if pipeline.predefined_vars_in_builder_enabled? + variables.concat(predefined_variables(job)) end end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 1aa3dbc5e47..296b0cfded2 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -29,10 +29,8 @@ module Gitlab run_logical_validations! Result.new(ci_config: @ci_config, warnings: @ci_config&.warnings) - rescue Gitlab::Ci::Config::ConfigError => e Result.new(ci_config: @ci_config, errors: [e.message], warnings: @ci_config&.warnings) - rescue ValidationError => e Result.new(ci_config: @ci_config, errors: [e.message], warnings: @ci_config&.warnings) end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 6215ba40ebe..f14279dca2d 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -92,6 +92,7 @@ module Gitlab script: job[:script], after_script: job[:after_script], environment: job[:environment], + resource_group_key: job[:resource_group], retry: job[:retry], parallel: job[:parallel], instance: job[:instance], |