diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-04-21 02:50:22 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-04-21 02:50:22 +0300 |
| commit | 9dc93a4519d9d5d7be48ff274127136236a3adb3 (patch) | |
| tree | 70467ae3692a0e35e5ea56bcb803eb512a10bedb /lib/gitlab/ci | |
| parent | 4b0f34b6d759d6299322b3a54453e930c6121ff0 (diff) | |
Add latest changes from gitlab-org/gitlab@13-11-stable-eev13.11.0-rc43
Diffstat (limited to 'lib/gitlab/ci')
45 files changed, 965 insertions, 222 deletions
diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index c5afb16ab1a..88d624503df 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -17,7 +17,9 @@ module Gitlab attr_reader :stream, :path, :full_version def initialize(stream, path, **opts) - @stream, @path, @opts = stream, path, opts + @stream = stream + @path = path + @opts = opts @full_version = read_version end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index d3f030c3b36..23b0c93a3ee 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -17,12 +17,14 @@ module Gitlab Config::Yaml::Tags::TagError ].freeze - attr_reader :root + attr_reader :root, :context, :ref - def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil) + def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, ref: nil) @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline) @context.set_deadline(TIMEOUT_SECONDS) + @ref = ref + @config = expand_config(config) @root = Entry::Root.new(@config) @@ -94,9 +96,7 @@ module Gitlab initial_config = Config::External::Processor.new(initial_config, @context).perform initial_config = Config::Extendable.new(initial_config).to_hash initial_config = Config::Yaml::Tags::Resolver.new(initial_config).to_hash - initial_config = Config::EdgeStagesInjector.new(initial_config).to_hash - - initial_config + Config::EdgeStagesInjector.new(initial_config).to_hash end def find_sha(project) diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index cf599ce5294..f9688c500d2 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -8,8 +8,8 @@ module Gitlab # Entry that represents a cache configuration # class Cache < ::Gitlab::Config::Entry::Simplifiable - strategy :Caches, if: -> (config) { Feature.enabled?(:multiple_cache_per_job) } - strategy :Cache, if: -> (config) { Feature.disabled?(:multiple_cache_per_job) } + strategy :Caches, if: -> (config) { Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml) } + strategy :Cache, if: -> (config) { Feature.disabled?(:multiple_cache_per_job, default_enabled: :yaml) } class Caches < ::Gitlab::Config::Entry::ComposableArray include ::Gitlab::Config::Entry::Validatable @@ -17,8 +17,6 @@ module Gitlab MULTIPLE_CACHE_LIMIT = 4 validations do - validates :config, presence: true - validate do unless config.is_a?(Hash) || config.is_a?(Array) errors.add(:config, 'can only be a Hash or an Array') diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 9584d19bdec..947b6787aa0 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -124,7 +124,9 @@ module Gitlab stage: stage_value, extends: extends, rules: rules_value, - variables: root_and_job_variables_value, + variables: root_and_job_variables_value, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581 + job_variables: job_variables, + root_variables_inheritance: root_variables_inheritance, only: only_value, except: except_value, resource_group: resource_group }.compact @@ -139,6 +141,14 @@ module Gitlab root_variables.merge(variables_value.to_h) end + def job_variables + variables_value.to_h + end + + def root_variables_inheritance + inherit_entry&.variables_entry&.value + end + def manual_action? self.when == 'manual' end diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb index aa34cfb3acc..e869e0bbb31 100644 --- a/lib/gitlab/ci/config/entry/product/variables.rb +++ b/lib/gitlab/ci/config/entry/product/variables.rb @@ -25,8 +25,7 @@ module Gitlab def value @config - .map { |key, value| [key.to_s, Array(value).map(&:to_s)] } - .to_h + .to_h { |key, value| [key.to_s, Array(value).map(&:to_s)] } end end end diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index dc164d752be..efb469ee32a 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -18,7 +18,7 @@ module Gitlab end def value - Hash[@config.map { |key, value| [key.to_s, expand_value(value)[:value]] }] + @config.to_h { |key, value| [key.to_s, expand_value(value)[:value]] } end def self.default(**) @@ -26,7 +26,7 @@ module Gitlab end def value_with_data - Hash[@config.map { |key, value| [key.to_s, expand_value(value)] }] + @config.to_h { |key, value| [key.to_s, expand_value(value)] } end def use_value_data? diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index b85b7a9edeb..3216d4eaac4 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -34,6 +34,7 @@ module Gitlab .compact .map(&method(:normalize_location)) .flat_map(&method(:expand_project_files)) + .flat_map(&method(:expand_wildcard_paths)) .map(&method(:expand_variables)) .each(&method(:verify_duplicates!)) .map(&method(:select_first_matching)) @@ -63,6 +64,17 @@ module Gitlab end end + def expand_wildcard_paths(location) + return location unless ::Feature.enabled?(:ci_wildcard_file_paths, context.project, default_enabled: :yaml) + + # We only support local files for wildcard paths + return location unless location[:local] && location[:local].include?('*') + + context.project.repository.search_files_by_wildcard_path(location[:local], context.sha).map do |path| + { local: path } + end + end + def normalize_location_string(location) if ::Gitlab::UrlSanitizer.valid?(location) { remote: location } diff --git a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb index 5a23836d8a0..5cabbc86d3e 100644 --- a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb +++ b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb @@ -43,9 +43,10 @@ module Gitlab { name: name, instance: instance, - variables: variables, + variables: variables, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581 + job_variables: variables, parallel: { total: total } - } + }.compact end def name diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index c811ef211d6..12e182b38fc 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -10,10 +10,6 @@ module Gitlab ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: true) end - def self.instance_variables_ui_enabled? - ::Feature.enabled?(:ci_instance_variables_ui, default_enabled: true) - end - def self.pipeline_latest? ::Feature.enabled?(:ci_pipeline_latest, default_enabled: true) end @@ -60,16 +56,12 @@ module Gitlab ::Feature.enabled?(:codequality_mr_diff, project, default_enabled: false) end - def self.display_codequality_backend_comparison?(project) - ::Feature.enabled?(:codequality_backend_comparison, project, default_enabled: :yaml) - end - def self.multiple_cache_per_job? ::Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml) end - def self.ci_commit_pipeline_mini_graph_vue_enabled?(project) - ::Feature.enabled?(:ci_commit_pipeline_mini_graph_vue, project, default_enabled: :yaml) + def self.gldropdown_tags_enabled? + ::Feature.enabled?(:gldropdown_tags, default_enabled: :yaml) end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index af06e124736..a6ae249fa58 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -72,16 +72,16 @@ module Gitlab def key @key ||= begin - key_data = if Feature.enabled?(:ci_jwt_signing_key, build.project, default_enabled: true) - Gitlab::CurrentSettings.ci_jwt_signing_key - else - Rails.application.secrets.openid_connect_signing_key - end + key_data = if Feature.enabled?(:ci_jwt_signing_key, build.project, default_enabled: true) + Gitlab::CurrentSettings.ci_jwt_signing_key + else + Rails.application.secrets.openid_connect_signing_key + end - raise NoSigningKeyError unless key_data + raise NoSigningKeyError unless key_data - OpenSSL::PKey::RSA.new(key_data) - end + OpenSSL::PKey::RSA.new(key_data) + end end def public_key diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 815fe6bac6d..c3c1728602c 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -12,7 +12,7 @@ module Gitlab :seeds_block, :variables_attributes, :push_options, :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, # These attributes are set by Chains during processing: - :config_content, :yaml_processor_result, :pipeline_seed + :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed ) do include Gitlab::Utils::StrongMemoize @@ -84,7 +84,7 @@ module Gitlab end def metrics - @metrics ||= ::Gitlab::Ci::Pipeline::Metrics.new + @metrics ||= ::Gitlab::Ci::Pipeline::Metrics end def observe_creation_duration(duration) @@ -97,6 +97,11 @@ module Gitlab .observe({ source: pipeline.source.to_s }, pipeline.total_size) end + def increment_pipeline_failure_reason_counter(reason) + metrics.pipeline_failure_reason_counter + .increment(reason: (reason || :unknown_failure).to_s) + end + def dangling_build? %i[ondemand_dast_scan webide].include?(source) end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index c3fbd0c9e24..8f1c49563f2 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -14,6 +14,7 @@ module Gitlab result = ::Gitlab::Ci::YamlProcessor.new( @command.config_content, { project: project, + ref: @pipeline.ref, sha: @pipeline.sha, user: current_user, parent_pipeline: parent_pipeline diff --git a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb index 3c910963a2a..cceaa52de16 100644 --- a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb +++ b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb @@ -9,6 +9,8 @@ module Gitlab include Chain::Helpers def perform! + @command.workflow_rules_result = workflow_rules_result + error('Pipeline filtered out by workflow rules.') unless workflow_passed? end @@ -19,27 +21,33 @@ module Gitlab private def workflow_passed? - strong_memoize(:workflow_passed) do - workflow_rules.evaluate(@pipeline, global_context).pass? + workflow_rules_result.pass? + end + + def workflow_rules_result + strong_memoize(:workflow_rules_result) do + workflow_rules.evaluate(@pipeline, global_context) end end def workflow_rules Gitlab::Ci::Build::Rules.new( - workflow_config[:rules], default_when: 'always') + workflow_rules_config, default_when: 'always') end def global_context Gitlab::Ci::Build::Context::Global.new( - @pipeline, yaml_variables: workflow_config[:yaml_variables]) + @pipeline, yaml_variables: @command.yaml_processor_result.root_variables) end def has_workflow_rules? - workflow_config[:rules].present? + workflow_rules_config.present? end - def workflow_config - @command.yaml_processor_result.workflow_attributes || {} + def workflow_rules_config + strong_memoize(:workflow_rules_config) do + @command.yaml_processor_result.workflow_rules + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index d7271df1694..9988b6f18ed 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -12,7 +12,8 @@ module Gitlab end pipeline.add_error_message(message) - pipeline.drop!(drop_reason) if drop_reason && persist_pipeline? + + drop_pipeline!(drop_reason) # TODO: consider not to rely on AR errors directly as they can be # polluted with other unrelated errors (e.g. state machine) @@ -24,8 +25,21 @@ module Gitlab pipeline.add_warning_message(message) end - def persist_pipeline? - command.save_incompleted && !pipeline.readonly? + private + + def drop_pipeline!(drop_reason) + return if pipeline.readonly? + + if drop_reason && command.save_incompleted + # Project iid must be called outside a transaction, so we ensure it is set here + # otherwise it may be set within the state transition transaction of the drop! call + # which it will lock the InternalId row for the whole transaction + pipeline.ensure_project_iid! + + pipeline.drop!(drop_reason) + else + command.increment_pipeline_failure_reason_counter(drop_reason) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/metrics.rb b/lib/gitlab/ci/pipeline/chain/metrics.rb index 0d7449813b4..b17ae77d445 100644 --- a/lib/gitlab/ci/pipeline/chain/metrics.rb +++ b/lib/gitlab/ci/pipeline/chain/metrics.rb @@ -14,7 +14,7 @@ module Gitlab end def counter - ::Gitlab::Ci::Pipeline::Metrics.new.pipelines_created_counter + ::Gitlab::Ci::Pipeline::Metrics.pipelines_created_counter end end end diff --git a/lib/gitlab/ci/pipeline/chain/pipeline/process.rb b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb index 1eb7474e915..c1b6dfb7e36 100644 --- a/lib/gitlab/ci/pipeline/chain/pipeline/process.rb +++ b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb @@ -8,9 +8,7 @@ module Gitlab # After pipeline has been successfully created we can start processing it. class Process < Chain::Base def perform! - ::Ci::ProcessPipelineService - .new(@pipeline) - .execute + ::Ci::InitialPipelineProcessWorker.perform_async(pipeline.id) end def break? diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index 7b537125b9b..66fc6741252 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -11,6 +11,10 @@ module Gitlab def perform! raise ArgumentError, 'missing YAML processor result' unless @command.yaml_processor_result + if ::Feature.enabled?(:ci_workflow_rules_variables, pipeline.project, default_enabled: :yaml) + raise ArgumentError, 'missing workflow rules result' unless @command.workflow_rules_result + end + # Allocate next IID. This operation must be outside of transactions of pipeline creations. pipeline.ensure_project_iid! pipeline.ensure_ci_ref! @@ -38,7 +42,21 @@ module Gitlab def pipeline_seed strong_memoize(:pipeline_seed) do stages_attributes = @command.yaml_processor_result.stages_attributes - Gitlab::Ci::Pipeline::Seed::Pipeline.new(pipeline, stages_attributes) + Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes) + end + end + + def context + Gitlab::Ci::Pipeline::Seed::Context.new(pipeline, root_variables: root_variables) + end + + def root_variables + if ::Feature.enabled?(:ci_workflow_rules_variables, pipeline.project, default_enabled: :yaml) + ::Gitlab::Ci::Variables::Helpers.merge_variables( + @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables + ) + else + @command.yaml_processor_result.root_variables end end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index d056501a6d3..6149d2f04d7 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -10,77 +10,116 @@ module Gitlab InvalidResponseCode = Class.new(StandardError) - VALIDATION_REQUEST_TIMEOUT = 5 + DEFAULT_VALIDATION_REQUEST_TIMEOUT = 5 + ACCEPTED_STATUS = 200 + DOT_COM_REJECTED_STATUS = 406 + GENERAL_REJECTED_STATUS = (400..499).freeze def perform! + return unless enabled? + pipeline_authorized = validate_external log_message = pipeline_authorized ? 'authorized' : 'not authorized' - Gitlab::AppLogger.info(message: "Pipeline #{log_message}", project_id: @pipeline.project.id, user_id: @pipeline.user.id) + Gitlab::AppLogger.info(message: "Pipeline #{log_message}", project_id: project.id, user_id: current_user.id) error('External validation failed', drop_reason: :external_validation_failure) unless pipeline_authorized end def break? - @pipeline.errors.any? + pipeline.errors.any? end private + def enabled? + return true unless Gitlab.com? + + ::Feature.enabled?(:ci_external_validation_service, project, default_enabled: :yaml) + end + def validate_external return true unless validation_service_url # 200 - accepted - # 4xx - not accepted + # 406 - not accepted on GitLab.com + # 4XX - not accepted for other installations # everything else - accepted and logged response_code = validate_service_request.code case response_code - when 200 + when ACCEPTED_STATUS true - when 400..499 + when rejected_status false else raise InvalidResponseCode, "Unsupported response code received from Validation Service: #{response_code}" end rescue => ex - Gitlab::ErrorTracking.track_exception(ex) + Gitlab::ErrorTracking.track_exception(ex, project_id: project.id) true end + def rejected_status + if Gitlab.com? + DOT_COM_REJECTED_STATUS + else + GENERAL_REJECTED_STATUS + end + end + def validate_service_request + headers = { + 'X-Gitlab-Correlation-id' => Labkit::Correlation::CorrelationId.current_id, + 'X-Gitlab-Token' => validation_service_token + }.compact + Gitlab::HTTP.post( - validation_service_url, timeout: VALIDATION_REQUEST_TIMEOUT, - body: validation_service_payload(@pipeline, @command.yaml_processor_result.stages_attributes) + validation_service_url, timeout: validation_service_timeout, + headers: headers, + body: validation_service_payload.to_json ) end + def validation_service_timeout + timeout = Gitlab::CurrentSettings.external_pipeline_validation_service_timeout || ENV['EXTERNAL_VALIDATION_SERVICE_TIMEOUT'].to_i + return timeout if timeout > 0 + + DEFAULT_VALIDATION_REQUEST_TIMEOUT + end + def validation_service_url - ENV['EXTERNAL_VALIDATION_SERVICE_URL'] + Gitlab::CurrentSettings.external_pipeline_validation_service_url || ENV['EXTERNAL_VALIDATION_SERVICE_URL'] + end + + def validation_service_token + Gitlab::CurrentSettings.external_pipeline_validation_service_token || ENV['EXTERNAL_VALIDATION_SERVICE_TOKEN'] end - def validation_service_payload(pipeline, stages_attributes) + def validation_service_payload { project: { - id: pipeline.project.id, - path: pipeline.project.full_path + id: project.id, + path: project.full_path, + created_at: project.created_at&.iso8601 }, user: { - id: pipeline.user.id, - username: pipeline.user.username, - email: pipeline.user.email + id: current_user.id, + username: current_user.username, + email: current_user.email, + created_at: current_user.created_at&.iso8601 }, pipeline: { sha: pipeline.sha, ref: pipeline.ref, type: pipeline.source }, - builds: builds_validation_payload(stages_attributes) - }.to_json + builds: builds_validation_payload + } end - def builds_validation_payload(stages_attributes) - stages_attributes.map { |stage| stage[:builds] }.flatten + def builds_validation_payload + stages_attributes.flat_map { |stage| stage[:builds] } .map(&method(:build_validation_payload)) end @@ -97,9 +136,15 @@ module Gitlab ].flatten.compact } end + + def stages_attributes + command.yaml_processor_result.stages_attributes + end end end end end end end + +Gitlab::Ci::Pipeline::Chain::Validate::External.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Validate::External') diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index c77f4dcca5a..6cb6fd3920d 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -4,55 +4,57 @@ module Gitlab module Ci module Pipeline class Metrics - include Gitlab::Utils::StrongMemoize + def self.pipeline_creation_duration_histogram + name = :gitlab_ci_pipeline_creation_duration_seconds + comment = 'Pipeline creation duration' + labels = {} + buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0] - def pipeline_creation_duration_histogram - strong_memoize(:pipeline_creation_duration_histogram) do - name = :gitlab_ci_pipeline_creation_duration_seconds - comment = 'Pipeline creation duration' - labels = {} - buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0] + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + + def self.pipeline_size_histogram + name = :gitlab_ci_pipeline_size_builds + comment = 'Pipeline size' + labels = { source: nil } + buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000] + + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + + def self.pipeline_processing_events_counter + name = :gitlab_ci_pipeline_processing_events_total + comment = 'Total amount of pipeline processing events' - ::Gitlab::Metrics.histogram(name, comment, labels, buckets) - end + Gitlab::Metrics.counter(name, comment) end - def pipeline_size_histogram - strong_memoize(:pipeline_size_histogram) do - name = :gitlab_ci_pipeline_size_builds - comment = 'Pipeline size' - labels = { source: nil } - buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000] + def self.pipelines_created_counter + name = :pipelines_created_total + comment = 'Counter of pipelines created' - ::Gitlab::Metrics.histogram(name, comment, labels, buckets) - end + Gitlab::Metrics.counter(name, comment) end - def pipeline_processing_events_counter - strong_memoize(:pipeline_processing_events_counter) do - name = :gitlab_ci_pipeline_processing_events_total - comment = 'Total amount of pipeline processing events' + def self.legacy_update_jobs_counter + name = :ci_legacy_update_jobs_as_retried_total + comment = 'Counter of occurrences when jobs were not being set as retried before update_retried' - Gitlab::Metrics.counter(name, comment) - end + Gitlab::Metrics.counter(name, comment) end - def pipelines_created_counter - strong_memoize(:pipelines_created_count) do - name = :pipelines_created_total - comment = 'Counter of pipelines created' + def self.pipeline_failure_reason_counter + name = :gitlab_ci_pipeline_failure_reasons + comment = 'Counter of pipeline failure reasons' - Gitlab::Metrics.counter(name, comment) - end + Gitlab::Metrics.counter(name, comment) end - def legacy_update_jobs_counter - strong_memoize(:legacy_update_jobs_counter) do - name = :ci_legacy_update_jobs_as_retried_total - comment = 'Counter of occurrences when jobs were not being set as retried before update_retried' + def self.job_failure_reason_counter + name = :gitlab_ci_job_failure_reasons + comment = 'Counter of job failure reasons' - Gitlab::Metrics.counter(name, comment) - end + Gitlab::Metrics.counter(name, comment) end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 11b01822e4b..39dee7750d6 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -11,12 +11,15 @@ module Gitlab delegate :dig, to: :@seed_attributes - def initialize(pipeline, attributes, previous_stages) - @pipeline = pipeline + def initialize(context, attributes, previous_stages) + @context = context + @pipeline = context.pipeline @seed_attributes = attributes @previous_stages = previous_stages @needs_attributes = dig(:needs_attributes) @resource_group_key = attributes.delete(:resource_group_key) + @job_variables = @seed_attributes.delete(:job_variables) + @root_variables_inheritance = @seed_attributes.delete(:root_variables_inheritance) { true } @using_rules = attributes.key?(:rules) @using_only = attributes.key?(:only) @@ -29,7 +32,9 @@ module Gitlab @rules = Gitlab::Ci::Build::Rules .new(attributes.delete(:rules), default_when: 'on_success') @cache = Gitlab::Ci::Build::Cache - .new(attributes.delete(:cache), pipeline) + .new(attributes.delete(:cache), @pipeline) + + recalculate_yaml_variables! end def name @@ -206,6 +211,14 @@ module Gitlab { options: { allow_failure_criteria: nil } } end + + def recalculate_yaml_variables! + return unless ::Feature.enabled?(:ci_workflow_rules_variables, @pipeline.project, default_enabled: :yaml) + + @seed_attributes[:yaml_variables] = Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( + from: @context.root_variables, to: @job_variables, inheritance: @root_variables_inheritance + ) + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/context.rb b/lib/gitlab/ci/pipeline/seed/context.rb new file mode 100644 index 00000000000..6194a78f682 --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/context.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Seed + class Context + attr_reader :pipeline, :root_variables + + def initialize(pipeline, root_variables: []) + @pipeline = pipeline + @root_variables = root_variables + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/pipeline.rb b/lib/gitlab/ci/pipeline/seed/pipeline.rb index da9d853cf68..e1a15fb8d5b 100644 --- a/lib/gitlab/ci/pipeline/seed/pipeline.rb +++ b/lib/gitlab/ci/pipeline/seed/pipeline.rb @@ -7,8 +7,8 @@ module Gitlab class Pipeline include Gitlab::Utils::StrongMemoize - def initialize(pipeline, stages_attributes) - @pipeline = pipeline + def initialize(context, stages_attributes) + @context = context @stages_attributes = stages_attributes end @@ -37,7 +37,7 @@ module Gitlab def stage_seeds strong_memoize(:stage_seeds) do seeds = @stages_attributes.inject([]) do |previous_stages, attributes| - seed = Gitlab::Ci::Pipeline::Seed::Stage.new(@pipeline, attributes, previous_stages) + seed = Gitlab::Ci::Pipeline::Seed::Stage.new(@context, attributes, previous_stages) previous_stages + [seed] end diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index b600df2f656..c988ea10e41 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -10,13 +10,14 @@ module Gitlab delegate :size, to: :seeds delegate :dig, to: :seeds - def initialize(pipeline, attributes, previous_stages) - @pipeline = pipeline + def initialize(context, attributes, previous_stages) + @context = context + @pipeline = context.pipeline @attributes = attributes @previous_stages = previous_stages @builds = attributes.fetch(:builds).map do |attributes| - Seed::Build.new(@pipeline, attributes, previous_stages) + Seed::Build.new(context, attributes, previous_stages) end end diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 5398c19e536..7ecb9a1db16 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -9,12 +9,12 @@ module Gitlab QUEUE_DURATION_SECONDS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze QUEUE_ACTIVE_RUNNERS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze QUEUE_DEPTH_TOTAL_BUCKETS = [1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze - QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000].freeze - QUEUE_ITERATION_DURATION_SECONDS_BUCKETS = [0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze + QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000, 7500, 10000, 15000, 20000].freeze + QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze METRICS_SHARD_TAG_PREFIX = 'metrics_shard::' DEFAULT_METRICS_SHARD = 'default' - JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET = 5.freeze + JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET = 5 OPERATION_COUNTERS = [ :build_can_pick, @@ -94,13 +94,13 @@ module Gitlab self.class.queue_depth_total.observe({ queue: queue }, size.to_f) end - def observe_queue_size(size_proc) + def observe_queue_size(size_proc, runner_type) return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) - self.class.queue_size_total.observe({}, size_proc.call.to_f) + self.class.queue_size_total.observe({ runner_type: runner_type }, size_proc.call.to_f) end - def observe_queue_time + def observe_queue_time(metric, runner_type) start_time = ::Gitlab::Metrics::System.monotonic_time result = yield @@ -108,7 +108,15 @@ module Gitlab return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) seconds = ::Gitlab::Metrics::System.monotonic_time - start_time - self.class.queue_iteration_duration_seconds.observe({}, seconds.to_f) + + case metric + when :process + self.class.queue_iteration_duration_seconds.observe({ runner_type: runner_type }, seconds.to_f) + when :retrieve + self.class.queue_retrieval_duration_seconds.observe({ runner_type: runner_type }, seconds.to_f) + else + raise ArgumentError unless Rails.env.production? + end result end @@ -187,7 +195,18 @@ module Gitlab strong_memoize(:queue_iteration_duration_seconds) do name = :gitlab_ci_queue_iteration_duration_seconds comment = 'Time it takes to find a build in CI/CD queue' - buckets = QUEUE_ITERATION_DURATION_SECONDS_BUCKETS + buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS + labels = {} + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def self.queue_retrieval_duration_seconds + strong_memoize(:queue_retrieval_duration_seconds) do + name = :gitlab_ci_queue_retrieval_duration_seconds + comment = 'Time it takes to execute a SQL query to retrieve builds queue' + buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS labels = {} Gitlab::Metrics.histogram(name, comment, labels, buckets) diff --git a/lib/gitlab/ci/reports/codequality_reports.rb b/lib/gitlab/ci/reports/codequality_reports.rb index 060a1e2399b..27c41c384b8 100644 --- a/lib/gitlab/ci/reports/codequality_reports.rb +++ b/lib/gitlab/ci/reports/codequality_reports.rb @@ -6,6 +6,7 @@ module Gitlab class CodequalityReports attr_reader :degradations, :error_message + SEVERITY_PRIORITIES = %w(blocker critical major minor info).map.with_index.to_h.freeze # { "blocker" => 0, "critical" => 1 ... } CODECLIMATE_SCHEMA_PATH = Rails.root.join('app', 'validators', 'json_schemas', 'codeclimate.json').to_s def initialize @@ -29,12 +30,17 @@ module Gitlab @degradations.values end + def sort_degradations! + @degradations = @degradations.sort_by do |_fingerprint, degradation| + SEVERITY_PRIORITIES[degradation.dig(:severity)] + end.to_h + end + private def valid_degradation?(degradation) - JSON::Validator.validate!(CODECLIMATE_SCHEMA_PATH, degradation) - rescue JSON::Schema::ValidationError => e - set_error_message("Invalid degradation format: #{e.message}") + JSONSchemer.schema(Pathname.new(CODECLIMATE_SCHEMA_PATH)).valid?(degradation) + rescue StandardError => _ false end end diff --git a/lib/gitlab/ci/reports/codequality_reports_comparer.rb b/lib/gitlab/ci/reports/codequality_reports_comparer.rb index 10748b8ca02..e34d9675c10 100644 --- a/lib/gitlab/ci/reports/codequality_reports_comparer.rb +++ b/lib/gitlab/ci/reports/codequality_reports_comparer.rb @@ -7,6 +7,11 @@ module Gitlab def initialize(base_report, head_report) @base_report = base_report @head_report = head_report + + unless not_found? + @base_report.sort_degradations! + @head_report.sort_degradations! + end end def success? diff --git a/lib/gitlab/ci/reports/test_failure_history.rb b/lib/gitlab/ci/reports/test_failure_history.rb index c024e794ad5..37d0da38065 100644 --- a/lib/gitlab/ci/reports/test_failure_history.rb +++ b/lib/gitlab/ci/reports/test_failure_history.rb @@ -6,32 +6,32 @@ module Gitlab class TestFailureHistory include Gitlab::Utils::StrongMemoize - def initialize(failed_test_cases, project) - @failed_test_cases = build_map(failed_test_cases) + def initialize(failed_junit_tests, project) + @failed_junit_tests = build_map(failed_junit_tests) @project = project end def load! recent_failures_count.each do |key_hash, count| - failed_test_cases[key_hash].set_recent_failures(count, project.default_branch_or_master) + failed_junit_tests[key_hash].set_recent_failures(count, project.default_branch_or_master) end end private - attr_reader :report, :project, :failed_test_cases + attr_reader :report, :project, :failed_junit_tests def recent_failures_count - ::Ci::TestCaseFailure.recent_failures_count( + ::Ci::UnitTestFailure.recent_failures_count( project: project, - test_case_keys: failed_test_cases.keys + unit_test_keys: failed_junit_tests.keys ) end - def build_map(test_cases) + def build_map(junit_tests) {}.tap do |hash| - test_cases.each do |test_case| - hash[test_case.key] = test_case + junit_tests.each do |test| + hash[test.key] = test end end end diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb index dd0bfa768a8..365864d3317 100644 --- a/lib/gitlab/ci/runner_instructions.rb +++ b/lib/gitlab/ci/runner_instructions.rb @@ -51,10 +51,7 @@ module Gitlab attr_reader :errors - def initialize(current_user:, group: nil, project: nil, os:, arch:) - @current_user = current_user - @group = group - @project = project + def initialize(os:, arch:) @os = os @arch = arch @errors = [] @@ -77,7 +74,7 @@ module Gitlab server_url = Gitlab::Routing.url_helpers.root_url(only_path: false) runner_executable = environment[:runner_executable] - "#{runner_executable} register --url #{server_url} --registration-token #{registration_token}" + "#{runner_executable} register --url #{server_url} --registration-token $REGISTRATION_TOKEN" end end @@ -108,30 +105,6 @@ module Gitlab def get_file(path) File.read(Rails.root.join(path).to_s) end - - def registration_token - project_token || group_token || instance_token - end - - def project_token - return unless @project - raise Gitlab::Access::AccessDeniedError unless can?(@current_user, :admin_pipeline, @project) - - @project.runners_token - end - - def group_token - return unless @group - raise Gitlab::Access::AccessDeniedError unless can?(@current_user, :admin_group, @group) - - @group.runners_token - end - - def instance_token - raise Gitlab::Access::AccessDeniedError unless @current_user&.admin? - - Gitlab::CurrentSettings.runners_registration_token - end end end end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index f6562737838..787dee3b267 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -26,7 +26,9 @@ module Gitlab bridge_pipeline_is_child_pipeline: 'creation of child pipeline not allowed from another child pipeline', downstream_pipeline_creation_failed: 'downstream pipeline can not be created', secrets_provider_not_found: 'secrets provider can not be found', - reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines' + reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines', + project_deleted: 'pipeline project was deleted', + user_blocked: 'pipeline user was blocked' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml index 5ebbbf15682..2ff36bcc657 100644 --- a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml @@ -113,9 +113,10 @@ promoteBeta: promoteProduction: extends: .promote_job stage: production - # We only allow production promotion on `master` because - # it has its own production scoped secret variables + # We only allow production promotion on the default branch because + # it has its own production scoped secret variables. only: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH script: - bundle exec fastlane promote_beta_to_production diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml index 15cdbf63cb1..d0c63ab6edf 100644 --- a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml @@ -1,27 +1,31 @@ -docker-build-master: - # Official docker image. - image: docker:latest - stage: build - services: - - docker:dind - before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - script: - - docker build --pull -t "$CI_REGISTRY_IMAGE" . - - docker push "$CI_REGISTRY_IMAGE" - only: - - master - +# Build a Docker image with CI/CD and push to the GitLab registry. +# Docker-in-Docker documentation: https://docs.gitlab.com/ee/ci/docker/using_docker_build.html +# +# This template uses one generic job with conditional builds +# for the default branch and all other (MR) branches. docker-build: - # Official docker image. + # Use the official docker image. image: docker:latest stage: build services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + # Default branch leaves tag empty (= latest tag) + # All other branches are tagged with the escaped branch name (commit ref slug) script: - - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" . - - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" - except: - - master + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + tag="" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" + else + tag=":$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi + - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . + - docker push "$CI_REGISTRY_IMAGE${tag}" + # Run this job in a branch where a Dockerfile exists + rules: + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile diff --git a/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml b/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml new file mode 100644 index 00000000000..90812083917 --- /dev/null +++ b/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml @@ -0,0 +1,9 @@ +# This file is a template demonstrating the `script` keyword. +# Learn more about this keyword here: https://docs.gitlab.com/ee/ci/yaml/README.html#script + +# After committing this template, visit CI/CD > Jobs to see the script output. + +job: + script: + # provide a shell script as argument for this keyword. + - echo "Hello World" diff --git a/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml new file mode 100644 index 00000000000..c7fb1321055 --- /dev/null +++ b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml @@ -0,0 +1,91 @@ +# This template is provided and maintained by Indeni, an official Technology Partner with GitLab. +# See https://about.gitlab.com/partners/technology-partners/#security for more information. + +# For more information about Indeni Cloudrail: https://indeni.com/cloudrail/ +# +# This file shows an example of using Indeni Cloudrail with GitLab CI/CD. +# It is not designed to be included in an existing CI/CD configuration with the "include:" keyword. +# Documentation about this integration: https://indeni.com/doc-indeni-cloudrail/integrate-with-ci-cd/gitlab-instructions +# +# For an example of this used in a GitLab repository, see: https://gitlab.com/indeni/cloudrail-demo/-/blob/master/.gitlab-ci.yml + +# The sast-report output complies with GitLab's format. This report displays Cloudrail's +# results in the Security tab in the pipeline view, if you have that feature enabled +# (GitLab Ultimate only). Otherwise, Cloudrail generates a JUnit report, which displays +# in the "Test summary" in merge requests. + +# Note that Cloudrail's input is the Terraform plan. That is why we've included in this +# template an example of doing that. You are welcome to replace it with your own way +# of generating a Terraform plan. + +# Before you can use this template, get a Cloudrail API key from the Cloudrail web +# user interface. Save it as a CI/CD variable named CLOUDRAIL_API_KEY in your project +# settings. + +variables: + TEST_ROOT: ${CI_PROJECT_DIR}/my_folder_with_terraform_content + +default: + before_script: + - cd ${CI_PROJECT_DIR}/my_folder_with_terraform_content + +stages: + - init_and_plan + - cloudrail + +init_and_plan: + stage: init_and_plan + image: registry.gitlab.com/gitlab-org/terraform-images/releases/0.13 + rules: + - if: $SAST_DISABLED + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.tf' + script: + - terraform init + - terraform plan -out=plan.out + artifacts: + name: "$CI_COMMIT_BRANCH-terraform_plan" + paths: + - ./**/plan.out + - ./**/.terraform + +cloudrail_scan: + stage: cloudrail + image: indeni/cloudrail-cli:1.2.44 + rules: + - if: $SAST_DISABLED + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.tf' + script: + - | + if [[ "${GITLAB_FEATURES}" == *"security_dashboard"* ]]; then + echo "You are licensed for GitLab Security Dashboards. Your scan results will display in the Security Dashboard." + cloudrail run --tf-plan plan.out \ + --directory . \ + --api-key ${CLOUDRAIL_API_KEY} \ + --origin ci \ + --build-link "$CI_PROJECT_URL/-/jobs/$CI_JOB_ID" \ + --execution-source-identifier "$CI_COMMIT_BRANCH - $CI_JOB_ID" \ + --output-format json-gitlab-sast \ + --output-file ${CI_PROJECT_DIR}/cloudrail-sast-report.json \ + --auto-approve + else + echo "Your scan results will display in the GitLab Test results visualization panel." + cloudrail run --tf-plan plan.out \ + --directory . \ + --api-key ${CLOUDRAIL_API_KEY} \ + --origin ci \ + --build-link "$CI_PROJECT_URL/-/jobs/$CI_JOB_ID" \ + --execution-source-identifier "$CI_COMMIT_BRANCH - $CI_JOB_ID" \ + --output-format junit \ + --output-file ${CI_PROJECT_DIR}/cloudrail-junit-report.xml \ + --auto-approve + fi + artifacts: + reports: + sast: cloudrail-sast-report.json + junit: cloudrail-junit-report.xml diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index 5edb26a0b56..01907ef9e2e 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -20,15 +20,48 @@ performance: fi - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) - mkdir gitlab-exporter + # Busybox wget does not support proxied HTTPS, get the real thing. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. + - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - | + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } + - | if [ -f .gitlab-urls.txt ] then sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS else - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS fi - mv sitespeed-results/data/performance.json browser-performance.json artifacts: diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 1c25d9d583b..196d42f3e3a 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,10 +1,10 @@ build: stage: build - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.4.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.6.0" variables: DOCKER_TLS_CERTDIR: "" services: - - docker:19.03.12-dind + - docker:20.10.6-dind script: - | if [[ -z "$CI_COMMIT_TAG" ]]; then diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index fd6c51ea350..b29342216fc 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -36,6 +36,7 @@ code_quality: REPORT_STDOUT \ REPORT_FORMAT \ ENGINE_MEMORY_LIMIT_BYTES \ + CODECLIMATE_PREFIX \ ) \ --volume "$PWD":/code \ --volume /var/run/docker.sock:/var/run/docker.sock \ diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index 654a03ced5f..bf42cd52605 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -12,7 +12,7 @@ stages: variables: FUZZAPI_PROFILE: Quick - FUZZAPI_VERSION: latest + FUZZAPI_VERSION: "1.6" FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml FUZZAPI_TIMEOUT: 30 FUZZAPI_REPORT: gl-api-fuzzing-report.json @@ -45,7 +45,7 @@ apifuzzer_fuzz: entrypoint: ["/bin/bash", "-l", "-c"] variables: FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://localhost:80 + FUZZAPI_API: http://localhost:5000 FUZZAPI_NEW_REPORT: 1 FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log TZ: America/Los_Angeles @@ -107,7 +107,7 @@ apifuzzer_fuzz_dnd: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://apifuzzer:80 + FUZZAPI_API: http://apifuzzer:5000 allow_failure: true rules: - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null @@ -142,6 +142,7 @@ apifuzzer_fuzz_dnd: -e TZ=America/Los_Angeles \ -e GITLAB_FEATURES \ -p 80:80 \ + -p 5000:5000 \ -p 8000:8000 \ -p 514:514 \ --restart=no \ @@ -168,7 +169,7 @@ apifuzzer_fuzz_dnd: docker run \ --name worker \ --network $FUZZAPI_D_NETWORK \ - -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_API=http://apifuzzer:5000 \ -e FUZZAPI_PROJECT \ -e FUZZAPI_PROFILE \ -e FUZZAPI_CONFIG \ @@ -211,7 +212,7 @@ apifuzzer_fuzz_dnd: --name worker \ --network $FUZZAPI_D_NETWORK \ -e TZ=America/Los_Angeles \ - -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_API=http://apifuzzer:5000 \ -e FUZZAPI_PROJECT \ -e FUZZAPI_PROFILE \ -e FUZZAPI_CONFIG \ @@ -237,6 +238,7 @@ apifuzzer_fuzz_dnd: -v $CI_PROJECT_DIR:/app \ -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \ -p 81:80 \ + -p 5001:5000 \ -p 8001:8000 \ -p 515:514 \ --restart=no \ diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml new file mode 100644 index 00000000000..215029dc952 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -0,0 +1,270 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ + +# Configure the scanning tool through the environment variables. +# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables +# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables + +variables: + FUZZAPI_PROFILE: Quick + FUZZAPI_VERSION: latest + FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml + FUZZAPI_TIMEOUT: 30 + FUZZAPI_REPORT: gl-api-fuzzing-report.json + FUZZAPI_REPORT_ASSET_PATH: assets + # + FUZZAPI_D_NETWORK: testing-net + # + # Wait up to 5 minutes for API Fuzzer and target url to become + # available (non 500 response to HTTP(s)) + FUZZAPI_SERVICE_START_TIMEOUT: "300" + # + FUZZAPI_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine + # + +apifuzzer_fuzz_unlicensed: + stage: fuzz + allow_failure: true + rules: + - if: '$GITLAB_FEATURES !~ /\bapi_fuzzing\b/ && $API_FUZZING_DISABLED == null' + - when: never + script: + - | + echo "Error: Your GitLab project is not licensed for API Fuzzing." + - exit 1 + +apifuzzer_fuzz: + stage: fuzz + image: + name: $FUZZAPI_IMAGE + entrypoint: ["/bin/bash", "-l", "-c"] + variables: + FUZZAPI_PROJECT: $CI_PROJECT_PATH + FUZZAPI_API: http://localhost:80 + FUZZAPI_NEW_REPORT: 1 + FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log + TZ: America/Los_Angeles + allow_failure: true + rules: + - if: $FUZZAPI_D_TARGET_IMAGE + when: never + - if: $FUZZAPI_D_WORKER_IMAGE + when: never + - if: $API_FUZZING_DISABLED + when: never + - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ + script: + # + # Validate options + - | + if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ + echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ + echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ + exit 1; \ + fi + # + # Run user provided pre-script + - sh -c "$FUZZAPI_PRE_SCRIPT" + # + # Make sure asset path exists + - mkdir -p $FUZZAPI_REPORT_ASSET_PATH + # + # Start API Security background process + - dotnet /peach/Peach.Web.dll &> $FUZZAPI_LOG_SCANNER & + - APISEC_PID=$! + # + # Start scanning + - worker-entry + # + # Run user provided post-script + - sh -c "$FUZZAPI_POST_SCRIPT" + # + # Shutdown API Security + - kill $APISEC_PID + - wait $APISEC_PID + # + artifacts: + when: always + paths: + - $FUZZAPI_REPORT_ASSET_PATH + - $FUZZAPI_REPORT + - $FUZZAPI_LOG_SCANNER + reports: + api_fuzzing: $FUZZAPI_REPORT + +apifuzzer_fuzz_dnd: + stage: fuzz + image: docker:19.03.12 + variables: + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + FUZZAPI_PROJECT: $CI_PROJECT_PATH + FUZZAPI_API: http://apifuzzer:80 + allow_failure: true + rules: + - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null + when: never + - if: $API_FUZZING_DISABLED + when: never + - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ + services: + - docker:19.03.12-dind + script: + # + # + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + # + - docker network create --driver bridge $FUZZAPI_D_NETWORK + # + # Run user provided pre-script + - sh -c "$FUZZAPI_PRE_SCRIPT" + # + # Make sure asset path exists + - mkdir -p $FUZZAPI_REPORT_ASSET_PATH + # + # Start peach testing engine container + - | + docker run -d \ + --name apifuzzer \ + --network $FUZZAPI_D_NETWORK \ + -e Proxy:Port=8000 \ + -e TZ=America/Los_Angeles \ + -e GITLAB_FEATURES \ + -p 80:80 \ + -p 8000:8000 \ + -p 514:514 \ + --restart=no \ + $FUZZAPI_IMAGE \ + dotnet /peach/Peach.Web.dll + # + # Start target container + - | + if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \ + docker run -d \ + --name target \ + --network $FUZZAPI_D_NETWORK \ + $FUZZAPI_D_TARGET_ENV \ + $FUZZAPI_D_TARGET_PORTS \ + $FUZZAPI_D_TARGET_VOLUME \ + --restart=no \ + $FUZZAPI_D_TARGET_IMAGE \ + ; fi + # + # Start worker container if provided + - | + if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \ + echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE"; \ + docker run \ + --name worker \ + --network $FUZZAPI_D_NETWORK \ + -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_PROJECT \ + -e FUZZAPI_PROFILE \ + -e FUZZAPI_CONFIG \ + -e FUZZAPI_REPORT \ + -e FUZZAPI_REPORT_ASSET_PATH \ + -e FUZZAPI_NEW_REPORT=1 \ + -e FUZZAPI_HAR \ + -e FUZZAPI_OPENAPI \ + -e FUZZAPI_POSTMAN_COLLECTION \ + -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ + -e FUZZAPI_TARGET_URL \ + -e FUZZAPI_OVERRIDES_FILE \ + -e FUZZAPI_OVERRIDES_ENV \ + -e FUZZAPI_OVERRIDES_CMD \ + -e FUZZAPI_OVERRIDES_INTERVAL \ + -e FUZZAPI_TIMEOUT \ + -e FUZZAPI_VERBOSE \ + -e FUZZAPI_SERVICE_START_TIMEOUT \ + -e FUZZAPI_HTTP_USERNAME \ + -e FUZZAPI_HTTP_PASSWORD \ + -e CI_PROJECT_URL \ + -e CI_JOB_ID \ + -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \ + $FUZZAPI_D_WORKER_ENV \ + $FUZZAPI_D_WORKER_PORTS \ + $FUZZAPI_D_WORKER_VOLUME \ + --restart=no \ + $FUZZAPI_D_WORKER_IMAGE \ + ; fi + # + # Start API Fuzzing provided worker if no other worker present + - | + if [ "$FUZZAPI_D_WORKER_IMAGE" == "" ]; then \ + if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ + echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ + echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ + exit 1; \ + fi; \ + docker run \ + --name worker \ + --network $FUZZAPI_D_NETWORK \ + -e TZ=America/Los_Angeles \ + -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_PROJECT \ + -e FUZZAPI_PROFILE \ + -e FUZZAPI_CONFIG \ + -e FUZZAPI_REPORT \ + -e FUZZAPI_REPORT_ASSET_PATH \ + -e FUZZAPI_NEW_REPORT=1 \ + -e FUZZAPI_HAR \ + -e FUZZAPI_OPENAPI \ + -e FUZZAPI_POSTMAN_COLLECTION \ + -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ + -e FUZZAPI_TARGET_URL \ + -e FUZZAPI_OVERRIDES_FILE \ + -e FUZZAPI_OVERRIDES_ENV \ + -e FUZZAPI_OVERRIDES_CMD \ + -e FUZZAPI_OVERRIDES_INTERVAL \ + -e FUZZAPI_TIMEOUT \ + -e FUZZAPI_VERBOSE \ + -e FUZZAPI_SERVICE_START_TIMEOUT \ + -e FUZZAPI_HTTP_USERNAME \ + -e FUZZAPI_HTTP_PASSWORD \ + -e CI_PROJECT_URL \ + -e CI_JOB_ID \ + -v $CI_PROJECT_DIR:/app \ + -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \ + -p 81:80 \ + -p 8001:8000 \ + -p 515:514 \ + --restart=no \ + $FUZZAPI_IMAGE \ + worker-entry \ + ; fi + # + # Propagate exit code from api fuzzing scanner (if any) + - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing scanner exited with an error. Logs are available as job artifacts."; exit 1; fi + # + # Run user provided post-script + - sh -c "$FUZZAPI_POST_SCRIPT" + # + after_script: + # + # Shutdown all containers + - echo "Stopping all containers" + - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi + - docker stop worker + - docker stop apifuzzer + # + # Save docker logs + - docker logs apifuzzer &> gl-api_fuzzing-logs.log + - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi + - docker logs worker &> gl-api_fuzzing-worker-logs.log + # + artifacts: + when: always + paths: + - ./gl-api_fuzzing*.log + - ./gl-api_fuzzing*.zip + - $FUZZAPI_REPORT_ASSET_PATH + - $FUZZAPI_REPORT + reports: + api_fuzzing: $FUZZAPI_REPORT + +# end diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 64001c2828a..c628e30b2c7 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -6,14 +6,10 @@ variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" CS_MAJOR_VERSION: 3 -container_scanning: +.cs_common: stage: test image: "$CS_ANALYZER_IMAGE" variables: - # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image - # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes - CLAIR_DB_IMAGE_TAG: "latest" - CLAIR_DB_IMAGE: "$SECURE_ANALYZERS_PREFIX/clair-vulnerabilities-db:$CLAIR_DB_IMAGE_TAG" # Override the GIT_STRATEGY variable in your `.gitlab-ci.yml` file and set it to `fetch` if you want to provide a `clair-whitelist.yml` # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template # for details @@ -21,19 +17,44 @@ container_scanning: # CS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to # override the analyzer image with a custom value. This may be subject to change or # breakage across GitLab releases. - CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION + CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/$CS_PROJECT:$CS_MAJOR_VERSION allow_failure: true + artifacts: + reports: + container_scanning: gl-container-scanning-report.json + dependencies: [] + +container_scanning: + extends: .cs_common + variables: + # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image + # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes + CLAIR_DB_IMAGE_TAG: "latest" + CLAIR_DB_IMAGE: "$SECURE_ANALYZERS_PREFIX/clair-vulnerabilities-db:$CLAIR_DB_IMAGE_TAG" + CS_PROJECT: 'klar' services: - name: $CLAIR_DB_IMAGE alias: clair-vulnerabilities-db script: - /analyzer run + rules: + - if: $CONTAINER_SCANNING_DISABLED + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ && + $CS_MAJOR_VERSION =~ /^[0-3]$/ + +container_scanning_new: + extends: .cs_common + variables: + CS_PROJECT: 'container-scanning' + script: + - gtcs scan artifacts: - reports: - container_scanning: gl-container-scanning-report.json - dependencies: [] + paths: [gl-container-scanning-report.json] rules: - if: $CONTAINER_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ + $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ && + $CS_MAJOR_VERSION !~ /^[0-3]$/ diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index fc1acd09714..533f8bb25f8 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -1,3 +1,16 @@ +# To use this template, add the following to your .gitlab-ci.yml file: +# +# include: +# template: DAST.latest.gitlab-ci.yml +# +# You also need to add a `dast` stage to your `stages:` configuration. A sample configuration for DAST: +# +# stages: +# - build +# - test +# - deploy +# - dast + # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ # Configure the scanning tool through the environment variables. @@ -9,6 +22,19 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + # + DAST_API_PROFILE: Full + DAST_API_VERSION: latest + DAST_API_CONFIG: .gitlab-dast-api.yml + DAST_API_TIMEOUT: 30 + DAST_API_REPORT: gl-dast-api-report.json + DAST_API_REPORT_ASSET_PATH: assets + # + # Wait up to 5 minutes for API Security and target url to become + # available (non 500 response to HTTP(s)) + DAST_API_SERVICE_START_TIMEOUT: "300" + # + DAST_API_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${DAST_API_VERSION}-engine dast: stage: dast @@ -25,6 +51,11 @@ dast: reports: dast: gl-dast-report.json rules: + - if: $DAST_API_BETA && ( $DAST_API_SPECIFICATION || + $DAST_API_OPENAPI || + $DAST_API_POSTMAN_COLLECTION || + $DAST_API_HAR ) + when: never - if: $DAST_DISABLED when: never - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && @@ -40,4 +71,72 @@ dast: - if: $CI_COMMIT_BRANCH && $DAST_WEBSITE - if: $CI_COMMIT_BRANCH && + $DAST_API_BETA == null && $DAST_API_SPECIFICATION + +dast_api: + stage: dast + image: + name: $DAST_API_IMAGE + entrypoint: ["/bin/bash", "-l", "-c"] + variables: + API_SECURITY_MODE: DAST + DAST_API_NEW_REPORT: 1 + DAST_API_PROJECT: $CI_PROJECT_PATH + DAST_API_API: http://127.0.0.1:5000 + DAST_API_LOG_SCANNER: gl-dast-api-scanner.log + TZ: America/Los_Angeles + allow_failure: true + rules: + - if: $DAST_API_BETA == null + when: never + - if: $DAST_DISABLED + when: never + - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && + $REVIEW_DISABLED && + $DAST_API_SPECIFICATION == null && + $DAST_API_OPENAPI == null && + $DAST_API_POSTMAN_COLLECTION == null && + $DAST_API_HAR == null + when: never + - if: $DAST_API_SPECIFICATION == null && + $DAST_API_OPENAPI == null && + $DAST_API_POSTMAN_COLLECTION == null && + $DAST_API_HAR == null + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdast\b/ + script: + # + # Run user provided pre-script + - sh -c "$DAST_API_PRE_SCRIPT" + # + # Make sure asset path exists + - mkdir -p $DAST_API_REPORT_ASSET_PATH + # + # Start API Security background process + - dotnet /peach/Peach.Web.dll &> $DAST_API_LOG_SCANNER & + - APISEC_PID=$! + # + # Start scanning + - worker-entry + # + # Run user provided post-script + - sh -c "$DAST_API_POST_SCRIPT" + # + # Shutdown API Security + - kill $APISEC_PID + - wait $APISEC_PID + # + artifacts: + when: always + paths: + - $DAST_API_REPORT_ASSET_PATH + - $DAST_API_REPORT + - $DAST_API_LOG_SCANNER + - gl-*.log + reports: + dast: $DAST_API_REPORT diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 9693a4fbca2..3ebccfbba4a 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -160,7 +160,7 @@ mobsf-android-sast: services: # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile # Unfortunately, we need to keep track of mobsf version in 2 different places for now. - - name: opensecurity/mobile-security-framework-mobsf:v3.3.3 + - name: opensecurity/mobile-security-framework-mobsf:v3.4.0 alias: mobsf image: name: "$SAST_ANALYZER_IMAGE" @@ -186,7 +186,7 @@ mobsf-ios-sast: services: # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile # Unfortunately, we need to keep track of mobsf version in 2 different places for now. - - name: opensecurity/mobile-security-framework-mobsf:v3.3.3 + - name: opensecurity/mobile-security-framework-mobsf:v3.4.0 alias: mobsf image: name: "$SAST_ANALYZER_IMAGE" @@ -303,6 +303,10 @@ semgrep-sast: $SAST_EXPERIMENTAL_FEATURES == 'true' exists: - '**/*.py' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' sobelow-sast: extends: .sast-analyzer @@ -348,3 +352,4 @@ spotbugs-sast: - '**/*.groovy' - '**/*.java' - '**/*.scala' + - '**/*.kt' diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index e591e3cc1e2..404d4a4c6db 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -18,9 +18,32 @@ performance: - docker:stable-dind script: - mkdir gitlab-exporter + # Busybox wget does not support proxied HTTPS, get the real thing. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. + - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS + - | + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } + - | + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS - mv sitespeed-results/data/performance.json browser-performance.json artifacts: paths: diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 3258d965c93..c25c4339c35 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -11,7 +11,7 @@ module Gitlab LOCK_SLEEP = 0.001.seconds WATCH_FLAG_TTL = 10.seconds - UPDATE_FREQUENCY_DEFAULT = 30.seconds + UPDATE_FREQUENCY_DEFAULT = 60.seconds UPDATE_FREQUENCY_WHEN_BEING_WATCHED = 3.seconds ArchiveError = Class.new(StandardError) @@ -93,6 +93,10 @@ module Gitlab end end + def erase_trace_chunks! + job.trace_chunks.fast_destroy_all # Destroy chunks of a live trace + end + def erase! ## # Erase the archived trace @@ -100,7 +104,7 @@ module Gitlab ## # Erase the live trace - job.trace_chunks.fast_destroy_all # Destroy chunks of a live trace + erase_trace_chunks! FileUtils.rm_f(current_path) if current_path # Remove a trace file of a live trace job.erase_old_trace! if job.has_old_trace? # Remove a trace in database of a live trace ensure @@ -114,7 +118,11 @@ module Gitlab end def update_interval - being_watched? ? UPDATE_FREQUENCY_WHEN_BEING_WATCHED : UPDATE_FREQUENCY_DEFAULT + if being_watched? + UPDATE_FREQUENCY_WHEN_BEING_WATCHED + else + UPDATE_FREQUENCY_DEFAULT + end end def being_watched! @@ -176,9 +184,14 @@ module Gitlab end def unsafe_archive! - raise AlreadyArchivedError, 'Could not archive again' if trace_artifact raise ArchiveError, 'Job is not finished yet' unless job.complete? + if trace_artifact + unsafe_trace_cleanup! if Feature.enabled?(:erase_traces_from_already_archived_jobs_when_archiving_again, job.project, default_enabled: :yaml) + + raise AlreadyArchivedError, 'Could not archive again' + end + if job.trace_chunks.any? Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream| archive_stream!(stream) @@ -197,6 +210,18 @@ module Gitlab end end + def unsafe_trace_cleanup! + return unless trace_artifact + + if trace_artifact.archived_trace_exists? + # An archive already exists, so make sure to remove the trace chunks + erase_trace_chunks! + else + # An archive already exists, but its associated file does not, so remove it + trace_artifact.destroy! + end + end + def in_write_lock(&blk) lock_key = "trace:write:lock:#{job.id}" in_lock(lock_key, ttl: LOCK_TTL, retries: LOCK_RETRIES, sleep_sec: LOCK_SLEEP, &blk) diff --git a/lib/gitlab/ci/variables/helpers.rb b/lib/gitlab/ci/variables/helpers.rb index e2a54f90ecb..3a62f01e2e3 100644 --- a/lib/gitlab/ci/variables/helpers.rb +++ b/lib/gitlab/ci/variables/helpers.rb @@ -23,7 +23,21 @@ module Gitlab def transform_from_yaml_variables(vars) return vars.stringify_keys if vars.is_a?(Hash) - vars.to_a.map { |var| [var[:key].to_s, var[:value]] }.to_h + vars.to_a.to_h { |var| [var[:key].to_s, var[:value]] } + end + + def inherit_yaml_variables(from:, to:, inheritance:) + merge_variables(apply_inheritance(from, inheritance), to) + end + + private + + def apply_inheritance(variables, inheritance) + case inheritance + when true then variables + when false then {} + when Array then variables.select { |var| inheritance.include?(var[:key]) } + end end end end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 3459b69bebc..f96a6629849 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -38,11 +38,12 @@ module Gitlab .map { |job| build_attributes(job[:name]) } end - def workflow_attributes - { - rules: hash_config.dig(:workflow, :rules), - yaml_variables: transform_to_yaml_variables(variables) - } + def workflow_rules + @workflow_rules ||= hash_config.dig(:workflow, :rules) + end + + def root_variables + @root_variables ||= transform_to_yaml_variables(variables) end def jobs @@ -68,7 +69,9 @@ module Gitlab when: job[:when] || 'on_success', environment: job[:environment_name], coverage_regex: job[:coverage], - yaml_variables: transform_to_yaml_variables(job[:variables]), + yaml_variables: transform_to_yaml_variables(job[:variables]), # https://gitlab.com/gitlab-org/gitlab/-/issues/300581 + job_variables: transform_to_yaml_variables(job[:job_variables]), + root_variables_inheritance: job[:root_variables_inheritance], needs_attributes: job.dig(:needs, :job), interruptible: job[:interruptible], only: job[:only], @@ -101,7 +104,7 @@ module Gitlab end def merged_yaml - @ci_config&.to_hash&.to_yaml + @ci_config&.to_hash&.deep_stringify_keys&.to_yaml end def variables_with_data |