diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-03-16 21:18:33 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-03-16 21:18:33 +0300 |
| commit | f64a639bcfa1fc2bc89ca7db268f594306edfd7c (patch) | |
| tree | a2c3c2ebcc3b45e596949db485d6ed18ffaacfa1 /lib/gitlab/ci | |
| parent | bfbc3e0d6583ea1a91f627528bedc3d65ba4b10f (diff) | |
Add latest changes from gitlab-org/gitlab@13-10-stable-eev13.10.0-rc40
Diffstat (limited to 'lib/gitlab/ci')
63 files changed, 840 insertions, 293 deletions
diff --git a/lib/gitlab/ci/artifacts/metrics.rb b/lib/gitlab/ci/artifacts/metrics.rb new file mode 100644 index 00000000000..656f4d2cc13 --- /dev/null +++ b/lib/gitlab/ci/artifacts/metrics.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Artifacts + class Metrics + include Gitlab::Utils::StrongMemoize + + def increment_destroyed_artifacts(size) + destroyed_artifacts_counter.increment({}, size.to_i) + end + + private + + def destroyed_artifacts_counter + strong_memoize(:destroyed_artifacts_counter) do + name = :destroyed_job_artifacts_count_total + comment = 'Counter of destroyed expired job artifacts' + + ::Gitlab::Metrics.counter(name, comment) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/cache.rb b/lib/gitlab/ci/build/cache.rb new file mode 100644 index 00000000000..4fcb5168847 --- /dev/null +++ b/lib/gitlab/ci/build/cache.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Cache + include ::Gitlab::Utils::StrongMemoize + + def initialize(cache, pipeline) + if multiple_cache_per_job? + cache = Array.wrap(cache) + @cache = cache.map do |cache| + Gitlab::Ci::Pipeline::Seed::Build::Cache + .new(pipeline, cache) + end + else + @cache = Gitlab::Ci::Pipeline::Seed::Build::Cache + .new(pipeline, cache) + end + end + + def cache_attributes + strong_memoize(:cache_attributes) do + if multiple_cache_per_job? + if @cache.empty? + {} + else + { options: { cache: @cache.map(&:attributes) } } + end + else + @cache.build_attributes + end + end + end + + private + + def multiple_cache_per_job? + strong_memoize(:multiple_cache_per_job) do + ::Gitlab::Ci::Features.multiple_cache_per_job? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/context/build.rb b/lib/gitlab/ci/build/context/build.rb index dfd86d3ad72..641aa71fb4e 100644 --- a/lib/gitlab/ci/build/context/build.rb +++ b/lib/gitlab/ci/build/context/build.rb @@ -21,7 +21,7 @@ module Gitlab # to the CI variables to evaluate rules before we persist a Build # with the result. We should refactor away the extra Build.new, # but be able to get CI Variables directly from the Seed::Build. - stub_build.scoped_variables_hash + stub_build.scoped_variables end end diff --git a/lib/gitlab/ci/build/context/global.rb b/lib/gitlab/ci/build/context/global.rb index fdd3ac358d5..dd0bc54d8b2 100644 --- a/lib/gitlab/ci/build/context/global.rb +++ b/lib/gitlab/ci/build/context/global.rb @@ -19,8 +19,7 @@ module Gitlab # to the CI variables to evaluate workflow:rules # with the result. We should refactor away the extra Build.new, # but be able to get CI Variables directly from the Seed::Build. - stub_build.scoped_variables_hash - .reject { |key, _value| key =~ /\ACI_(JOB|BUILD)/ } + stub_build.scoped_variables.reject { |var| var[:key] =~ /\ACI_(JOB|BUILD)/ } end end diff --git a/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb b/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb index b6ac06cfb53..76eec2172b1 100644 --- a/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb +++ b/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb @@ -7,7 +7,7 @@ module Gitlab module Registry class DependencyProxy < GitlabRegistry def url - "#{Gitlab.config.gitlab.host}:#{Gitlab.config.gitlab.port}" + Gitlab.host_with_port end def valid? diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index dbb48a81030..d3f030c3b36 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -99,10 +99,18 @@ module Gitlab initial_config end + def find_sha(project) + branches = project&.repository&.branches || [] + + unless branches.empty? + project.repository.root_ref_sha + end + end + def build_context(project:, sha:, user:, parent_pipeline:) Config::External::Context.new( project: project, - sha: sha || project&.repository&.root_ref_sha, + sha: sha || find_sha(project), user: user, parent_pipeline: parent_pipeline, variables: project&.predefined_variables&.to_runner_variables) diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb index e8e2eef281e..73742298628 100644 --- a/lib/gitlab/ci/config/entry/bridge.rb +++ b/lib/gitlab/ci/config/entry/bridge.rb @@ -12,7 +12,7 @@ module Gitlab include ::Gitlab::Ci::Config::Entry::Processable ALLOWED_WHEN = %w[on_success on_failure always manual].freeze - ALLOWED_KEYS = %i[trigger].freeze + ALLOWED_KEYS = %i[trigger parallel].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS @@ -48,7 +48,12 @@ module Gitlab inherit: false, metadata: { allowed_needs: %i[job bridge] } - attributes :when, :allow_failure + entry :parallel, Entry::Product::Parallel, + description: 'Parallel configuration for this job.', + inherit: false, + metadata: { allowed_strategies: %i(matrix) } + + attributes :when, :allow_failure, :parallel def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -66,7 +71,8 @@ module Gitlab needs: (needs_value if needs_defined?), ignore: ignored?, when: self.when, - scheduling_type: needs_defined? && !bridge_needs ? :dag : :stage + scheduling_type: needs_defined? && !bridge_needs ? :dag : :stage, + parallel: has_parallel? ? parallel_value : nil ).compact end diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index 6b036182706..cf599ce5294 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -7,52 +7,90 @@ module Gitlab ## # Entry that represents a cache configuration # - class Cache < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Configurable - include ::Gitlab::Config::Entry::Validatable - include ::Gitlab::Config::Entry::Attributable - - ALLOWED_KEYS = %i[key untracked paths when policy].freeze - ALLOWED_POLICY = %w[pull-push push pull].freeze - DEFAULT_POLICY = 'pull-push' - ALLOWED_WHEN = %w[on_success on_failure always].freeze - DEFAULT_WHEN = 'on_success' - - validations do - validates :config, type: Hash, allowed_keys: ALLOWED_KEYS - validates :policy, - inclusion: { in: ALLOWED_POLICY, message: 'should be pull-push, push, or pull' }, - allow_blank: true - - with_options allow_nil: true do - validates :when, - inclusion: { - in: ALLOWED_WHEN, - message: 'should be on_success, on_failure or always' - } + class Cache < ::Gitlab::Config::Entry::Simplifiable + strategy :Caches, if: -> (config) { Feature.enabled?(:multiple_cache_per_job) } + strategy :Cache, if: -> (config) { Feature.disabled?(:multiple_cache_per_job) } + + class Caches < ::Gitlab::Config::Entry::ComposableArray + include ::Gitlab::Config::Entry::Validatable + + MULTIPLE_CACHE_LIMIT = 4 + + validations do + validates :config, presence: true + + validate do + unless config.is_a?(Hash) || config.is_a?(Array) + errors.add(:config, 'can only be a Hash or an Array') + end + + if config.is_a?(Array) && config.count > MULTIPLE_CACHE_LIMIT + errors.add(:config, "no more than #{MULTIPLE_CACHE_LIMIT} caches can be created") + end + end + end + + def initialize(*args) + super + + @key = nil + end + + def composable_class + Entry::Cache::Cache end end - entry :key, Entry::Key, - description: 'Cache key used to define a cache affinity.' + class Cache < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[key untracked paths when policy].freeze + ALLOWED_POLICY = %w[pull-push push pull].freeze + DEFAULT_POLICY = 'pull-push' + ALLOWED_WHEN = %w[on_success on_failure always].freeze + DEFAULT_WHEN = 'on_success' + + validations do + validates :config, type: Hash, allowed_keys: ALLOWED_KEYS + validates :policy, + inclusion: { in: ALLOWED_POLICY, message: 'should be pull-push, push, or pull' }, + allow_blank: true + + with_options allow_nil: true do + validates :when, + inclusion: { + in: ALLOWED_WHEN, + message: 'should be on_success, on_failure or always' + } + end + end - entry :untracked, ::Gitlab::Config::Entry::Boolean, - description: 'Cache all untracked files.' + entry :key, Entry::Key, + description: 'Cache key used to define a cache affinity.' - entry :paths, Entry::Paths, - description: 'Specify which paths should be cached across builds.' + entry :untracked, ::Gitlab::Config::Entry::Boolean, + description: 'Cache all untracked files.' - attributes :policy, :when + entry :paths, Entry::Paths, + description: 'Specify which paths should be cached across builds.' - def value - result = super + attributes :policy, :when - result[:key] = key_value - result[:policy] = policy || DEFAULT_POLICY - # Use self.when to avoid conflict with reserved word - result[:when] = self.when || DEFAULT_WHEN + def value + result = super + + result[:key] = key_value + result[:policy] = policy || DEFAULT_POLICY + # Use self.when to avoid conflict with reserved word + result[:when] = self.when || DEFAULT_WHEN + + result + end + end - result + class UnknownStrategy < ::Gitlab::Config::Entry::Node end end end diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index 64e6d48133f..2066e9be3b1 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -10,7 +10,7 @@ module Gitlab class Environment < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Configurable - ALLOWED_KEYS = %i[name url action on_stop auto_stop_in kubernetes].freeze + ALLOWED_KEYS = %i[name url action on_stop auto_stop_in kubernetes deployment_tier].freeze entry :kubernetes, Entry::Kubernetes, description: 'Kubernetes deployment configuration.' @@ -47,6 +47,11 @@ module Gitlab inclusion: { in: %w[start stop prepare], message: 'should be start, stop or prepare' }, allow_nil: true + validates :deployment_tier, + type: String, + inclusion: { in: ::Environment.tiers.keys, message: "must be one of #{::Environment.tiers.keys.join(', ')}" }, + allow_nil: true + validates :on_stop, type: String, allow_nil: true validates :kubernetes, type: Hash, allow_nil: true validates :auto_stop_in, duration: true, allow_nil: true @@ -85,6 +90,10 @@ module Gitlab value[:auto_stop_in] end + def deployment_tier + value[:deployment_tier] + end + def value case @config when String then { name: @config, action: 'start' } diff --git a/lib/gitlab/ci/config/entry/need.rb b/lib/gitlab/ci/config/entry/need.rb index 46191eca842..b3cf0f9e0fd 100644 --- a/lib/gitlab/ci/config/entry/need.rb +++ b/lib/gitlab/ci/config/entry/need.rb @@ -35,7 +35,14 @@ module Gitlab end def value - { name: @config, artifacts: true } + if ::Feature.enabled?(:ci_needs_optional, default_enabled: :yaml) + { name: @config, + artifacts: true, + optional: false } + else + { name: @config, + artifacts: true } + end end end @@ -43,14 +50,15 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[job artifacts].freeze - attributes :job, :artifacts + ALLOWED_KEYS = %i[job artifacts optional].freeze + attributes :job, :artifacts, :optional validations do validates :config, presence: true validates :config, allowed_keys: ALLOWED_KEYS validates :job, type: String, presence: true validates :artifacts, boolean: true, allow_nil: true + validates :optional, boolean: true, allow_nil: true end def type @@ -58,7 +66,14 @@ module Gitlab end def value - { name: job, artifacts: artifacts || artifacts.nil? } + if ::Feature.enabled?(:ci_needs_optional, default_enabled: :yaml) + { name: job, + artifacts: artifacts || artifacts.nil?, + optional: !!optional } + else + { name: job, + artifacts: artifacts || artifacts.nil? } + end end end diff --git a/lib/gitlab/ci/config/entry/product/parallel.rb b/lib/gitlab/ci/config/entry/product/parallel.rb index cd9eabbbc66..5c78a8f68c7 100644 --- a/lib/gitlab/ci/config/entry/product/parallel.rb +++ b/lib/gitlab/ci/config/entry/product/parallel.rb @@ -22,6 +22,13 @@ module Gitlab greater_than_or_equal_to: 2, less_than_or_equal_to: Entry::Product::Parallel::PARALLEL_LIMIT }, allow_nil: true + + validate do + next unless opt(:allowed_strategies) + next if opt(:allowed_strategies).include?(:numeric) + + errors.add(:config, 'cannot use "parallel: <number>".') + end end def value @@ -38,6 +45,13 @@ module Gitlab validations do validates :config, allowed_keys: PERMITTED_KEYS validates :config, required_keys: PERMITTED_KEYS + + validate do + next unless opt(:allowed_strategies) + next if opt(:allowed_strategies).include?(:matrix) + + errors.add(:config, 'cannot use "parallel: matrix".') + end end entry :matrix, Entry::Product::Matrix, diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index d1a366125ef..c811ef211d6 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -38,10 +38,6 @@ module Gitlab ::Feature.enabled?(:ci_disallow_to_create_merge_request_pipelines_in_target_project, target_project) end - def self.project_transactionless_destroy?(project) - Feature.enabled?(:project_transactionless_destroy, project, default_enabled: false) - end - def self.trace_overwrite? ::Feature.enabled?(:ci_trace_overwrite, type: :ops, default_enabled: false) end @@ -55,14 +51,6 @@ module Gitlab ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) end - def self.ci_pipeline_editor_page_enabled?(project) - ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: :yaml) - end - - def self.rules_variables_enabled?(project) - ::Feature.enabled?(:ci_rules_variables, project, default_enabled: true) - end - def self.validate_build_dependencies?(project) ::Feature.enabled?(:ci_validate_build_dependencies, project, default_enabled: :yaml) && ::Feature.disabled?(:ci_validate_build_dependencies_override, project) @@ -76,8 +64,12 @@ module Gitlab ::Feature.enabled?(:codequality_backend_comparison, project, default_enabled: :yaml) end - def self.use_coverage_data_new_finder?(record) - ::Feature.enabled?(:coverage_data_new_finder, record, default_enabled: :yaml) + def self.multiple_cache_per_job? + ::Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml) + end + + def self.ci_commit_pipeline_mini_graph_vue_enabled?(project) + ::Feature.enabled?(:ci_commit_pipeline_mini_graph_vue, project, default_enabled: :yaml) end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index 0870c74053a..af06e124736 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -60,7 +60,7 @@ module Gitlab ref_protected: build.protected.to_s } - if include_environment_claims? + if environment.present? fields.merge!( environment: environment.name, environment_protected: environment_protected?.to_s @@ -119,10 +119,6 @@ module Gitlab def environment_protected? false # Overridden in EE end - - def include_environment_claims? - Feature.enabled?(:ci_jwt_include_environment) && environment.present? - end end end end diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index 364e67db02b..4a7c11ee26e 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -21,7 +21,7 @@ module Gitlab def initialize(project:, current_user:, sha: nil) @project = project @current_user = current_user - @sha = sha || project.repository.commit.sha + @sha = sha || project.repository.commit&.sha end def validate(content, dry_run: false) diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index f0214bb4e38..1c0dfbdbee3 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -10,7 +10,7 @@ module Gitlab def perform! return unless project.auto_cancel_pending_pipelines? - Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines) do |cancelables| + Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines, name: 'cancel_pending_pipelines') do |cancelables| cancelables.find_each do |cancelable| cancelable.auto_cancel_running(pipeline) end diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index 0e81e1bd34c..5f3310dd668 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -7,9 +7,9 @@ module Gitlab class Statement StatementError = Class.new(Expression::ExpressionError) - def initialize(statement, variables = {}) + def initialize(statement, variables = nil) @lexer = Expression::Lexer.new(statement) - @variables = variables.with_indifferent_access + @variables = variables&.to_hash end def parse_tree diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 3770bb4b328..11b01822e4b 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -28,8 +28,8 @@ module Gitlab .fabricate(attributes.delete(:except)) @rules = Gitlab::Ci::Build::Rules .new(attributes.delete(:rules), default_when: 'on_success') - @cache = Seed::Build::Cache - .new(pipeline, attributes.delete(:cache)) + @cache = Gitlab::Ci::Build::Cache + .new(attributes.delete(:cache), pipeline) end def name @@ -52,7 +52,7 @@ module Gitlab return unless included? strong_memoize(:errors) do - needs_errors + [needs_errors, variable_expansion_errors].compact.flatten end end @@ -61,7 +61,7 @@ module Gitlab .deep_merge(pipeline_attributes) .deep_merge(rules_attributes) .deep_merge(allow_failure_criteria_attributes) - .deep_merge(cache_attributes) + .deep_merge(@cache.cache_attributes) end def bridge? @@ -141,6 +141,8 @@ module Gitlab end @needs_attributes.flat_map do |need| + next if ::Feature.enabled?(:ci_needs_optional, default_enabled: :yaml) && need[:optional] + result = @previous_stages.any? do |stage| stage.seeds_names.include?(need[:name]) end @@ -153,6 +155,12 @@ module Gitlab @pipeline.project.actual_limits.ci_needs_size_limit end + def variable_expansion_errors + expanded_collection = evaluate_context.variables.sort_and_expand_all(@pipeline.project) + errors = expanded_collection.errors + ["#{name}: #{errors}"] if errors + end + def pipeline_attributes { pipeline: @pipeline, @@ -169,15 +177,11 @@ module Gitlab strong_memoize(:rules_attributes) do next {} unless @using_rules - if ::Gitlab::Ci::Features.rules_variables_enabled?(@pipeline.project) - rules_variables_result = ::Gitlab::Ci::Variables::Helpers.merge_variables( - @seed_attributes[:yaml_variables], rules_result.variables - ) + rules_variables_result = ::Gitlab::Ci::Variables::Helpers.merge_variables( + @seed_attributes[:yaml_variables], rules_result.variables + ) - rules_result.build_attributes.merge(yaml_variables: rules_variables_result) - else - rules_result.build_attributes - end + rules_result.build_attributes.merge(yaml_variables: rules_variables_result) end end @@ -193,12 +197,6 @@ module Gitlab end end - def cache_attributes - strong_memoize(:cache_attributes) do - @cache.build_attributes - end - end - # If a job uses `allow_failure:exit_codes` and `rules:allow_failure` # we need to prevent the exit codes from being persisted because they # would break the behavior defined by `rules:allow_failure`. diff --git a/lib/gitlab/ci/pipeline/seed/build/cache.rb b/lib/gitlab/ci/pipeline/seed/build/cache.rb index 8d6fe13c3b9..78ffaaa7e81 100644 --- a/lib/gitlab/ci/pipeline/seed/build/cache.rb +++ b/lib/gitlab/ci/pipeline/seed/build/cache.rb @@ -18,18 +18,18 @@ module Gitlab raise ArgumentError, "unknown cache keys: #{local_cache.keys}" if local_cache.any? end - def build_attributes + def attributes { - options: { - cache: { - key: key_string, - paths: @paths, - policy: @policy, - untracked: @untracked, - when: @when - }.compact.presence - }.compact - } + key: key_string, + paths: @paths, + policy: @policy, + untracked: @untracked, + when: @when + }.compact + end + + def build_attributes + { options: { cache: attributes.presence }.compact } end private diff --git a/lib/gitlab/ci/pipeline/seed/environment.rb b/lib/gitlab/ci/pipeline/seed/environment.rb index 5dff0788ec9..c8795840e5f 100644 --- a/lib/gitlab/ci/pipeline/seed/environment.rb +++ b/lib/gitlab/ci/pipeline/seed/environment.rb @@ -13,7 +13,9 @@ module Gitlab def to_resource environments.safe_find_or_create_by(name: expanded_environment_name) do |environment| + # Initialize the attributes at creation environment.auto_stop_in = auto_stop_in + environment.tier = deployment_tier end end @@ -27,6 +29,10 @@ module Gitlab job.environment_auto_stop_in end + def deployment_tier + job.environment_deployment_tier + end + def expanded_environment_name job.expanded_environment_name end diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb new file mode 100644 index 00000000000..5398c19e536 --- /dev/null +++ b/lib/gitlab/ci/queue/metrics.rb @@ -0,0 +1,210 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Queue + class Metrics + extend Gitlab::Utils::StrongMemoize + + QUEUE_DURATION_SECONDS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze + QUEUE_ACTIVE_RUNNERS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze + QUEUE_DEPTH_TOTAL_BUCKETS = [1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze + QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000].freeze + QUEUE_ITERATION_DURATION_SECONDS_BUCKETS = [0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze + + METRICS_SHARD_TAG_PREFIX = 'metrics_shard::' + DEFAULT_METRICS_SHARD = 'default' + JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET = 5.freeze + + OPERATION_COUNTERS = [ + :build_can_pick, + :build_not_pick, + :build_not_pending, + :build_temporary_locked, + :build_conflict_lock, + :build_conflict_exception, + :build_conflict_transition, + :queue_attempt, + :queue_conflict, + :queue_iteration, + :queue_depth_limit, + :queue_replication_lag, + :runner_pre_assign_checks_failed, + :runner_pre_assign_checks_success, + :runner_queue_tick + ].to_set.freeze + + QUEUE_DEPTH_HISTOGRAMS = [ + :found, + :not_found, + :conflict + ].to_set.freeze + + attr_reader :runner + + def initialize(runner) + @runner = runner + end + + def register_failure + self.class.failed_attempt_counter.increment + self.class.attempt_counter.increment + end + + def register_success(job) + labels = { shared_runner: runner.instance_type?, + jobs_running_for_project: jobs_running_for_project(job), + shard: DEFAULT_METRICS_SHARD } + + if runner.instance_type? + shard = runner.tag_list.sort.find { |name| name.starts_with?(METRICS_SHARD_TAG_PREFIX) } + labels[:shard] = shard.gsub(METRICS_SHARD_TAG_PREFIX, '') if shard + end + + self.class.job_queue_duration_seconds.observe(labels, Time.current - job.queued_at) unless job.queued_at.nil? + self.class.attempt_counter.increment + end + + # rubocop: disable CodeReuse/ActiveRecord + def jobs_running_for_project(job) + return '+Inf' unless runner.instance_type? + + # excluding currently started job + running_jobs_count = job.project.builds.running.where(runner: ::Ci::Runner.instance_type) + .limit(JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET + 1).count - 1 + running_jobs_count < JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET ? running_jobs_count : "#{JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET}+" + end + # rubocop: enable CodeReuse/ActiveRecord + + def increment_queue_operation(operation) + if !Rails.env.production? && !OPERATION_COUNTERS.include?(operation) + raise ArgumentError, "unknown queue operation: #{operation}" + end + + self.class.queue_operations_total.increment(operation: operation) + end + + def observe_queue_depth(queue, size) + return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + + if !Rails.env.production? && !QUEUE_DEPTH_HISTOGRAMS.include?(queue) + raise ArgumentError, "unknown queue depth label: #{queue}" + end + + self.class.queue_depth_total.observe({ queue: queue }, size.to_f) + end + + def observe_queue_size(size_proc) + return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + + self.class.queue_size_total.observe({}, size_proc.call.to_f) + end + + def observe_queue_time + start_time = ::Gitlab::Metrics::System.monotonic_time + + result = yield + + return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + + seconds = ::Gitlab::Metrics::System.monotonic_time - start_time + self.class.queue_iteration_duration_seconds.observe({}, seconds.to_f) + + result + end + + def self.observe_active_runners(runners_proc) + return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + + queue_active_runners_total.observe({}, runners_proc.call.to_f) + end + + def self.increment_runner_tick(runner) + self.new(runner).increment_queue_operation(:runner_queue_tick) + end + + def self.failed_attempt_counter + strong_memoize(:failed_attempt_counter) do + name = :job_register_attempts_failed_total + comment = 'Counts the times a runner tries to register a job' + + Gitlab::Metrics.counter(name, comment) + end + end + + def self.attempt_counter + strong_memoize(:attempt_counter) do + name = :job_register_attempts_total + comment = 'Counts the times a runner tries to register a job' + + Gitlab::Metrics.counter(name, comment) + end + end + + def self.job_queue_duration_seconds + strong_memoize(:job_queue_duration_seconds) do + name = :job_queue_duration_seconds + comment = 'Request handling execution time' + buckets = QUEUE_DURATION_SECONDS_BUCKETS + labels = {} + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def self.queue_operations_total + strong_memoize(:queue_operations_total) do + name = :gitlab_ci_queue_operations_total + comment = 'Counts all the operations that are happening inside a queue' + + Gitlab::Metrics.counter(name, comment) + end + end + + def self.queue_depth_total + strong_memoize(:queue_depth_total) do + name = :gitlab_ci_queue_depth_total + comment = 'Size of a CI/CD builds queue in relation to the operation result' + buckets = QUEUE_DEPTH_TOTAL_BUCKETS + labels = {} + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def self.queue_size_total + strong_memoize(:queue_size_total) do + name = :gitlab_ci_queue_size_total + comment = 'Size of initialized CI/CD builds queue' + buckets = QUEUE_SIZE_TOTAL_BUCKETS + labels = {} + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def self.queue_iteration_duration_seconds + strong_memoize(:queue_iteration_duration_seconds) do + name = :gitlab_ci_queue_iteration_duration_seconds + comment = 'Time it takes to find a build in CI/CD queue' + buckets = QUEUE_ITERATION_DURATION_SECONDS_BUCKETS + labels = {} + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def self.queue_active_runners_total + strong_memoize(:queue_active_runners_total) do + name = :gitlab_ci_queue_active_runners_total + comment = 'The amount of active runners that can process queue in a project' + buckets = QUEUE_ACTIVE_RUNNERS_BUCKETS + labels = {} + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/codequality_reports_comparer.rb b/lib/gitlab/ci/reports/codequality_reports_comparer.rb index 88e02cd9004..10748b8ca02 100644 --- a/lib/gitlab/ci/reports/codequality_reports_comparer.rb +++ b/lib/gitlab/ci/reports/codequality_reports_comparer.rb @@ -5,7 +5,7 @@ module Gitlab module Reports class CodequalityReportsComparer < ReportsComparer def initialize(base_report, head_report) - @base_report = base_report || CodequalityReports.new + @base_report = base_report @head_report = head_report end @@ -15,12 +15,16 @@ module Gitlab def existing_errors strong_memoize(:existing_errors) do + next [] if not_found? + base_report.all_degradations & head_report.all_degradations end end def new_errors strong_memoize(:new_errors) do + next [] if not_found? + fingerprints = head_report.degradations.keys - base_report.degradations.keys head_report.degradations.fetch_values(*fingerprints) end @@ -28,6 +32,8 @@ module Gitlab def resolved_errors strong_memoize(:resolved_errors) do + next [] if not_found? + fingerprints = base_report.degradations.keys - head_report.degradations.keys base_report.degradations.fetch_values(*fingerprints) end diff --git a/lib/gitlab/ci/reports/reports_comparer.rb b/lib/gitlab/ci/reports/reports_comparer.rb index d413d3a74f6..16a7f6478b7 100644 --- a/lib/gitlab/ci/reports/reports_comparer.rb +++ b/lib/gitlab/ci/reports/reports_comparer.rb @@ -8,6 +8,7 @@ module Gitlab STATUS_SUCCESS = 'success' STATUS_FAILED = 'failed' + STATUS_NOT_FOUND = 'not_found' attr_reader :base_report, :head_report @@ -17,7 +18,13 @@ module Gitlab end def status - success? ? STATUS_SUCCESS : STATUS_FAILED + if base_report.nil? || head_report.nil? + STATUS_NOT_FOUND + elsif success? + STATUS_SUCCESS + else + STATUS_FAILED + end end def success? @@ -47,6 +54,10 @@ module Gitlab def total_count existing_errors.size + new_errors.size end + + def not_found? + status == STATUS_NOT_FOUND + end end end end diff --git a/lib/gitlab/ci/reports/test_suite_summary.rb b/lib/gitlab/ci/reports/test_suite_summary.rb index 32b06d0ad49..461aefc6fa9 100644 --- a/lib/gitlab/ci/reports/test_suite_summary.rb +++ b/lib/gitlab/ci/reports/test_suite_summary.rb @@ -4,6 +4,8 @@ module Gitlab module Ci module Reports class TestSuiteSummary + include Gitlab::Utils::StrongMemoize + def initialize(build_report_results) @build_report_results = build_report_results end @@ -42,6 +44,12 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord + def suite_error + strong_memoize(:suite_error) do + @build_report_results.map(&:suite_error).compact.first + end + end + def to_h { time: total_time, @@ -49,7 +57,8 @@ module Gitlab success: success_count, failed: failed_count, skipped: skipped_count, - error: error_count + error: error_count, + suite_error: suite_error } end end diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb index 9a4f5644f7d..5368e020a50 100644 --- a/lib/gitlab/ci/status/composite.rb +++ b/lib/gitlab/ci/status/composite.rb @@ -7,7 +7,10 @@ module Gitlab include Gitlab::Utils::StrongMemoize # This class accepts an array of arrays/hashes/or objects - def initialize(all_statuses, with_allow_failure: true, dag: false) + # + # The parameter `project` is only used for the feature flag check, and will be removed with + # https://gitlab.com/gitlab-org/gitlab/-/issues/321972 + def initialize(all_statuses, with_allow_failure: true, dag: false, project: nil) unless all_statuses.respond_to?(:pluck) raise ArgumentError, "all_statuses needs to respond to `.pluck`" end @@ -16,6 +19,7 @@ module Gitlab @status_key = 0 @allow_failure_key = 1 if with_allow_failure @dag = dag + @project = project consume_all_statuses(all_statuses) end @@ -32,7 +36,7 @@ module Gitlab return if none? strong_memoize(:status) do - if @dag && any_of?(:skipped) + if @dag && any_skipped_or_ignored? # The DAG job is skipped if one of the needs does not run at all. 'skipped' elsif @dag && !only_of?(:success, :failed, :canceled, :skipped, :success_with_warnings) @@ -90,6 +94,14 @@ module Gitlab matching == @status_set.size end + def any_skipped_or_ignored? + if ::Feature.enabled?(:ci_fix_pipeline_status_for_dag_needs_manual, @project, default_enabled: :yaml) + any_of?(:skipped) || any_of?(:ignored) + else + any_of?(:skipped) + end + end + def consume_all_statuses(all_statuses) columns = [] columns[@status_key] = :status diff --git a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml index 5f17c93b853..d879e27dfcb 100644 --- a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml @@ -20,11 +20,6 @@ stages: - functional - deploy -foodcritic: - stage: lint - script: - - chef exec foodcritic . - cookstyle: stage: lint script: diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index daed75a42ee..fd6c51ea350 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.22" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.23" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index c4e194bd658..29edada4041 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -23,7 +23,7 @@ review: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$REVIEW_DISABLED' when: never @@ -44,7 +44,7 @@ stop_review: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$REVIEW_DISABLED' when: never @@ -73,7 +73,7 @@ staging: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$STAGING_ENABLED' @@ -98,7 +98,7 @@ canary: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$CANARY_ENABLED' when: manual @@ -136,7 +136,7 @@ production: when: never - if: '$INCREMENTAL_ROLLOUT_MODE' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' production_manual: <<: *production_template @@ -148,12 +148,12 @@ production_manual: when: never - if: '$INCREMENTAL_ROLLOUT_MODE' when: never - - if: '$CI_COMMIT_BRANCH == "master" && $STAGING_ENABLED' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $STAGING_ENABLED' when: manual - - if: '$CI_COMMIT_BRANCH == "master" && $CANARY_ENABLED' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CANARY_ENABLED' when: manual -# This job implements incremental rollout on for every push to `master`. +# This job implements incremental rollout on for every push to the default branch. .rollout: &rollout_template extends: .auto-deploy @@ -184,7 +184,7 @@ production_manual: when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never # $INCREMENTAL_ROLLOUT_ENABLED is for compatibility with pre-GitLab 11.4 syntax - if: '$INCREMENTAL_ROLLOUT_MODE == "manual" || $INCREMENTAL_ROLLOUT_ENABLED' @@ -197,7 +197,7 @@ production_manual: when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "manual"' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' when: delayed diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index e5b40e5f49a..530ab1d0f99 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.0.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.6.0" dependencies: [] review: @@ -23,7 +23,7 @@ review: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$REVIEW_DISABLED' when: never @@ -44,7 +44,7 @@ stop_review: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$REVIEW_DISABLED' when: never @@ -73,7 +73,7 @@ staging: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$STAGING_ENABLED' @@ -98,7 +98,7 @@ canary: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$CANARY_ENABLED' when: manual @@ -135,7 +135,7 @@ production: when: never - if: '$INCREMENTAL_ROLLOUT_MODE' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' production_manual: <<: *production_template @@ -147,12 +147,12 @@ production_manual: when: never - if: '$INCREMENTAL_ROLLOUT_MODE' when: never - - if: '$CI_COMMIT_BRANCH == "master" && $STAGING_ENABLED' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $STAGING_ENABLED' when: manual - - if: '$CI_COMMIT_BRANCH == "master" && $CANARY_ENABLED' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CANARY_ENABLED' when: manual -# This job implements incremental rollout on for every push to `master`. +# This job implements incremental rollout on for every push to the default branch. .rollout: &rollout_template extends: .auto-deploy @@ -181,7 +181,7 @@ production_manual: when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never # $INCREMENTAL_ROLLOUT_ENABLED is for compatibility with pre-GitLab 11.4 syntax - if: '$INCREMENTAL_ROLLOUT_MODE == "manual" || $INCREMENTAL_ROLLOUT_ENABLED' @@ -194,7 +194,7 @@ production_manual: when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "manual"' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' when: delayed diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml index ed2172ef7f5..7efbcab221b 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml @@ -20,7 +20,7 @@ review_ec2: when: never - if: '$REVIEW_DISABLED' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' @@ -34,6 +34,6 @@ production_ec2: when: never - if: '$CI_KUBERNETES_ACTIVE' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml index 0289ba1c473..332c58c8695 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -46,7 +46,7 @@ review_ecs: when: never - if: '$REVIEW_DISABLED' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' @@ -62,7 +62,7 @@ stop_review_ecs: when: never - if: '$REVIEW_DISABLED' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' when: manual @@ -81,7 +81,7 @@ review_fargate: when: never - if: '$REVIEW_DISABLED' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' @@ -97,7 +97,7 @@ stop_review_fargate: when: never - if: '$REVIEW_DISABLED' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' when: manual @@ -109,7 +109,7 @@ production_ecs: when: never - if: '$CI_KUBERNETES_ACTIVE' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' @@ -120,6 +120,6 @@ production_fargate: when: never - if: '$CI_KUBERNETES_ACTIVE' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml index 32d4e07d398..be0efc9180b 100644 --- a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml @@ -10,11 +10,11 @@ # Below is the template to run your tests in Julia .test_template: &test_definition - # Uncomment below if you would like to run the tests on specific references - # only, such as the branches `master`, `development`, etc. - # only: - # - master - # - development + # Uncomment below (and adjust as needed) to run the tests for specific references + # only, such as the default branch, a `development` branch, and so on: + # rules: + # - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + # - if: $CI_COMMIT_BRANCH == "development" script: # Let's run the tests. Substitute `coverage = false` below, if you do not # want coverage results. @@ -63,8 +63,8 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # WARNING: This template is using the `julia` images from [Docker # Hub][3]. One can use custom Julia images and/or the official ones found diff --git a/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml index a4aed36889e..e4ed7fadfaa 100644 --- a/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml @@ -1,11 +1,20 @@ -# use docker image with latex preinstalled -# since there is no official latex image, use https://github.com/blang/latex-docker -# possible alternative: https://github.com/natlownes/docker-latex -image: blang/latex +--- +variables: + # Feel free to choose the image that suits you best. + # blang/latex:latest ... Former image used in this template. No longer maintained by author. + # listx/texlive:2020 ... The default, referring to TexLive 2020. Current at least to 2021-02-02. + + # Additional alternatives with high Docker pull counts: + # thomasweise/docker-texlive-full + # thomasweise/texlive + # adnrv/texlive + LATEX_IMAGE: listx/texlive:2020 build: + image: $LATEX_IMAGE script: - latexmk -pdf + artifacts: paths: - "*.pdf" diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml index 8f64da24410..97d0f611f47 100644 --- a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -33,7 +33,8 @@ cache: script: - 'mvn $MAVEN_CLI_OPTS verify' except: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Verify merge requests using JDK8 verify:jdk8: @@ -51,4 +52,5 @@ deploy:jdk8: fi - 'mvn $MAVEN_CLI_OPTS deploy -s ci_settings.xml' only: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml index 10fb6be6c39..36fe27f54c2 100644 --- a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml @@ -24,8 +24,8 @@ before_script: release: stage: deploy - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH artifacts: paths: - build/release/MyProject.exe diff --git a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml index 0a3cf3dcf77..0b03ba6c3d8 100644 --- a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml @@ -23,6 +23,6 @@ build: environment: production script: - find . -maxdepth 1 -name '*.json' -print0 | xargs -t0n1 packer build - when: manual - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: manual diff --git a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml index d2dd3fbfb75..90cd8472916 100644 --- a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml @@ -11,5 +11,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml index ba422c08614..7435afef572 100644 --- a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml @@ -9,5 +9,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml index 3a6eac63892..708c5063cc6 100644 --- a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml @@ -13,5 +13,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml index 92f25280c6e..694446dd6c9 100644 --- a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -8,5 +8,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml index 0e206423fa5..a2fd6620909 100644 --- a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml @@ -11,5 +11,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml index d91a8d7421f..fd75e47e899 100644 --- a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml @@ -13,5 +13,5 @@ pages: paths: - node_modules key: project - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml index 975cb3b7698..a6a605e35f0 100644 --- a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -10,7 +10,8 @@ test: script: - hugo except: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH pages: script: @@ -19,4 +20,5 @@ pages: paths: - public only: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml index 7a441a2f70f..1be2f4bad76 100644 --- a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml @@ -11,7 +11,8 @@ test: - pip install hyde - hyde gen except: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH pages: stage: deploy @@ -22,4 +23,5 @@ pages: paths: - public only: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml index f2f92fe0704..01e063c50ad 100644 --- a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -18,7 +18,8 @@ test: paths: - test except: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH pages: stage: deploy @@ -28,4 +29,5 @@ pages: paths: - public only: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml index 2d26b86a328..e39aa8a2063 100644 --- a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml @@ -33,5 +33,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml index 93ab8e0be0d..13d3089f4fa 100644 --- a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml @@ -8,5 +8,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml index 6524405133a..e65cf3928f2 100644 --- a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml @@ -12,5 +12,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml index 462b4737c4e..377fd8c396e 100644 --- a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml @@ -12,7 +12,8 @@ test: - bundle install --path vendor - bundle exec middleman build except: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH pages: script: @@ -23,5 +24,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml index b512f8d77e9..89281b41b66 100644 --- a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml @@ -8,5 +8,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml index 4318aadcaa6..8fd4702b90d 100644 --- a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml @@ -11,5 +11,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml index 8fd08ea7995..9fa8b07f7cb 100644 --- a/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml @@ -25,5 +25,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index 00b8b94b574..abce887d45b 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -47,5 +47,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index 135f0df99fe..654a03ced5f 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -45,13 +45,10 @@ apifuzzer_fuzz: entrypoint: ["/bin/bash", "-l", "-c"] variables: FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://apifuzzer:80 + FUZZAPI_API: http://localhost:80 FUZZAPI_NEW_REPORT: 1 + FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log TZ: America/Los_Angeles - services: - - name: $FUZZAPI_IMAGE - alias: apifuzzer - entrypoint: ["dotnet", "/peach/Peach.Web.dll"] allow_failure: true rules: - if: $FUZZAPI_D_TARGET_IMAGE @@ -80,17 +77,26 @@ apifuzzer_fuzz: # Make sure asset path exists - mkdir -p $FUZZAPI_REPORT_ASSET_PATH # + # Start API Security background process + - dotnet /peach/Peach.Web.dll &> $FUZZAPI_LOG_SCANNER & + - APISEC_PID=$! + # # Start scanning - worker-entry # # Run user provided post-script - sh -c "$FUZZAPI_POST_SCRIPT" # + # Shutdown API Security + - kill $APISEC_PID + - wait $APISEC_PID + # artifacts: when: always paths: - $FUZZAPI_REPORT_ASSET_PATH - $FUZZAPI_REPORT + - $FUZZAPI_LOG_SCANNER reports: api_fuzzing: $FUZZAPI_REPORT @@ -172,6 +178,7 @@ apifuzzer_fuzz_dnd: -e FUZZAPI_HAR \ -e FUZZAPI_OPENAPI \ -e FUZZAPI_POSTMAN_COLLECTION \ + -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ -e FUZZAPI_TARGET_URL \ -e FUZZAPI_OVERRIDES_FILE \ -e FUZZAPI_OVERRIDES_ENV \ @@ -214,6 +221,7 @@ apifuzzer_fuzz_dnd: -e FUZZAPI_HAR \ -e FUZZAPI_OPENAPI \ -e FUZZAPI_POSTMAN_COLLECTION \ + -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ -e FUZZAPI_TARGET_URL \ -e FUZZAPI_OVERRIDES_FILE \ -e FUZZAPI_OVERRIDES_ENV \ diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 5ea2363a0c5..64001c2828a 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -18,6 +18,9 @@ container_scanning: # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template # for details GIT_STRATEGY: none + # CS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION allow_failure: true services: diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index b534dad9593..3039d64514b 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -38,6 +38,9 @@ gemnasium-dependency_scanning: image: name: "$DS_ANALYZER_IMAGE" variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium:$DS_MAJOR_VERSION" rules: - if: $DEPENDENCY_SCANNING_DISABLED @@ -61,6 +64,9 @@ gemnasium-maven-dependency_scanning: image: name: "$DS_ANALYZER_IMAGE" variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-maven:$DS_MAJOR_VERSION" rules: - if: $DEPENDENCY_SCANNING_DISABLED @@ -79,6 +85,9 @@ gemnasium-python-dependency_scanning: image: name: "$DS_ANALYZER_IMAGE" variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-python:$DS_MAJOR_VERSION" rules: - if: $DEPENDENCY_SCANNING_DISABLED @@ -104,6 +113,9 @@ bundler-audit-dependency_scanning: image: name: "$DS_ANALYZER_IMAGE" variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bundler-audit:$DS_MAJOR_VERSION" rules: - if: $DEPENDENCY_SCANNING_DISABLED @@ -119,6 +131,9 @@ retire-js-dependency_scanning: image: name: "$DS_ANALYZER_IMAGE" variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/retire.js:$DS_MAJOR_VERSION" rules: - if: $DEPENDENCY_SCANNING_DISABLED diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 828352743b4..9693a4fbca2 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -41,6 +41,9 @@ bandit-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -57,6 +60,9 @@ brakeman-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -74,6 +80,9 @@ eslint-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -94,6 +103,9 @@ flawfinder-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -111,6 +123,9 @@ kubesec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -126,6 +141,9 @@ gosec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -140,11 +158,16 @@ gosec-sast: mobsf-android-sast: extends: .sast-analyzer services: - - name: opensecurity/mobile-security-framework-mobsf:latest + # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile + # Unfortunately, we need to keep track of mobsf version in 2 different places for now. + - name: opensecurity/mobile-security-framework-mobsf:v3.3.3 alias: mobsf image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" MOBSF_API_KEY: key rules: @@ -161,11 +184,16 @@ mobsf-android-sast: mobsf-ios-sast: extends: .sast-analyzer services: - - name: opensecurity/mobile-security-framework-mobsf:latest + # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile + # Unfortunately, we need to keep track of mobsf version in 2 different places for now. + - name: opensecurity/mobile-security-framework-mobsf:v3.3.3 alias: mobsf image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" MOBSF_API_KEY: key rules: @@ -184,6 +212,9 @@ nodejs-scan-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -200,6 +231,9 @@ phpcs-security-audit-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -216,6 +250,9 @@ pmd-apex-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -232,6 +269,9 @@ security-code-scan-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -249,6 +289,9 @@ semgrep-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:latest" rules: - if: $SAST_DISABLED @@ -266,6 +309,9 @@ sobelow-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -282,6 +328,9 @@ spotbugs-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/ diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index d2a6fa06dd8..c255fb4707a 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -1,7 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/secret_detection # # Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/secret_detection#available-variables +# List of the variables: https://docs.gitlab.com/ee/user/application_security/secret_detection/#available-variables # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: diff --git a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml index ffed7a0fec2..cca0ba5d38e 100644 --- a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml @@ -21,8 +21,8 @@ archive_project: script: - xcodebuild clean archive -archivePath build/ProjectName -scheme SchemeName - xcodebuild -exportArchive -exportFormat ipa -archivePath "build/ProjectName.xcarchive" -exportPath "build/ProjectName.ipa" -exportProvisioningProfile "ProvisioningProfileName" - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH artifacts: paths: - build/ProjectName.ipa diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index c2db0fc44f1..200388a274c 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -52,7 +52,8 @@ cache: - gitlab-terraform apply when: manual only: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH .destroy: &destroy stage: cleanup diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index f7bbb58df7e..e2a8af9c26b 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -6,14 +6,22 @@ module Gitlab class Collection include Enumerable - def initialize(variables = []) + attr_reader :errors + + def initialize(variables = [], errors = nil) @variables = [] + @variables_by_key = {} + @errors = errors variables.each { |variable| self.append(variable) } end def append(resource) - tap { @variables.append(Collection::Item.fabricate(resource)) } + item = Collection::Item.fabricate(resource) + @variables.append(item) + @variables_by_key[item[:key]] = item + + self end def concat(resources) @@ -33,15 +41,67 @@ module Gitlab end end + def [](key) + @variables_by_key[key] + end + + def size + @variables.size + end + def to_runner_variables self.map(&:to_runner_variable) end def to_hash self.to_runner_variables - .map { |env| [env.fetch(:key), env.fetch(:value)] } - .to_h.with_indifferent_access + .to_h { |env| [env.fetch(:key), env.fetch(:value)] } + .with_indifferent_access end + + def reject(&block) + Collection.new(@variables.reject(&block)) + end + + def expand_value(value, keep_undefined: false) + value.gsub(ExpandVariables::VARIABLES_REGEXP) do + match = Regexp.last_match + result = @variables_by_key[match[1] || match[2]]&.value + result ||= match[0] if keep_undefined + result + end + end + + def sort_and_expand_all(project, keep_undefined: false) + return self if Feature.disabled?(:variable_inside_variable, project) + + sorted = Sort.new(self) + return self.class.new(self, sorted.errors) unless sorted.valid? + + new_collection = self.class.new + + sorted.tsort.each do |item| + unless item.depends_on + new_collection.append(item) + next + end + + # expand variables as they are added + variable = item.to_runner_variable + variable[:value] = new_collection.expand_value(variable[:value], keep_undefined: keep_undefined) + new_collection.append(variable) + end + + new_collection + end + + def to_s + "#{@variables_by_key.keys}, @errors='#{@errors}'" + end + + protected + + attr_reader :variables end end end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 84a9280e507..77da2c4cb91 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -5,13 +5,21 @@ module Gitlab module Variables class Collection class Item - def initialize(key:, value:, public: true, file: false, masked: false) + include Gitlab::Utils::StrongMemoize + + def initialize(key:, value:, public: true, file: false, masked: false, raw: false) raise ArgumentError, "`#{key}` must be of type String or nil value, while it was: #{value.class}" unless value.is_a?(String) || value.nil? - @variable = { - key: key, value: value, public: public, file: file, masked: masked - } + @variable = { key: key, value: value, public: public, file: file, masked: masked, raw: raw } + end + + def value + @variable.fetch(:value) + end + + def raw + @variable.fetch(:raw) end def [](key) @@ -22,6 +30,16 @@ module Gitlab to_runner_variable == self.class.fabricate(other).to_runner_variable end + def depends_on + strong_memoize(:depends_on) do + next if raw + + next unless ExpandVariables.possible_var_reference?(value) + + value.scan(ExpandVariables::VARIABLES_REGEXP).map(&:first) + end + end + ## # If `file: true` has been provided we expose it, otherwise we # don't expose `file` attribute at all (stems from what the runner @@ -29,7 +47,7 @@ module Gitlab # def to_runner_variable @variable.reject do |hash_key, hash_value| - hash_key == :file && hash_value == false + (hash_key == :file || hash_key == :raw) && hash_value == false end end @@ -45,6 +63,12 @@ module Gitlab raise ArgumentError, "Unknown `#{resource.class}` variable resource!" end end + + def to_s + return to_runner_variable.to_s unless depends_on + + "#{to_runner_variable}, depends_on=#{depends_on}" + end end end end diff --git a/lib/gitlab/ci/variables/collection/sort.rb b/lib/gitlab/ci/variables/collection/sort.rb new file mode 100644 index 00000000000..90a929b8a07 --- /dev/null +++ b/lib/gitlab/ci/variables/collection/sort.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + class Collection + class Sort + include TSort + include Gitlab::Utils::StrongMemoize + + def initialize(collection) + raise(ArgumentError, "A Gitlab::Ci::Variables::Collection object was expected") unless + collection.is_a?(Collection) + + @collection = collection + end + + def valid? + errors.nil? + end + + # errors sorts an array of variables, ignoring unknown variable references, + # and returning an error string if a circular variable reference is found + def errors + strong_memoize(:errors) do + # Check for cyclic dependencies and build error message in that case + cyclic_vars = each_strongly_connected_component.filter_map do |component| + component.map { |v| v[:key] }.inspect if component.size > 1 + end + + "circular variable reference detected: #{cyclic_vars.join(', ')}" if cyclic_vars.any? + end + end + + private + + def tsort_each_node(&block) + @collection.each(&block) + end + + def tsort_each_child(var_item, &block) + depends_on = var_item.depends_on + return unless depends_on + + depends_on.filter_map { |var_ref_name| @collection[var_ref_name] }.each(&block) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/variables/collection/sorted.rb b/lib/gitlab/ci/variables/collection/sorted.rb deleted file mode 100644 index e641df10462..00000000000 --- a/lib/gitlab/ci/variables/collection/sorted.rb +++ /dev/null @@ -1,78 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Variables - class Collection - class Sorted - include TSort - include Gitlab::Utils::StrongMemoize - - def initialize(variables, project) - @variables = variables - @project = project - end - - def valid? - errors.nil? - end - - # errors sorts an array of variables, ignoring unknown variable references, - # and returning an error string if a circular variable reference is found - def errors - return if Feature.disabled?(:variable_inside_variable, @project) - - strong_memoize(:errors) do - # Check for cyclic dependencies and build error message in that case - errors = each_strongly_connected_component.filter_map do |component| - component.map { |v| v[:key] }.inspect if component.size > 1 - end - - "circular variable reference detected: #{errors.join(', ')}" if errors.any? - end - end - - # sort sorts an array of variables, ignoring unknown variable references. - # If a circular variable reference is found, the original array is returned - def sort - return @variables if Feature.disabled?(:variable_inside_variable, @project) - return @variables if errors - - tsort - end - - private - - def tsort_each_node(&block) - @variables.each(&block) - end - - def tsort_each_child(variable, &block) - each_variable_reference(variable[:value], &block) - end - - def input_vars - strong_memoize(:input_vars) do - @variables.index_by { |env| env.fetch(:key) } - end - end - - def walk_references(value) - return unless ExpandVariables.possible_var_reference?(value) - - value.scan(ExpandVariables::VARIABLES_REGEXP) do |var_ref| - yield(input_vars, var_ref.first) - end - end - - def each_variable_reference(value) - walk_references(value) do |vars_hash, ref_var_name| - variable = vars_hash.dig(ref_var_name) - yield variable if variable - end - end - end - end - end - end -end |