diff options
Diffstat (limited to 'lib')
22 files changed, 264 insertions, 64 deletions
diff --git a/lib/gitlab/ci/artifact_file_reader.rb b/lib/gitlab/ci/artifact_file_reader.rb new file mode 100644 index 00000000000..c2d17cc176e --- /dev/null +++ b/lib/gitlab/ci/artifact_file_reader.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +# This class takes in input a Ci::Build object and an artifact path to read. +# It downloads and extracts the artifacts archive, then returns the content +# of the artifact, if found. +module Gitlab + module Ci + class ArtifactFileReader + Error = Class.new(StandardError) + + MAX_ARCHIVE_SIZE = 5.megabytes + + def initialize(job) + @job = job + + raise ArgumentError, 'Job does not have artifacts' unless @job.artifacts? + + validate! + end + + def read(path) + return unless job.artifacts_metadata + + metadata_entry = job.artifacts_metadata_entry(path) + + if metadata_entry.total_size > MAX_ARCHIVE_SIZE + raise Error, "Artifacts archive for job `#{job.name}` is too large: max #{max_archive_size_in_mb}" + end + + read_zip_file!(path) + end + + private + + attr_reader :job + + def validate! + if job.job_artifacts_archive.size > MAX_ARCHIVE_SIZE + raise Error, "Artifacts archive for job `#{job.name}` is too large: max #{max_archive_size_in_mb}" + end + + unless job.artifacts_metadata? + raise Error, "Job `#{job.name}` has missing artifacts metadata and cannot be extracted!" + end + end + + def read_zip_file!(file_path) + job.artifacts_file.use_file do |archive_path| + Zip::File.open(archive_path) do |zip_file| + entry = zip_file.find_entry(file_path) + unless entry + raise Error, "Path `#{file_path}` does not exist inside the `#{job.name}` artifacts archive!" + end + + if entry.name_is_directory? + raise Error, "Path `#{file_path}` was expected to be a file but it was a directory!" + end + + zip_file.get_input_stream(entry) do |is| + is.read + end + end + end + end + + def max_archive_size_in_mb + ActiveSupport::NumberHelper.number_to_human_size(MAX_ARCHIVE_SIZE) + end + end + end +end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 38ab3475d01..e069f734e32 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -18,12 +18,9 @@ module Gitlab attr_reader :root - def initialize(config, project: nil, sha: nil, user: nil) - @context = build_context(project: project, sha: sha, user: user) - - if Feature.enabled?(:ci_limit_yaml_expansion, project, default_enabled: true) - @context.set_deadline(TIMEOUT_SECONDS) - end + def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil) + @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline) + @context.set_deadline(TIMEOUT_SECONDS) @config = expand_config(config) @@ -87,11 +84,12 @@ module Gitlab initial_config end - def build_context(project:, sha:, user:) + def build_context(project:, sha:, user:, parent_pipeline:) Config::External::Context.new( project: project, sha: sha || project&.repository&.root_ref_sha, - user: user) + user: user, + parent_pipeline: parent_pipeline) end def track_and_raise_for_dev_exception(error) diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb index 6fdaa373170..f4362d3b0ce 100644 --- a/lib/gitlab/ci/config/entry/bridge.rb +++ b/lib/gitlab/ci/config/entry/bridge.rb @@ -11,7 +11,7 @@ module Gitlab class Bridge < ::Gitlab::Config::Entry::Node include ::Gitlab::Ci::Config::Entry::Processable - ALLOWED_KEYS = %i[trigger allow_failure when variables needs].freeze + ALLOWED_KEYS = %i[trigger allow_failure when needs].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS @@ -45,10 +45,6 @@ module Gitlab inherit: false, metadata: { allowed_needs: %i[job bridge] } - entry :variables, ::Gitlab::Ci::Config::Entry::Variables, - description: 'Environment variables available for this job.', - inherit: false - attributes :when, :allow_failure def self.matching?(name, config) @@ -67,7 +63,6 @@ module Gitlab needs: (needs_value if needs_defined?), ignore: !!allow_failure, when: self.when, - variables: (variables_value if variables_defined?), scheduling_type: needs_defined? && !bridge_needs ? :dag : :stage ).compact end diff --git a/lib/gitlab/ci/config/entry/include.rb b/lib/gitlab/ci/config/entry/include.rb index f2f3dd84eda..cd09d83b728 100644 --- a/lib/gitlab/ci/config/entry/include.rb +++ b/lib/gitlab/ci/config/entry/include.rb @@ -10,7 +10,7 @@ module Gitlab class Include < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable - ALLOWED_KEYS = %i[local file remote template].freeze + ALLOWED_KEYS = %i[local file remote template artifact job].freeze validations do validates :config, hash_or_string: true diff --git a/lib/gitlab/ci/config/entry/inherit.rb b/lib/gitlab/ci/config/entry/inherit.rb new file mode 100644 index 00000000000..540f1e62c6c --- /dev/null +++ b/lib/gitlab/ci/config/entry/inherit.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # This class represents a inherit entry + # + class Inherit < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + + ALLOWED_KEYS = %i[default variables].freeze + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + end + + entry :default, ::Gitlab::Config::Entry::Boolean, + description: 'Indicates whether to inherit `default:`.', + default: true + + entry :variables, ::Gitlab::Config::Entry::Boolean, + description: 'Indicates whether to inherit `variables:`.', + default: true + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 8db21b116eb..1ea59491378 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -13,7 +13,7 @@ module Gitlab ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze ALLOWED_KEYS = %i[tags script type image services allow_failure type when start_in artifacts cache - dependencies before_script needs after_script variables + dependencies before_script needs after_script environment coverage retry parallel interruptible timeout resource_group release].freeze @@ -112,10 +112,6 @@ module Gitlab metadata: { allowed_needs: %i[job cross_dependency] }, inherit: false - entry :variables, Entry::Variables, - description: 'Environment variables available for this job.', - inherit: false - entry :environment, Entry::Environment, description: 'Environment configuration for this job.', inherit: false @@ -174,7 +170,6 @@ module Gitlab when: self.when, start_in: self.start_in, dependencies: dependencies, - variables: variables_defined? ? variables_value : {}, environment: environment_defined? ? environment_value : nil, environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index bfa2905ed77..b4da48957b0 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -14,7 +14,7 @@ module Gitlab include ::Gitlab::Config::Entry::Attributable include ::Gitlab::Config::Entry::Inheritable - PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules].freeze + PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables inherit].freeze included do validations do @@ -54,12 +54,21 @@ module Gitlab allowed_when: %w[on_success on_failure always never manual delayed].freeze } + entry :variables, ::Gitlab::Ci::Config::Entry::Variables, + description: 'Environment variables available for this job.', + inherit: false + + entry :inherit, ::Gitlab::Ci::Config::Entry::Inherit, + description: 'Indicates whether to inherit defaults or not.', + inherit: false, + default: {} + attributes :extends, :rules end def compose!(deps = nil) super do - has_workflow_rules = deps&.workflow&.has_rules? + has_workflow_rules = deps&.workflow_entry&.has_rules? # If workflow:rules: or rules: are used # they are considered not compatible @@ -73,6 +82,9 @@ module Gitlab @entries.delete(:except) unless except_defined? # rubocop:disable Gitlab/ModuleWithInstanceVariables end + # inherit root variables + @root_variables_value = deps&.variables_value # rubocop:disable Gitlab/ModuleWithInstanceVariables + yield if block_given? end end @@ -82,7 +94,10 @@ module Gitlab end def overwrite_entry(deps, key, current_entry) - deps.default[key] unless current_entry.specified? + return unless inherit_entry&.default_value + return unless deps.default_entry + + deps.default_entry[key] unless current_entry.specified? end def value @@ -90,9 +105,18 @@ module Gitlab stage: stage_value, extends: extends, rules: rules_value, + variables: root_and_job_variables_value, only: only_value, except: except_value }.compact end + + def root_and_job_variables_value + if inherit_entry&.variables_value + @root_variables_value.to_h.merge(variables_value.to_h) # rubocop:disable Gitlab/ModuleWithInstanceVariables + else + variables_value.to_h + end + end end end end diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index caa0725c4bd..19d6a470941 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -65,7 +65,8 @@ module Gitlab reserved: true entry :workflow, Entry::Workflow, - description: 'List of evaluable rules to determine Pipeline status' + description: 'List of evaluable rules to determine Pipeline status', + default: {} dynamic_helpers :jobs @@ -73,7 +74,7 @@ module Gitlab :image_value, :services_value, :after_script_value, - :cache_value, to: :default + :cache_value, to: :default_entry attr_reader :jobs_config @@ -102,14 +103,6 @@ module Gitlab end end - def default - self[:default] - end - - def workflow - self[:workflow] if workflow_defined? - end - private # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/config/entry/workflow.rb b/lib/gitlab/ci/config/entry/workflow.rb index 1d9007c9b9b..5bc992a38a0 100644 --- a/lib/gitlab/ci/config/entry/workflow.rb +++ b/lib/gitlab/ci/config/entry/workflow.rb @@ -12,7 +12,6 @@ module Gitlab validations do validates :config, type: Hash validates :config, allowed_keys: ALLOWED_KEYS - validates :config, presence: true end entry :rules, Entry::Rules, diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index bb4439cd069..814dcc66362 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -7,13 +7,14 @@ module Gitlab class Context TimeoutError = Class.new(StandardError) - attr_reader :project, :sha, :user + attr_reader :project, :sha, :user, :parent_pipeline attr_reader :expandset, :execution_deadline - def initialize(project: nil, sha: nil, user: nil) + def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil) @project = project @sha = sha @user = user + @parent_pipeline = parent_pipeline @expandset = Set.new @execution_deadline = 0 diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb new file mode 100644 index 00000000000..fcfdda21c08 --- /dev/null +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + module File + class Artifact < Base + extend ::Gitlab::Utils::Override + include Gitlab::Utils::StrongMemoize + + attr_reader :job_name + + def initialize(params, context) + @location = params[:artifact] + @job_name = params[:job] + + super + end + + def content + strong_memoize(:content) do + next unless artifact_job + + Gitlab::Ci::ArtifactFileReader.new(artifact_job).read(location) + rescue Gitlab::Ci::ArtifactFileReader::Error => error + errors.push(error.message) + end + end + + def matching? + super && + Feature.enabled?(:ci_dynamic_child_pipeline, project) + end + + private + + def project + context&.parent_pipeline&.project + end + + def validate_content! + return unless ensure_preconditions_satisfied! + + errors.push("File `#{location}` is empty!") unless content.present? + end + + def ensure_preconditions_satisfied! + unless creating_child_pipeline? + errors.push('Including configs from artifacts is only allowed when triggering child pipelines') + return false + end + + unless job_name.present? + errors.push("Job must be provided when including configs from artifacts") + return false + end + + unless artifact_job.present? + errors.push("Job `#{job_name}` not found in parent pipeline or does not have artifacts!") + return false + end + + true + end + + def artifact_job + strong_memoize(:artifact_job) do + next unless creating_child_pipeline? + + context.parent_pipeline.find_job_with_archive_artifacts(job_name) + end + end + + def creating_child_pipeline? + context.parent_pipeline.present? + end + + override :expand_context_attrs + def expand_context_attrs + { + project: context.project, + sha: context.sha, + user: context.user, + parent_pipeline: context.parent_pipeline + } + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index 8cb1575a3e1..e74f5b33de7 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -40,7 +40,8 @@ module Gitlab { project: context.project, sha: context.sha, - user: context.user + user: context.user, + parent_pipeline: context.parent_pipeline } end end diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index c7b49b495fa..be479741784 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -71,7 +71,8 @@ module Gitlab { project: project, sha: sha, - user: context.user + user: context.user, + parent_pipeline: context.parent_pipeline } end end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 0143d7784fa..97ae6c4ceba 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -13,7 +13,8 @@ module Gitlab External::File::Remote, External::File::Template, External::File::Local, - External::File::Project + External::File::Project, + External::File::Artifact ].freeze Error = Class.new(StandardError) diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb index aabdf7ce47d..9b494f3a7ec 100644 --- a/lib/gitlab/ci/pipeline/chain/base.rb +++ b/lib/gitlab/ci/pipeline/chain/base.rb @@ -7,7 +7,7 @@ module Gitlab class Base attr_reader :pipeline, :command, :config - delegate :project, :current_user, to: :command + delegate :project, :current_user, :parent_pipeline, to: :command def initialize(pipeline, command) @pipeline = pipeline diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 6a16e6df23d..fa46114615c 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -72,6 +72,10 @@ module Gitlab project.repository.ambiguous_ref?(origin_ref) end end + + def parent_pipeline + bridge&.parent_pipeline + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 09d1b0edc93..1e47be21b93 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -15,7 +15,8 @@ module Gitlab @command.config_content, { project: project, sha: @pipeline.sha, - user: current_user + user: current_user, + parent_pipeline: parent_pipeline } ) rescue Gitlab::Ci::YamlProcessor::ValidationError => ex diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index f708e95c2cf..6efb6b4e273 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -5,9 +5,7 @@ variables: container_scanning: stage: test - image: - name: registry.gitlab.com/gitlab-org/security-products/analyzers/klar:$CS_MAJOR_VERSION - entrypoint: [] + image: registry.gitlab.com/gitlab-org/security-products/analyzers/klar:$CS_MAJOR_VERSION variables: # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes @@ -22,10 +20,7 @@ container_scanning: - name: $CLAIR_DB_IMAGE alias: clair-vulnerabilities-db script: - # the kubernetes executor currently ignores the Docker image entrypoint value, so the start.sh script must - # be explicitly executed here in order for this to work with both the kubernetes and docker executors - # see this issue for more details https://gitlab.com/gitlab-org/gitlab-runner/issues/4125 - - /container-scanner/start.sh + - /analyzer run artifacts: reports: container_scanning: gl-container-scanning-report.json diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 5ff6413898f..4a9fa3091be 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -59,6 +59,8 @@ dependency_scanning: BUNDLER_AUDIT_UPDATE_DISABLED \ BUNDLER_AUDIT_ADVISORY_DB_URL \ BUNDLER_AUDIT_ADVISORY_DB_REF_NAME \ + RETIREJS_JS_ADVISORY_DB \ + RETIREJS_NODE_ADVISORY_DB \ ) \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index ae3ff4a51e2..764047dae6d 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -57,7 +57,7 @@ module Gitlab when: job[:when] || 'on_success', environment: job[:environment_name], coverage_regex: job[:coverage], - yaml_variables: transform_to_yaml_variables(job_variables(name)), + yaml_variables: transform_to_yaml_variables(job[:variables]), needs_attributes: job.dig(:needs, :job), interruptible: job[:interruptible], only: job[:only], @@ -146,13 +146,6 @@ module Gitlab end end - def job_variables(name) - job_variables = @jobs.dig(name.to_sym, :variables) - - @variables.to_h - .merge(job_variables.to_h) - end - def transform_to_yaml_variables(variables) variables.to_h.map do |key, value| { key: key.to_s, value: value, public: true } diff --git a/lib/gitlab/config/entry/attributable.rb b/lib/gitlab/config/entry/attributable.rb index 4deb233d10e..d266d5218de 100644 --- a/lib/gitlab/config/entry/attributable.rb +++ b/lib/gitlab/config/entry/attributable.rb @@ -10,7 +10,7 @@ module Gitlab def attributes(*attributes) attributes.flatten.each do |attribute| if method_defined?(attribute) - raise ArgumentError, "Method already defined: #{attribute}" + raise ArgumentError, "Method '#{attribute}' already defined in '#{name}'" end define_method(attribute) do diff --git a/lib/gitlab/config/entry/configurable.rb b/lib/gitlab/config/entry/configurable.rb index 3fd562c2904..571e7a5127e 100644 --- a/lib/gitlab/config/entry/configurable.rb +++ b/lib/gitlab/config/entry/configurable.rb @@ -76,7 +76,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def entry(key, entry, description: nil, default: nil, inherit: nil, reserved: nil, metadata: {}) entry_name = key.to_sym - raise ArgumentError, "Entry #{key} already defined" if @nodes.to_h[entry_name] + raise ArgumentError, "Entry '#{key}' already defined in '#{name}'" if @nodes.to_h[entry_name] factory = ::Gitlab::Config::Entry::Factory.new(entry) .with(description: description) @@ -98,8 +98,8 @@ module Gitlab def helpers(*nodes, dynamic: false) nodes.each do |symbol| - if method_defined?("#{symbol}_defined?") || method_defined?("#{symbol}_value") - raise ArgumentError, "Method #{symbol}_defined? or #{symbol}_value already defined" + if method_defined?("#{symbol}_defined?") || method_defined?("#{symbol}_entry") || method_defined?("#{symbol}_value") + raise ArgumentError, "Method '#{symbol}_defined?', '#{symbol}_entry' or '#{symbol}_value' already defined in '#{name}'" end unless @nodes.to_h[symbol] @@ -110,10 +110,13 @@ module Gitlab entries[symbol]&.specified? end - define_method("#{symbol}_value") do - return unless entries[symbol] && entries[symbol].valid? + define_method("#{symbol}_entry") do + entries[symbol] + end - entries[symbol].value + define_method("#{symbol}_value") do + entry = entries[symbol] + entry.value if entry&.valid? end end end |