diff options
Diffstat (limited to 'lib/gitlab/ci')
54 files changed, 1015 insertions, 136 deletions
diff --git a/lib/gitlab/ci/badge/base.rb b/lib/gitlab/ci/badge/base.rb new file mode 100644 index 00000000000..c65f120753d --- /dev/null +++ b/lib/gitlab/ci/badge/base.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab::Ci + module Badge + class Base + def entity + raise NotImplementedError + end + + def status + raise NotImplementedError + end + + def metadata + raise NotImplementedError + end + + def template + raise NotImplementedError + end + end + end +end diff --git a/lib/gitlab/ci/badge/coverage/metadata.rb b/lib/gitlab/ci/badge/coverage/metadata.rb new file mode 100644 index 00000000000..7654b6d6fc5 --- /dev/null +++ b/lib/gitlab/ci/badge/coverage/metadata.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab::Ci + module Badge + module Coverage + ## + # Class that describes coverage badge metadata + # + class Metadata < Badge::Metadata + def initialize(badge) + @project = badge.project + @ref = badge.ref + @job = badge.job + end + + def title + 'coverage report' + end + + def image_url + coverage_project_badges_url(@project, @ref, format: :svg) + end + + def link_url + project_commits_url(@project, @ref) + end + end + end + end +end diff --git a/lib/gitlab/ci/badge/coverage/report.rb b/lib/gitlab/ci/badge/coverage/report.rb new file mode 100644 index 00000000000..28863a0703b --- /dev/null +++ b/lib/gitlab/ci/badge/coverage/report.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab::Ci + module Badge + module Coverage + ## + # Test coverage report badge + # + class Report < Badge::Base + attr_reader :project, :ref, :job, :customization + + def initialize(project, ref, opts: { job: nil }) + @project = project + @ref = ref + @job = opts[:job] + @customization = { + key_width: opts[:key_width].to_i, + key_text: opts[:key_text] + } + end + + def entity + 'coverage' + end + + def status + @coverage ||= raw_coverage + return unless @coverage + + @coverage.to_f.round(2) + end + + def metadata + @metadata ||= Coverage::Metadata.new(self) + end + + def template + @template ||= Coverage::Template.new(self) + end + + private + + def successful_pipeline + @successful_pipeline ||= @project.ci_pipelines.latest_successful_for_ref(@ref) + end + + def failed_pipeline + @failed_pipeline ||= @project.ci_pipelines.latest_failed_for_ref(@ref) + end + + def running_pipeline + @running_pipeline ||= @project.ci_pipelines.latest_running_for_ref(@ref) + end + + def raw_coverage + latest = + if @job.present? + builds = ::Ci::Build + .in_pipelines([successful_pipeline, running_pipeline, failed_pipeline]) + .latest + .success + .for_ref(@ref) + .by_name(@job) + + builds.max_by(&:created_at) + else + successful_pipeline + end + + latest&.coverage + end + end + end + end +end diff --git a/lib/gitlab/ci/badge/coverage/template.rb b/lib/gitlab/ci/badge/coverage/template.rb new file mode 100644 index 00000000000..7589fa5ff8b --- /dev/null +++ b/lib/gitlab/ci/badge/coverage/template.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab::Ci + module Badge + module Coverage + ## + # Class that represents a coverage badge template. + # + # Template object will be passed to badge.svg.erb template. + # + class Template < Badge::Template + STATUS_COLOR = { + good: '#4c1', + acceptable: '#a3c51c', + medium: '#dfb317', + low: '#e05d44', + unknown: '#9f9f9f' + }.freeze + + def initialize(badge) + @entity = badge.entity + @status = badge.status + @key_text = badge.customization.dig(:key_text) + @key_width = badge.customization.dig(:key_width) + end + + def key_text + if @key_text && @key_text.size <= MAX_KEY_TEXT_SIZE + @key_text + else + @entity.to_s + end + end + + def value_text + @status ? ("%.2f%%" % @status) : 'unknown' + end + + def key_width + if @key_width && @key_width.between?(1, MAX_KEY_WIDTH) + @key_width + else + 62 + end + end + + def value_width + @status ? 54 : 58 + end + + def value_color + case @status + when 95..100 then STATUS_COLOR[:good] + when 90..95 then STATUS_COLOR[:acceptable] + when 75..90 then STATUS_COLOR[:medium] + when 0..75 then STATUS_COLOR[:low] + else + STATUS_COLOR[:unknown] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/badge/metadata.rb b/lib/gitlab/ci/badge/metadata.rb new file mode 100644 index 00000000000..eec9fedfaa9 --- /dev/null +++ b/lib/gitlab/ci/badge/metadata.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab::Ci + module Badge + ## + # Abstract class for badge metadata + # + class Metadata + include Gitlab::Routing + include ActionView::Helpers::AssetTagHelper + include ActionView::Helpers::UrlHelper + + def initialize(badge) + @badge = badge + end + + def to_html + link_to(image_tag(image_url, alt: title), link_url) + end + + def to_markdown + "[](#{link_url})" + end + + def to_asciidoc + "image:#{image_url}[link=\"#{link_url}\",title=\"#{title}\"]" + end + + def title + raise NotImplementedError + end + + def image_url + raise NotImplementedError + end + + def link_url + raise NotImplementedError + end + end + end +end diff --git a/lib/gitlab/ci/badge/pipeline/metadata.rb b/lib/gitlab/ci/badge/pipeline/metadata.rb new file mode 100644 index 00000000000..2aa08476336 --- /dev/null +++ b/lib/gitlab/ci/badge/pipeline/metadata.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab::Ci + module Badge + module Pipeline + ## + # Class that describes pipeline badge metadata + # + class Metadata < Badge::Metadata + def initialize(badge) + @project = badge.project + @ref = badge.ref + end + + def title + 'pipeline status' + end + + def image_url + pipeline_project_badges_url(@project, @ref, format: :svg) + end + + def link_url + project_commits_url(@project, id: @ref) + end + end + end + end +end diff --git a/lib/gitlab/ci/badge/pipeline/status.rb b/lib/gitlab/ci/badge/pipeline/status.rb new file mode 100644 index 00000000000..a2ee2642872 --- /dev/null +++ b/lib/gitlab/ci/badge/pipeline/status.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab::Ci + module Badge + module Pipeline + ## + # Pipeline status badge + # + class Status < Badge::Base + attr_reader :project, :ref, :customization + + def initialize(project, ref, opts: {}) + @project = project + @ref = ref + @ignore_skipped = Gitlab::Utils.to_boolean(opts[:ignore_skipped], default: false) + @customization = { + key_width: opts[:key_width].to_i, + key_text: opts[:key_text] + } + + @sha = @project.commit(@ref).try(:sha) + end + + def entity + 'pipeline' + end + + # rubocop: disable CodeReuse/ActiveRecord + def status + pipelines = @project.ci_pipelines + .where(sha: @sha) + + relation = @ignore_skipped ? pipelines.without_statuses([:skipped]) : pipelines + relation.latest_status(@ref) || 'unknown' + end + # rubocop: enable CodeReuse/ActiveRecord + + def metadata + @metadata ||= Pipeline::Metadata.new(self) + end + + def template + @template ||= Pipeline::Template.new(self) + end + end + end + end +end diff --git a/lib/gitlab/ci/badge/pipeline/template.rb b/lib/gitlab/ci/badge/pipeline/template.rb new file mode 100644 index 00000000000..8430b01fc9a --- /dev/null +++ b/lib/gitlab/ci/badge/pipeline/template.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab::Ci + module Badge + module Pipeline + ## + # Class that represents a pipeline badge template. + # + # Template object will be passed to badge.svg.erb template. + # + class Template < Badge::Template + STATUS_RENAME = { 'success' => 'passed' }.freeze + STATUS_COLOR = { + success: '#4c1', + failed: '#e05d44', + running: '#dfb317', + pending: '#dfb317', + preparing: '#a7a7a7', + canceled: '#9f9f9f', + skipped: '#9f9f9f', + unknown: '#9f9f9f' + }.freeze + + def initialize(badge) + @entity = badge.entity + @status = badge.status + @key_text = badge.customization.dig(:key_text) + @key_width = badge.customization.dig(:key_width) + end + + def key_text + if @key_text && @key_text.size <= MAX_KEY_TEXT_SIZE + @key_text + else + @entity.to_s + end + end + + def value_text + STATUS_RENAME[@status.to_s] || @status.to_s + end + + def key_width + if @key_width && @key_width.between?(1, MAX_KEY_WIDTH) + @key_width + else + 62 + end + end + + def value_width + 54 + end + + def value_color + STATUS_COLOR[@status.to_sym] || STATUS_COLOR[:unknown] + end + end + end + end +end diff --git a/lib/gitlab/ci/badge/template.rb b/lib/gitlab/ci/badge/template.rb new file mode 100644 index 00000000000..0580dad72ba --- /dev/null +++ b/lib/gitlab/ci/badge/template.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab::Ci + module Badge + ## + # Abstract template class for badges + # + class Template + MAX_KEY_TEXT_SIZE = 64 + MAX_KEY_WIDTH = 512 + + def initialize(badge) + @entity = badge.entity + @status = badge.status + end + + def key_text + raise NotImplementedError + end + + def value_text + raise NotImplementedError + end + + def key_width + raise NotImplementedError + end + + def value_width + raise NotImplementedError + end + + def value_color + raise NotImplementedError + end + + def key_color + '#555' + end + + def key_text_anchor + key_width / 2 + end + + def value_text_anchor + key_width + (value_width / 2) + end + + def width + key_width + value_width + end + end + end +end diff --git a/lib/gitlab/ci/build/credentials/base.rb b/lib/gitlab/ci/build/credentials/base.rb index 58adf6e506d..2aeb8453703 100644 --- a/lib/gitlab/ci/build/credentials/base.rb +++ b/lib/gitlab/ci/build/credentials/base.rb @@ -6,7 +6,7 @@ module Gitlab module Credentials class Base def type - self.class.name.demodulize.underscore + raise NotImplementedError end end end diff --git a/lib/gitlab/ci/build/credentials/factory.rb b/lib/gitlab/ci/build/credentials/factory.rb index fa805abb8bb..e8996cb9dc4 100644 --- a/lib/gitlab/ci/build/credentials/factory.rb +++ b/lib/gitlab/ci/build/credentials/factory.rb @@ -20,7 +20,7 @@ module Gitlab end def providers - [Registry] + [Registry::GitlabRegistry, Registry::DependencyProxy] end end end diff --git a/lib/gitlab/ci/build/credentials/registry.rb b/lib/gitlab/ci/build/credentials/registry.rb deleted file mode 100644 index 1c8588d9913..00000000000 --- a/lib/gitlab/ci/build/credentials/registry.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Build - module Credentials - class Registry < Base - attr_reader :username, :password - - def initialize(build) - @username = 'gitlab-ci-token' - @password = build.token - end - - def url - Gitlab.config.registry.host_port - end - - def valid? - Gitlab.config.registry.enabled - end - end - end - end - end -end diff --git a/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb b/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb new file mode 100644 index 00000000000..b6ac06cfb53 --- /dev/null +++ b/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Credentials + module Registry + class DependencyProxy < GitlabRegistry + def url + "#{Gitlab.config.gitlab.host}:#{Gitlab.config.gitlab.port}" + end + + def valid? + Gitlab.config.dependency_proxy.enabled + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/credentials/registry/gitlab_registry.rb b/lib/gitlab/ci/build/credentials/registry/gitlab_registry.rb new file mode 100644 index 00000000000..5bd30e677e9 --- /dev/null +++ b/lib/gitlab/ci/build/credentials/registry/gitlab_registry.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Credentials + module Registry + class GitlabRegistry < Credentials::Base + attr_reader :username, :password + + def initialize(build) + @username = Gitlab::Auth::CI_JOB_USER + @password = build.token + end + + def url + Gitlab.config.registry.host_port + end + + def valid? + Gitlab.config.registry.enabled + end + + def type + 'registry' + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb index a39afee194c..2d4f9cf635b 100644 --- a/lib/gitlab/ci/build/rules.rb +++ b/lib/gitlab/ci/build/rules.rb @@ -7,30 +7,17 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize Result = Struct.new(:when, :start_in, :allow_failure, :variables) do - def build_attributes(seed_attributes = {}) + def build_attributes { when: self.when, options: { start_in: start_in }.compact, - allow_failure: allow_failure, - yaml_variables: yaml_variables(seed_attributes[:yaml_variables]) + allow_failure: allow_failure }.compact end def pass? self.when != 'never' end - - private - - def yaml_variables(seed_variables) - return unless variables && seed_variables - - indexed_seed_variables = seed_variables.deep_dup.index_by { |var| var[:key] } - - variables.each_with_object(indexed_seed_variables) do |var, hash| - hash[var[0].to_s] = { key: var[0].to_s, value: var[1], public: true } - end.values - end end def initialize(rule_hashes, default_when:) diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb index 25fb9c0ca97..797193a6be5 100644 --- a/lib/gitlab/ci/charts.rb +++ b/lib/gitlab/ci/charts.rb @@ -31,9 +31,10 @@ module Gitlab current = @from while current <= @to - @labels << current.strftime(@format) - @total << (totals_count[current] || 0) - @success << (success_count[current] || 0) + label = current.strftime(@format) + @labels << label + @total << (totals_count[label] || 0) + @success << (success_count[label] || 0) current += interval_step end @@ -45,6 +46,7 @@ module Gitlab query .group("date_trunc('#{interval}', #{::Ci::Pipeline.table_name}.created_at)") .count(:created_at) + .transform_keys { |date| date.strftime(@format) } end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 8ed4dc61920..dbb48a81030 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -13,7 +13,8 @@ module Gitlab RESCUE_ERRORS = [ Gitlab::Config::Loader::FormatError, Extendable::ExtensionError, - External::Processor::IncludeError + External::Processor::IncludeError, + Config::Yaml::Tags::TagError ].freeze attr_reader :root @@ -89,9 +90,10 @@ module Gitlab end def build_config(config) - initial_config = Gitlab::Config::Loader::Yaml.new(config).load! + initial_config = Config::Yaml.load!(config) initial_config = Config::External::Processor.new(initial_config, @context).perform initial_config = Config::Extendable.new(initial_config).to_hash + initial_config = Config::Yaml::Tags::Resolver.new(initial_config).to_hash initial_config = Config::EdgeStagesInjector.new(initial_config).to_hash initial_config diff --git a/lib/gitlab/ci/config/entry/commands.rb b/lib/gitlab/ci/config/entry/commands.rb index 7a86fca3056..341f87b44ab 100644 --- a/lib/gitlab/ci/config/entry/commands.rb +++ b/lib/gitlab/ci/config/entry/commands.rb @@ -10,12 +10,14 @@ module Gitlab class Commands < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable + MAX_NESTING_LEVEL = 10 + validations do - validates :config, string_or_nested_array_of_strings: true + validates :config, string_or_nested_array_of_strings: { max_level: MAX_NESTING_LEVEL } end def value - Array(@config).flatten(1) + Array(@config).flatten(MAX_NESTING_LEVEL) end end end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 85e3514499c..a20b802be58 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -14,7 +14,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script type image services start_in artifacts cache dependencies before_script after_script environment coverage retry parallel interruptible timeout - resource_group release secrets].freeze + release secrets].freeze REQUIRED_BY_NEEDS = %i[stage].freeze @@ -30,7 +30,6 @@ module Gitlab } validates :dependencies, array_of_strings: true - validates :resource_group, type: String validates :allow_failure, hash_or_boolean: true end @@ -124,7 +123,7 @@ module Gitlab attributes :script, :tags, :when, :dependencies, :needs, :retry, :parallel, :start_in, - :interruptible, :timeout, :resource_group, + :interruptible, :timeout, :release, :allow_failure def self.matching?(name, config) @@ -174,7 +173,6 @@ module Gitlab ignore: ignored?, allow_failure_criteria: allow_failure_criteria, needs: needs_defined? ? needs_value : nil, - resource_group: resource_group, scheduling_type: needs_defined? ? :dag : :stage ).compact end @@ -186,8 +184,6 @@ module Gitlab private def allow_failure_criteria - return unless ::Gitlab::Ci::Features.allow_failure_with_exit_codes_enabled? - if allow_failure_defined? && allow_failure_value.is_a?(Hash) allow_failure_value end diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 5ef8cfbddb7..9584d19bdec 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -15,7 +15,7 @@ module Gitlab include ::Gitlab::Config::Entry::Inheritable PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables - inherit allow_failure when needs].freeze + inherit allow_failure when needs resource_group].freeze included do validations do @@ -32,6 +32,7 @@ module Gitlab with_options allow_nil: true do validates :extends, array_of_strings_or_string: true validates :rules, array_of_hashes: true + validates :resource_group, type: String end end @@ -64,7 +65,7 @@ module Gitlab inherit: false, default: {} - attributes :extends, :rules + attributes :extends, :rules, :resource_group end def compose!(deps = nil) @@ -125,7 +126,8 @@ module Gitlab rules: rules_value, variables: root_and_job_variables_value, only: only_value, - except: except_value }.compact + except: except_value, + resource_group: resource_group }.compact end def root_and_job_variables_value diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 4684a9eb981..7d3fddd850d 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -60,7 +60,7 @@ module Gitlab def content_hash strong_memoize(:content_yaml) do - Gitlab::Config::Loader::Yaml.new(content).load! + ::Gitlab::Ci::Config::Yaml.load!(content) end rescue Gitlab::Config::Loader::FormatError nil diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 4d91cfd4c57..b85b7a9edeb 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -99,8 +99,6 @@ module Gitlab end def expand_variables(data) - return data unless ::Feature.enabled?(:variables_in_include_section_ci) - if data.is_a?(String) expand(data) else diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb new file mode 100644 index 00000000000..de833619c8d --- /dev/null +++ b/lib/gitlab/ci/config/yaml.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze + + class << self + def load!(content) + ensure_custom_tags + + Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load! + end + + private + + def ensure_custom_tags + @ensure_custom_tags ||= begin + AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) } + + true + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml/tags.rb b/lib/gitlab/ci/config/yaml/tags.rb new file mode 100644 index 00000000000..1575edad3b0 --- /dev/null +++ b/lib/gitlab/ci/config/yaml/tags.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + module Tags + TagError = Class.new(StandardError) + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml/tags/base.rb b/lib/gitlab/ci/config/yaml/tags/base.rb new file mode 100644 index 00000000000..13416a4afb6 --- /dev/null +++ b/lib/gitlab/ci/config/yaml/tags/base.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + module Tags + class Base + CircularReferenceError = Class.new(Tags::TagError) + NotValidError = Class.new(Tags::TagError) + + extend ::Gitlab::Utils::Override + + attr_accessor :resolved_status, :resolved_value, :data + + def self.tag + raise NotImplementedError + end + + # Only one of the `seq`, `scalar`, `map` fields is available. + def init_with(coder) + @data = { + tag: coder.tag, # This is the custom YAML tag, like !reference or !flatten + style: coder.style, + seq: coder.seq, # This holds Array data + scalar: coder.scalar, # This holds data of basic types, like String. + map: coder.map # This holds Hash data. + } + end + + def valid? + raise NotImplementedError + end + + def resolve(resolver) + raise NotValidError, validation_error_message unless valid? + raise CircularReferenceError, circular_error_message if resolving? + return resolved_value if resolved? + + self.resolved_status = :in_progress + self.resolved_value = _resolve(resolver) + self.resolved_status = :done + resolved_value + end + + private + + def _resolve(resolver) + raise NotImplementedError + end + + def resolved? + resolved_status == :done + end + + def resolving? + resolved_status == :in_progress + end + + def circular_error_message + "#{data[:tag]} #{data[:seq].inspect} is part of a circular chain" + end + + def validation_error_message + "#{data[:tag]} #{(data[:scalar].presence || data[:map].presence || data[:seq]).inspect} is not valid" + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml/tags/reference.rb b/lib/gitlab/ci/config/yaml/tags/reference.rb new file mode 100644 index 00000000000..22822614b67 --- /dev/null +++ b/lib/gitlab/ci/config/yaml/tags/reference.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + module Tags + class Reference < Base + MissingReferenceError = Class.new(Tags::TagError) + + def self.tag + '!reference' + end + + override :valid? + def valid? + data[:seq].is_a?(Array) && + !data[:seq].empty? && + data[:seq].all? { |identifier| identifier.is_a?(String) } + end + + private + + def location + data[:seq].to_a.map(&:to_sym) + end + + override :_resolve + def _resolve(resolver) + object = resolver.config.dig(*location) + value = resolver.deep_resolve(object) + + raise MissingReferenceError, missing_ref_error_message unless value + + value + end + + def missing_ref_error_message + "#{data[:tag]} #{data[:seq].inspect} could not be found" + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml/tags/resolver.rb b/lib/gitlab/ci/config/yaml/tags/resolver.rb new file mode 100644 index 00000000000..e207ec296b6 --- /dev/null +++ b/lib/gitlab/ci/config/yaml/tags/resolver.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + module Tags + # This class is the entry point for transforming custom YAML tags back + # into primitive objects. + # Usage: `Resolver.new(a_hash_including_custom_tag_objects).to_hash` + # + class Resolver + attr_reader :config + + def initialize(config) + @config = config.deep_dup + end + + def to_hash + deep_resolve(config) + end + + def deep_resolve(object) + case object + when Array + object.map(&method(:resolve_wrapper)) + when Hash + object.deep_transform_values(&method(:resolve_wrapper)) + else + resolve_wrapper(object) + end + end + + def resolve_wrapper(object) + if object.respond_to?(:resolve) + object.resolve(self) + else + object + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index 7956cf14203..d1a366125ef 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -55,21 +55,30 @@ module Gitlab ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) end - def self.pipeline_open_merge_requests?(project) - ::Feature.enabled?(:ci_pipeline_open_merge_requests, project, default_enabled: true) - end - def self.ci_pipeline_editor_page_enabled?(project) ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: :yaml) end - def self.allow_failure_with_exit_codes_enabled? - ::Feature.enabled?(:ci_allow_failure_with_exit_codes, default_enabled: :yaml) - end - def self.rules_variables_enabled?(project) ::Feature.enabled?(:ci_rules_variables, project, default_enabled: true) end + + def self.validate_build_dependencies?(project) + ::Feature.enabled?(:ci_validate_build_dependencies, project, default_enabled: :yaml) && + ::Feature.disabled?(:ci_validate_build_dependencies_override, project) + end + + def self.display_quality_on_mr_diff?(project) + ::Feature.enabled?(:codequality_mr_diff, project, default_enabled: false) + end + + def self.display_codequality_backend_comparison?(project) + ::Feature.enabled?(:codequality_backend_comparison, project, default_enabled: :yaml) + end + + def self.use_coverage_data_new_finder?(record) + ::Feature.enabled?(:coverage_data_new_finder, record, default_enabled: :yaml) + end end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index a8943eadf4f..0870c74053a 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -45,7 +45,7 @@ module Gitlab end def custom_claims - { + fields = { namespace_id: namespace.id.to_s, namespace_path: namespace.full_path, project_id: project.id.to_s, @@ -59,6 +59,15 @@ module Gitlab ref_type: ref_type, ref_protected: build.protected.to_s } + + if include_environment_claims? + fields.merge!( + environment: environment.name, + environment_protected: environment_protected?.to_s + ) + end + + fields end def key @@ -102,6 +111,20 @@ module Gitlab def ref_type ::Ci::BuildRunnerPresenter.new(build).ref_type end + + def environment + build.persisted_environment + end + + def environment_protected? + false # Overridden in EE + end + + def include_environment_claims? + Feature.enabled?(:ci_jwt_include_environment) && environment.present? + end end end end + +Gitlab::Ci::Jwt.prepend_if_ee('::EE::Gitlab::Ci::Jwt') diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index 985639982aa..2baa8faf849 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -20,6 +20,10 @@ module Gitlab rescue KeyError raise ParserNotFoundError, "Cannot find any parser matching file type '#{file_type}'" end + + def self.instrument! + parsers.values.each { |parser_class| parser_class.prepend(Parsers::Instrumentation) } + end end end end diff --git a/lib/gitlab/ci/parsers/instrumentation.rb b/lib/gitlab/ci/parsers/instrumentation.rb new file mode 100644 index 00000000000..ab4a923d9aa --- /dev/null +++ b/lib/gitlab/ci/parsers/instrumentation.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Instrumentation + BUCKETS = [0.25, 1, 5, 10].freeze + + def parse!(*args) + parser_result = nil + + duration = Benchmark.realtime do + parser_result = super + end + + labels = {} + + histogram = Gitlab::Metrics.histogram( + :ci_report_parser_duration_seconds, + 'Duration of parsing a CI report artifact', + labels, + BUCKETS + ) + + histogram.observe({ parser: self.class.name }, duration) + + parser_result + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index f0548284001..d3bc3a38f1f 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -23,7 +23,7 @@ module Gitlab pipeline_schedule: @command.schedule, merge_request: @command.merge_request, external_pull_request: @command.external_pull_request, - locked: @command.project.latest_pipeline_locked, + locked: @command.project.default_pipeline_lock, variables_attributes: variables_attributes ) end diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index 2ca51930c19..f0214bb4e38 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -25,7 +25,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def auto_cancelable_pipelines - pipelines + project.all_pipelines.ci_and_parent_sources .where(ref: pipeline.ref) .where.not(id: pipeline.same_family_pipeline_ids) .where.not(sha: project.commit(pipeline.ref).try(:id)) @@ -33,14 +33,6 @@ module Gitlab .with_only_interruptible_builds end # rubocop: enable CodeReuse/ActiveRecord - - def pipelines - if ::Feature.enabled?(:ci_auto_cancel_all_pipelines, project, default_enabled: true) - project.all_pipelines.ci_and_parent_sources - else - project.ci_pipelines - end - end end end end diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb index 5314fd471c3..a7680f6e593 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -34,16 +34,22 @@ module Gitlab private def find_config - SOURCES.each do |source| + sources.each do |source| config = source.new(@pipeline, @command) return config if config.exists? end nil end + + def sources + SOURCES + end end end end end end end + +Gitlab::Ci::Pipeline::Chain::Config::Content.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Config::Content') diff --git a/lib/gitlab/ci/pipeline/chain/template_usage.rb b/lib/gitlab/ci/pipeline/chain/template_usage.rb index c1a7b4ed453..2fcf1740b5f 100644 --- a/lib/gitlab/ci/pipeline/chain/template_usage.rb +++ b/lib/gitlab/ci/pipeline/chain/template_usage.rb @@ -19,7 +19,7 @@ module Gitlab def track_event(template) Gitlab::UsageDataCounters::CiTemplateUniqueCounter - .track_unique_project_event(project_id: pipeline.project_id, template: template) + .track_unique_project_event(project_id: pipeline.project_id, template: template, config_source: pipeline.config_source) end def included_templates diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index db6cca27f1c..c77f4dcca5a 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -45,6 +45,15 @@ module Gitlab Gitlab::Metrics.counter(name, comment) end end + + def legacy_update_jobs_counter + strong_memoize(:legacy_update_jobs_counter) do + name = :ci_legacy_update_jobs_as_retried_total + comment = 'Counter of occurrences when jobs were not being set as retried before update_retried' + + Gitlab::Metrics.counter(name, comment) + end + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index fe3c2bca551..3770bb4b328 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -73,17 +73,28 @@ module Gitlab def to_resource strong_memoize(:resource) do - if bridge? - ::Ci::Bridge.new(attributes) - else - ::Ci::Build.new(attributes).tap do |build| - build.assign_attributes(self.class.environment_attributes_for(build)) - build.resource_group = Seed::Build::ResourceGroup.new(build, @resource_group_key).to_resource - end + processable = initialize_processable + assign_resource_group(processable) + processable + end + end + + def initialize_processable + if bridge? + ::Ci::Bridge.new(attributes) + else + ::Ci::Build.new(attributes).tap do |build| + build.assign_attributes(self.class.environment_attributes_for(build)) end end end + def assign_resource_group(processable) + processable.resource_group = + Seed::Processable::ResourceGroup.new(processable, @resource_group_key) + .to_resource + end + def self.environment_attributes_for(build) return {} unless build.has_environment? @@ -159,7 +170,11 @@ module Gitlab next {} unless @using_rules if ::Gitlab::Ci::Features.rules_variables_enabled?(@pipeline.project) - rules_result.build_attributes(@seed_attributes) + rules_variables_result = ::Gitlab::Ci::Variables::Helpers.merge_variables( + @seed_attributes[:yaml_variables], rules_result.variables + ) + + rules_result.build_attributes.merge(yaml_variables: rules_variables_result) else rules_result.build_attributes end @@ -188,7 +203,6 @@ module Gitlab # we need to prevent the exit codes from being persisted because they # would break the behavior defined by `rules:allow_failure`. def allow_failure_criteria_attributes - return {} unless ::Gitlab::Ci::Features.allow_failure_with_exit_codes_enabled? return {} if rules_attributes[:allow_failure].nil? return {} unless @seed_attributes.dig(:options, :allow_failure_criteria) diff --git a/lib/gitlab/ci/pipeline/seed/build/resource_group.rb b/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb index c0641d9ff0a..f8ea6d4184c 100644 --- a/lib/gitlab/ci/pipeline/seed/build/resource_group.rb +++ b/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb @@ -4,21 +4,21 @@ module Gitlab module Ci module Pipeline module Seed - class Build + module Processable class ResourceGroup < Seed::Base include Gitlab::Utils::StrongMemoize - attr_reader :build, :resource_group_key + attr_reader :processable, :resource_group_key - def initialize(build, resource_group_key) - @build = build + def initialize(processable, resource_group_key) + @processable = processable @resource_group_key = resource_group_key end def to_resource return unless resource_group_key.present? - resource_group = build.project.resource_groups + resource_group = processable.project.resource_groups .safe_find_or_create_by(key: expanded_resource_group_key) resource_group if resource_group.persisted? @@ -28,7 +28,7 @@ module Gitlab def expanded_resource_group_key strong_memoize(:expanded_resource_group_key) do - ExpandVariables.expand(resource_group_key, -> { build.simple_variables }) + ExpandVariables.expand(resource_group_key, -> { processable.simple_variables }) end end end diff --git a/lib/gitlab/ci/reports/codequality_mr_diff.rb b/lib/gitlab/ci/reports/codequality_mr_diff.rb new file mode 100644 index 00000000000..e60a075e3f5 --- /dev/null +++ b/lib/gitlab/ci/reports/codequality_mr_diff.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class CodequalityMrDiff + attr_reader :files + + def initialize(raw_report) + @raw_report = raw_report + @files = {} + build_report! + end + + private + + def build_report! + codequality_files = @raw_report.all_degradations.each_with_object({}) do |degradation, codequality_files| + unless codequality_files[degradation.dig(:location, :path)].present? + codequality_files[degradation.dig(:location, :path)] = [] + end + + build_mr_diff_payload(codequality_files, degradation) + end + + @files = codequality_files + end + + def build_mr_diff_payload(codequality_files, degradation) + codequality_files[degradation.dig(:location, :path)] << { + line: degradation.dig(:location, :lines, :begin) || degradation.dig(:location, :positions, :begin, :line), + description: degradation[:description], + severity: degradation[:severity] + } + end + end + end + end +end diff --git a/lib/gitlab/ci/status/bridge/factory.rb b/lib/gitlab/ci/status/bridge/factory.rb index b9bd66cee71..4d5a94a3beb 100644 --- a/lib/gitlab/ci/status/bridge/factory.rb +++ b/lib/gitlab/ci/status/bridge/factory.rb @@ -8,6 +8,7 @@ module Gitlab def self.extended_statuses [[Status::Bridge::Failed], [Status::Bridge::Manual], + [Status::Bridge::WaitingForResource], [Status::Bridge::Play], [Status::Bridge::Action]] end diff --git a/lib/gitlab/ci/status/bridge/waiting_for_resource.rb b/lib/gitlab/ci/status/bridge/waiting_for_resource.rb new file mode 100644 index 00000000000..d2c8f71a609 --- /dev/null +++ b/lib/gitlab/ci/status/bridge/waiting_for_resource.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Bridge + class WaitingForResource < Status::Processable::WaitingForResource + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/waiting_for_resource.rb b/lib/gitlab/ci/status/build/waiting_for_resource.rb index 008e6a17bdd..5dcc060a990 100644 --- a/lib/gitlab/ci/status/build/waiting_for_resource.rb +++ b/lib/gitlab/ci/status/build/waiting_for_resource.rb @@ -4,22 +4,7 @@ module Gitlab module Ci module Status module Build - class WaitingForResource < Status::Extended - ## - # TODO: image is shared with 'pending' - # until we get a dedicated one - # - def illustration - { - image: 'illustrations/pending_job_empty.svg', - size: 'svg-430', - title: _('This job is waiting for resource: ') + subject.resource_group.key - } - end - - def self.matches?(build, _) - build.waiting_for_resource? - end + class WaitingForResource < Status::Processable::WaitingForResource end end end diff --git a/lib/gitlab/ci/status/processable/waiting_for_resource.rb b/lib/gitlab/ci/status/processable/waiting_for_resource.rb new file mode 100644 index 00000000000..c9b1dd795d0 --- /dev/null +++ b/lib/gitlab/ci/status/processable/waiting_for_resource.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Processable + class WaitingForResource < Status::Extended + ## + # TODO: image is shared with 'pending' + # until we get a dedicated one + # + def illustration + { + image: 'illustrations/pending_job_empty.svg', + size: 'svg-430', + title: _('This job is waiting for resource: ') + subject.resource_group.key + } + end + + def self.matches?(processable, _) + processable.waiting_for_resource? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 501d8737acd..daed75a42ee 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.19" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.22" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 192b1509fdc..6f30fc2dcd5 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,6 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.37.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.40.0" environment: name: production variables: diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml index 84bb0ff3b33..8f64da24410 100644 --- a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -40,14 +40,14 @@ verify:jdk8: <<: *verify # To deploy packages from CI, create a ci_settings.xml file -# For deploying packages to GitLab's Maven Repository: See https://docs.gitlab.com/ee/user/project/packages/maven_repository.html#creating-maven-packages-with-gitlab-cicd for more details. +# For deploying packages to GitLab's Maven Repository: See https://docs.gitlab.com/ee/user/packages/maven_repository/index.html#create-maven-packages-with-gitlab-cicd for more details. # Please note: The GitLab Maven Repository is currently only available in GitLab Premium / Ultimate. # For `master` branch run `mvn deploy` automatically. deploy:jdk8: stage: deploy script: - if [ ! -f ci_settings.xml ]; - then echo "CI settings missing\! If deploying to GitLab Maven Repository, please see https://docs.gitlab.com/ee/user/project/packages/maven_repository.html#creating-maven-packages-with-gitlab-cicd for instructions."; + then echo "CI settings missing\! If deploying to GitLab Maven Repository, please see https://docs.gitlab.com/ee/user/packages/maven_repository/index.html#create-maven-packages-with-gitlab-cicd for instructions."; fi - 'mvn $MAVEN_CLI_OPTS deploy -s ci_settings.xml' only: diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml index 63237e41376..21e926ef275 100644 --- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml @@ -1,4 +1,4 @@ -# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/license_compliance/ +# Read more about this feature here: https://docs.gitlab.com/ee/user/compliance/license_compliance/index.html # # Configure the scanning tool through the environment variables. # List of the variables: https://gitlab.com/gitlab-org/security-products/analyzers/license-finder#settings @@ -21,7 +21,6 @@ license_scanning: LM_REPORT_VERSION: '2.1' SETUP_CMD: $LICENSE_MANAGEMENT_SETUP_CMD allow_failure: true - needs: [] script: - /run.sh analyze . artifacts: diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 56c6fbd96bc..828352743b4 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -9,7 +9,7 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec, mobsf" + SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec, mobsf, semgrep" SAST_EXCLUDED_ANALYZERS: "" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" SAST_ANALYZER_IMAGE_TAG: 2 @@ -66,7 +66,8 @@ brakeman-sast: - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /brakeman/ exists: - - 'config/routes.rb' + - '**/*.rb' + - '**/Gemfile' eslint-sast: extends: .sast-analyzer @@ -243,6 +244,23 @@ security-code-scan-sast: - '**/*.csproj' - '**/*.vbproj' +semgrep-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:latest" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ + when: never + - if: $CI_COMMIT_BRANCH && + $SAST_DEFAULT_ANALYZERS =~ /semgrep/ && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.py' + sobelow-sast: extends: .sast-analyzer image: diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 0222ca021b7..3258d965c93 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -182,7 +182,7 @@ module Gitlab if job.trace_chunks.any? Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream| archive_stream!(stream) - stream.destroy! + destroy_stream(job) { stream.destroy! } end elsif current_path File.open(current_path) do |stream| @@ -268,7 +268,21 @@ module Gitlab end def trace_artifact - job.job_artifacts_trace + read_trace_artifact(job) { job.job_artifacts_trace } + end + + ## + # Overridden in EE + # + def destroy_stream(job) + yield + end + + ## + # Overriden in EE + # + def read_trace_artifact(job) + yield end def being_watched_cache_key @@ -277,3 +291,5 @@ module Gitlab end end end + +::Gitlab::Ci::Trace.prepend_if_ee('EE::Gitlab::Ci::Trace') diff --git a/lib/gitlab/ci/trace/checksum.rb b/lib/gitlab/ci/trace/checksum.rb index 7cdb6a6c03c..92bed817875 100644 --- a/lib/gitlab/ci/trace/checksum.rb +++ b/lib/gitlab/ci/trace/checksum.rb @@ -30,7 +30,11 @@ module Gitlab end def state_crc32 - strong_memoize(:state_crc32) { build.pending_state&.crc32 } + strong_memoize(:state_crc32) do + ::Gitlab::Database::Consistency.with_read_consistency do + build.pending_state&.crc32 + end + end end def chunks_crc32 @@ -59,8 +63,10 @@ module Gitlab # def trace_chunks strong_memoize(:trace_chunks) do - build.trace_chunks.persisted - .select(::Ci::BuildTraceChunk.metadata_attributes) + ::Ci::BuildTraceChunk.with_read_consistency(build) do + build.trace_chunks.persisted + .select(::Ci::BuildTraceChunk.metadata_attributes) + end end end diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 6f3e4ccf48d..7c2e39b1e53 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -227,12 +227,20 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def build_chunk - @chunks_cache[chunk_index] = ::Ci::BuildTraceChunk.new(build: build, chunk_index: chunk_index) + def next_chunk + @chunks_cache[chunk_index] = begin + if ::Ci::BuildTraceChunk.consistent_reads_enabled?(build) + ::Ci::BuildTraceChunk + .safe_find_or_create_by(build: build, chunk_index: chunk_index) + else + ::Ci::BuildTraceChunk + .new(build: build, chunk_index: chunk_index) + end + end end def ensure_chunk - current_chunk || build_chunk + current_chunk || next_chunk || current_chunk end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/variables/collection/sorted.rb b/lib/gitlab/ci/variables/collection/sorted.rb index 6abc6a5644f..e641df10462 100644 --- a/lib/gitlab/ci/variables/collection/sorted.rb +++ b/lib/gitlab/ci/variables/collection/sorted.rb @@ -8,8 +8,9 @@ module Gitlab include TSort include Gitlab::Utils::StrongMemoize - def initialize(variables) + def initialize(variables, project) @variables = variables + @project = project end def valid? @@ -19,7 +20,7 @@ module Gitlab # errors sorts an array of variables, ignoring unknown variable references, # and returning an error string if a circular variable reference is found def errors - return if Feature.disabled?(:variable_inside_variable) + return if Feature.disabled?(:variable_inside_variable, @project) strong_memoize(:errors) do # Check for cyclic dependencies and build error message in that case @@ -34,7 +35,7 @@ module Gitlab # sort sorts an array of variables, ignoring unknown variable references. # If a circular variable reference is found, the original array is returned def sort - return @variables if Feature.disabled?(:variable_inside_variable) + return @variables if Feature.disabled?(:variable_inside_variable, @project) return @variables if errors tsort diff --git a/lib/gitlab/ci/variables/helpers.rb b/lib/gitlab/ci/variables/helpers.rb new file mode 100644 index 00000000000..e2a54f90ecb --- /dev/null +++ b/lib/gitlab/ci/variables/helpers.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + module Helpers + class << self + def merge_variables(current_vars, new_vars) + current_vars = transform_from_yaml_variables(current_vars) + new_vars = transform_from_yaml_variables(new_vars) + + transform_to_yaml_variables( + current_vars.merge(new_vars) + ) + end + + def transform_to_yaml_variables(vars) + vars.to_h.map do |key, value| + { key: key.to_s, value: value, public: true } + end + end + + def transform_from_yaml_variables(vars) + return vars.stringify_keys if vars.is_a?(Hash) + + vars.to_a.map { |var| [var[:key].to_s, var[:value]] }.to_h + end + end + end + end + end +end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 86749cda9c7..3459b69bebc 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -123,9 +123,7 @@ module Gitlab end def transform_to_yaml_variables(variables) - variables.to_h.map do |key, value| - { key: key.to_s, value: value, public: true } - end + ::Gitlab::Ci::Variables::Helpers.transform_to_yaml_variables(variables) end end end |