diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-10-21 10:08:36 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-10-21 10:08:36 +0300 |
| commit | 48aff82709769b098321c738f3444b9bdaa694c6 (patch) | |
| tree | e00c7c43e2d9b603a5a6af576b1685e400410dee /lib/gitlab/ci | |
| parent | 879f5329ee916a948223f8f43d77fba4da6cd028 (diff) | |
Add latest changes from gitlab-org/gitlab@13-5-stable-eev13.5.0-rc42
Diffstat (limited to 'lib/gitlab/ci')
77 files changed, 705 insertions, 237 deletions
diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb index 0373a12ab69..6d152c052dc 100644 --- a/lib/gitlab/ci/ansi2json/converter.rb +++ b/lib/gitlab/ci/ansi2json/converter.rb @@ -104,23 +104,24 @@ module Gitlab action = scanner[1] timestamp = scanner[2] section = scanner[3] + options = parse_section_options(scanner[4]) section_name = sanitize_section_name(section) - if action == "start" - handle_section_start(scanner, section_name, timestamp) - elsif action == "end" + if action == 'start' + handle_section_start(scanner, section_name, timestamp, options) + elsif action == 'end' handle_section_end(scanner, section_name, timestamp) else raise 'unsupported action' end end - def handle_section_start(scanner, section, timestamp) + def handle_section_start(scanner, section, timestamp, options) # We make a new line for new section flush_current_line - @state.open_section(section, timestamp) + @state.open_section(section, timestamp, options) # we need to consume match after handling # the open of section, as we want the section @@ -157,6 +158,18 @@ module Gitlab def sanitize_section_name(section) section.to_s.downcase.gsub(/[^a-z0-9]/, '-') end + + def parse_section_options(raw_options) + return unless raw_options + + # We need to remove the square brackets and split + # by comma to get a list of the options + options = raw_options[1...-1].split ',' + + # Now split each option by equals to separate + # each in the format [key, value] + options.to_h { |option| option.split '=' } + end end end end diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb index 21aa1f84353..b1dee0e1ecc 100644 --- a/lib/gitlab/ci/ansi2json/line.rb +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -32,7 +32,7 @@ module Gitlab end attr_reader :offset, :sections, :segments, :current_segment, - :section_header, :section_duration + :section_header, :section_duration, :section_options def initialize(offset:, style:, sections: []) @offset = offset @@ -68,6 +68,10 @@ module Gitlab @sections << section end + def set_section_options(options) + @section_options = options + end + def set_as_section_header @section_header = true end @@ -90,6 +94,7 @@ module Gitlab result[:section] = sections.last if sections.any? result[:section_header] = true if @section_header result[:section_duration] = @section_duration if @section_duration + result[:section_options] = @section_options if @section_options end end end diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb index 38d36e6950c..b2b6ce649ed 100644 --- a/lib/gitlab/ci/ansi2json/state.rb +++ b/lib/gitlab/ci/ansi2json/state.rb @@ -26,10 +26,11 @@ module Gitlab Base64.urlsafe_encode64(state.to_json) end - def open_section(section, timestamp) + def open_section(section, timestamp, options) @open_sections[section] = timestamp @current_line.add_section(section) + @current_line.set_section_options(options) @current_line.set_as_section_header end diff --git a/lib/gitlab/ci/artifact_file_reader.rb b/lib/gitlab/ci/artifact_file_reader.rb index 6395a20ca99..b0fad026ec5 100644 --- a/lib/gitlab/ci/artifact_file_reader.rb +++ b/lib/gitlab/ci/artifact_file_reader.rb @@ -45,14 +45,6 @@ module Gitlab end def read_zip_file!(file_path) - if ::Gitlab::Ci::Features.new_artifact_file_reader_enabled?(job.project) - read_with_new_artifact_file_reader(file_path) - else - read_with_legacy_artifact_file_reader(file_path) - end - end - - def read_with_new_artifact_file_reader(file_path) job.artifacts_file.use_open_file do |file| zip_file = Zip::File.new(file, false, true) entry = zip_file.find_entry(file_path) @@ -69,25 +61,6 @@ module Gitlab end end - def read_with_legacy_artifact_file_reader(file_path) - job.artifacts_file.use_file do |archive_path| - Zip::File.open(archive_path) do |zip_file| - entry = zip_file.find_entry(file_path) - unless entry - raise Error, "Path `#{file_path}` does not exist inside the `#{job.name}` artifacts archive!" - end - - if entry.name_is_directory? - raise Error, "Path `#{file_path}` was expected to be a file but it was a directory!" - end - - zip_file.get_input_stream(entry) do |is| - is.read - end - end - end - end - def max_archive_size_in_mb ActiveSupport::NumberHelper.number_to_human_size(MAX_ARCHIVE_SIZE) end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 9d269831679..071a8ef830f 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -54,6 +54,10 @@ module Gitlab root.variables_value end + def variables_with_data + root.variables_entry.value_with_data + end + def stages root.stages_value end diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb index a8b67a1db4f..1740032e5c7 100644 --- a/lib/gitlab/ci/config/entry/bridge.rb +++ b/lib/gitlab/ci/config/entry/bridge.rb @@ -11,15 +11,18 @@ module Gitlab class Bridge < ::Gitlab::Config::Entry::Node include ::Gitlab::Ci::Config::Entry::Processable + ALLOWED_WHEN = %w[on_success on_failure always manual].freeze ALLOWED_KEYS = %i[trigger].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS with_options allow_nil: true do - validates :when, - inclusion: { in: %w[on_success on_failure always], - message: 'should be on_success, on_failure or always' } + validates :allow_failure, boolean: true + validates :when, inclusion: { + in: ALLOWED_WHEN, + message: "should be one of: #{ALLOWED_WHEN.join(', ')}" + } end validate on: :composed do @@ -57,11 +60,19 @@ module Gitlab true end + def manual_action? + self.when == 'manual' + end + + def ignored? + allow_failure.nil? ? manual_action? : allow_failure + end + def value super.merge( trigger: (trigger_value if trigger_defined?), needs: (needs_value if needs_defined?), - ignore: !!allow_failure, + ignore: ignored?, when: self.when, scheduling_type: needs_defined? && !bridge_needs ? :dag : :stage ).compact diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index a304d9b724f..6b036182706 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -9,14 +9,28 @@ module Gitlab # class Cache < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[key untracked paths policy].freeze + ALLOWED_KEYS = %i[key untracked paths when policy].freeze + ALLOWED_POLICY = %w[pull-push push pull].freeze DEFAULT_POLICY = 'pull-push' + ALLOWED_WHEN = %w[on_success on_failure always].freeze + DEFAULT_WHEN = 'on_success' validations do - validates :config, allowed_keys: ALLOWED_KEYS - validates :policy, inclusion: { in: %w[pull-push push pull], message: 'should be pull-push, push, or pull' }, allow_blank: true + validates :config, type: Hash, allowed_keys: ALLOWED_KEYS + validates :policy, + inclusion: { in: ALLOWED_POLICY, message: 'should be pull-push, push, or pull' }, + allow_blank: true + + with_options allow_nil: true do + validates :when, + inclusion: { + in: ALLOWED_WHEN, + message: 'should be on_success, on_failure or always' + } + end end entry :key, Entry::Key, @@ -28,13 +42,15 @@ module Gitlab entry :paths, Entry::Paths, description: 'Specify which paths should be cached across builds.' - attributes :policy + attributes :policy, :when def value result = super result[:key] = key_value result[:policy] = policy || DEFAULT_POLICY + # Use self.when to avoid conflict with reserved word + result[:when] = self.when || DEFAULT_WHEN result end diff --git a/lib/gitlab/ci/config/entry/include.rb b/lib/gitlab/ci/config/entry/include.rb index 9c2e5f641d0..ad0ed00aa6f 100644 --- a/lib/gitlab/ci/config/entry/include.rb +++ b/lib/gitlab/ci/config/entry/include.rb @@ -10,7 +10,7 @@ module Gitlab class Include < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable - ALLOWED_KEYS = %i[local file remote template artifact job].freeze + ALLOWED_KEYS = %i[local file remote template artifact job project ref].freeze validations do validates :config, hash_or_string: true @@ -22,6 +22,10 @@ module Gitlab if config[:artifact] && config[:job].blank? errors.add(:config, "must specify the job where to fetch the artifact from") end + + if config[:project] && config[:file].blank? + errors.add(:config, "must specify the file where to fetch the config from") + end end end end diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb index b5ce42969a5..b0fd9cef10b 100644 --- a/lib/gitlab/ci/config/entry/jobs.rb +++ b/lib/gitlab/ci/config/entry/jobs.rb @@ -7,7 +7,7 @@ module Gitlab ## # Entry that represents a set of jobs. # - class Jobs < ::Gitlab::Config::Entry::Node + class Jobs < ::Gitlab::Config::Entry::ComposableHash include ::Gitlab::Config::Entry::Validatable validations do @@ -36,6 +36,10 @@ module Gitlab end end + def composable_class(name, config) + self.class.find_type(name, config) + end + TYPES = [Entry::Hidden, Entry::Job, Entry::Bridge].freeze private_constant :TYPES @@ -49,29 +53,6 @@ module Gitlab type.matching?(name, config) end end - - # rubocop: disable CodeReuse/ActiveRecord - def compose!(deps = nil) - super do - @config.each do |name, config| - node = self.class.find_type(name, config) - next unless node - - factory = ::Gitlab::Config::Entry::Factory.new(node) - .value(config || {}) - .metadata(name: name) - .with(key: name, parent: self, - description: "#{name} job definition.") - - @entries[name] = factory.create! - end - - @entries.each_value do |entry| - entry.compose!(deps) - end - end - end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/config/entry/needs.rb b/lib/gitlab/ci/config/entry/needs.rb index d7ba8624882..66cd57b8cf3 100644 --- a/lib/gitlab/ci/config/entry/needs.rb +++ b/lib/gitlab/ci/config/entry/needs.rb @@ -7,7 +7,7 @@ module Gitlab ## # Entry that represents a set of needs dependencies. # - class Needs < ::Gitlab::Config::Entry::Node + class Needs < ::Gitlab::Config::Entry::ComposableArray include ::Gitlab::Config::Entry::Validatable validations do @@ -29,27 +29,16 @@ module Gitlab end end - def compose!(deps = nil) - super(deps) do - [@config].flatten.each_with_index do |need, index| - @entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Need) - .value(need) - .with(key: "need", parent: self, description: "need definition.") # rubocop:disable CodeReuse/ActiveRecord - .create! - end - - @entries.each_value do |entry| - entry.compose!(deps) - end - end - end - def value - values = @entries.values.select(&:type) + values = @entries.select(&:type) values.group_by(&:type).transform_values do |values| values.map(&:value) end end + + def composable_class + Entry::Need + end end end end diff --git a/lib/gitlab/ci/config/entry/ports.rb b/lib/gitlab/ci/config/entry/ports.rb index 01ffcc7dd87..d26b31deca8 100644 --- a/lib/gitlab/ci/config/entry/ports.rb +++ b/lib/gitlab/ci/config/entry/ports.rb @@ -7,7 +7,7 @@ module Gitlab ## # Entry that represents a configuration of the ports of a Docker service. # - class Ports < ::Gitlab::Config::Entry::Node + class Ports < ::Gitlab::Config::Entry::ComposableArray include ::Gitlab::Config::Entry::Validatable validations do @@ -16,28 +16,8 @@ module Gitlab validates :config, port_unique: true end - def compose!(deps = nil) - super do - @entries = [] - @config.each do |config| - @entries << ::Gitlab::Config::Entry::Factory.new(Entry::Port) - .value(config || {}) - .with(key: "port", parent: self, description: "port definition.") # rubocop:disable CodeReuse/ActiveRecord - .create! - end - - @entries.each do |entry| - entry.compose!(deps) - end - end - end - - def value - @entries.map(&:value) - end - - def descendants - @entries + def composable_class + Entry::Port end end end diff --git a/lib/gitlab/ci/config/entry/product/matrix.rb b/lib/gitlab/ci/config/entry/product/matrix.rb index 6af809d46c1..d4ee0978e1b 100644 --- a/lib/gitlab/ci/config/entry/product/matrix.rb +++ b/lib/gitlab/ci/config/entry/product/matrix.rb @@ -46,13 +46,11 @@ module Gitlab end end - # rubocop:disable CodeReuse/ActiveRecord def number_of_generated_jobs value.sum do |config| config.values.reduce(1) { |acc, values| acc * values.size } end end - # rubocop:enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb index ac4f70fb69e..2481989060e 100644 --- a/lib/gitlab/ci/config/entry/product/variables.rb +++ b/lib/gitlab/ci/config/entry/product/variables.rb @@ -14,7 +14,7 @@ module Gitlab validations do validates :config, variables: { array_values: true } validates :config, length: { - minimum: 2, + minimum: :minimum, too_short: 'requires at least %{count} items' } end @@ -28,6 +28,10 @@ module Gitlab .map { |key, value| [key.to_s, Array(value).map(&:to_s)] } .to_h end + + def minimum + ::Gitlab::Ci::Features.one_dimensional_matrix_enabled? ? 1 : 2 + end end end end diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 0ae65f43723..f2fd8ac7fd9 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -15,7 +15,7 @@ module Gitlab %i[junit codequality sast secret_detection dependency_scanning container_scanning dast performance browser_performance load_performance license_management license_scanning metrics lsif dotenv cobertura terraform accessibility cluster_applications - requirements coverage_fuzzing].freeze + requirements coverage_fuzzing api_fuzzing].freeze attributes ALLOWED_KEYS @@ -25,6 +25,7 @@ module Gitlab with_options allow_nil: true do validates :junit, array_of_strings_or_string: true + validates :api_fuzzing, array_of_strings_or_string: true validates :coverage_fuzzing, array_of_strings_or_string: true validates :sast, array_of_strings_or_string: true validates :sast, array_of_strings_or_string: true diff --git a/lib/gitlab/ci/config/entry/rules.rb b/lib/gitlab/ci/config/entry/rules.rb index 2fbc3d9e367..bf74f995e80 100644 --- a/lib/gitlab/ci/config/entry/rules.rb +++ b/lib/gitlab/ci/config/entry/rules.rb @@ -4,7 +4,7 @@ module Gitlab module Ci class Config module Entry - class Rules < ::Gitlab::Config::Entry::Node + class Rules < ::Gitlab::Config::Entry::ComposableArray include ::Gitlab::Config::Entry::Validatable validations do @@ -12,24 +12,13 @@ module Gitlab validates :config, type: Array end - def compose!(deps = nil) - super(deps) do - @config.each_with_index do |rule, index| - @entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Rules::Rule) - .value(rule) - .with(key: "rule", parent: self, description: "rule definition.") # rubocop:disable CodeReuse/ActiveRecord - .create! - end - - @entries.each_value do |entry| - entry.compose!(deps) - end - end - end - def value @config end + + def composable_class + Entry::Rules::Rule + end end end end diff --git a/lib/gitlab/ci/config/entry/services.rb b/lib/gitlab/ci/config/entry/services.rb index 83baa83711f..44e2903a300 100644 --- a/lib/gitlab/ci/config/entry/services.rb +++ b/lib/gitlab/ci/config/entry/services.rb @@ -7,7 +7,7 @@ module Gitlab ## # Entry that represents a configuration of Docker services. # - class Services < ::Gitlab::Config::Entry::Node + class Services < ::Gitlab::Config::Entry::ComposableArray include ::Gitlab::Config::Entry::Validatable validations do @@ -15,28 +15,8 @@ module Gitlab validates :config, services_with_ports_alias_unique: true, if: ->(record) { record.opt(:with_image_ports) } end - def compose!(deps = nil) - super do - @entries = [] - @config.each do |config| - @entries << ::Gitlab::Config::Entry::Factory.new(Entry::Service) - .value(config || {}) - .with(key: "service", parent: self, description: "service definition.") # rubocop:disable CodeReuse/ActiveRecord - .create! - end - - @entries.each do |entry| - entry.compose!(deps) - end - end - end - - def value - @entries.map(&:value) - end - - def descendants - @entries + def composable_class + Entry::Service end end end diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index c9d0c7cb568..e258f7128fc 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -10,16 +10,32 @@ module Gitlab class Variables < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable + ALLOWED_VALUE_DATA = %i[value description].freeze + validations do - validates :config, variables: true + validates :config, variables: { allowed_value_data: ALLOWED_VALUE_DATA } + end + + def value + Hash[@config.map { |key, value| [key.to_s, expand_value(value)[:value]] }] end def self.default(**) {} end - def value - Hash[@config.map { |key, value| [key.to_s, value.to_s] }] + def value_with_data + Hash[@config.map { |key, value| [key.to_s, expand_value(value)] }] + end + + private + + def expand_value(value) + if value.is_a?(Hash) + { value: value[:value].to_s, description: value[:description] } + else + { value: value.to_s, description: nil } + end end end end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index e770187b124..1b58e3ec71a 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -10,10 +10,6 @@ module Gitlab ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: true) end - def self.job_heartbeats_runner?(project) - ::Feature.enabled?(:ci_job_heartbeats_runner, project, default_enabled: true) - end - def self.instance_variables_ui_enabled? ::Feature.enabled?(:ci_instance_variables_ui, default_enabled: true) end @@ -35,10 +31,6 @@ module Gitlab ::Feature.enabled?(:ci_raise_job_rules_without_workflow_rules_warning, default_enabled: true) end - def self.bulk_insert_on_create?(project) - ::Feature.enabled?(:ci_bulk_insert_on_create, project, default_enabled: true) - end - # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` # is a safe switch to disable the feature for a parituclar project when something went wrong, # therefore it's not supposed to be enabled by default. @@ -54,25 +46,25 @@ module Gitlab Feature.enabled?(:project_transactionless_destroy, project, default_enabled: false) end - def self.coverage_report_view?(project) - ::Feature.enabled?(:coverage_report_view, project, default_enabled: true) - end - - def self.child_of_child_pipeline_enabled?(project) - ::Feature.enabled?(:ci_child_of_child_pipeline, project, default_enabled: true) - end - def self.trace_overwrite? ::Feature.enabled?(:ci_trace_overwrite, type: :ops, default_enabled: false) end def self.accept_trace?(project) ::Feature.enabled?(:ci_enable_live_trace, project) && - ::Feature.enabled?(:ci_accept_trace, project, type: :ops, default_enabled: false) + ::Feature.enabled?(:ci_accept_trace, project, type: :ops, default_enabled: true) + end + + def self.log_invalid_trace_chunks?(project) + ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) + end + + def self.one_dimensional_matrix_enabled? + ::Feature.enabled?(:one_dimensional_matrix, default_enabled: true) end - def self.new_artifact_file_reader_enabled?(project) - ::Feature.enabled?(:ci_new_artifact_file_reader, project, default_enabled: false) + def self.manual_bridges_enabled?(project) + ::Feature.enabled?(:ci_manual_bridges, project, default_enabled: true) end end end diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index 86a9ebfa451..44f2ac23ce3 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -4,10 +4,11 @@ module Gitlab module Ci class Lint class Result - attr_reader :jobs, :errors, :warnings + attr_reader :jobs, :merged_yaml, :errors, :warnings - def initialize(jobs:, errors:, warnings:) + def initialize(jobs:, merged_yaml:, errors:, warnings:) @jobs = jobs + @merged_yaml = merged_yaml @errors = errors @warnings = warnings end @@ -39,6 +40,7 @@ module Gitlab Result.new( jobs: dry_run_convert_to_jobs(pipeline.stages), + merged_yaml: pipeline.merged_yaml, errors: pipeline.error_messages.map(&:content), warnings: pipeline.warning_messages(limit: ::Gitlab::Ci::Warnings::MAX_LIMIT).map(&:content) ) @@ -54,6 +56,7 @@ module Gitlab Result.new( jobs: static_validation_convert_to_jobs(result), + merged_yaml: result.merged_yaml, errors: result.errors, warnings: result.warnings.take(::Gitlab::Ci::Warnings::MAX_LIMIT) # rubocop: disable CodeReuse/ActiveRecord ) diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb index 5746f38ae5b..50cd703da4a 100644 --- a/lib/gitlab/ci/parsers/test/junit.rb +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -8,12 +8,17 @@ module Gitlab JunitParserError = Class.new(Gitlab::Ci::Parsers::ParserError) ATTACHMENT_TAG_REGEX = /\[\[ATTACHMENT\|(?<path>.+?)\]\]/.freeze - def parse!(xml_data, test_suite, **args) + def parse!(xml_data, test_suite, job:) root = Hash.from_xml(xml_data) + total_parsed = 0 + max_test_cases = job.max_test_cases_per_report all_cases(root) do |test_case| - test_case = create_test_case(test_case, args) + test_case = create_test_case(test_case, test_suite, job) test_suite.add_test_case(test_case) + total_parsed += 1 + + ensure_test_cases_limited!(total_parsed, max_test_cases) end rescue Nokogiri::XML::SyntaxError => e test_suite.set_suite_error("JUnit XML parsing failed: #{e}") @@ -23,6 +28,12 @@ module Gitlab private + def ensure_test_cases_limited!(total_parsed, limit) + return unless limit > 0 && total_parsed > limit + + raise JunitParserError.new("number of test cases exceeded the limit of #{limit}") + end + def all_cases(root, parent = nil, &blk) return unless root.present? @@ -33,20 +44,24 @@ module Gitlab all_cases(node['testsuites'], root, &blk) unless parent # we require at least one level of testsuites or testsuite - each_case(node['testcase'], &blk) if parent + each_case(node['testcase'], node['name'], &blk) if parent # we allow multiple nested 'testsuite' (eg. PHPUnit) all_cases(node['testsuite'], root, &blk) end end - def each_case(testcase, &blk) + def each_case(testcase, testsuite_name, &blk) return unless testcase.present? - [testcase].flatten.compact.map(&blk) + [testcase].flatten.compact.each do |tc| + tc['suite_name'] = testsuite_name + + yield(tc) + end end - def create_test_case(data, args) + def create_test_case(data, test_suite, job) if data.key?('failure') status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED system_output = data['failure'] @@ -63,6 +78,7 @@ module Gitlab end ::Gitlab::Ci::Reports::TestCase.new( + suite_name: data['suite_name'] || test_suite.name, classname: data['classname'], name: data['name'], file: data['file'], @@ -70,10 +86,14 @@ module Gitlab status: status, system_output: system_output, attachment: attachment, - job: args.fetch(:job) + job: job ) end + def suite_name(parent, test_suite) + parent.dig('testsuite', 'name') || test_suite.name + end + def attachment_path(data) return unless data diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index d1882059dd8..06096a33f27 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -16,7 +16,7 @@ module Gitlab ) do include Gitlab::Utils::StrongMemoize - def initialize(**params) + def initialize(params = {}) params.each do |key, value| self[key] = value end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 8ccb33ffd34..c3fbd0c9e24 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -28,6 +28,8 @@ module Gitlab error(result.errors.first, config_error: true) end + @pipeline.merged_yaml = result.merged_yaml + rescue => ex Gitlab::ErrorTracking.track_exception(ex, project_id: project.id, diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 34649fe16f3..81ef3bb074d 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -8,7 +8,7 @@ module Gitlab include Chain::Helpers def perform! - BulkInsertableAssociations.with_bulk_insert(enabled: ::Gitlab::Ci::Features.bulk_insert_on_create?(project)) do + BulkInsertableAssociations.with_bulk_insert do pipeline.save! end rescue ActiveRecord::RecordInvalid => e diff --git a/lib/gitlab/ci/pipeline/seed/build/cache.rb b/lib/gitlab/ci/pipeline/seed/build/cache.rb index a4127ea0be2..8d6fe13c3b9 100644 --- a/lib/gitlab/ci/pipeline/seed/build/cache.rb +++ b/lib/gitlab/ci/pipeline/seed/build/cache.rb @@ -13,6 +13,7 @@ module Gitlab @paths = local_cache.delete(:paths) @policy = local_cache.delete(:policy) @untracked = local_cache.delete(:untracked) + @when = local_cache.delete(:when) raise ArgumentError, "unknown cache keys: #{local_cache.keys}" if local_cache.any? end @@ -24,7 +25,8 @@ module Gitlab key: key_string, paths: @paths, policy: @policy, - untracked: @untracked + untracked: @untracked, + when: @when }.compact.presence }.compact } diff --git a/lib/gitlab/ci/reports/test_case.rb b/lib/gitlab/ci/reports/test_case.rb index 15a3c862c9e..8c70dbb6931 100644 --- a/lib/gitlab/ci/reports/test_case.rb +++ b/lib/gitlab/ci/reports/test_case.rb @@ -10,9 +10,10 @@ module Gitlab STATUS_ERROR = 'error' STATUS_TYPES = [STATUS_ERROR, STATUS_FAILED, STATUS_SUCCESS, STATUS_SKIPPED].freeze - attr_reader :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key, :attachment, :job + attr_reader :suite_name, :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key, :attachment, :job def initialize(params) + @suite_name = params.fetch(:suite_name) @name = params.fetch(:name) @classname = params.fetch(:classname) @file = params.fetch(:file, nil) @@ -23,7 +24,7 @@ module Gitlab @attachment = params.fetch(:attachment, nil) @job = params.fetch(:job, nil) - @key = sanitize_key_name("#{classname}_#{name}") + @key = hash_key("#{suite_name}_#{classname}_#{name}") end def has_attachment? @@ -42,8 +43,8 @@ module Gitlab private - def sanitize_key_name(key) - key.gsub(/[^0-9A-Za-z]/, '-') + def hash_key(key) + Digest::SHA256.hexdigest(key) end end end diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb index e9b78b841e4..00920dfbd54 100644 --- a/lib/gitlab/ci/reports/test_suite.rb +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -12,18 +12,24 @@ module Gitlab def initialize(name = nil) @name = name @test_cases = {} + @all_test_cases = [] @total_time = 0.0 - @duplicate_cases = [] end def add_test_case(test_case) - @duplicate_cases << test_case if existing_key?(test_case) - @test_cases[test_case.status] ||= {} @test_cases[test_case.status][test_case.key] = test_case @total_time += test_case.execution_time end + def each_test_case + @test_cases.each do |status, test_cases| + test_cases.values.each do |test_case| + yield test_case + end + end + end + # rubocop: disable CodeReuse/ActiveRecord def total_count return 0 if suite_error @@ -86,10 +92,6 @@ module Gitlab private - def existing_key?(test_case) - @test_cases[test_case.status]&.key?(test_case.key) - end - def sort_by_status @test_cases = @test_cases.sort_by { |status, _| Gitlab::Ci::Reports::TestCase::STATUS_TYPES.index(status) }.to_h end diff --git a/lib/gitlab/ci/runner/backoff.rb b/lib/gitlab/ci/runner/backoff.rb new file mode 100644 index 00000000000..95d7719e9cb --- /dev/null +++ b/lib/gitlab/ci/runner/backoff.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Runner + ## + # Runner Backoff class is an implementation of an exponential backoff + # used when a runner communicates with GitLab. We typically use it when a + # runner retries sending a build status after we created a build pending + # state. + # + # Backoff is calculated based on the backoff slot which is always a power + # of 2: + # + # 0s - 3s duration -> 1 second backoff + # 4s - 7s duration -> 2 seconds backoff + # 8s - 15s duration -> 4 seconds backoff + # 16s - 31s duration -> 8 seconds backoff + # 32s - 63s duration -> 16 seconds backoff + # 64s - 127s duration -> 32 seconds backoff + # 127s - 256s+ duration -> 64 seconds backoff + # + # It means that first 15 requests made by a runner will need to respect + # following backoffs: + # + # 0s -> 1 second backoff (backoff started, slot 0, 2^0 backoff) + # 1s -> 1 second backoff + # 2s -> 1 second backoff + # 3s -> 1 seconds backoff + # (slot 1 - 2^1 backoff) + # 4s -> 2 seconds backoff + # 6s -> 2 seconds backoff + # (slot 2 - 2^2 backoff) + # 8s -> 4 seconds backoff + # 12s -> 4 seconds backoff + # (slot 3 - 2^3 backoff) + # 16s -> 8 seconds backoff + # 24s -> 8 seconds backoff + # (slot 4 - 2^4 backoff) + # 32s -> 16 seconds backoff + # 48s -> 16 seconds backoff + # (slot 5 - 2^5 backoff) + # 64s -> 32 seconds backoff + # 96s -> 32 seconds backoff + # (slot 6 - 2^6 backoff) + # 128s -> 64 seconds backoff + # + # There is a cap on the backoff - it will never exceed 64 seconds. + # + class Backoff + def initialize(started) + @started = started + + if duration < 0 + raise ArgumentError, 'backoff duration negative' + end + end + + def duration + (Time.current - @started).ceil + end + + def slot + return 0 if duration < 2 + + Math.log(duration, 2).floor - 1 + end + + def to_seconds + 2**[slot, 6].min + end + end + end + end +end diff --git a/lib/gitlab/ci/status/bridge/action.rb b/lib/gitlab/ci/status/bridge/action.rb new file mode 100644 index 00000000000..1ba4700d9b0 --- /dev/null +++ b/lib/gitlab/ci/status/bridge/action.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Bridge + class Action < Status::Build::Action + end + end + end + end +end diff --git a/lib/gitlab/ci/status/bridge/common.rb b/lib/gitlab/ci/status/bridge/common.rb index b95565b5e09..d66d4b20bba 100644 --- a/lib/gitlab/ci/status/bridge/common.rb +++ b/lib/gitlab/ci/status/bridge/common.rb @@ -14,7 +14,6 @@ module Gitlab end def details_path - return unless Feature.enabled?(:ci_bridge_pipeline_details, subject.project, default_enabled: true) return unless can?(user, :read_pipeline, downstream_pipeline) project_pipeline_path(downstream_project, downstream_pipeline) diff --git a/lib/gitlab/ci/status/bridge/factory.rb b/lib/gitlab/ci/status/bridge/factory.rb index 5d397dba0de..b9bd66cee71 100644 --- a/lib/gitlab/ci/status/bridge/factory.rb +++ b/lib/gitlab/ci/status/bridge/factory.rb @@ -6,7 +6,10 @@ module Gitlab module Bridge class Factory < Status::Factory def self.extended_statuses - [Status::Bridge::Failed] + [[Status::Bridge::Failed], + [Status::Bridge::Manual], + [Status::Bridge::Play], + [Status::Bridge::Action]] end def self.common_helpers diff --git a/lib/gitlab/ci/status/bridge/manual.rb b/lib/gitlab/ci/status/bridge/manual.rb new file mode 100644 index 00000000000..e07e645a34d --- /dev/null +++ b/lib/gitlab/ci/status/bridge/manual.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Bridge + class Manual < Status::Build::Manual + end + end + end + end +end diff --git a/lib/gitlab/ci/status/bridge/play.rb b/lib/gitlab/ci/status/bridge/play.rb new file mode 100644 index 00000000000..ae00ef6c2ad --- /dev/null +++ b/lib/gitlab/ci/status/bridge/play.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Bridge + class Play < Status::Build::Play + def has_action? + can?(user, :play_job, subject) + end + + def self.matches?(bridge, user) + bridge.playable? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/canceled.rb b/lib/gitlab/ci/status/canceled.rb index 07f37732023..f173964b36c 100644 --- a/lib/gitlab/ci/status/canceled.rb +++ b/lib/gitlab/ci/status/canceled.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_canceled' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/created.rb b/lib/gitlab/ci/status/created.rb index fface4bb97b..33e67314d93 100644 --- a/lib/gitlab/ci/status/created.rb +++ b/lib/gitlab/ci/status/created.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_created' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/failed.rb b/lib/gitlab/ci/status/failed.rb index 770ed7d4d5a..215d27734a7 100644 --- a/lib/gitlab/ci/status/failed.rb +++ b/lib/gitlab/ci/status/failed.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_failed' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/manual.rb b/lib/gitlab/ci/status/manual.rb index 50c92add400..eb376df5f22 100644 --- a/lib/gitlab/ci/status/manual.rb +++ b/lib/gitlab/ci/status/manual.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_manual' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/pending.rb b/lib/gitlab/ci/status/pending.rb index cea7e6ed938..4280ad84534 100644 --- a/lib/gitlab/ci/status/pending.rb +++ b/lib/gitlab/ci/status/pending.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_pending' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/preparing.rb b/lib/gitlab/ci/status/preparing.rb index 1ebdbc482b7..e59d1d2eed1 100644 --- a/lib/gitlab/ci/status/preparing.rb +++ b/lib/gitlab/ci/status/preparing.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_preparing' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/running.rb b/lib/gitlab/ci/status/running.rb index ac7dd74cdce..eed1983e60e 100644 --- a/lib/gitlab/ci/status/running.rb +++ b/lib/gitlab/ci/status/running.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_running' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/scheduled.rb b/lib/gitlab/ci/status/scheduled.rb index 16ad1da89e3..e9068c326cf 100644 --- a/lib/gitlab/ci/status/scheduled.rb +++ b/lib/gitlab/ci/status/scheduled.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_scheduled' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/skipped.rb b/lib/gitlab/ci/status/skipped.rb index aaec1e1d201..238aa3ab4f9 100644 --- a/lib/gitlab/ci/status/skipped.rb +++ b/lib/gitlab/ci/status/skipped.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_skipped' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/success.rb b/lib/gitlab/ci/status/success.rb index 020f2c5b89f..2a10e60414e 100644 --- a/lib/gitlab/ci/status/success.rb +++ b/lib/gitlab/ci/status/success.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_success' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/waiting_for_resource.rb b/lib/gitlab/ci/status/waiting_for_resource.rb index 4c9e706bc51..2026148f752 100644 --- a/lib/gitlab/ci/status/waiting_for_resource.rb +++ b/lib/gitlab/ci/status/waiting_for_resource.rb @@ -23,6 +23,10 @@ module Gitlab def group 'waiting-for-resource' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml b/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml new file mode 100644 index 00000000000..267027a1b8a --- /dev/null +++ b/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml @@ -0,0 +1,11 @@ +stages: + - provision + - review + - production + +variables: + AUTO_DEVOPS_PLATFORM_TARGET: EC2 + +include: + - template: Jobs/CF-Provision.gitlab-ci.yml + - template: Jobs/Deploy/EC2.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 6966ce88b30..cba13f374f4 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -20,6 +20,7 @@ # * dast: DAST_DISABLED # * review: REVIEW_DISABLED # * stop_review: REVIEW_DISABLED +# * code_intelligence: CODE_INTELLIGENCE_DISABLED # # In order to deploy, you must have a Kubernetes cluster configured either # via a project integration, or via group/project variables. @@ -159,6 +160,7 @@ include: - template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml - template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml - template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml + - template: Jobs/Code-Intelligence.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml - template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml - template: Jobs/Deploy/ECS.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml - template: Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml index 368069844ea..67e58d9ee99 100644 --- a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml @@ -1,4 +1,4 @@ -# see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options +# see https://docs.gitlab.com/ee/ci/yaml/README.html for all available options # you can delete this line if you're not using Docker image: busybox:latest diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml index c3568c0d2c8..0c5850bdb8e 100644 --- a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml @@ -1,7 +1,7 @@ # Based on openjdk:8, already includes lein image: clojure:lein-2.7.0 # If you need to configure a database, add a `services` section here -# See https://docs.gitlab.com/ce/ci/services/postgres.html +# See https://docs.gitlab.com/ee/ci/services/postgres.html # Make sure you configure the connection as well before_script: diff --git a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml index e9301a2638d..538f96c4084 100644 --- a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml @@ -4,7 +4,7 @@ image: "crystallang/crystal:latest" # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service # services: # - mysql:latest # - redis:latest diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml index d35fcb0f807..c657c7e8eb1 100644 --- a/lib/gitlab/ci/templates/Django.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml @@ -4,7 +4,7 @@ image: python:latest # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service services: - mysql:latest - postgres:latest @@ -13,7 +13,7 @@ variables: POSTGRES_DB: database_name # This folder is cached between builds -# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +# http://docs.gitlab.com/ee/ci/yaml/README.html#cache cache: paths: - ~/.cache/pip/ diff --git a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml index 4d4c6a64cd5..7271526ab1b 100644 --- a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml @@ -2,7 +2,7 @@ image: elixir:latest # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service services: - mysql:latest - redis:latest diff --git a/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml new file mode 100644 index 00000000000..31ca68c57d7 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml @@ -0,0 +1,14 @@ +stages: + - provision + +cloud_formation: + image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-cloudformation:latest' + stage: provision + script: + - gl-cloudformation create-stack + rules: + - if: '($AUTO_DEVOPS_PLATFORM_TARGET != "EC2") || ($AUTO_DEVOPS_PLATFORM_TARGET != "ECS")' + when: never + - if: '$CI_KUBERNETES_ACTIVE' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml new file mode 100644 index 00000000000..83bc5548614 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml @@ -0,0 +1,16 @@ +code_intelligence_go: + stage: test + needs: [] + allow_failure: true + image: sourcegraph/lsif-go:v1 + rules: + - if: $CODE_INTELLIGENCE_DISABLED + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.go' + script: + - lsif-go + artifacts: + reports: + lsif: dump.lsif diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 568ceceeaa2..ec33020205b 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -9,9 +9,8 @@ code_quality: DOCKER_TLS_CERTDIR: "" CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.10-gitlab.1" needs: [] - before_script: - - export SOURCE_CODE=$PWD script: + - export SOURCE_CODE=$PWD - | if ! docker info &>/dev/null; then if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 86f946aafda..77216a6e404 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.5" dast_environment_deploy: extends: .dast-auto-deploy diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 829fd7a722f..32a207a85d1 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.5" dependencies: [] review: diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 829fd7a722f..8b921305c11 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.0.0-beta.2" dependencies: [] review: @@ -91,7 +91,7 @@ canary: - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret - - auto-deploy deploy canary + - auto-deploy deploy canary 50 environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -114,7 +114,6 @@ canary: - auto-deploy create_secret - auto-deploy deploy - auto-deploy delete canary - - auto-deploy delete rollout - auto-deploy persist_environment_url environment: name: production @@ -163,9 +162,7 @@ production_manual: - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret - - auto-deploy deploy rollout $ROLLOUT_PERCENTAGE - - auto-deploy scale stable $((100-ROLLOUT_PERCENTAGE)) - - auto-deploy delete canary + - auto-deploy deploy canary $ROLLOUT_PERCENTAGE - auto-deploy persist_environment_url environment: name: production diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml new file mode 100644 index 00000000000..ed2172ef7f5 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml @@ -0,0 +1,39 @@ +stages: + - review + - production + +.push-and-deploy: + image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ec2:latest' + script: + - gl-ec2 push-to-s3 + - gl-ec2 deploy-to-ec2 + +review_ec2: + extends: .push-and-deploy + stage: review + environment: + name: review/$CI_COMMIT_REF_NAME + rules: + - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "EC2"' + when: never + - if: '$CI_KUBERNETES_ACTIVE' + when: never + - if: '$REVIEW_DISABLED' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + +production_ec2: + extends: .push-and-deploy + stage: production + environment: + name: production + rules: + - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "EC2"' + when: never + - if: '$CI_KUBERNETES_ACTIVE' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml index da474f8ac88..317e8bfab0e 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -10,6 +10,7 @@ .deploy_to_ecs: image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest' + dependencies: [] script: - ecs update-task-definition diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml index 9bde04dff19..5d2c8024524 100644 --- a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -4,7 +4,7 @@ image: php:latest # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service services: - mysql:latest @@ -13,7 +13,7 @@ variables: MYSQL_ROOT_PASSWORD: secret # This folder is cached between builds -# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +# http://docs.gitlab.com/ee/ci/yaml/README.html#cache cache: paths: - vendor/ diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 7050b41e045..a9638f564f3 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,6 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.29.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.33.0" environment: name: production variables: diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml index b87178141a1..92379ded77c 100644 --- a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml @@ -4,14 +4,14 @@ image: node:latest # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service services: - mysql:latest - redis:latest - postgres:latest # This folder is cached between builds -# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +# http://docs.gitlab.com/ee/ci/yaml/README.html#cache cache: paths: - node_modules/ diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml index 25ea20e454f..84e8223e69b 100644 --- a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml @@ -19,7 +19,7 @@ before_script: - php composer.phar install # Bring in any services we need http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service -# See http://docs.gitlab.com/ce/ci/services/README.html for examples. +# See http://docs.gitlab.com/ee/ci/services/README.html for examples. services: - mysql:5.7 diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml index a683561a455..3a6eac63892 100644 --- a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml @@ -1,7 +1,7 @@ image: node:latest # This folder is cached between builds -# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +# http://docs.gitlab.com/ee/ci/yaml/README.html#cache cache: paths: - node_modules/ diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml index 0c8859dc779..f2f92fe0704 100644 --- a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -1,5 +1,5 @@ # Template project: https://gitlab.com/pages/jekyll -# Docs: https://docs.gitlab.com/ce/pages/ +# Docs: https://docs.gitlab.com/ee/pages/ image: ruby:2.6 variables: diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index b3cad8b858a..275364afae4 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -4,7 +4,7 @@ image: "ruby:2.5" # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service services: - mysql:latest - redis:latest diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml index f35470367cc..94117a79d1c 100644 --- a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml @@ -4,7 +4,7 @@ image: "rust:latest" # Optional: Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service # services: # - mysql:latest # - redis:latest diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index 4b957a8f771..e268b48d133 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -35,4 +35,3 @@ variables: - if: $COVFUZZ_DISABLED when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bcoverage_fuzzing\b/ - - if: $CI_RUNNER_EXECUTABLE_ARCH == "linux" diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml index cc34d23decc..63237e41376 100644 --- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml @@ -1,7 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/license_compliance/ # # Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/license-management#settings +# List of the variables: https://gitlab.com/gitlab-org/security-products/analyzers/license-finder#settings # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 77ea11d01d1..4418ff18d73 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -9,7 +9,7 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec" + SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec, mobsf" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" SAST_ANALYZER_IMAGE_TAG: 2 SCAN_KUBERNETES_MANIFESTS: "false" @@ -125,6 +125,42 @@ gosec-sast: exists: - '**/*.go' +mobsf-android-sast: + extends: .sast-analyzer + services: + - name: opensecurity/mobile-security-framework-mobsf:latest + alias: mobsf + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $CI_COMMIT_BRANCH && + $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/AndroidManifest.xml' + +mobsf-ios-sast: + extends: .sast-analyzer + services: + - name: opensecurity/mobile-security-framework-mobsf:latest + alias: mobsf + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $CI_COMMIT_BRANCH && + $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.xcodeproj/*' + nodejs-scan-sast: extends: .sast-analyzer image: @@ -203,6 +239,11 @@ spotbugs-sast: variables: SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" rules: + - if: $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/AndroidManifest.xml' + when: never - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index bde6a0fbebb..6ebff102ccb 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -34,8 +34,8 @@ secret_detection: when: never - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH script: - - git fetch origin $CI_DEFAULT_BRANCH $CI_BUILD_REF_NAME - - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_BUILD_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt + - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME + - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt - /analyzer run - rm "$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml new file mode 100644 index 00000000000..b08ccf18b58 --- /dev/null +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -0,0 +1,22 @@ +include: + - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml + +stages: + - init + - validate + - build + - deploy + +init: + extends: .init + +validate: + extends: .validate + +build: + extends: .build + +deploy: + extends: .deploy + dependencies: + - build diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml new file mode 100644 index 00000000000..000a1a7f580 --- /dev/null +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -0,0 +1,53 @@ +# Terraform/Base.latest +# +# The purpose of this template is to provide flexibility to the user so +# they are able to only include the jobs that they find interesting. +# +# Therefore, this template is not supposed to run any jobs. The idea is to only +# create hidden jobs. See: https://docs.gitlab.com/ee/ci/yaml/#hide-jobs +# +# There is a more opinionated template which we suggest the users to abide, +# which is the lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml + +image: + name: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + +before_script: + - cd ${TF_ROOT} + +variables: + TF_ROOT: ${CI_PROJECT_DIR} + +cache: + key: "${TF_ROOT}" + paths: + - ${TF_ROOT}/.terraform/ + +.init: &init + stage: init + script: + - gitlab-terraform init + +.validate: &validate + stage: validate + script: + - gitlab-terraform validate + +.build: &build + stage: build + script: + - gitlab-terraform plan + - gitlab-terraform plan-json + artifacts: + paths: + - ${TF_ROOT}/plan.cache + reports: + terraform: ${TF_ROOT}/plan.json + +.deploy: &deploy + stage: deploy + script: + - gitlab-terraform apply + when: manual + only: + - master diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 348e5472cb4..0222ca021b7 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -16,6 +16,7 @@ module Gitlab ArchiveError = Class.new(StandardError) AlreadyArchivedError = Class.new(StandardError) + LockedError = Class.new(StandardError) attr_reader :job @@ -79,11 +80,9 @@ module Gitlab job.trace_chunks.any? || current_path.present? || old_trace.present? end - def read(should_retry: true, &block) + def read(&block) read_stream(&block) - rescue Errno::ENOENT - raise unless should_retry - + rescue Errno::ENOENT, ChunkedIO::FailedToGetChunkError job.reset read_stream(&block) end @@ -130,6 +129,12 @@ module Gitlab end end + def lock(&block) + in_write_lock(&block) + rescue FailedToObtainLockError + raise LockedError, "build trace `#{job.id}` is locked" + end + private def read_stream diff --git a/lib/gitlab/ci/trace/checksum.rb b/lib/gitlab/ci/trace/checksum.rb new file mode 100644 index 00000000000..62532ef1cd2 --- /dev/null +++ b/lib/gitlab/ci/trace/checksum.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Trace + ## + # Trace::Checksum class is responsible for calculating a CRC32 checksum + # of an entire build trace using partial build trace chunks stored in a + # database. + # + # CRC32 checksum can be easily calculated by combining partial checksums + # in a right order. + # + # Then we compare CRC32 checksum provided by a GitLab Runner and expect + # it to be the same as the CRC32 checksum derived from partial chunks. + # + class Checksum + include Gitlab::Utils::StrongMemoize + + attr_reader :build + + def initialize(build) + @build = build + end + + def valid? + return false unless state_crc32.present? + + state_crc32 == chunks_crc32 + end + + def state_crc32 + strong_memoize(:state_crc32) { build.pending_state&.crc32 } + end + + def chunks_crc32 + strong_memoize(:chunks_crc32) do + trace_chunks.reduce(0) do |crc32, chunk| + Zlib.crc32_combine(crc32, chunk.crc32, chunk_size(chunk)) + end + end + end + + def last_chunk + strong_memoize(:last_chunk) { trace_chunks.max } + end + + ## + # Trace chunks will be persisted in a database if an object store is + # not configured - in that case we do not want to load entire raw data + # of all the chunks into memory. + # + # We ignore `raw_data` attribute instead, and rely on internal build + # trace chunk database adapter to handle + # `ActiveModel::MissingAttributeError` exception. + # + # Alternative solution would be separating chunk data from chunk + # metadata on the database level too. + # + def trace_chunks + strong_memoize(:trace_chunks) do + build.trace_chunks.persisted + .select(::Ci::BuildTraceChunk.metadata_attributes) + end + end + + def chunks_count + trace_chunks.to_a.size + end + + private + + def chunk_size(chunk) + if chunk == last_chunk + chunk.size + else + ::Ci::BuildTraceChunk::CHUNK_SIZE + end + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb index 82a7d5fb83c..097436d84ea 100644 --- a/lib/gitlab/ci/trace/metrics.rb +++ b/lib/gitlab/ci/trace/metrics.rb @@ -6,8 +6,20 @@ module Gitlab class Metrics extend Gitlab::Utils::StrongMemoize - OPERATIONS = [:appended, :streamed, :chunked, :mutated, :overwrite, - :accepted, :finalized, :discarded, :conflict].freeze + OPERATIONS = [ + :appended, # new trace data has been written to a chunk + :streamed, # new trace data has been sent by a runner + :chunked, # new trace chunk has been created + :mutated, # trace has been mutated when removing secrets + :overwrite, # runner requested overwritting a build trace + :accepted, # scheduled chunks for migration and responded with 202 + :finalized, # all live build trace chunks have been persisted + :discarded, # failed to persist live chunks before timeout + :conflict, # runner has sent unrecognized build state details + :locked, # build trace has been locked by a different mechanism + :stalled, # failed to migrate chunk due to a worker duplication + :invalid # malformed build trace has been detected using CRC32 + ].freeze def increment_trace_operation(operation: :unknown) unless OPERATIONS.include?(operation) @@ -18,7 +30,11 @@ module Gitlab end def increment_trace_bytes(size) - self.class.trace_bytes.increment(by: size.to_i) + self.class.trace_bytes.increment({}, size.to_i) + end + + def observe_migration_duration(seconds) + self.class.finalize_histogram.observe({}, seconds.to_f) end def self.trace_operations @@ -38,6 +54,17 @@ module Gitlab Gitlab::Metrics.counter(name, comment) end end + + def self.finalize_histogram + strong_memoize(:finalize_histogram) do + name = :gitlab_ci_trace_finalize_duration_seconds + comment = 'Duration of build trace chunks migration to object storage' + buckets = [0.1, 0.5, 1.0, 2.0, 3.0, 5.0, 7.0, 10.0, 20.0, 30.0, 60.0, 300.0] + labels = {} + + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end end end end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index a072036daa8..84a9280e507 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -36,9 +36,9 @@ module Gitlab def self.fabricate(resource) case resource when Hash - self.new(resource.symbolize_keys) + self.new(**resource.symbolize_keys) when ::Ci::HasVariable - self.new(resource.to_runner_variable) + self.new(**resource.to_runner_variable) when self resource.dup else diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 68f61e52df7..52a00e41214 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -95,6 +95,14 @@ module Gitlab }.compact }.compact end + def merged_yaml + @ci_config&.to_hash&.to_yaml + end + + def variables_with_data + @ci_config.variables_with_data + end + private def variables |