diff options
Diffstat (limited to 'lib/gitlab/ci')
-rw-r--r-- | lib/gitlab/ci/ansi2json/converter.rb | 23 | ||||
-rw-r--r-- | lib/gitlab/ci/ansi2json/line.rb | 7 | ||||
-rw-r--r-- | lib/gitlab/ci/ansi2json/state.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/ci/config/entry/product/matrix.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/ci/config/entry/product/variables.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/ci/features.rb | 18 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/command.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/create.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/ci/runner/backoff.rb | 75 | ||||
-rw-r--r-- | lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml | 9 | ||||
-rw-r--r-- | lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml | 1 | ||||
-rw-r--r-- | lib/gitlab/ci/trace.rb | 7 | ||||
-rw-r--r-- | lib/gitlab/ci/trace/checksum.rb | 77 | ||||
-rw-r--r-- | lib/gitlab/ci/trace/metrics.rb | 5 | ||||
-rw-r--r-- | lib/gitlab/ci/variables/collection/item.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/ci/yaml_processor/result.rb | 4 |
16 files changed, 210 insertions, 35 deletions
diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb index 0373a12ab69..6d152c052dc 100644 --- a/lib/gitlab/ci/ansi2json/converter.rb +++ b/lib/gitlab/ci/ansi2json/converter.rb @@ -104,23 +104,24 @@ module Gitlab action = scanner[1] timestamp = scanner[2] section = scanner[3] + options = parse_section_options(scanner[4]) section_name = sanitize_section_name(section) - if action == "start" - handle_section_start(scanner, section_name, timestamp) - elsif action == "end" + if action == 'start' + handle_section_start(scanner, section_name, timestamp, options) + elsif action == 'end' handle_section_end(scanner, section_name, timestamp) else raise 'unsupported action' end end - def handle_section_start(scanner, section, timestamp) + def handle_section_start(scanner, section, timestamp, options) # We make a new line for new section flush_current_line - @state.open_section(section, timestamp) + @state.open_section(section, timestamp, options) # we need to consume match after handling # the open of section, as we want the section @@ -157,6 +158,18 @@ module Gitlab def sanitize_section_name(section) section.to_s.downcase.gsub(/[^a-z0-9]/, '-') end + + def parse_section_options(raw_options) + return unless raw_options + + # We need to remove the square brackets and split + # by comma to get a list of the options + options = raw_options[1...-1].split ',' + + # Now split each option by equals to separate + # each in the format [key, value] + options.to_h { |option| option.split '=' } + end end end end diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb index 21aa1f84353..b1dee0e1ecc 100644 --- a/lib/gitlab/ci/ansi2json/line.rb +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -32,7 +32,7 @@ module Gitlab end attr_reader :offset, :sections, :segments, :current_segment, - :section_header, :section_duration + :section_header, :section_duration, :section_options def initialize(offset:, style:, sections: []) @offset = offset @@ -68,6 +68,10 @@ module Gitlab @sections << section end + def set_section_options(options) + @section_options = options + end + def set_as_section_header @section_header = true end @@ -90,6 +94,7 @@ module Gitlab result[:section] = sections.last if sections.any? result[:section_header] = true if @section_header result[:section_duration] = @section_duration if @section_duration + result[:section_options] = @section_options if @section_options end end end diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb index 38d36e6950c..b2b6ce649ed 100644 --- a/lib/gitlab/ci/ansi2json/state.rb +++ b/lib/gitlab/ci/ansi2json/state.rb @@ -26,10 +26,11 @@ module Gitlab Base64.urlsafe_encode64(state.to_json) end - def open_section(section, timestamp) + def open_section(section, timestamp, options) @open_sections[section] = timestamp @current_line.add_section(section) + @current_line.set_section_options(options) @current_line.set_as_section_header end diff --git a/lib/gitlab/ci/config/entry/product/matrix.rb b/lib/gitlab/ci/config/entry/product/matrix.rb index 6af809d46c1..d4ee0978e1b 100644 --- a/lib/gitlab/ci/config/entry/product/matrix.rb +++ b/lib/gitlab/ci/config/entry/product/matrix.rb @@ -46,13 +46,11 @@ module Gitlab end end - # rubocop:disable CodeReuse/ActiveRecord def number_of_generated_jobs value.sum do |config| config.values.reduce(1) { |acc, values| acc * values.size } end end - # rubocop:enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb index ac4f70fb69e..2481989060e 100644 --- a/lib/gitlab/ci/config/entry/product/variables.rb +++ b/lib/gitlab/ci/config/entry/product/variables.rb @@ -14,7 +14,7 @@ module Gitlab validations do validates :config, variables: { array_values: true } validates :config, length: { - minimum: 2, + minimum: :minimum, too_short: 'requires at least %{count} items' } end @@ -28,6 +28,10 @@ module Gitlab .map { |key, value| [key.to_s, Array(value).map(&:to_s)] } .to_h end + + def minimum + ::Gitlab::Ci::Features.one_dimensional_matrix_enabled? ? 1 : 2 + end end end end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index e770187b124..e14d56af978 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -10,10 +10,6 @@ module Gitlab ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: true) end - def self.job_heartbeats_runner?(project) - ::Feature.enabled?(:ci_job_heartbeats_runner, project, default_enabled: true) - end - def self.instance_variables_ui_enabled? ::Feature.enabled?(:ci_instance_variables_ui, default_enabled: true) end @@ -35,10 +31,6 @@ module Gitlab ::Feature.enabled?(:ci_raise_job_rules_without_workflow_rules_warning, default_enabled: true) end - def self.bulk_insert_on_create?(project) - ::Feature.enabled?(:ci_bulk_insert_on_create, project, default_enabled: true) - end - # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` # is a safe switch to disable the feature for a parituclar project when something went wrong, # therefore it's not supposed to be enabled by default. @@ -54,10 +46,6 @@ module Gitlab Feature.enabled?(:project_transactionless_destroy, project, default_enabled: false) end - def self.coverage_report_view?(project) - ::Feature.enabled?(:coverage_report_view, project, default_enabled: true) - end - def self.child_of_child_pipeline_enabled?(project) ::Feature.enabled?(:ci_child_of_child_pipeline, project, default_enabled: true) end @@ -72,7 +60,11 @@ module Gitlab end def self.new_artifact_file_reader_enabled?(project) - ::Feature.enabled?(:ci_new_artifact_file_reader, project, default_enabled: false) + ::Feature.enabled?(:ci_new_artifact_file_reader, project, default_enabled: true) + end + + def self.one_dimensional_matrix_enabled? + ::Feature.enabled?(:one_dimensional_matrix, default_enabled: false) end end end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index d1882059dd8..06096a33f27 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -16,7 +16,7 @@ module Gitlab ) do include Gitlab::Utils::StrongMemoize - def initialize(**params) + def initialize(params = {}) params.each do |key, value| self[key] = value end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 34649fe16f3..81ef3bb074d 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -8,7 +8,7 @@ module Gitlab include Chain::Helpers def perform! - BulkInsertableAssociations.with_bulk_insert(enabled: ::Gitlab::Ci::Features.bulk_insert_on_create?(project)) do + BulkInsertableAssociations.with_bulk_insert do pipeline.save! end rescue ActiveRecord::RecordInvalid => e diff --git a/lib/gitlab/ci/runner/backoff.rb b/lib/gitlab/ci/runner/backoff.rb new file mode 100644 index 00000000000..95d7719e9cb --- /dev/null +++ b/lib/gitlab/ci/runner/backoff.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Runner + ## + # Runner Backoff class is an implementation of an exponential backoff + # used when a runner communicates with GitLab. We typically use it when a + # runner retries sending a build status after we created a build pending + # state. + # + # Backoff is calculated based on the backoff slot which is always a power + # of 2: + # + # 0s - 3s duration -> 1 second backoff + # 4s - 7s duration -> 2 seconds backoff + # 8s - 15s duration -> 4 seconds backoff + # 16s - 31s duration -> 8 seconds backoff + # 32s - 63s duration -> 16 seconds backoff + # 64s - 127s duration -> 32 seconds backoff + # 127s - 256s+ duration -> 64 seconds backoff + # + # It means that first 15 requests made by a runner will need to respect + # following backoffs: + # + # 0s -> 1 second backoff (backoff started, slot 0, 2^0 backoff) + # 1s -> 1 second backoff + # 2s -> 1 second backoff + # 3s -> 1 seconds backoff + # (slot 1 - 2^1 backoff) + # 4s -> 2 seconds backoff + # 6s -> 2 seconds backoff + # (slot 2 - 2^2 backoff) + # 8s -> 4 seconds backoff + # 12s -> 4 seconds backoff + # (slot 3 - 2^3 backoff) + # 16s -> 8 seconds backoff + # 24s -> 8 seconds backoff + # (slot 4 - 2^4 backoff) + # 32s -> 16 seconds backoff + # 48s -> 16 seconds backoff + # (slot 5 - 2^5 backoff) + # 64s -> 32 seconds backoff + # 96s -> 32 seconds backoff + # (slot 6 - 2^6 backoff) + # 128s -> 64 seconds backoff + # + # There is a cap on the backoff - it will never exceed 64 seconds. + # + class Backoff + def initialize(started) + @started = started + + if duration < 0 + raise ArgumentError, 'backoff duration negative' + end + end + + def duration + (Time.current - @started).ceil + end + + def slot + return 0 if duration < 2 + + Math.log(duration, 2).floor - 1 + end + + def to_seconds + 2**[slot, 6].min + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 829fd7a722f..8b921305c11 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.0.0-beta.2" dependencies: [] review: @@ -91,7 +91,7 @@ canary: - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret - - auto-deploy deploy canary + - auto-deploy deploy canary 50 environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -114,7 +114,6 @@ canary: - auto-deploy create_secret - auto-deploy deploy - auto-deploy delete canary - - auto-deploy delete rollout - auto-deploy persist_environment_url environment: name: production @@ -163,9 +162,7 @@ production_manual: - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret - - auto-deploy deploy rollout $ROLLOUT_PERCENTAGE - - auto-deploy scale stable $((100-ROLLOUT_PERCENTAGE)) - - auto-deploy delete canary + - auto-deploy deploy canary $ROLLOUT_PERCENTAGE - auto-deploy persist_environment_url environment: name: production diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml index da474f8ac88..317e8bfab0e 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -10,6 +10,7 @@ .deploy_to_ecs: image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest' + dependencies: [] script: - ecs update-task-definition diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 348e5472cb4..6fd32b3f1a0 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -16,6 +16,7 @@ module Gitlab ArchiveError = Class.new(StandardError) AlreadyArchivedError = Class.new(StandardError) + LockedError = Class.new(StandardError) attr_reader :job @@ -130,6 +131,12 @@ module Gitlab end end + def lock(&block) + in_write_lock(&block) + rescue FailedToObtainLockError + raise LockedError, "build trace `#{job.id}` is locked" + end + private def read_stream diff --git a/lib/gitlab/ci/trace/checksum.rb b/lib/gitlab/ci/trace/checksum.rb new file mode 100644 index 00000000000..b01136a6d24 --- /dev/null +++ b/lib/gitlab/ci/trace/checksum.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Trace + ## + # Trace::Checksum class is responsible for calculating a CRC32 checksum + # of an entire build trace using partial build trace chunks stored in a + # database. + # + # CRC32 checksum can be easily calculated by combining partial checksums + # in a right order. + # + # Then we compare CRC32 checksum provided by a GitLab Runner and expect + # it to be the same as the CRC32 checksum derived from partial chunks. + # + class Checksum + include Gitlab::Utils::StrongMemoize + + attr_reader :build + + def initialize(build) + @build = build + end + + def valid? + return false unless state_crc32.present? + + state_crc32 == chunks_crc32 + end + + def state_crc32 + strong_memoize(:crc32) { build.pending_state&.crc32 } + end + + def chunks_crc32 + trace_chunks.reduce(0) do |crc32, chunk| + Zlib.crc32_combine(crc32, chunk.crc32, chunk_size(chunk)) + end + end + + def last_chunk + strong_memoize(:last_chunk) { trace_chunks.max } + end + + ## + # Trace chunks will be persisted in a database if an object store is + # not configured - in that case we do not want to load entire raw data + # of all the chunks into memory. + # + # We ignore `raw_data` attribute instead, and rely on internal build + # trace chunk database adapter to handle + # `ActiveModel::MissingAttributeError` exception. + # + # Alternative solution would be separating chunk data from chunk + # metadata on the database level too. + # + def trace_chunks + strong_memoize(:trace_chunks) do + build.trace_chunks.persisted + .select(::Ci::BuildTraceChunk.metadata_attributes) + end + end + + private + + def chunk_size(chunk) + if chunk == last_chunk + chunk.size + else + ::Ci::BuildTraceChunk::CHUNK_SIZE + end + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb index 82a7d5fb83c..51372871f39 100644 --- a/lib/gitlab/ci/trace/metrics.rb +++ b/lib/gitlab/ci/trace/metrics.rb @@ -7,7 +7,8 @@ module Gitlab extend Gitlab::Utils::StrongMemoize OPERATIONS = [:appended, :streamed, :chunked, :mutated, :overwrite, - :accepted, :finalized, :discarded, :conflict].freeze + :accepted, :finalized, :discarded, :conflict, :locked, + :invalid].freeze def increment_trace_operation(operation: :unknown) unless OPERATIONS.include?(operation) @@ -18,7 +19,7 @@ module Gitlab end def increment_trace_bytes(size) - self.class.trace_bytes.increment(by: size.to_i) + self.class.trace_bytes.increment({}, size.to_i) end def self.trace_operations diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index a072036daa8..84a9280e507 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -36,9 +36,9 @@ module Gitlab def self.fabricate(resource) case resource when Hash - self.new(resource.symbolize_keys) + self.new(**resource.symbolize_keys) when ::Ci::HasVariable - self.new(resource.to_runner_variable) + self.new(**resource.to_runner_variable) when self resource.dup else diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 68f61e52df7..6c771b220ad 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -95,6 +95,10 @@ module Gitlab }.compact }.compact end + def merged_yaml + @ci_config&.to_hash&.to_yaml + end + private def variables |