Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/lib/gitlab/ci
diff options
context:
space:
mode:
Diffstat (limited to 'lib/gitlab/ci')
-rw-r--r--lib/gitlab/ci/ansi2json/converter.rb23
-rw-r--r--lib/gitlab/ci/ansi2json/line.rb7
-rw-r--r--lib/gitlab/ci/ansi2json/state.rb3
-rw-r--r--lib/gitlab/ci/config/entry/product/matrix.rb2
-rw-r--r--lib/gitlab/ci/config/entry/product/variables.rb6
-rw-r--r--lib/gitlab/ci/features.rb18
-rw-r--r--lib/gitlab/ci/pipeline/chain/command.rb2
-rw-r--r--lib/gitlab/ci/pipeline/chain/create.rb2
-rw-r--r--lib/gitlab/ci/runner/backoff.rb75
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml9
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml1
-rw-r--r--lib/gitlab/ci/trace.rb7
-rw-r--r--lib/gitlab/ci/trace/checksum.rb77
-rw-r--r--lib/gitlab/ci/trace/metrics.rb5
-rw-r--r--lib/gitlab/ci/variables/collection/item.rb4
-rw-r--r--lib/gitlab/ci/yaml_processor/result.rb4
16 files changed, 210 insertions, 35 deletions
diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb
index 0373a12ab69..6d152c052dc 100644
--- a/lib/gitlab/ci/ansi2json/converter.rb
+++ b/lib/gitlab/ci/ansi2json/converter.rb
@@ -104,23 +104,24 @@ module Gitlab
action = scanner[1]
timestamp = scanner[2]
section = scanner[3]
+ options = parse_section_options(scanner[4])
section_name = sanitize_section_name(section)
- if action == "start"
- handle_section_start(scanner, section_name, timestamp)
- elsif action == "end"
+ if action == 'start'
+ handle_section_start(scanner, section_name, timestamp, options)
+ elsif action == 'end'
handle_section_end(scanner, section_name, timestamp)
else
raise 'unsupported action'
end
end
- def handle_section_start(scanner, section, timestamp)
+ def handle_section_start(scanner, section, timestamp, options)
# We make a new line for new section
flush_current_line
- @state.open_section(section, timestamp)
+ @state.open_section(section, timestamp, options)
# we need to consume match after handling
# the open of section, as we want the section
@@ -157,6 +158,18 @@ module Gitlab
def sanitize_section_name(section)
section.to_s.downcase.gsub(/[^a-z0-9]/, '-')
end
+
+ def parse_section_options(raw_options)
+ return unless raw_options
+
+ # We need to remove the square brackets and split
+ # by comma to get a list of the options
+ options = raw_options[1...-1].split ','
+
+ # Now split each option by equals to separate
+ # each in the format [key, value]
+ options.to_h { |option| option.split '=' }
+ end
end
end
end
diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb
index 21aa1f84353..b1dee0e1ecc 100644
--- a/lib/gitlab/ci/ansi2json/line.rb
+++ b/lib/gitlab/ci/ansi2json/line.rb
@@ -32,7 +32,7 @@ module Gitlab
end
attr_reader :offset, :sections, :segments, :current_segment,
- :section_header, :section_duration
+ :section_header, :section_duration, :section_options
def initialize(offset:, style:, sections: [])
@offset = offset
@@ -68,6 +68,10 @@ module Gitlab
@sections << section
end
+ def set_section_options(options)
+ @section_options = options
+ end
+
def set_as_section_header
@section_header = true
end
@@ -90,6 +94,7 @@ module Gitlab
result[:section] = sections.last if sections.any?
result[:section_header] = true if @section_header
result[:section_duration] = @section_duration if @section_duration
+ result[:section_options] = @section_options if @section_options
end
end
end
diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb
index 38d36e6950c..b2b6ce649ed 100644
--- a/lib/gitlab/ci/ansi2json/state.rb
+++ b/lib/gitlab/ci/ansi2json/state.rb
@@ -26,10 +26,11 @@ module Gitlab
Base64.urlsafe_encode64(state.to_json)
end
- def open_section(section, timestamp)
+ def open_section(section, timestamp, options)
@open_sections[section] = timestamp
@current_line.add_section(section)
+ @current_line.set_section_options(options)
@current_line.set_as_section_header
end
diff --git a/lib/gitlab/ci/config/entry/product/matrix.rb b/lib/gitlab/ci/config/entry/product/matrix.rb
index 6af809d46c1..d4ee0978e1b 100644
--- a/lib/gitlab/ci/config/entry/product/matrix.rb
+++ b/lib/gitlab/ci/config/entry/product/matrix.rb
@@ -46,13 +46,11 @@ module Gitlab
end
end
- # rubocop:disable CodeReuse/ActiveRecord
def number_of_generated_jobs
value.sum do |config|
config.values.reduce(1) { |acc, values| acc * values.size }
end
end
- # rubocop:enable CodeReuse/ActiveRecord
end
end
end
diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb
index ac4f70fb69e..2481989060e 100644
--- a/lib/gitlab/ci/config/entry/product/variables.rb
+++ b/lib/gitlab/ci/config/entry/product/variables.rb
@@ -14,7 +14,7 @@ module Gitlab
validations do
validates :config, variables: { array_values: true }
validates :config, length: {
- minimum: 2,
+ minimum: :minimum,
too_short: 'requires at least %{count} items'
}
end
@@ -28,6 +28,10 @@ module Gitlab
.map { |key, value| [key.to_s, Array(value).map(&:to_s)] }
.to_h
end
+
+ def minimum
+ ::Gitlab::Ci::Features.one_dimensional_matrix_enabled? ? 1 : 2
+ end
end
end
end
diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb
index e770187b124..e14d56af978 100644
--- a/lib/gitlab/ci/features.rb
+++ b/lib/gitlab/ci/features.rb
@@ -10,10 +10,6 @@ module Gitlab
::Feature.enabled?(:ci_artifacts_exclude, default_enabled: true)
end
- def self.job_heartbeats_runner?(project)
- ::Feature.enabled?(:ci_job_heartbeats_runner, project, default_enabled: true)
- end
-
def self.instance_variables_ui_enabled?
::Feature.enabled?(:ci_instance_variables_ui, default_enabled: true)
end
@@ -35,10 +31,6 @@ module Gitlab
::Feature.enabled?(:ci_raise_job_rules_without_workflow_rules_warning, default_enabled: true)
end
- def self.bulk_insert_on_create?(project)
- ::Feature.enabled?(:ci_bulk_insert_on_create, project, default_enabled: true)
- end
-
# NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project`
# is a safe switch to disable the feature for a parituclar project when something went wrong,
# therefore it's not supposed to be enabled by default.
@@ -54,10 +46,6 @@ module Gitlab
Feature.enabled?(:project_transactionless_destroy, project, default_enabled: false)
end
- def self.coverage_report_view?(project)
- ::Feature.enabled?(:coverage_report_view, project, default_enabled: true)
- end
-
def self.child_of_child_pipeline_enabled?(project)
::Feature.enabled?(:ci_child_of_child_pipeline, project, default_enabled: true)
end
@@ -72,7 +60,11 @@ module Gitlab
end
def self.new_artifact_file_reader_enabled?(project)
- ::Feature.enabled?(:ci_new_artifact_file_reader, project, default_enabled: false)
+ ::Feature.enabled?(:ci_new_artifact_file_reader, project, default_enabled: true)
+ end
+
+ def self.one_dimensional_matrix_enabled?
+ ::Feature.enabled?(:one_dimensional_matrix, default_enabled: false)
end
end
end
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index d1882059dd8..06096a33f27 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -16,7 +16,7 @@ module Gitlab
) do
include Gitlab::Utils::StrongMemoize
- def initialize(**params)
+ def initialize(params = {})
params.each do |key, value|
self[key] = value
end
diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb
index 34649fe16f3..81ef3bb074d 100644
--- a/lib/gitlab/ci/pipeline/chain/create.rb
+++ b/lib/gitlab/ci/pipeline/chain/create.rb
@@ -8,7 +8,7 @@ module Gitlab
include Chain::Helpers
def perform!
- BulkInsertableAssociations.with_bulk_insert(enabled: ::Gitlab::Ci::Features.bulk_insert_on_create?(project)) do
+ BulkInsertableAssociations.with_bulk_insert do
pipeline.save!
end
rescue ActiveRecord::RecordInvalid => e
diff --git a/lib/gitlab/ci/runner/backoff.rb b/lib/gitlab/ci/runner/backoff.rb
new file mode 100644
index 00000000000..95d7719e9cb
--- /dev/null
+++ b/lib/gitlab/ci/runner/backoff.rb
@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Runner
+ ##
+ # Runner Backoff class is an implementation of an exponential backoff
+ # used when a runner communicates with GitLab. We typically use it when a
+ # runner retries sending a build status after we created a build pending
+ # state.
+ #
+ # Backoff is calculated based on the backoff slot which is always a power
+ # of 2:
+ #
+ # 0s - 3s duration -> 1 second backoff
+ # 4s - 7s duration -> 2 seconds backoff
+ # 8s - 15s duration -> 4 seconds backoff
+ # 16s - 31s duration -> 8 seconds backoff
+ # 32s - 63s duration -> 16 seconds backoff
+ # 64s - 127s duration -> 32 seconds backoff
+ # 127s - 256s+ duration -> 64 seconds backoff
+ #
+ # It means that first 15 requests made by a runner will need to respect
+ # following backoffs:
+ #
+ # 0s -> 1 second backoff (backoff started, slot 0, 2^0 backoff)
+ # 1s -> 1 second backoff
+ # 2s -> 1 second backoff
+ # 3s -> 1 seconds backoff
+ # (slot 1 - 2^1 backoff)
+ # 4s -> 2 seconds backoff
+ # 6s -> 2 seconds backoff
+ # (slot 2 - 2^2 backoff)
+ # 8s -> 4 seconds backoff
+ # 12s -> 4 seconds backoff
+ # (slot 3 - 2^3 backoff)
+ # 16s -> 8 seconds backoff
+ # 24s -> 8 seconds backoff
+ # (slot 4 - 2^4 backoff)
+ # 32s -> 16 seconds backoff
+ # 48s -> 16 seconds backoff
+ # (slot 5 - 2^5 backoff)
+ # 64s -> 32 seconds backoff
+ # 96s -> 32 seconds backoff
+ # (slot 6 - 2^6 backoff)
+ # 128s -> 64 seconds backoff
+ #
+ # There is a cap on the backoff - it will never exceed 64 seconds.
+ #
+ class Backoff
+ def initialize(started)
+ @started = started
+
+ if duration < 0
+ raise ArgumentError, 'backoff duration negative'
+ end
+ end
+
+ def duration
+ (Time.current - @started).ceil
+ end
+
+ def slot
+ return 0 if duration < 2
+
+ Math.log(duration, 2).floor - 1
+ end
+
+ def to_seconds
+ 2**[slot, 6].min
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
index 829fd7a722f..8b921305c11 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
@@ -1,5 +1,5 @@
.auto-deploy:
- image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.0.0-beta.2"
dependencies: []
review:
@@ -91,7 +91,7 @@ canary:
- auto-deploy ensure_namespace
- auto-deploy initialize_tiller
- auto-deploy create_secret
- - auto-deploy deploy canary
+ - auto-deploy deploy canary 50
environment:
name: production
url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN
@@ -114,7 +114,6 @@ canary:
- auto-deploy create_secret
- auto-deploy deploy
- auto-deploy delete canary
- - auto-deploy delete rollout
- auto-deploy persist_environment_url
environment:
name: production
@@ -163,9 +162,7 @@ production_manual:
- auto-deploy ensure_namespace
- auto-deploy initialize_tiller
- auto-deploy create_secret
- - auto-deploy deploy rollout $ROLLOUT_PERCENTAGE
- - auto-deploy scale stable $((100-ROLLOUT_PERCENTAGE))
- - auto-deploy delete canary
+ - auto-deploy deploy canary $ROLLOUT_PERCENTAGE
- auto-deploy persist_environment_url
environment:
name: production
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
index da474f8ac88..317e8bfab0e 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
@@ -10,6 +10,7 @@
.deploy_to_ecs:
image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest'
+ dependencies: []
script:
- ecs update-task-definition
diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
index 348e5472cb4..6fd32b3f1a0 100644
--- a/lib/gitlab/ci/trace.rb
+++ b/lib/gitlab/ci/trace.rb
@@ -16,6 +16,7 @@ module Gitlab
ArchiveError = Class.new(StandardError)
AlreadyArchivedError = Class.new(StandardError)
+ LockedError = Class.new(StandardError)
attr_reader :job
@@ -130,6 +131,12 @@ module Gitlab
end
end
+ def lock(&block)
+ in_write_lock(&block)
+ rescue FailedToObtainLockError
+ raise LockedError, "build trace `#{job.id}` is locked"
+ end
+
private
def read_stream
diff --git a/lib/gitlab/ci/trace/checksum.rb b/lib/gitlab/ci/trace/checksum.rb
new file mode 100644
index 00000000000..b01136a6d24
--- /dev/null
+++ b/lib/gitlab/ci/trace/checksum.rb
@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Trace
+ ##
+ # Trace::Checksum class is responsible for calculating a CRC32 checksum
+ # of an entire build trace using partial build trace chunks stored in a
+ # database.
+ #
+ # CRC32 checksum can be easily calculated by combining partial checksums
+ # in a right order.
+ #
+ # Then we compare CRC32 checksum provided by a GitLab Runner and expect
+ # it to be the same as the CRC32 checksum derived from partial chunks.
+ #
+ class Checksum
+ include Gitlab::Utils::StrongMemoize
+
+ attr_reader :build
+
+ def initialize(build)
+ @build = build
+ end
+
+ def valid?
+ return false unless state_crc32.present?
+
+ state_crc32 == chunks_crc32
+ end
+
+ def state_crc32
+ strong_memoize(:crc32) { build.pending_state&.crc32 }
+ end
+
+ def chunks_crc32
+ trace_chunks.reduce(0) do |crc32, chunk|
+ Zlib.crc32_combine(crc32, chunk.crc32, chunk_size(chunk))
+ end
+ end
+
+ def last_chunk
+ strong_memoize(:last_chunk) { trace_chunks.max }
+ end
+
+ ##
+ # Trace chunks will be persisted in a database if an object store is
+ # not configured - in that case we do not want to load entire raw data
+ # of all the chunks into memory.
+ #
+ # We ignore `raw_data` attribute instead, and rely on internal build
+ # trace chunk database adapter to handle
+ # `ActiveModel::MissingAttributeError` exception.
+ #
+ # Alternative solution would be separating chunk data from chunk
+ # metadata on the database level too.
+ #
+ def trace_chunks
+ strong_memoize(:trace_chunks) do
+ build.trace_chunks.persisted
+ .select(::Ci::BuildTraceChunk.metadata_attributes)
+ end
+ end
+
+ private
+
+ def chunk_size(chunk)
+ if chunk == last_chunk
+ chunk.size
+ else
+ ::Ci::BuildTraceChunk::CHUNK_SIZE
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb
index 82a7d5fb83c..51372871f39 100644
--- a/lib/gitlab/ci/trace/metrics.rb
+++ b/lib/gitlab/ci/trace/metrics.rb
@@ -7,7 +7,8 @@ module Gitlab
extend Gitlab::Utils::StrongMemoize
OPERATIONS = [:appended, :streamed, :chunked, :mutated, :overwrite,
- :accepted, :finalized, :discarded, :conflict].freeze
+ :accepted, :finalized, :discarded, :conflict, :locked,
+ :invalid].freeze
def increment_trace_operation(operation: :unknown)
unless OPERATIONS.include?(operation)
@@ -18,7 +19,7 @@ module Gitlab
end
def increment_trace_bytes(size)
- self.class.trace_bytes.increment(by: size.to_i)
+ self.class.trace_bytes.increment({}, size.to_i)
end
def self.trace_operations
diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb
index a072036daa8..84a9280e507 100644
--- a/lib/gitlab/ci/variables/collection/item.rb
+++ b/lib/gitlab/ci/variables/collection/item.rb
@@ -36,9 +36,9 @@ module Gitlab
def self.fabricate(resource)
case resource
when Hash
- self.new(resource.symbolize_keys)
+ self.new(**resource.symbolize_keys)
when ::Ci::HasVariable
- self.new(resource.to_runner_variable)
+ self.new(**resource.to_runner_variable)
when self
resource.dup
else
diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb
index 68f61e52df7..6c771b220ad 100644
--- a/lib/gitlab/ci/yaml_processor/result.rb
+++ b/lib/gitlab/ci/yaml_processor/result.rb
@@ -95,6 +95,10 @@ module Gitlab
}.compact }.compact
end
+ def merged_yaml
+ @ci_config&.to_hash&.to_yaml
+ end
+
private
def variables
generated by cgit v1.2.3 (git 2.25.1) at 2024-09-01 12:35:11 +0300