1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
|
# frozen_string_literal: true
module Ci
class ArchiveTraceService
include ::Gitlab::ExclusiveLeaseHelpers
EXCLUSIVE_LOCK_KEY = 'archive_trace_service:batch_execute:lock'
LOCK_TIMEOUT = 56.minutes
LOOP_TIMEOUT = 55.minutes
LOOP_LIMIT = 2000
BATCH_SIZE = 100
# rubocop: disable CodeReuse/ActiveRecord
def batch_execute(worker_name:)
start_time = Time.current
in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do
Ci::Build.with_stale_live_trace.find_each(batch_size: BATCH_SIZE).with_index do |build, index|
break if Time.current - start_time > LOOP_TIMEOUT
if index > LOOP_LIMIT
Sidekiq.logger.warn(class: worker_name, message: 'Loop limit reached.', job_id: build.id)
break
end
begin
execute(build, worker_name: worker_name)
rescue StandardError
next
end
end
end
end
# rubocop: enable CodeReuse/ActiveRecord
def execute(job, worker_name:)
unless job.trace.archival_attempts_available?
Sidekiq.logger.warn(class: worker_name, message: 'The job is out of archival attempts.', job_id: job.id)
job.trace.attempt_archive_cleanup!
return
end
unless job.trace.can_attempt_archival_now?
Sidekiq.logger.warn(class: worker_name, message: 'The job can not be archived right now.', job_id: job.id)
return
end
# TODO: Remove this logging once we confirmed new live trace architecture is functional.
# See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667.
unless job.has_live_trace?
Sidekiq.logger.warn(class: worker_name,
message: 'The job does not have live trace but going to be archived.',
job_id: job.id)
return
end
job.trace.archive!
job.remove_pending_state!
if job.job_artifacts_trace.present?
job.project.execute_integrations(Gitlab::DataBuilder::ArchiveTrace.build(job), :archive_trace_hooks)
end
# TODO: Remove this logging once we confirmed new live trace architecture is functional.
# See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667.
unless job.has_archived_trace?
Sidekiq.logger.warn(class: worker_name,
message: 'The job does not have archived trace after archiving.',
job_id: job.id)
end
rescue ::Gitlab::Ci::Trace::AlreadyArchivedError
# It's already archived, thus we can safely ignore this exception.
rescue StandardError => e
job.trace.increment_archival_attempts!
# Tracks this error with application logs, Sentry, and Prometheus.
# If `archive!` keeps failing for over a week, that could incur data loss.
# (See more https://docs.gitlab.com/ee/administration/job_logs.html#new-incremental-logging-architecture)
# In order to avoid interrupting the system, we do not raise an exception here.
archive_error(e, job, worker_name)
end
private
def failed_archive_counter
@failed_archive_counter ||=
Gitlab::Metrics.counter(:job_trace_archive_failed_total,
"Counter of failed attempts of trace archiving")
end
def archive_error(error, job, worker_name)
failed_archive_counter.increment
Sidekiq.logger.warn(class: worker_name,
message: "Failed to archive trace. message: #{error.message}.",
job_id: job.id)
Gitlab::ErrorTracking
.track_and_raise_for_dev_exception(error,
issue_url: 'https://gitlab.com/gitlab-org/gitlab-foss/issues/51502',
job_id: job.id)
end
end
end
|
