diff options
author | Shinya Maeda <shinya@gitlab.com> | 2019-07-18 12:22:46 +0300 |
---|---|---|
committer | Shinya Maeda <shinya@gitlab.com> | 2019-07-23 13:26:08 +0300 |
commit | c2e0e689f355555db231ac6db40ab1b654c90233 (patch) | |
tree | 1ab8b2e6561598a61a10a0197a975019ed13f464 /app/services | |
parent | 1a3fda63a5f9756cde19bc7e221651b0c33cb5dc (diff) |
Validate the existence of archived traces before removing live trace
Often live traces are removed even though the archived trace
doesn't exist. This commit checkes the existence strictly.
Diffstat (limited to 'app/services')
-rw-r--r-- | app/services/ci/archive_trace_service.rb | 28 |
1 files changed, 24 insertions, 4 deletions
diff --git a/app/services/ci/archive_trace_service.rb b/app/services/ci/archive_trace_service.rb index b5cfa1d019c..700d78361a4 100644 --- a/app/services/ci/archive_trace_service.rb +++ b/app/services/ci/archive_trace_service.rb @@ -2,8 +2,25 @@ module Ci class ArchiveTraceService - def execute(job) + def execute(job, worker_name:) + # TODO: Remove this logging once we confirmed new live trace architecture is functional. + # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667. + unless job.has_live_trace? + Sidekiq.logger.warn(class: worker_name, + message: 'The job does not have live trace but going to be archived.', + job_id: job.id) + return + end + job.trace.archive! + + # TODO: Remove this logging once we confirmed new live trace architecture is functional. + # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667. + unless job.has_archived_trace? + Sidekiq.logger.warn(class: worker_name, + message: 'The job does not have archived trace after archiving.', + job_id: job.id) + end rescue ::Gitlab::Ci::Trace::AlreadyArchivedError # It's already archived, thus we can safely ignore this exception. rescue => e @@ -11,7 +28,7 @@ module Ci # If `archive!` keeps failing for over a week, that could incur data loss. # (See more https://docs.gitlab.com/ee/administration/job_traces.html#new-live-trace-architecture) # In order to avoid interrupting the system, we do not raise an exception here. - archive_error(e, job) + archive_error(e, job, worker_name) end private @@ -22,9 +39,12 @@ module Ci "Counter of failed attempts of trace archiving") end - def archive_error(error, job) + def archive_error(error, job, worker_name) failed_archive_counter.increment - Rails.logger.error "Failed to archive trace. id: #{job.id} message: #{error.message}" # rubocop:disable Gitlab/RailsLogger + + Sidekiq.logger.warn(class: worker_name, + message: "Failed to archive trace. message: #{error.message}.", + job_id: job.id) Gitlab::Sentry .track_exception(error, |