diff options
author | Stan Hu <stanhu@gmail.com> | 2018-11-27 23:56:40 +0300 |
---|---|---|
committer | Stan Hu <stanhu@gmail.com> | 2018-11-27 23:56:40 +0300 |
commit | 6c83c2d8b9305fe67fe31cf944c9d83cdbb00b74 (patch) | |
tree | f127e4af5333c63c2b23a2be6f99c566b4829e58 /lib/gitlab/ci | |
parent | 1524a19302cea096ddf2c008abe1307527ae6938 (diff) | |
parent | 31a1ce2132d83e333e84f365b9aada2e17c61b43 (diff) |
Merge branch 'lock-trace-writes' into 'master'
Lock writes to trace stream
Closes #51502
See merge request gitlab-org/gitlab-ce!23332
Diffstat (limited to 'lib/gitlab/ci')
-rw-r--r-- | lib/gitlab/ci/trace.rb | 63 | ||||
-rw-r--r-- | lib/gitlab/ci/trace/stream.rb | 11 |
2 files changed, 36 insertions, 38 deletions
diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 8eccd262db9..bf5f2a31f0e 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -3,9 +3,11 @@ module Gitlab module Ci class Trace - include ExclusiveLeaseGuard + include ::Gitlab::ExclusiveLeaseHelpers - LEASE_TIMEOUT = 1.hour + LOCK_TTL = 1.minute + LOCK_RETRIES = 2 + LOCK_SLEEP = 0.001.seconds ArchiveError = Class.new(StandardError) AlreadyArchivedError = Class.new(StandardError) @@ -82,24 +84,10 @@ module Gitlab stream&.close end - def write(mode) - stream = Gitlab::Ci::Trace::Stream.new do - if trace_artifact - raise AlreadyArchivedError, 'Could not write to the archived trace' - elsif current_path - File.open(current_path, mode) - elsif Feature.enabled?('ci_enable_live_trace') - Gitlab::Ci::Trace::ChunkedIO.new(job) - else - File.open(ensure_path, mode) - end + def write(mode, &blk) + in_write_lock do + unsafe_write!(mode, &blk) end - - yield(stream).tap do - job.touch if job.needs_touch? - end - ensure - stream&.close end def erase! @@ -117,13 +105,33 @@ module Gitlab end def archive! - try_obtain_lease do + in_write_lock do unsafe_archive! end end private + def unsafe_write!(mode, &blk) + stream = Gitlab::Ci::Trace::Stream.new do + if trace_artifact + raise AlreadyArchivedError, 'Could not write to the archived trace' + elsif current_path + File.open(current_path, mode) + elsif Feature.enabled?('ci_enable_live_trace') + Gitlab::Ci::Trace::ChunkedIO.new(job) + else + File.open(ensure_path, mode) + end + end + + yield(stream).tap do + job.touch if job.needs_touch? + end + ensure + stream&.close + end + def unsafe_archive! raise AlreadyArchivedError, 'Could not archive again' if trace_artifact raise ArchiveError, 'Job is not finished yet' unless job.complete? @@ -146,6 +154,11 @@ module Gitlab end end + def in_write_lock(&blk) + lock_key = "trace:write:lock:#{job.id}" + in_lock(lock_key, ttl: LOCK_TTL, retries: LOCK_RETRIES, sleep_sec: LOCK_SLEEP, &blk) + end + def archive_stream!(stream) clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path| create_build_trace!(job, clone_path) @@ -226,16 +239,6 @@ module Gitlab def trace_artifact job.job_artifacts_trace end - - # For ExclusiveLeaseGuard concern - def lease_key - @lease_key ||= "trace:archive:#{job.id}" - end - - # For ExclusiveLeaseGuard concern - def lease_timeout - LEASE_TIMEOUT - end end end end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index bd40fdf59b1..0f23b95ba15 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -43,19 +43,14 @@ module Gitlab def append(data, offset) data = data.force_encoding(Encoding::BINARY) - stream.truncate(offset) - stream.seek(0, IO::SEEK_END) + stream.seek(offset, IO::SEEK_SET) stream.write(data) + stream.truncate(offset + data.bytesize) stream.flush() end def set(data) - data = data.force_encoding(Encoding::BINARY) - - stream.seek(0, IO::SEEK_SET) - stream.write(data) - stream.truncate(data.bytesize) - stream.flush() + append(data, 0) end def raw(last_lines: nil) |