diff options
author | Kamil TrzciĆski <ayufan@ayufan.eu> | 2018-04-04 13:48:30 +0300 |
---|---|---|
committer | Shinya Maeda <shinya@gitlab.com> | 2018-04-05 08:14:54 +0300 |
commit | de5194cdb8136d424e0ba88914645cb7936299be (patch) | |
tree | fca9340aba9ffcf495396943bcab654d1d7eacb6 /lib/gitlab/ci/trace | |
parent | cb3a0e33a051be1422c13494b282ea8c93f29e2a (diff) |
Removed unused code
Diffstat (limited to 'lib/gitlab/ci/trace')
-rw-r--r-- | lib/gitlab/ci/trace/chunked_file/chunk_store/base.rb | 63 | ||||
-rw-r--r-- | lib/gitlab/ci/trace/chunked_file/chunk_store/database.rb | 92 | ||||
-rw-r--r-- | lib/gitlab/ci/trace/chunked_file/chunk_store/redis.rb | 132 | ||||
-rw-r--r-- | lib/gitlab/ci/trace/chunked_file/chunked_io.rb | 274 | ||||
-rw-r--r-- | lib/gitlab/ci/trace/chunked_file/concerns/callbacks.rb | 37 | ||||
-rw-r--r-- | lib/gitlab/ci/trace/chunked_file/concerns/errors.rb | 18 | ||||
-rw-r--r-- | lib/gitlab/ci/trace/chunked_file/concerns/opener.rb | 23 | ||||
-rw-r--r-- | lib/gitlab/ci/trace/chunked_file/concerns/permissions.rb | 93 | ||||
-rw-r--r-- | lib/gitlab/ci/trace/chunked_file/live_trace.rb | 63 |
9 files changed, 0 insertions, 795 deletions
diff --git a/lib/gitlab/ci/trace/chunked_file/chunk_store/base.rb b/lib/gitlab/ci/trace/chunked_file/chunk_store/base.rb deleted file mode 100644 index 6e104a6d764..00000000000 --- a/lib/gitlab/ci/trace/chunked_file/chunk_store/base.rb +++ /dev/null @@ -1,63 +0,0 @@ -module Gitlab - module Ci - class Trace - module ChunkedFile - module ChunkStore - class Base - attr_reader :params - - def initialize(*identifiers, **params) - @params = params - end - - def close - raise NotImplementedError - end - - def get - raise NotImplementedError - end - - def size - raise NotImplementedError - end - - # Write data to chunk store. Always overwrite. - # - # @param [String] data - # @return [Fixnum] length of the data after writing - def write!(data) - raise NotImplementedError - end - - # Append data to chunk store - # - # @param [String] data - # @return [Fixnum] length of the appended - def append!(data) - raise NotImplementedError - end - - # Truncate data to chunk store - # - # @param [String] offset - def truncate!(offset) - raise NotImplementedError - end - - # Delete data from chunk store - # - # @param [String] offset - def delete! - raise NotImplementedError - end - - def filled? - size == params[:buffer_size] - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/trace/chunked_file/chunk_store/database.rb b/lib/gitlab/ci/trace/chunked_file/chunk_store/database.rb deleted file mode 100644 index d3665031e9d..00000000000 --- a/lib/gitlab/ci/trace/chunked_file/chunk_store/database.rb +++ /dev/null @@ -1,92 +0,0 @@ -module Gitlab - module Ci - class Trace - module ChunkedFile - module ChunkStore - class Database < Base - class << self - def open(job_id, chunk_index, **params) - raise ArgumentError unless job_id && chunk_index - - job_trace_chunk = ::Ci::JobTraceChunk - .find_or_initialize_by(job_id: job_id, chunk_index: chunk_index) - store = self.new(job_trace_chunk, params) - - yield store - ensure - store&.close - end - - def exist?(job_id, chunk_index) - ::Ci::JobTraceChunk.exists?(job_id: job_id, chunk_index: chunk_index) - end - - def chunks_count(job_id) - ::Ci::JobTraceChunk.where(job_id: job_id).count - end - - def chunks_size(job_id) - ::Ci::JobTraceChunk.where(job_id: job_id).pluck('data') - .inject(0) { |sum, data| sum + data.bytesize } - end - - def delete_all(job_id) - ::Ci::JobTraceChunk.destroy_all(job_id: job_id) - end - end - - attr_reader :job_trace_chunk - - def initialize(job_trace_chunk, **params) - super - - @job_trace_chunk = job_trace_chunk - end - - def close - @job_trace_chunk = nil - end - - def get - puts "#{self.class.name} - #{__callee__}: params[:chunk_index]: #{params[:chunk_index]}" - - job_trace_chunk.data - end - - def size - job_trace_chunk.data&.bytesize || 0 - end - - def write!(data) - raise NotImplementedError, 'Partial writing is not supported' unless params[:buffer_size] == data&.bytesize - raise NotImplementedError, 'UPDATE (Overwriting data) is not supported' if job_trace_chunk.data - - puts "#{self.class.name} - #{__callee__}: data.bytesize: #{data.bytesize.inspect} params[:chunk_index]: #{params[:chunk_index]}" - - job_trace_chunk.data = data - job_trace_chunk.save! - - data.bytesize - end - - def append!(data) - raise NotImplementedError - end - - def truncate!(offset) - raise NotImplementedError - end - - def delete! - raise ActiveRecord::RecordNotFound, 'Could not find deletable record' unless job_trace_chunk.persisted? - - puts "#{self.class.name} - #{__callee__}: params[:chunk_index]: #{params[:chunk_index]}" - - job_trace_chunk.destroy! - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/trace/chunked_file/chunk_store/redis.rb b/lib/gitlab/ci/trace/chunked_file/chunk_store/redis.rb deleted file mode 100644 index 5d3b43f8291..00000000000 --- a/lib/gitlab/ci/trace/chunked_file/chunk_store/redis.rb +++ /dev/null @@ -1,132 +0,0 @@ -module Gitlab - module Ci - class Trace - module ChunkedFile - module ChunkStore - class Redis < Base - class << self - def open(job_id, chunk_index, **params) - raise ArgumentError unless job_id && chunk_index - - buffer_key = self.buffer_key(job_id, chunk_index) - store = self.new(buffer_key, params) - - yield store - ensure - store&.close - end - - def exist?(job_id, chunk_index) - Gitlab::Redis::Cache.with do |redis| - redis.exists(self.buffer_key(job_id, chunk_index)) - end - end - - def chunks_count(job_id) - Gitlab::Redis::Cache.with do |redis| - redis.scan_each(match: buffer_key(job_id, '?')).inject(0) do |sum, key| - sum + 1 - end - end - end - - def chunks_size(job_id) - Gitlab::Redis::Cache.with do |redis| - redis.scan_each(match: buffer_key(job_id, '?')).inject(0) do |sum, key| - sum + redis.strlen(key) - end - end - end - - def delete_all(job_id) - Gitlab::Redis::Cache.with do |redis| - redis.scan_each(match: buffer_key(job_id, '?')) do |key| - redis.del(key) - end - end - end - - def buffer_key(job_id, chunk_index) - "live_trace_buffer:#{job_id}:#{chunk_index}" - end - end - - BufferKeyNotFoundError = Class.new(StandardError) - WriteError = Class.new(StandardError) - - attr_reader :buffer_key - - def initialize(buffer_key, **params) - super - - @buffer_key = buffer_key - end - - def close - @buffer_key = nil - end - - def get - puts "#{self.class.name} - #{__callee__}: params[:chunk_index]: #{params[:chunk_index]}" - - Gitlab::Redis::Cache.with do |redis| - redis.get(buffer_key) - end - end - - def size - Gitlab::Redis::Cache.with do |redis| - redis.strlen(buffer_key) - end - end - - def write!(data) - raise ArgumentError, 'Could not write empty data' unless data.present? - - puts "#{self.class.name} - #{__callee__}: data.bytesize: #{data.bytesize.inspect} params[:chunk_index]: #{params[:chunk_index]}" - Gitlab::Redis::Cache.with do |redis| - unless redis.set(buffer_key, data) == 'OK' - raise WriteError, 'Failed to write' - end - - redis.strlen(buffer_key) - end - end - - def append!(data) - raise ArgumentError, 'Could not write empty data' unless data.present? - - puts "#{self.class.name} - #{__callee__}: data.bytesize: #{data.bytesize.inspect} params[:chunk_index]: #{params[:chunk_index]}" - Gitlab::Redis::Cache.with do |redis| - raise BufferKeyNotFoundError, 'Buffer key is not found' unless redis.exists(buffer_key) - - original_size = size - new_size = redis.append(buffer_key, data) - appended_size = new_size - original_size - - raise WriteError, 'Failed to append' unless appended_size == data.bytesize - - appended_size - end - end - - def truncate!(offset) - raise NotImplementedError - end - - def delete! - puts "#{self.class.name} - #{__callee__}: params[:chunk_index]: #{params[:chunk_index]}" - Gitlab::Redis::Cache.with do |redis| - raise BufferKeyNotFoundError, 'Buffer key is not found' unless redis.exists(buffer_key) - - unless redis.del(buffer_key) == 1 - raise WriteError, 'Failed to delete' - end - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/trace/chunked_file/chunked_io.rb b/lib/gitlab/ci/trace/chunked_file/chunked_io.rb deleted file mode 100644 index 55e04322cea..00000000000 --- a/lib/gitlab/ci/trace/chunked_file/chunked_io.rb +++ /dev/null @@ -1,274 +0,0 @@ -## -# ChunkedIO Engine -# -# Choose a chunk_store with your purpose -# This class is designed that it's compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html) -module Gitlab - module Ci - class Trace - module ChunkedFile - class ChunkedIO - # extend ChunkedFile::Concerns::Opener - include ChunkedFile::Concerns::Errors - include ChunkedFile::Concerns::Hooks - include ChunkedFile::Concerns::Callbacks - prepend ChunkedFile::Concerns::Permissions - - attr_reader :size - attr_reader :tell - attr_reader :chunk, :chunk_range - attr_reader :job_id - attr_reader :mode - - alias_method :pos, :tell - - def initialize(job_id, size = nil, mode = 'rb', &block) - raise NotImplementedError, "Mode 'w' is not supported" if mode.include?('w') - - @size = size || calculate_size(job_id) - @tell = 0 - @job_id = job_id - @mode = mode - - if block_given? - begin - yield self - ensure - self.close - end - end - end - - def close - end - - def binmode - # no-op - end - - def binmode? - true - end - - def seek(amount, where = IO::SEEK_SET) - new_pos = - case where - when IO::SEEK_END - size + amount - when IO::SEEK_SET - amount - when IO::SEEK_CUR - tell + amount - else - -1 - end - - raise ArgumentError, 'new position is outside of file' if new_pos < 0 || new_pos > size - - @tell = new_pos - end - - def eof? - tell == size - end - - def each_line - until eof? - line = readline - break if line.nil? - - yield(line) - end - end - - def read(length = nil, outbuf = nil) - out = "" - - until eof? || (length && out.bytesize >= length) - data = get_chunk - break if data.empty? - - out << data - @tell += data.bytesize - end - - out = out.byteslice(0, length) if length && out.bytesize > length - - out - end - - def readline - out = "" - - until eof? - data = get_chunk - break if data.empty? - - new_line_pos = byte_position(data, "\n") - - if new_line_pos.nil? - out << data - @tell += data.bytesize - else - out << data.byteslice(0..new_line_pos) - @tell += new_line_pos + 1 - break - end - end - - out - end - - def write(data) - raise ArgumentError, 'Could not write empty data' unless data.present? - - if mode.include?('w') - raise NotImplementedError, "Overwrite is not supported" - elsif mode.include?('a') - write_as_append(data) - end - end - - def truncate(offset) - raise NotImplementedError - end - - def flush - # no-op - end - - def present? - chunks_count > 0 - end - - def delete - chunk_store.delete_all - end - - private - - def in_range? - @chunk_range&.include?(tell) - end - - def get_chunk - return '' if size <= 0 || eof? - - unless in_range? - chunk_store.open(job_id, chunk_index, params_for_store) do |store| - @chunk = store.get - - raise ReadError, 'Could not get a chunk' unless chunk && chunk.present? - - @chunk_range = (chunk_start...(chunk_start + chunk.bytesize)) - end - end - - @chunk.byteslice(chunk_offset, buffer_size) - end - - def write_as_append(data) - @tell = size - - data_size = data.bytesize - new_tell = tell + data_size - data_offset = 0 - - until tell == new_tell - writable_size = buffer_size - chunk_offset - writable_data = data.byteslice(data_offset, writable_size) - written_size = write_chunk(writable_data) - - data_offset += written_size - @tell += written_size - @size = [tell, size].max - end - - data_size - end - - def write_chunk(data) - written_size = 0 - - chunk_store.open(job_id, chunk_index, params_for_store) do |store| - with_callbacks(:write_chunk, store) do - written_size = if store.size > 0 # # rubocop:disable ZeroLengthPredicate - store.append!(data) - else - store.write!(data) - end - - raise WriteError, 'Written size mismatch' unless data.bytesize == written_size - end - end - - written_size - end - - def params_for_store(c_index = chunk_index) - { - buffer_size: buffer_size, - chunk_start: c_index * buffer_size, - chunk_index: c_index - } - end - - def chunk_offset - tell % buffer_size - end - - def chunk_start - chunk_index * buffer_size - end - - def chunk_end - [chunk_start + buffer_size, size].min - end - - def chunk_index - (tell / buffer_size) - end - - def chunks_count - (size / buffer_size.to_f).ceil - end - - def last_chunk? - ((size / buffer_size) * buffer_size..size).include?(tell) - end - - def chunk_store - raise NotImplementedError - end - - def buffer_size - raise NotImplementedError - end - - def calculate_size(job_id) - chunk_store.chunks_size(job_id) - end - - def byte_position(data, pattern_byte) - index_as_string = data.index(pattern_byte) - return nil unless index_as_string - - if data.getbyte(index_as_string) == pattern_byte.getbyte(0) - index_as_string - else - data2 = data.byteslice(index_as_string, 100) - additional_pos = 0 - data2.each_byte do |b| - break if b == pattern_byte.getbyte(0) - - additional_pos += 1 - end - - index_as_string + additional_pos - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/trace/chunked_file/concerns/callbacks.rb b/lib/gitlab/ci/trace/chunked_file/concerns/callbacks.rb deleted file mode 100644 index 3990a492612..00000000000 --- a/lib/gitlab/ci/trace/chunked_file/concerns/callbacks.rb +++ /dev/null @@ -1,37 +0,0 @@ -module Gitlab - module Ci - class Trace - module ChunkedFile - module Concerns - module Callbacks - extend ActiveSupport::Concern - - included do - class_attribute :_before_callbacks, :_after_callbacks, instance_writer: false - self._before_callbacks = Hash.new [] - self._after_callbacks = Hash.new [] - end - - def with_callbacks(kind, *args) - self.class._before_callbacks[kind].each { |c| send c, *args } # rubocop:disable GitlabSecurity/PublicSend - yield - self.class._after_callbacks[kind].each { |c| send c, *args } # rubocop:disable GitlabSecurity/PublicSend - end - - module ClassMethods - def before_callback(kind, callback) - self._before_callbacks = self._before_callbacks - .merge kind => _before_callbacks[kind] + [callback] - end - - def after_callback(kind, callback) - self._after_callbacks = self._after_callbacks - .merge kind => _after_callbacks[kind] + [callback] - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/trace/chunked_file/concerns/errors.rb b/lib/gitlab/ci/trace/chunked_file/concerns/errors.rb deleted file mode 100644 index 5fba9605585..00000000000 --- a/lib/gitlab/ci/trace/chunked_file/concerns/errors.rb +++ /dev/null @@ -1,18 +0,0 @@ -module Gitlab - module Ci - class Trace - module ChunkedFile - module Concerns - module Errors - extend ActiveSupport::Concern - - included do - WriteError = Class.new(StandardError) - ReadError = Class.new(StandardError) - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/trace/chunked_file/concerns/opener.rb b/lib/gitlab/ci/trace/chunked_file/concerns/opener.rb deleted file mode 100644 index 9f1f6eefcbc..00000000000 --- a/lib/gitlab/ci/trace/chunked_file/concerns/opener.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Gitlab - module Ci - class Trace - module ChunkedFile - module Concerns - module Opener - extend ActiveSupport::Concern - - class_methods do - def open(*args) - stream = self.new(*args) - - yield stream - ensure - stream&.close - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/trace/chunked_file/concerns/permissions.rb b/lib/gitlab/ci/trace/chunked_file/concerns/permissions.rb deleted file mode 100644 index 016b796afc2..00000000000 --- a/lib/gitlab/ci/trace/chunked_file/concerns/permissions.rb +++ /dev/null @@ -1,93 +0,0 @@ -module Gitlab - module Ci - class Trace - module ChunkedFile - module Concerns - module Permissions - extend ActiveSupport::Concern - - WRITABLE_MODE = %w[a].freeze - READABLE_MODE = %w[r +].freeze - - included do - attr_reader :write_lock_uuid - end - - def initialize(job_id, size, mode = 'rb') - if WRITABLE_MODE.any? { |m| mode.include?(m) } - @write_lock_uuid = Gitlab::ExclusiveLease - .new(write_lock_key(job_id), timeout: 1.hour.to_i).try_obtain - - raise IOError, 'Already opened by another process' unless write_lock_uuid - end - - super - end - - def close - if write_lock_uuid - Gitlab::ExclusiveLease.cancel(write_lock_key(job_id), write_lock_uuid) - end - - super - end - - def read(*args) - can_read! - - super - end - - def readline(*args) - can_read! - - super - end - - def each_line(*args) - can_read! - - super - end - - def write(*args) - can_write! - - super - end - - def truncate(*args) - can_write! - - super - end - - def delete(*args) - can_write! - - super - end - - private - - def can_read! - unless READABLE_MODE.any? { |m| mode.include?(m) } - raise IOError, 'not opened for reading' - end - end - - def can_write! - unless WRITABLE_MODE.any? { |m| mode.include?(m) } - raise IOError, 'not opened for writing' - end - end - - def write_lock_key(job_id) - "live_trace:operation:write:#{job_id}" - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/trace/chunked_file/live_trace.rb b/lib/gitlab/ci/trace/chunked_file/live_trace.rb deleted file mode 100644 index 1d3433d88ce..00000000000 --- a/lib/gitlab/ci/trace/chunked_file/live_trace.rb +++ /dev/null @@ -1,63 +0,0 @@ -module Gitlab - module Ci - class Trace - module ChunkedFile - class LiveTrace < ChunkedIO - class << self - def exist?(job_id) - ChunkedFile::ChunkStore::Redis.chunks_count(job_id) > 0 || ChunkedFile::ChunkStore::Database.chunks_count(job_id) > 0 - end - end - - after_callback :write_chunk, :stash_to_database - - def stash_to_database(store) - # Once data is filled into redis, move the data to database - if store.filled? - ChunkedFile::ChunkStore::Database.open(job_id, chunk_index, params_for_store) do |to_store| - to_store.write!(store.get) - store.delete! - end - end - end - - # This is more efficient than iterating each chunk store and deleting - def truncate(offset) - if offset == 0 - delete - @size = @tell = 0 - elsif offset == size - # no-op - else - raise NotImplementedError, 'Unexpected operation' - end - end - - def delete - ChunkedFile::ChunkStore::Redis.delete_all(job_id) - ChunkedFile::ChunkStore::Database.delete_all(job_id) - end - - private - - def calculate_size(job_id) - ChunkedFile::ChunkStore::Redis.chunks_size(job_id) + - ChunkedFile::ChunkStore::Database.chunks_size(job_id) - end - - def chunk_store - if last_chunk? - ChunkedFile::ChunkStore::Redis - else - ChunkedFile::ChunkStore::Database - end - end - - def buffer_size - 128.kilobytes - end - end - end - end - end -end |