diff options
Diffstat (limited to 'app/models/ci/build_trace_chunks')
-rw-r--r-- | app/models/ci/build_trace_chunks/database.rb | 4 | ||||
-rw-r--r-- | app/models/ci/build_trace_chunks/fog.rb | 12 | ||||
-rw-r--r-- | app/models/ci/build_trace_chunks/redis.rb | 87 | ||||
-rw-r--r-- | app/models/ci/build_trace_chunks/redis_base.rb | 90 | ||||
-rw-r--r-- | app/models/ci/build_trace_chunks/redis_trace_chunks.rb | 13 |
5 files changed, 116 insertions, 90 deletions
diff --git a/app/models/ci/build_trace_chunks/database.rb b/app/models/ci/build_trace_chunks/database.rb index 7448afba4c2..895028778a9 100644 --- a/app/models/ci/build_trace_chunks/database.rb +++ b/app/models/ci/build_trace_chunks/database.rb @@ -3,10 +3,6 @@ module Ci module BuildTraceChunks class Database - def available? - true - end - def keys(relation) [] end diff --git a/app/models/ci/build_trace_chunks/fog.rb b/app/models/ci/build_trace_chunks/fog.rb index cbf0c0a1696..fab85fae33d 100644 --- a/app/models/ci/build_trace_chunks/fog.rb +++ b/app/models/ci/build_trace_chunks/fog.rb @@ -3,10 +3,18 @@ module Ci module BuildTraceChunks class Fog - def available? + def self.available? object_store.enabled end + def self.object_store + Gitlab.config.artifacts.object_store + end + + def available? + self.class.available? + end + def data(model) files.get(key(model))&.body rescue Excon::Error::NotFound @@ -85,7 +93,7 @@ module Ci end def object_store - Gitlab.config.artifacts.object_store + self.class.object_store end def object_store_raw_config diff --git a/app/models/ci/build_trace_chunks/redis.rb b/app/models/ci/build_trace_chunks/redis.rb index 003ec107895..46f275636e1 100644 --- a/app/models/ci/build_trace_chunks/redis.rb +++ b/app/models/ci/build_trace_chunks/redis.rb @@ -2,92 +2,11 @@ module Ci module BuildTraceChunks - class Redis - CHUNK_REDIS_TTL = 1.week - LUA_APPEND_CHUNK = <<~EOS - local key, new_data, offset = KEYS[1], ARGV[1], ARGV[2] - local length = new_data:len() - local expire = #{CHUNK_REDIS_TTL.seconds} - local current_size = redis.call("strlen", key) - offset = tonumber(offset) - - if offset == 0 then - -- overwrite everything - redis.call("set", key, new_data, "ex", expire) - return redis.call("strlen", key) - elseif offset > current_size then - -- offset range violation - return -1 - elseif offset + length >= current_size then - -- efficiently append or overwrite and append - redis.call("expire", key, expire) - return redis.call("setrange", key, offset, new_data) - else - -- append and truncate - local current_data = redis.call("get", key) - new_data = current_data:sub(1, offset) .. new_data - redis.call("set", key, new_data, "ex", expire) - return redis.call("strlen", key) - end - EOS - - def available? - true - end - - def data(model) - Gitlab::Redis::SharedState.with do |redis| - redis.get(key(model)) - end - end - - def set_data(model, new_data) - Gitlab::Redis::SharedState.with do |redis| - redis.set(key(model), new_data, ex: CHUNK_REDIS_TTL) - end - end - - def append_data(model, new_data, offset) - Gitlab::Redis::SharedState.with do |redis| - redis.eval(LUA_APPEND_CHUNK, keys: [key(model)], argv: [new_data, offset]) - end - end - - def size(model) - Gitlab::Redis::SharedState.with do |redis| - redis.strlen(key(model)) - end - end - - def delete_data(model) - delete_keys([[model.build_id, model.chunk_index]]) - end - - def keys(relation) - relation.pluck(:build_id, :chunk_index) - end - - def delete_keys(keys) - return if keys.empty? - - keys = keys.map { |key| key_raw(*key) } - - Gitlab::Redis::SharedState.with do |redis| - # https://gitlab.com/gitlab-org/gitlab/-/issues/224171 - Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.del(keys) - end - end - end - + class Redis < RedisBase private - def key(model) - key_raw(model.build_id, model.chunk_index) - end - - def key_raw(build_id, chunk_index) - "gitlab:ci:trace:#{build_id.to_i}:chunks:#{chunk_index.to_i}" + def with_redis + Gitlab::Redis::SharedState.with { |redis| yield(redis) } end end end diff --git a/app/models/ci/build_trace_chunks/redis_base.rb b/app/models/ci/build_trace_chunks/redis_base.rb new file mode 100644 index 00000000000..3b7a844d122 --- /dev/null +++ b/app/models/ci/build_trace_chunks/redis_base.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +module Ci + module BuildTraceChunks + class RedisBase + CHUNK_REDIS_TTL = 1.week + LUA_APPEND_CHUNK = <<~EOS + local key, new_data, offset = KEYS[1], ARGV[1], ARGV[2] + local length = new_data:len() + local expire = #{CHUNK_REDIS_TTL.seconds} + local current_size = redis.call("strlen", key) + offset = tonumber(offset) + + if offset == 0 then + -- overwrite everything + redis.call("set", key, new_data, "ex", expire) + return redis.call("strlen", key) + elseif offset > current_size then + -- offset range violation + return -1 + elseif offset + length >= current_size then + -- efficiently append or overwrite and append + redis.call("expire", key, expire) + return redis.call("setrange", key, offset, new_data) + else + -- append and truncate + local current_data = redis.call("get", key) + new_data = current_data:sub(1, offset) .. new_data + redis.call("set", key, new_data, "ex", expire) + return redis.call("strlen", key) + end + EOS + + def data(model) + with_redis do |redis| + redis.get(key(model)) + end + end + + def set_data(model, new_data) + with_redis do |redis| + redis.set(key(model), new_data, ex: CHUNK_REDIS_TTL) + end + end + + def append_data(model, new_data, offset) + with_redis do |redis| + redis.eval(LUA_APPEND_CHUNK, keys: [key(model)], argv: [new_data, offset]) + end + end + + def size(model) + with_redis do |redis| + redis.strlen(key(model)) + end + end + + def delete_data(model) + delete_keys([[model.build_id, model.chunk_index]]) + end + + def keys(relation) + relation.pluck(:build_id, :chunk_index) + end + + def delete_keys(keys) + return if keys.empty? + + keys = keys.map { |key| key_raw(*key) } + + with_redis do |redis| + # https://gitlab.com/gitlab-org/gitlab/-/issues/224171 + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.del(keys) + end + end + end + + private + + def key(model) + key_raw(model.build_id, model.chunk_index) + end + + def key_raw(build_id, chunk_index) + "gitlab:ci:trace:#{build_id.to_i}:chunks:#{chunk_index.to_i}" + end + end + end +end diff --git a/app/models/ci/build_trace_chunks/redis_trace_chunks.rb b/app/models/ci/build_trace_chunks/redis_trace_chunks.rb new file mode 100644 index 00000000000..06e315b0aaf --- /dev/null +++ b/app/models/ci/build_trace_chunks/redis_trace_chunks.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Ci + module BuildTraceChunks + class RedisTraceChunks < RedisBase + private + + def with_redis + Gitlab::Redis::TraceChunks.with { |redis| yield(redis) } + end + end + end +end |