diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-03 23:34:32 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-03 23:34:32 +0300 |
commit | a47bbf7ce093a46ce83cd66da5d9ce3150324860 (patch) | |
tree | 36ee17fd93f50c371dba3e8829283fa4bfb98f99 /app/models | |
parent | 4bafeeda963a11ce4004bbe35a4ff2606bc4d10a (diff) |
Add latest changes from gitlab-org/gitlab@13-5-stable-ee
Diffstat (limited to 'app/models')
-rw-r--r-- | app/models/ci/build_trace_chunk.rb | 12 | ||||
-rw-r--r-- | app/models/ci/build_trace_chunks/fog.rb | 18 | ||||
-rw-r--r-- | app/models/ci/build_trace_chunks/legacy_fog.rb | 77 |
3 files changed, 104 insertions, 3 deletions
diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 6926ccd9438..cf6eb159f52 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -45,6 +45,10 @@ module Ci def get_store_class(store) @stores ||= {} + + # Can't memoize this because the feature flag may alter this + return fog_store_class.new if store.to_sym == :fog + @stores[store] ||= "Ci::BuildTraceChunks::#{store.capitalize}".constantize.new end @@ -74,6 +78,14 @@ module Ci def metadata_attributes attribute_names - %w[raw_data] end + + def fog_store_class + if Feature.enabled?(:ci_trace_new_fog_store, default_enabled: true) + Ci::BuildTraceChunks::Fog + else + Ci::BuildTraceChunks::LegacyFog + end + end end def data diff --git a/app/models/ci/build_trace_chunks/fog.rb b/app/models/ci/build_trace_chunks/fog.rb index b1e9fd1faeb..d3051e3dadc 100644 --- a/app/models/ci/build_trace_chunks/fog.rb +++ b/app/models/ci/build_trace_chunks/fog.rb @@ -8,13 +8,17 @@ module Ci end def data(model) - connection.get_object(bucket_name, key(model))[:body] + files.get(key(model))&.body rescue Excon::Error::NotFound # If the object does not exist in the object storage, this method returns nil. end def set_data(model, new_data) - connection.put_object(bucket_name, key(model), new_data) + # TODO: Support AWS S3 server side encryption + files.create({ + key: key(model), + body: new_data + }) end def append_data(model, new_data, offset) @@ -43,7 +47,7 @@ module Ci def delete_keys(keys) keys.each do |key| - connection.delete_object(bucket_name, key_raw(*key)) + files.destroy(key_raw(*key)) end end @@ -69,6 +73,14 @@ module Ci @connection ||= ::Fog::Storage.new(object_store.connection.to_hash.deep_symbolize_keys) end + def fog_directory + @fog_directory ||= connection.directories.new(key: bucket_name) + end + + def files + @files ||= fog_directory.files + end + def object_store Gitlab.config.artifacts.object_store end diff --git a/app/models/ci/build_trace_chunks/legacy_fog.rb b/app/models/ci/build_trace_chunks/legacy_fog.rb new file mode 100644 index 00000000000..b710ed2890b --- /dev/null +++ b/app/models/ci/build_trace_chunks/legacy_fog.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Ci + module BuildTraceChunks + class LegacyFog + def available? + object_store.enabled + end + + def data(model) + connection.get_object(bucket_name, key(model))[:body] + rescue Excon::Error::NotFound + # If the object does not exist in the object storage, this method returns nil. + end + + def set_data(model, new_data) + connection.put_object(bucket_name, key(model), new_data) + end + + def append_data(model, new_data, offset) + if offset > 0 + truncated_data = data(model).to_s.byteslice(0, offset) + new_data = truncated_data + new_data + end + + set_data(model, new_data) + new_data.bytesize + end + + def size(model) + data(model).to_s.bytesize + end + + def delete_data(model) + delete_keys([[model.build_id, model.chunk_index]]) + end + + def keys(relation) + return [] unless available? + + relation.pluck(:build_id, :chunk_index) + end + + def delete_keys(keys) + keys.each do |key| + connection.delete_object(bucket_name, key_raw(*key)) + end + end + + private + + def key(model) + key_raw(model.build_id, model.chunk_index) + end + + def key_raw(build_id, chunk_index) + "tmp/builds/#{build_id.to_i}/chunks/#{chunk_index.to_i}.log" + end + + def bucket_name + return unless available? + + object_store.remote_directory + end + + def connection + return unless available? + + @connection ||= ::Fog::Storage.new(object_store.connection.to_hash.deep_symbolize_keys) + end + + def object_store + Gitlab.config.artifacts.object_store + end + end + end +end |