diff options
Diffstat (limited to 'app/services')
| -rw-r--r-- | app/services/ci/job_artifacts/destroy_all_expired_service.rb | 2 | ||||
| -rw-r--r-- | app/services/ci/job_artifacts/destroy_batch_service.rb | 24 | ||||
| -rw-r--r-- | app/services/web_hook_service.rb | 44 |
3 files changed, 61 insertions, 9 deletions
diff --git a/app/services/ci/job_artifacts/destroy_all_expired_service.rb b/app/services/ci/job_artifacts/destroy_all_expired_service.rb index 4070875ffe1..b5dd5b843c6 100644 --- a/app/services/ci/job_artifacts/destroy_all_expired_service.rb +++ b/app/services/ci/job_artifacts/destroy_all_expired_service.rb @@ -60,7 +60,7 @@ module Ci end def destroy_batch(artifacts) - Ci::JobArtifacts::DestroyBatchService.new(artifacts).execute + Ci::JobArtifacts::DestroyBatchService.new(artifacts, skip_projects_on_refresh: true).execute end def loop_timeout? diff --git a/app/services/ci/job_artifacts/destroy_batch_service.rb b/app/services/ci/job_artifacts/destroy_batch_service.rb index e8d21f14ee6..49b65f13804 100644 --- a/app/services/ci/job_artifacts/destroy_batch_service.rb +++ b/app/services/ci/job_artifacts/destroy_batch_service.rb @@ -17,15 +17,20 @@ module Ci # +pick_up_at+:: When to pick up for deletion of files # Returns: # +Hash+:: A hash with status and destroyed_artifacts_count keys - def initialize(job_artifacts, pick_up_at: nil, fix_expire_at: fix_expire_at?) + def initialize(job_artifacts, pick_up_at: nil, fix_expire_at: fix_expire_at?, skip_projects_on_refresh: false) @job_artifacts = job_artifacts.with_destroy_preloads.to_a @pick_up_at = pick_up_at @fix_expire_at = fix_expire_at + @skip_projects_on_refresh = skip_projects_on_refresh end # rubocop: disable CodeReuse/ActiveRecord def execute(update_stats: true) - track_artifacts_undergoing_stats_refresh + if @skip_projects_on_refresh + exclude_artifacts_undergoing_stats_refresh + else + track_artifacts_undergoing_stats_refresh + end # Detect and fix artifacts that had `expire_at` wrongly backfilled by migration # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47723 @@ -169,6 +174,21 @@ module Ci ) end end + + def exclude_artifacts_undergoing_stats_refresh + project_ids = Set.new + + @job_artifacts.reject! do |artifact| + next unless artifact.project.refreshing_build_artifacts_size? + + project_ids << artifact.project_id + end + + Gitlab::ProjectStatsRefreshConflictsLogger.warn_skipped_artifact_deletion_during_stats_refresh( + method: 'Ci::JobArtifacts::DestroyBatchService#execute', + project_ids: project_ids + ) + end end end end diff --git a/app/services/web_hook_service.rb b/app/services/web_hook_service.rb index c0727e52cc3..6526e6a4c5e 100644 --- a/app/services/web_hook_service.rb +++ b/app/services/web_hook_service.rb @@ -26,6 +26,12 @@ class WebHookService end REQUEST_BODY_SIZE_LIMIT = 25.megabytes + # Response body is for UI display only. It does not make much sense to save + # whatever the receivers throw back at us + RESPONSE_BODY_SIZE_LIMIT = 8.kilobytes + # The headers are for debugging purpose. They are displayed on the UI only. + RESPONSE_HEADERS_COUNT_LIMIT = 50 + RESPONSE_HEADERS_SIZE_LIMIT = 1.kilobytes attr_accessor :hook, :data, :hook_name, :request_options attr_reader :uniqueness_token @@ -141,7 +147,7 @@ class WebHookService execution_duration: execution_duration, request_headers: build_headers, request_data: data, - response_headers: format_response_headers(response), + response_headers: safe_response_headers(response), response_body: safe_response_body(response), response_status: response.code, internal_error_message: error_message @@ -150,8 +156,21 @@ class WebHookService if @force # executed as part of test - run log-execution inline. ::WebHooks::LogExecutionService.new(hook: hook, log_data: log_data, response_category: category).execute else - ::WebHooks::LogExecutionWorker - .perform_async(hook.id, log_data, category, uniqueness_token) + queue_log_execution_with_retry(log_data, category) + end + end + + def queue_log_execution_with_retry(log_data, category) + retried = false + begin + ::WebHooks::LogExecutionWorker.perform_async(hook.id, log_data, category, uniqueness_token) + rescue Gitlab::SidekiqMiddleware::SizeLimiter::ExceedLimitError + raise if retried + + # Strip request data + log_data[:request_data] = ::WebHookLog::OVERSIZE_REQUEST_DATA + retried = true + retry end end @@ -181,14 +200,19 @@ class WebHookService # Make response headers more stylish # Net::HTTPHeader has downcased hash with arrays: { 'content-type' => ['text/html; charset=utf-8'] } # This method format response to capitalized hash with strings: { 'Content-Type' => 'text/html; charset=utf-8' } - def format_response_headers(response) - response.headers.each_capitalized.to_h + # rubocop:disable Style/HashTransformValues + def safe_response_headers(response) + response.headers.each_capitalized.first(RESPONSE_HEADERS_COUNT_LIMIT).to_h do |header_key, header_value| + [enforce_utf8(header_key), string_size_limit(enforce_utf8(header_value), RESPONSE_HEADERS_SIZE_LIMIT)] + end end + # rubocop:enable Style/HashTransformValues def safe_response_body(response) return '' unless response.body - response.body.encode('UTF-8', invalid: :replace, undef: :replace, replace: '') + response_body = enforce_utf8(response.body) + string_size_limit(response_body, RESPONSE_BODY_SIZE_LIMIT) end def rate_limited? @@ -229,4 +253,12 @@ class WebHookService **Gitlab::ApplicationContext.current ) end + + def string_size_limit(str, limit) + str.truncate_bytes(limit) + end + + def enforce_utf8(str) + Gitlab::EncodingHelper.encode_utf8(str) + end end |