diff options
author | Shinya Maeda <shinya@gitlab.com> | 2018-05-23 09:12:17 +0300 |
---|---|---|
committer | Shinya Maeda <shinya@gitlab.com> | 2018-05-23 09:12:17 +0300 |
commit | dd28d60c9c5b9a0c695ffd58a89d56af3c02caac (patch) | |
tree | 631264bb8d3abf933b5c695c4bc1823604389bcb | |
parent | 76e276cb433a7023cf8154f9d9555725b3b11e67 (diff) |
Add rake task to clean up remaining filesadd-rake-task-to-clean-up-remaing-files
-rw-r--r-- | lib/tasks/gitlab/traces.rake | 37 |
1 files changed, 37 insertions, 0 deletions
diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake index fd2a4f2d11a..376bea79512 100644 --- a/lib/tasks/gitlab/traces.rake +++ b/lib/tasks/gitlab/traces.rake @@ -20,5 +20,42 @@ namespace :gitlab do logger.info("Scheduled #{job_ids.count} jobs. From #{job_ids.min} to #{job_ids.max}") end end + + task cleanup: :environment do + logger = Logger.new(STDOUT) + logger.info('Cleanup remaining traces') + + # Create a directory to move duplicated files in + backup_path = File.join(Settings.shared.path, 'duplicated', 'traces') + FileUtils.mkdir_p(backup_path) + + Dir.glob("#{Settings.gitlab_ci.builds_path}/**/**/*.log") do |entry| + file_name = File.basename(entry) + job_id = file_name.scan(/(\d+)\.log/).first.first.to_i + + build = Ci::Build.find_by_id(job_id) + + if build.nil? + logger.warn("id: #{job_id.to_s.rjust(8)} msg: build is not found") + + FileUtils.mv(entry, File.join(backup_path, file_name)) + elsif build.job_artifacts_trace&.file&.file&.exists? + if build.job_artifacts_trace.file_store == ObjectStorage::Store::REMOTE + logger.warn("id: #{job_id.to_s.rjust(8)} msg: build has already had an archived trace in object storage") + + FileUtils.mv(entry, File.join(backup_path, file_name)) + elsif build.job_artifacts_trace.file_store == ObjectStorage::Store::LOCAL || !build.job_artifacts_trace.file_store + logger.warn("id: #{job_id.to_s.rjust(8)} msg: build has already had an archived trace in file storage") + + FileUtils.mv(entry, File.join(backup_path, file_name)) + build.job_artifacts_trace.file.schedule_background_upload # Schedule to upload the local file to the object storage + end + else + logger.warn("id: #{job_id.to_s.rjust(8)} file_size: #{File.size(entry).to_s.rjust(8)} msg: build has not had an archived trace yet") + + ArchiveTraceWorker.perform_async(job_id) # Schedule to archive the trace + end + end + end end end |