diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-10-09 21:07:58 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-10-09 21:07:58 +0300 |
commit | b3986e06460cc2bff35c1c1e7902579b46c1d4c6 (patch) | |
tree | 3646975b83dd94a772a5a943ca1c5e7872646813 /app/services/pages | |
parent | fa0535cda4c99da0ca078882c610fd3b3564f991 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'app/services/pages')
3 files changed, 0 insertions, 236 deletions
diff --git a/app/services/pages/migrate_from_legacy_storage_service.rb b/app/services/pages/migrate_from_legacy_storage_service.rb deleted file mode 100644 index d102f93e863..00000000000 --- a/app/services/pages/migrate_from_legacy_storage_service.rb +++ /dev/null @@ -1,90 +0,0 @@ -# frozen_string_literal: true - -module Pages - class MigrateFromLegacyStorageService - def initialize(logger, ignore_invalid_entries:, mark_projects_as_not_deployed:) - @logger = logger - @ignore_invalid_entries = ignore_invalid_entries - @mark_projects_as_not_deployed = mark_projects_as_not_deployed - - @migrated = 0 - @errored = 0 - @counters_lock = Mutex.new - end - - def execute_with_threads(threads:, batch_size:) - @queue = SizedQueue.new(1) - - migration_threads = start_migration_threads(threads) - - ProjectPagesMetadatum.only_on_legacy_storage.each_batch(of: batch_size) do |batch| - @queue.push(batch) - end - - @queue.close - - @logger.info(message: "Pages legacy storage migration: Waiting for threads to finish...") - migration_threads.each(&:join) - - { migrated: @migrated, errored: @errored } - end - - def execute_for_batch(project_ids) - batch = ProjectPagesMetadatum.only_on_legacy_storage.where(project_id: project_ids) # rubocop: disable CodeReuse/ActiveRecord - - process_batch(batch) - - { migrated: @migrated, errored: @errored } - end - - private - - def start_migration_threads(count) - Array.new(count) do - Thread.new do - while batch = @queue.pop - Rails.application.executor.wrap do - process_batch(batch) - end - end - end - end - end - - def process_batch(batch) - batch.with_project_route_and_deployment.each do |metadatum| - project = metadatum.project - - migrate_project(project) - end - - @logger.info(message: "Pages legacy storage migration: batch processed", migrated: @migrated, errored: @errored) - rescue StandardError => e - # This method should never raise exception otherwise all threads might be killed - # and this will result in queue starving (and deadlock) - Gitlab::ErrorTracking.track_exception(e) - @logger.error(message: "Pages legacy storage migration: failed processing a batch: #{e.message}") - end - - def migrate_project(project) - result = nil - time = Benchmark.realtime do - result = ::Pages::MigrateLegacyStorageToDeploymentService.new(project, - ignore_invalid_entries: @ignore_invalid_entries, - mark_projects_as_not_deployed: @mark_projects_as_not_deployed).execute - end - - if result[:status] == :success - @logger.info(message: "Pages legacy storage migration: project migrated: #{result[:message]}", project_id: project.id, pages_path: project.pages_path, duration: time.round(2)) - @counters_lock.synchronize { @migrated += 1 } - else - @logger.error(message: "Pages legacy storage migration: project failed to be migrated: #{result[:message]}", project_id: project.id, pages_path: project.pages_path, duration: time.round(2)) - @counters_lock.synchronize { @errored += 1 } - end - rescue StandardError => e - @counters_lock.synchronize { @errored += 1 } - @logger.error(message: "Pages legacy storage migration: project failed to be migrated: #{result[:message]}", project_id: project&.id, pages_path: project&.pages_path) - Gitlab::ErrorTracking.track_exception(e, project_id: project&.id) - end - end -end diff --git a/app/services/pages/migrate_legacy_storage_to_deployment_service.rb b/app/services/pages/migrate_legacy_storage_to_deployment_service.rb deleted file mode 100644 index 9c1671fbc15..00000000000 --- a/app/services/pages/migrate_legacy_storage_to_deployment_service.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -module Pages - class MigrateLegacyStorageToDeploymentService - include BaseServiceUtility - - attr_reader :project - - def initialize(project, ignore_invalid_entries: false, mark_projects_as_not_deployed: false) - @project = project - @ignore_invalid_entries = ignore_invalid_entries - @mark_projects_as_not_deployed = mark_projects_as_not_deployed - end - - def execute - zip_result = ::Pages::ZipDirectoryService.new(project.pages_path, ignore_invalid_entries: @ignore_invalid_entries).execute - - if zip_result[:status] == :error - return error("Can't create zip archive: #{zip_result[:message]}") - end - - archive_path = zip_result[:archive_path] - - unless archive_path - return error("Archive not created. Missing public directory in #{@project.pages_path}") unless @mark_projects_as_not_deployed - - project.set_first_pages_deployment!(nil) - - return success( - message: "Archive not created. Missing public directory in #{project.pages_path}? Marked project as not deployed") - end - - deployment = nil - File.open(archive_path) do |file| - deployment = project.pages_deployments.create!( - file: file, - file_count: zip_result[:entries_count], - file_sha256: Digest::SHA256.file(archive_path).hexdigest - ) - end - - project.set_first_pages_deployment!(deployment) - - success - ensure - FileUtils.rm_f(archive_path) if archive_path - end - end -end diff --git a/app/services/pages/zip_directory_service.rb b/app/services/pages/zip_directory_service.rb deleted file mode 100644 index c9029b9666a..00000000000 --- a/app/services/pages/zip_directory_service.rb +++ /dev/null @@ -1,97 +0,0 @@ -# frozen_string_literal: true - -module Pages - class ZipDirectoryService - include BaseServiceUtility - include Gitlab::Utils::StrongMemoize - - # used only to track exceptions in Sentry - InvalidEntryError = Class.new(StandardError) - - PUBLIC_DIR = 'public' - - attr_reader :public_dir, :real_dir - - def initialize(input_dir, ignore_invalid_entries: false) - @input_dir = input_dir - @ignore_invalid_entries = ignore_invalid_entries - end - - def execute - return success unless resolve_public_dir - - output_file = File.join(real_dir, "@migrated.zip") # '@' to avoid any name collision with groups or projects - - FileUtils.rm_f(output_file) - - entries_count = 0 - # Since we're writing not reading here, we can safely silence the cop. - # It currently cannot discern between opening for reading or writing. - ::Zip::File.open(output_file, ::Zip::File::CREATE) do |zipfile| # rubocop:disable Performance/Rubyzip - write_entry(zipfile, PUBLIC_DIR) - entries_count = zipfile.entries.count - end - - success(archive_path: output_file, entries_count: entries_count) - rescue StandardError => e - FileUtils.rm_f(output_file) if output_file - raise e - end - - private - - def resolve_public_dir - @real_dir = File.realpath(@input_dir) - @public_dir = File.join(real_dir, PUBLIC_DIR) - - valid_path?(public_dir) - rescue Errno::ENOENT - false - end - - def write_entry(zipfile, zipfile_path) - disk_file_path = File.join(real_dir, zipfile_path) - - unless valid_path?(disk_file_path) - # archive with invalid entry will just have this entry missing - raise InvalidEntryError, "#{disk_file_path} is invalid, input_dir: #{@input_dir}" - end - - ftype = File.lstat(disk_file_path).ftype - case ftype - when 'directory' - recursively_zip_directory(zipfile, disk_file_path, zipfile_path) - when 'file', 'link' - zipfile.add(zipfile_path, disk_file_path) - else - raise InvalidEntryError, "#{disk_file_path} has invalid ftype: #{ftype}, input_dir: #{@input_dir}" - end - rescue Errno::ENOENT, Errno::ELOOP, InvalidEntryError => e - Gitlab::ErrorTracking.track_exception(e, input_dir: @input_dir, disk_file_path: disk_file_path) - - raise e unless @ignore_invalid_entries - end - - def recursively_zip_directory(zipfile, disk_file_path, zipfile_path) - zipfile.mkdir(zipfile_path) - - entries = Dir.entries(disk_file_path) - %w[. ..] - entries = entries.map { |entry| File.join(zipfile_path, entry) } - - write_entries(zipfile, entries) - end - - def write_entries(zipfile, entries) - entries.each do |zipfile_path| - write_entry(zipfile, zipfile_path) - end - end - - # SafeZip was introduced only recently, - # so we have invalid entries on disk - def valid_path?(disk_file_path) - realpath = File.realpath(disk_file_path) - realpath == public_dir || realpath.start_with?(public_dir + "/") - end - end -end |