diff options
author | Gabriel Mazetto <gabriel@gitlab.com> | 2018-06-07 18:40:44 +0300 |
---|---|---|
committer | Nick Thomas <nick@gitlab.com> | 2018-06-07 18:40:44 +0300 |
commit | 36c337647591d964b7ef1e1fc61fc64a930fb6f4 (patch) | |
tree | dd00844e2b78592b453d04443367eb9879f18c42 /lib/gitlab/hashed_storage | |
parent | 1b06b8342cc75d948c40c9b74e4c4884a488484e (diff) |
Resolve "Hashed Storage: Make possible to migrate single project"
Diffstat (limited to 'lib/gitlab/hashed_storage')
-rw-r--r-- | lib/gitlab/hashed_storage/migrator.rb | 57 | ||||
-rw-r--r-- | lib/gitlab/hashed_storage/rake_helper.rb | 14 |
2 files changed, 70 insertions, 1 deletions
diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb new file mode 100644 index 00000000000..9251ed654cd --- /dev/null +++ b/lib/gitlab/hashed_storage/migrator.rb @@ -0,0 +1,57 @@ +module Gitlab + module HashedStorage + # Hashed Storage Migrator + # + # This is responsible for scheduling and flagging projects + # to be migrated from Legacy to Hashed storage, either one by one or in bulk. + class Migrator + BATCH_SIZE = 100 + + # Schedule a range of projects to be bulk migrated with #bulk_migrate asynchronously + # + # @param [Object] start first project id for the range + # @param [Object] finish last project id for the range + def bulk_schedule(start, finish) + StorageMigratorWorker.perform_async(start, finish) + end + + # Start migration of projects from specified range + # + # Flagging a project to be migrated is a synchronous action, + # but the migration runs through async jobs + # + # @param [Object] start first project id for the range + # @param [Object] finish last project id for the range + def bulk_migrate(start, finish) + projects = build_relation(start, finish) + + projects.with_route.find_each(batch_size: BATCH_SIZE) do |project| + migrate(project) + end + end + + # Flag a project to me migrated + # + # @param [Object] project that will be migrated + def migrate(project) + Rails.logger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..." + + project.migrate_to_hashed_storage! + rescue => err + Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") + end + + private + + def build_relation(start, finish) + relation = Project + table = Project.arel_table + + relation = relation.where(table[:id].gteq(start)) if start + relation = relation.where(table[:id].lteq(finish)) if finish + + relation + end + end + end +end diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb index 8aba42ccfce..303b05e6a9a 100644 --- a/lib/gitlab/hashed_storage/rake_helper.rb +++ b/lib/gitlab/hashed_storage/rake_helper.rb @@ -9,8 +9,20 @@ module Gitlab ENV.fetch('LIMIT', 500).to_i end + def self.range_from + ENV['ID_FROM'] + end + + def self.range_to + ENV['ID_TO'] + end + + def self.range_single_item? + !range_from.nil? && range_from == range_to + end + def self.project_id_batches(&block) - Project.with_unmigrated_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches + Project.with_unmigrated_storage.in_batches(of: batch_size, start: range_from, finish: range_to) do |relation| # rubocop: disable Cop/InBatches ids = relation.pluck(:id) yield ids.min, ids.max |