diff options
author | Douwe Maan <douwe@gitlab.com> | 2017-09-28 20:26:16 +0300 |
---|---|---|
committer | Douwe Maan <douwe@gitlab.com> | 2017-09-28 20:26:16 +0300 |
commit | ae03a52f0927f5f0881e3269faba90028e6d808b (patch) | |
tree | 56e4500d4e3ae5e67b16367d06898da389c35ba2 | |
parent | 0ab2ff72a4968ebf9f6e1eb41b46edbe3b5486ef (diff) | |
parent | f4de14d71f425dc14ee5837d96f4e9f42c7cc239 (diff) |
Merge branch 'hashed-storage-migration-path' into 'master'
Hashed storage migration path
Closes gitlab-ee#3118
See merge request gitlab-org/gitlab-ce!14067
20 files changed, 736 insertions, 14 deletions
diff --git a/app/models/project.rb b/app/models/project.rb index f7221e4f3b2..9b10401f5b8 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -245,6 +245,9 @@ class Project < ActiveRecord::Base scope :pending_delete, -> { where(pending_delete: true) } scope :without_deleted, -> { where(pending_delete: false) } + scope :with_hashed_storage, -> { where('storage_version >= 1') } + scope :with_legacy_storage, -> { where(storage_version: [nil, 0]) } + scope :sorted_by_activity, -> { reorder(last_activity_at: :desc) } scope :sorted_by_stars, -> { reorder('projects.star_count DESC') } @@ -1550,18 +1553,44 @@ class Project < ActiveRecord::Base end def legacy_storage? - self.storage_version.nil? + [nil, 0].include?(self.storage_version) + end + + def hashed_storage? + self.storage_version && self.storage_version >= 1 end def renamed? persisted? && path_changed? end + def migrate_to_hashed_storage! + return if hashed_storage? + + update!(repository_read_only: true) + + if repo_reference_count > 0 || wiki_reference_count > 0 + ProjectMigrateHashedStorageWorker.perform_in(Gitlab::ReferenceCounter::REFERENCE_EXPIRE_TIME, id) + else + ProjectMigrateHashedStorageWorker.perform_async(id) + end + end + + def storage_version=(value) + super + + @storage = nil if storage_version_changed? + end + + def gl_repository(is_wiki:) + Gitlab::GlRepository.gl_repository(self, is_wiki) + end + private def storage @storage ||= - if self.storage_version && self.storage_version >= 1 + if hashed_storage? Storage::HashedProject.new(self) else Storage::LegacyProject.new(self) @@ -1574,6 +1603,14 @@ class Project < ActiveRecord::Base end end + def repo_reference_count + Gitlab::ReferenceCounter.new(gl_repository(is_wiki: false)).value + end + + def wiki_reference_count + Gitlab::ReferenceCounter.new(gl_repository(is_wiki: true)).value + end + # set last_activity_at to the same as created_at def set_last_activity_at update_column(:last_activity_at, self.created_at) diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb index fae1b64961a..f025f40994e 100644 --- a/app/models/storage/hashed_project.rb +++ b/app/models/storage/hashed_project.rb @@ -4,6 +4,7 @@ module Storage delegate :gitlab_shell, :repository_storage_path, to: :project ROOT_PATH_PREFIX = '@hashed'.freeze + STORAGE_VERSION = 1 def initialize(project) @project = project diff --git a/app/services/projects/hashed_storage_migration_service.rb b/app/services/projects/hashed_storage_migration_service.rb new file mode 100644 index 00000000000..41259de3a16 --- /dev/null +++ b/app/services/projects/hashed_storage_migration_service.rb @@ -0,0 +1,68 @@ +module Projects + class HashedStorageMigrationService < BaseService + include Gitlab::ShellAdapter + + attr_reader :old_disk_path, :new_disk_path + + def initialize(project, logger = nil) + @project = project + @logger ||= Rails.logger + end + + def execute + return if project.hashed_storage? + + @old_disk_path = project.disk_path + has_wiki = project.wiki.repository_exists? + + project.storage_version = Storage::HashedProject::STORAGE_VERSION + project.ensure_storage_path_exists + + @new_disk_path = project.disk_path + + result = move_repository(@old_disk_path, @new_disk_path) + + if has_wiki + result &&= move_repository("#{@old_disk_path}.wiki", "#{@new_disk_path}.wiki") + end + + unless result + rollback_folder_move + return + end + + project.repository_read_only = false + project.save! + + block_given? ? yield : result + end + + private + + def move_repository(from_name, to_name) + from_exists = gitlab_shell.exists?(project.repository_storage_path, "#{from_name}.git") + to_exists = gitlab_shell.exists?(project.repository_storage_path, "#{to_name}.git") + + # If we don't find the repository on either original or target we should log that as it could be an issue if the + # project was not originally empty. + if !from_exists && !to_exists + logger.warn "Can't find a repository on either source or target paths for #{project.full_path} (ID=#{project.id}) ..." + return false + elsif !from_exists + # Repository have been moved already. + return true + end + + gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name) + end + + def rollback_folder_move + move_repository(@new_disk_path, @old_disk_path) + move_repository("#{@new_disk_path}.wiki", "#{@old_disk_path}.wiki") + end + + def logger + @logger + end + end +end diff --git a/app/workers/project_migrate_hashed_storage_worker.rb b/app/workers/project_migrate_hashed_storage_worker.rb new file mode 100644 index 00000000000..ca276d7801c --- /dev/null +++ b/app/workers/project_migrate_hashed_storage_worker.rb @@ -0,0 +1,11 @@ +class ProjectMigrateHashedStorageWorker + include Sidekiq::Worker + include DedicatedSidekiqQueue + + def perform(project_id) + project = Project.find_by(id: project_id) + return if project.nil? || project.pending_delete? + + ::Projects::HashedStorageMigrationService.new(project, logger).execute + end +end diff --git a/app/workers/storage_migrator_worker.rb b/app/workers/storage_migrator_worker.rb new file mode 100644 index 00000000000..b48ead799b9 --- /dev/null +++ b/app/workers/storage_migrator_worker.rb @@ -0,0 +1,30 @@ +class StorageMigratorWorker + include Sidekiq::Worker + include DedicatedSidekiqQueue + + BATCH_SIZE = 100 + + def perform(start, finish) + projects = build_relation(start, finish) + + projects.with_route.find_each(batch_size: BATCH_SIZE) do |project| + Rails.logger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..." + + begin + project.migrate_to_hashed_storage! + rescue => err + Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") + end + end + end + + def build_relation(start, finish) + relation = Project + table = Project.arel_table + + relation = relation.where(table[:id].gteq(start)) if start + relation = relation.where(table[:id].lteq(finish)) if finish + + relation + end +end diff --git a/changelogs/unreleased/hashed-storage-migration-path.yml b/changelogs/unreleased/hashed-storage-migration-path.yml new file mode 100644 index 00000000000..5890eb09c38 --- /dev/null +++ b/changelogs/unreleased/hashed-storage-migration-path.yml @@ -0,0 +1,5 @@ +--- +title: Script to migrate project's repositories to new Hashed Storage +merge_request: 14067 +author: +type: added diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index d169c38a693..8235e3853dc 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -62,3 +62,5 @@ - [update_user_activity, 1] - [propagate_service_template, 1] - [background_migration, 1] + - [project_migrate_hashed_storage, 1] + - [storage_migrator, 1] diff --git a/db/migrate/20160713200638_add_repository_read_only_to_projects.rb b/db/migrate/20160713200638_add_repository_read_only_to_projects.rb new file mode 100644 index 00000000000..8ee8b55f210 --- /dev/null +++ b/db/migrate/20160713200638_add_repository_read_only_to_projects.rb @@ -0,0 +1,9 @@ +class AddRepositoryReadOnlyToProjects < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :projects, :repository_read_only, :boolean + end +end diff --git a/db/schema.rb b/db/schema.rb index 63c8b8b3f49..d14d4d8b94d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1215,6 +1215,7 @@ ActiveRecord::Schema.define(version: 20170921115009) do t.datetime "last_repository_updated_at" t.integer "storage_version", limit: 2 t.boolean "resolve_outdated_diff_discussions" + t.boolean "repository_read_only" end add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree diff --git a/doc/administration/raketasks/storage.md b/doc/administration/raketasks/storage.md new file mode 100644 index 00000000000..bac8fa4bd9d --- /dev/null +++ b/doc/administration/raketasks/storage.md @@ -0,0 +1,107 @@ +# Repository Storage Rake Tasks + +This is a collection of rake tasks you can use to help you list and migrate +existing projects from Legacy storage to the new Hashed storage type. + +You can read more about the storage types [here][storage-types]. + +## List projects on Legacy storage + +To have a simple summary of projects using **Legacy** storage: + +**Omnibus Installation** + +```bash +gitlab-rake gitlab:storage:legacy_projects +``` + +**Source Installation** + +```bash +rake gitlab:storage:legacy_projects + +``` + +------ + +To list projects using **Legacy** storage: + +**Omnibus Installation** + +```bash +gitlab-rake gitlab:storage:list_legacy_projects +``` + +**Source Installation** + +```bash +rake gitlab:storage:list_legacy_projects + +``` + +## List projects on Hashed storage + +To have a simple summary of projects using **Hashed** storage: + +**Omnibus Installation** + +```bash +gitlab-rake gitlab:storage:hashed_projects +``` + +**Source Installation** + +```bash +rake gitlab:storage:hashed_projects + +``` + +------ + +To list projects using **Hashed** storage: + +**Omnibus Installation** + +```bash +gitlab-rake gitlab:storage:list_hashed_projects +``` + +**Source Installation** + +```bash +rake gitlab:storage:list_hashed_projects + +``` + +## Migrate existing projects to Hashed storage + +Before migrating your existing projects, you should +[enable hashed storage][storage-migration] for the new projects as well. + +This task will schedule all your existing projects to be migrated to the +**Hashed** storage type: + +**Omnibus Installation** + +```bash +gitlab-rake gitlab:storage:migrate_to_hashed +``` + +**Source Installation** + +```bash +rake gitlab:storage:migrate_to_hashed + +``` + +You can monitor the progress in the _Admin > Monitoring > Background jobs_ screen. +There is a specific Queue you can watch to see how long it will take to finish: **project_migrate_hashed_storage** + +After it reaches zero, you can confirm every project has been migrated by running the commands above. +If you find it necessary, you can run this migration script again to schedule missing projects. + +Any error or warning will be logged in the sidekiq log file. + + +[storage-types]: ../repository_storage_types.md +[storage-migration]: ../repository_storage_types.md#how-to-migrate-to-hashed-storage diff --git a/doc/administration/repository_storage_types.md b/doc/administration/repository_storage_types.md new file mode 100644 index 00000000000..fa882bbe28a --- /dev/null +++ b/doc/administration/repository_storage_types.md @@ -0,0 +1,69 @@ +# Repository Storage Types + +> [Introduced][ce-28283] in GitLab 10.0. + +## Legacy Storage + +Legacy Storage is the storage behavior prior to version 10.0. For historical reasons, GitLab replicated the same +mapping structure from the projects URLs: + + * Project's repository: `#{namespace}/#{project_name}.git` + * Project's wiki: `#{namespace}/#{project_name}.wiki.git` + +This structure made simple to migrate from existing solutions to GitLab and easy for Administrators to find where the +repository is stored. + +On the other hand this has some drawbacks: + +Storage location will concentrate huge amount of top-level namespaces. The impact can be reduced by the introduction of [multiple storage paths][storage-paths]. + +Because Backups are a snapshot of the same URL mapping, if you try to recover a very old backup, you need to verify +if any project has taken the place of an old removed project sharing the same URL. This means that `mygroup/myproject` +from your backup may not be the same original project that is today in the same URL. + +Any change in the URL will need to be reflected on disk (when groups / users or projects are renamed). This can add a lot +of load in big installations, and can be even worst if they are using any type of network based filesystem. + +Last, for GitLab Geo, this storage type means we have to synchronize the disk state, replicate renames in the correct +order or we may end-up with wrong repository or missing data temporarily. + +## Hashed Storage + +Hashed Storage is the new storage behavior we are rolling out with 10.0. It's not enabled by default yet, but we +encourage everyone to try-it and take the time to fix any script you may have that depends on the old behavior. + +Instead of coupling project URL and the folder structure where the repository will be stored on disk, we are coupling +a hash, based on the project's ID. + +This makes the folder structure immutable, and therefore eliminates any requirement to synchronize state from URLs to +disk structure. This means that renaming a group, user or project will cost only the database transaction, and will take +effect immediately. + +The hash also helps to spread the repositories more evenly on the disk, so the top-level directory will contain less +folders than the total amount of top-level namespaces. + +Hash format is based on hexadecimal representation of SHA256: `SHA256(project.id)`. +Top-level folder uses first 2 characters, followed by another folder with the next 2 characters. They are both stored in +a special folder `@hashed`, to co-exist with existing Legacy projects: + +```ruby +# Project's repository: +"@hashed/#{hash[0..1]}/#{hash[2..3]}/#{hash}.git" + +# Wiki's repository: +"@hashed/#{hash[0..1]}/#{hash[2..3]}/#{hash}.wiki.git" +``` + +This new format also makes possible to restore backups with confidence, as when restoring a repository from the backup, +you will never mistakenly restore a repository in the wrong project (considering the backup is made after the migration). + +### How to migrate to Hashed Storage + +In GitLab, go to **Admin > Settings**, find the **Repository Storage** section and select +"_Create new projects using hashed storage paths_". + +To migrate your existing projects to the new storage type, check the specific [rake tasks]. + +[ce-28283]: https://gitlab.com/gitlab-org/gitlab-ce/issues/28283 +[rake tasks]: raketasks/storage.md#migrate-existing-projects-to-hashed-storage +[storage-paths]: repository_storage_types.md diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 62d1ecae676..db67ede9d9e 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -16,7 +16,8 @@ module Gitlab account_blocked: 'Your account has been blocked.', command_not_allowed: "The command you're trying to execute is not allowed.", upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', - receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.' + receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.', + readonly: 'The repository is temporarily read-only. Please try again later.' }.freeze DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze @@ -159,6 +160,10 @@ module Gitlab end def check_push_access!(changes) + if project.repository_read_only? + raise UnauthorizedError, ERROR_MESSAGES[:readonly] + end + if deploy_key check_deploy_key_push_access! elsif user diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake new file mode 100644 index 00000000000..e05be4a3405 --- /dev/null +++ b/lib/tasks/gitlab/storage.rake @@ -0,0 +1,85 @@ +namespace :gitlab do + namespace :storage do + desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' + task migrate_to_hashed: :environment do + legacy_projects_count = Project.with_legacy_storage.count + + if legacy_projects_count == 0 + puts 'There are no projects using legacy storage. Nothing to do!' + + next + end + + print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{batch_size}" + + project_id_batches do |start, finish| + StorageMigratorWorker.perform_async(start, finish) + + print '.' + end + + puts ' Done!' + end + + desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage' + task legacy_projects: :environment do + projects_summary(Project.with_legacy_storage) + end + + desc 'Gitlab | Storage | List existing projects using Legacy Storage' + task list_legacy_projects: :environment do + projects_list(Project.with_legacy_storage) + end + + desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage' + task hashed_projects: :environment do + projects_summary(Project.with_hashed_storage) + end + + desc 'Gitlab | Storage | List existing projects using Hashed Storage' + task list_hashed_projects: :environment do + projects_list(Project.with_hashed_storage) + end + + def batch_size + ENV.fetch('BATCH', 200).to_i + end + + def project_id_batches(&block) + Project.with_legacy_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches + ids = relation.pluck(:id) + + yield ids.min, ids.max + end + end + + def projects_summary(relation) + projects_count = relation.count + puts "* Found #{projects_count} projects".color(:green) + + projects_count + end + + def projects_list(relation) + projects_count = projects_summary(relation) + + projects = relation.with_route + limit = ENV.fetch('LIMIT', 500).to_i + + return unless projects_count > 0 + + puts " ! Displaying first #{limit} projects..." if projects_count > limit + + counter = 0 + projects.find_in_batches(batch_size: batch_size) do |batch| + batch.each do |project| + counter += 1 + + puts " - #{project.full_path} (id: #{project.id})".color(:red) + + return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator + end + end + end + end +end diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb index 7493b0a8b35..958d62181a2 100644 --- a/spec/factories/projects.rb +++ b/spec/factories/projects.rb @@ -143,6 +143,16 @@ FactoryGirl.define do end end + trait :wiki_repo do + after(:create) do |project| + raise 'Failed to create wiki repository!' unless project.create_wiki + end + end + + trait :readonly do + repository_read_only true + end + trait :broken_repo do after(:create) do |project| raise "Failed to create repository!" unless project.create_repository diff --git a/spec/lib/gitlab/shell_spec.rb b/spec/lib/gitlab/shell_spec.rb index c7930378240..8edf83864da 100644 --- a/spec/lib/gitlab/shell_spec.rb +++ b/spec/lib/gitlab/shell_spec.rb @@ -48,14 +48,35 @@ describe Gitlab::Shell do end end - describe '#add_key' do - it 'removes trailing garbage' do - allow(gitlab_shell).to receive(:gitlab_shell_keys_path).and_return(:gitlab_shell_keys_path) - expect(gitlab_shell).to receive(:gitlab_shell_fast_execute).with( - [:gitlab_shell_keys_path, 'add-key', 'key-123', 'ssh-rsa foobar'] - ) - - gitlab_shell.add_key('key-123', 'ssh-rsa foobar trailing garbage') + describe 'projects commands' do + let(:gitlab_shell_path) { File.expand_path('tmp/tests/gitlab-shell') } + let(:projects_path) { File.join(gitlab_shell_path, 'bin/gitlab-projects') } + let(:gitlab_shell_hooks_path) { File.join(gitlab_shell_path, 'hooks') } + + before do + allow(Gitlab.config.gitlab_shell).to receive(:path).and_return(gitlab_shell_path) + allow(Gitlab.config.gitlab_shell).to receive(:hooks_path).and_return(gitlab_shell_hooks_path) + allow(Gitlab.config.gitlab_shell).to receive(:git_timeout).and_return(800) + end + + describe '#mv_repository' do + it 'executes the command' do + expect(gitlab_shell).to receive(:gitlab_shell_fast_execute).with( + [projects_path, 'mv-project', 'storage/path', 'project/path.git', 'new/path.git'] + ) + gitlab_shell.mv_repository('storage/path', 'project/path', 'new/path') + end + end + + describe '#add_key' do + it 'removes trailing garbage' do + allow(gitlab_shell).to receive(:gitlab_shell_keys_path).and_return(:gitlab_shell_keys_path) + expect(gitlab_shell).to receive(:gitlab_shell_fast_execute).with( + [:gitlab_shell_keys_path, 'add-key', 'key-123', 'ssh-rsa foobar'] + ) + + gitlab_shell.add_key('key-123', 'ssh-rsa foobar trailing garbage') + end end end @@ -136,7 +157,7 @@ describe Gitlab::Shell do it 'returns true when the command succeeds' do expect(Gitlab::Popen).to receive(:popen) .with([projects_path, 'rm-project', 'current/storage', 'project/path.git'], - nil, popen_vars).and_return([nil, 0]) + nil, popen_vars).and_return([nil, 0]) expect(gitlab_shell.remove_repository('current/storage', 'project/path')).to be true end @@ -144,7 +165,7 @@ describe Gitlab::Shell do it 'returns false when the command fails' do expect(Gitlab::Popen).to receive(:popen) .with([projects_path, 'rm-project', 'current/storage', 'project/path.git'], - nil, popen_vars).and_return(["error", 1]) + nil, popen_vars).and_return(["error", 1]) expect(gitlab_shell.remove_repository('current/storage', 'project/path')).to be false end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 78226c6c3fa..3ca88071ced 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -2363,10 +2363,22 @@ describe Project do describe '#legacy_storage?' do it 'returns true when storage_version is nil' do - project = build(:project) + project = build(:project, storage_version: nil) expect(project.legacy_storage?).to be_truthy end + + it 'returns true when the storage_version is 0' do + project = build(:project, storage_version: 0) + + expect(project.legacy_storage?).to be_truthy + end + end + + describe '#hashed_storage?' do + it 'returns false' do + expect(project.hashed_storage?).to be_falsey + end end describe '#rename_repo' do @@ -2425,6 +2437,38 @@ describe Project do expect(project.pages_path).to eq(File.join(Settings.pages.path, project.namespace.full_path, project.path)) end end + + describe '#migrate_to_hashed_storage!' do + it 'returns true' do + expect(project.migrate_to_hashed_storage!).to be_truthy + end + + it 'flags as readonly' do + expect { project.migrate_to_hashed_storage! }.to change { project.repository_read_only }.to(true) + end + + it 'schedules ProjectMigrateHashedStorageWorker with delayed start when the project repo is in use' do + Gitlab::ReferenceCounter.new(project.gl_repository(is_wiki: false)).increase + + expect(ProjectMigrateHashedStorageWorker).to receive(:perform_in) + + project.migrate_to_hashed_storage! + end + + it 'schedules ProjectMigrateHashedStorageWorker with delayed start when the wiki repo is in use' do + Gitlab::ReferenceCounter.new(project.gl_repository(is_wiki: true)).increase + + expect(ProjectMigrateHashedStorageWorker).to receive(:perform_in) + + project.migrate_to_hashed_storage! + end + + it 'schedules ProjectMigrateHashedStorageWorker' do + expect(ProjectMigrateHashedStorageWorker).to receive(:perform_async).with(project.id) + + project.migrate_to_hashed_storage! + end + end end context 'hashed storage' do @@ -2438,6 +2482,18 @@ describe Project do allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) end + describe '#legacy_storage?' do + it 'returns false' do + expect(project.legacy_storage?).to be_falsey + end + end + + describe '#hashed_storage?' do + it 'returns true' do + expect(project.hashed_storage?).to be_truthy + end + end + describe '#base_dir' do it 'returns base_dir based on hash of project id' do expect(project.base_dir).to eq('@hashed/6b/86') @@ -2508,6 +2564,26 @@ describe Project do expect(project.pages_path).to eq(File.join(Settings.pages.path, project.namespace.full_path, project.path)) end end + + describe '#migrate_to_hashed_storage!' do + it 'returns nil' do + expect(project.migrate_to_hashed_storage!).to be_nil + end + + it 'does not flag as readonly' do + expect { project.migrate_to_hashed_storage! }.not_to change { project.repository_read_only } + end + end + end + + describe '#gl_repository' do + let(:project) { create(:project) } + + it 'delegates to Gitlab::GlRepository.gl_repository' do + expect(Gitlab::GlRepository).to receive(:gl_repository).with(project, true) + + project.gl_repository(is_wiki: true) + end end describe '#has_ci?' do diff --git a/spec/services/projects/hashed_storage_migration_service_spec.rb b/spec/services/projects/hashed_storage_migration_service_spec.rb new file mode 100644 index 00000000000..1b61207b550 --- /dev/null +++ b/spec/services/projects/hashed_storage_migration_service_spec.rb @@ -0,0 +1,74 @@ +require 'spec_helper' + +describe Projects::HashedStorageMigrationService do + let(:gitlab_shell) { Gitlab::Shell.new } + let(:project) { create(:project, :empty_repo, :wiki_repo) } + let(:service) { described_class.new(project) } + let(:legacy_storage) { Storage::LegacyProject.new(project) } + let(:hashed_storage) { Storage::HashedProject.new(project) } + + describe '#execute' do + before do + allow(service).to receive(:gitlab_shell) { gitlab_shell } + end + + context 'when succeeds' do + it 'renames project and wiki repositories' do + service.execute + + expect(gitlab_shell.exists?(project.repository_storage_path, "#{hashed_storage.disk_path}.git")).to be_truthy + expect(gitlab_shell.exists?(project.repository_storage_path, "#{hashed_storage.disk_path}.wiki.git")).to be_truthy + end + + it 'updates project to be hashed and not readonly' do + service.execute + + expect(project.hashed_storage?).to be_truthy + expect(project.repository_read_only).to be_falsey + end + + it 'move operation is called for both repositories' do + expect_move_repository(project.disk_path, hashed_storage.disk_path) + expect_move_repository("#{project.disk_path}.wiki", "#{hashed_storage.disk_path}.wiki") + + service.execute + end + end + + context 'when one move fails' do + it 'rollsback repositories to original name' do + from_name = project.disk_path + to_name = hashed_storage.disk_path + allow(service).to receive(:move_repository).and_call_original + allow(service).to receive(:move_repository).with(from_name, to_name).once { false } # will disable first move only + + expect(service).to receive(:rollback_folder_move).and_call_original + + service.execute + + expect(gitlab_shell.exists?(project.repository_storage_path, "#{hashed_storage.disk_path}.git")).to be_falsey + expect(gitlab_shell.exists?(project.repository_storage_path, "#{hashed_storage.disk_path}.wiki.git")).to be_falsey + end + + context 'when rollback fails' do + before do + from_name = legacy_storage.disk_path + to_name = hashed_storage.disk_path + + hashed_storage.ensure_storage_path_exists + gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name) + end + + it 'does not try to move nil repository over hashed' do + expect_move_repository("#{project.disk_path}.wiki", "#{hashed_storage.disk_path}.wiki") + + service.execute + end + end + end + + def expect_move_repository(from_name, to_name) + expect(gitlab_shell).to receive(:mv_repository).with(project.repository_storage_path, from_name, to_name).and_call_original + end + end +end diff --git a/spec/tasks/gitlab/storage_rake_spec.rb b/spec/tasks/gitlab/storage_rake_spec.rb new file mode 100644 index 00000000000..f59792c3d36 --- /dev/null +++ b/spec/tasks/gitlab/storage_rake_spec.rb @@ -0,0 +1,52 @@ +require 'rake_helper' + +describe 'gitlab:storage rake tasks' do + before do + Rake.application.rake_require 'tasks/gitlab/storage' + + stub_warn_user_is_not_gitlab + end + + describe 'migrate_to_hashed rake task' do + context '0 legacy projects' do + it 'does nothing' do + expect(StorageMigratorWorker).not_to receive(:perform_async) + + run_rake_task('gitlab:storage:migrate_to_hashed') + end + end + + context '5 legacy projects' do + let(:projects) { create_list(:project, 5, storage_version: 0) } + + context 'in batches of 1' do + before do + stub_env('BATCH' => 1) + end + + it 'enqueues one StorageMigratorWorker per project' do + projects.each do |project| + expect(StorageMigratorWorker).to receive(:perform_async).with(project.id, project.id) + end + + run_rake_task('gitlab:storage:migrate_to_hashed') + end + end + + context 'in batches of 2' do + before do + stub_env('BATCH' => 2) + end + + it 'enqueues one StorageMigratorWorker per 2 projects' do + projects.map(&:id).sort.each_slice(2) do |first, last| + last ||= first + expect(StorageMigratorWorker).to receive(:perform_async).with(first, last) + end + + run_rake_task('gitlab:storage:migrate_to_hashed') + end + end + end + end +end diff --git a/spec/workers/project_migrate_hashed_storage_worker_spec.rb b/spec/workers/project_migrate_hashed_storage_worker_spec.rb new file mode 100644 index 00000000000..f5226dee0ad --- /dev/null +++ b/spec/workers/project_migrate_hashed_storage_worker_spec.rb @@ -0,0 +1,29 @@ +require 'spec_helper' + +describe ProjectMigrateHashedStorageWorker do + describe '#perform' do + let(:project) { create(:project, :empty_repo) } + let(:pending_delete_project) { create(:project, :empty_repo, pending_delete: true) } + + it 'skips when project no longer exists' do + nonexistent_id = 999999999999 + + expect(::Projects::HashedStorageMigrationService).not_to receive(:new) + subject.perform(nonexistent_id) + end + + it 'skips when project is pending delete' do + expect(::Projects::HashedStorageMigrationService).not_to receive(:new) + + subject.perform(pending_delete_project.id) + end + + it 'delegates removal to service class' do + service = double('service') + expect(::Projects::HashedStorageMigrationService).to receive(:new).with(project, subject.logger).and_return(service) + expect(service).to receive(:execute) + + subject.perform(project.id) + end + end +end diff --git a/spec/workers/storage_migrator_worker_spec.rb b/spec/workers/storage_migrator_worker_spec.rb new file mode 100644 index 00000000000..8619ff2f7da --- /dev/null +++ b/spec/workers/storage_migrator_worker_spec.rb @@ -0,0 +1,30 @@ +require 'spec_helper' + +describe StorageMigratorWorker do + subject(:worker) { described_class.new } + let(:projects) { create_list(:project, 2) } + + describe '#perform' do + let(:ids) { projects.map(&:id) } + + it 'enqueue jobs to ProjectMigrateHashedStorageWorker' do + expect(ProjectMigrateHashedStorageWorker).to receive(:perform_async).twice + + worker.perform(ids.min, ids.max) + end + + it 'sets projects as read only' do + allow(ProjectMigrateHashedStorageWorker).to receive(:perform_async).twice + worker.perform(ids.min, ids.max) + + projects.each do |project| + expect(project.reload.repository_read_only?).to be_truthy + end + end + + it 'rescues and log exceptions' do + allow_any_instance_of(Project).to receive(:migrate_to_hashed_storage!).and_raise(StandardError) + expect { worker.perform(ids.min, ids.max) }.not_to raise_error + end + end +end |