diff options
| -rw-r--r-- | changelogs/unreleased/51571-wrapper-rake-task-uploads-migrate-os.yml | 5 | ||||
| -rw-r--r-- | doc/administration/raketasks/uploads/migrate.md | 37 | ||||
| -rw-r--r-- | doc/development/file_storage.md | 7 | ||||
| -rw-r--r-- | lib/tasks/gitlab/uploads/migrate.rake | 26 |
4 files changed, 71 insertions, 4 deletions
diff --git a/changelogs/unreleased/51571-wrapper-rake-task-uploads-migrate-os.yml b/changelogs/unreleased/51571-wrapper-rake-task-uploads-migrate-os.yml new file mode 100644 index 00000000000..50710ca0aa8 --- /dev/null +++ b/changelogs/unreleased/51571-wrapper-rake-task-uploads-migrate-os.yml @@ -0,0 +1,5 @@ +--- +title: Add wrapper rake task to migrate all uploads to OS +merge_request: 21779 +author: +type: other diff --git a/doc/administration/raketasks/uploads/migrate.md b/doc/administration/raketasks/uploads/migrate.md index 0cd33ffc122..b5c40478ea5 100644 --- a/doc/administration/raketasks/uploads/migrate.md +++ b/doc/administration/raketasks/uploads/migrate.md @@ -7,10 +7,32 @@ After [configuring the object storage](../../uploads.md#using-object-storage) fo >**Note:** All of the processing will be done in a background worker and requires **no downtime**. -This tasks uses 3 parameters to find uploads to migrate. +### All-in-one rake task + +GitLab provides a wrapper rake task that migrates all uploaded files - avatars, +logos, attachments, favicon, etc. - to object storage in one go. Under the hood, +it invokes individual rake tasks to migrate files falling under each of this +category one by one. The specifications of these individual rake tasks are +described in the next section. + +**Omnibus Installation** + +```bash +gitlab-rake "gitlab:uploads:migrate:all" +``` + +**Source Installation** + +```bash +sudo RAILS_ENV=production -u git -H bundle exec rake gitlab:uploads:migrate:all +``` + +### Individual rake tasks >**Note:** -These parameters are mainly internal to GitLab's structure, you may want to refer to the task list instead below. +If you already ran the rake task mentioned above, no need to run these individual rake tasks as that has been done automatically. + +The rake task uses 3 parameters to find uploads to migrate. Parameter | Type | Description --------- | ---- | ----------- @@ -18,6 +40,9 @@ Parameter | Type | Description `model_class` | string | Type of the model to migrate from `mount_point` | string/symbol | Name of the model's column on which the uploader is mounted on. +>**Note:** +These parameters are mainly internal to GitLab's structure, you may want to refer to the task list instead below. + This task also accepts some environment variables which you can use to override certain values: @@ -25,7 +50,7 @@ Variable | Type | Description -------- | ---- | ----------- `BATCH` | integer | Specifies the size of the batch. Defaults to 200. -** Omnibus Installation** +**Omnibus Installation** ```bash # gitlab-rake gitlab:uploads:migrate[uploader_class, model_class, mount_point] @@ -40,6 +65,9 @@ gitlab-rake "gitlab:uploads:migrate[AttachmentUploader, Note, :attachment]" gitlab-rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :logo]" gitlab-rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :header_logo]" +# Favicon +gitlab-rake "gitlab:uploads:migrate[FaviconUploader, Appearance, :favicon]" + # Markdown gitlab-rake "gitlab:uploads:migrate[FileUploader, Project]" gitlab-rake "gitlab:uploads:migrate[PersonalFileUploader, Snippet]" @@ -65,6 +93,9 @@ sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AttachmentUploader, Note sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :logo]" sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :header_logo]" +# Favicon +sudo -u git -H bundle exec rake "gitlab:uploads:migrate[FaviconUploader, Appearance, :favicon]" + # Markdown sudo -u git -H bundle exec rake "gitlab:uploads:migrate[FileUploader, Project]" sudo -u git -H bundle exec rake "gitlab:uploads:migrate[PersonalFileUploader, Snippet]" diff --git a/doc/development/file_storage.md b/doc/development/file_storage.md index fdbd7f1fa37..6e014e8c751 100644 --- a/doc/development/file_storage.md +++ b/doc/development/file_storage.md @@ -45,6 +45,11 @@ In the case of Issues/MR/Notes Markdown attachments, there is a different approa instead of basing the path into a mutable variable `:project_path_with_namespace`, it's possible to use the hash of the project ID instead, if project migrates to the new approach (introduced in 10.2). +> Note: We provide an [all-in-one rake task] to migrate all uploads to object +> storage in one go. If a new Uploader class or model type is introduced, make +> sure you add a rake task invocation corresponding to it to the [category +> list]. + ### Path segments Files are stored at multiple locations and use different path schemes. @@ -137,3 +142,5 @@ end [CarrierWave]: https://github.com/carrierwaveuploader/carrierwave [Hashed Storage]: ../administration/repository_storage_types.md +[all-in-one rake task]: ../administration/raketasks/uploads/migrate.md +[category list]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/tasks/gitlab/uploads/migrate.rake diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake index f548a266b99..1c93609a006 100644 --- a/lib/tasks/gitlab/uploads/migrate.rake +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -1,6 +1,30 @@ namespace :gitlab do namespace :uploads do - desc 'GitLab | Uploads | Migrate the uploaded files to object storage' + namespace :migrate do + desc "GitLab | Uploads | Migrate all uploaded files to object storage" + task all: :environment do + categories = [%w(AvatarUploader Project :avatar), + %w(AvatarUploader Group :avatar), + %w(AvatarUploader User :avatar), + %w(AttachmentUploader Note :attachment), + %w(AttachmentUploader Appearance :logo), + %w(AttachmentUploader Appearance :header_logo), + %w(FaviconUploader Appearance :favicon), + %w(FileUploader Project), + %w(PersonalFileUploader Snippet), + %w(NamespaceFileUploader Snippet), + %w(FileUploader MergeRequest)] + + categories.each do |args| + Rake::Task["gitlab:uploads:migrate"].invoke(*args) + Rake::Task["gitlab:uploads:migrate"].reenable + end + end + end + + # The following is the actual rake task that migrates uploads of specified + # category to object storage + desc 'GitLab | Uploads | Migrate the uploaded files of specified type to object storage' task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args| batch_size = ENV.fetch('BATCH', 200).to_i @to_store = ObjectStorage::Store::REMOTE |