Add latest changes from gitlab-org/gitlab@master

7 files changed, 157 insertions, 84 deletions

diff --git a/lib/gitlab/artifacts/migration_helper.rb b/lib/gitlab/artifacts/migration_helper.rb
new file mode 100644
index 00000000000..4f047ab3ea8
--- /dev/null
+++ b/lib/gitlab/artifacts/migration_helper.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Artifacts
+    class MigrationHelper
+      def migrate_to_remote_storage(&block)
+        artifacts = ::Ci::JobArtifact.with_files_stored_locally
+        migrate(artifacts, ObjectStorage::Store::REMOTE, &block)
+      end
+
+      def migrate_to_local_storage(&block)
+        artifacts = ::Ci::JobArtifact.with_files_stored_remotely
+        migrate(artifacts, ObjectStorage::Store::LOCAL, &block)
+      end
+
+      private
+
+      def batch_size
+        ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i
+      end
+
+      def migrate(artifacts, store, &block)
+        artifacts.find_each(batch_size: batch_size) do |artifact| # rubocop:disable CodeReuse/ActiveRecord
+          artifact.file.migrate!(store)
+
+          yield artifact if block
+        rescue => e
+          raise StandardError.new("Failed to transfer artifact of type #{artifact.file_type} and ID #{artifact.id} with error: #{e.message}")
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb
index 9d925581441..fe3a8d4aea5 100644
--- a/lib/gitlab/github_import/importer/releases_importer.rb
+++ b/lib/gitlab/github_import/importer/releases_importer.rb
@@ -32,10 +32,11 @@ module Gitlab
 
         def build(release)
           {
+            name: release.name,
             tag: release.tag_name,
             description: description_for(release),
             created_at: release.created_at,
-            updated_at: release.updated_at,
+            updated_at: release.created_at,
             released_at: release.published_at,
             project_id: project.id
           }
diff --git a/lib/gitlab/uploads/migration_helper.rb b/lib/gitlab/uploads/migration_helper.rb
new file mode 100644
index 00000000000..4ff064007f1
--- /dev/null
+++ b/lib/gitlab/uploads/migration_helper.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Uploads
+    class MigrationHelper
+      attr_reader :logger
+
+      CATEGORIES = [%w(AvatarUploader Project :avatar),
+                    %w(AvatarUploader Group :avatar),
+                    %w(AvatarUploader User :avatar),
+                    %w(AttachmentUploader Note :attachment),
+                    %w(AttachmentUploader Appearance :logo),
+                    %w(AttachmentUploader Appearance :header_logo),
+                    %w(FaviconUploader Appearance :favicon),
+                    %w(FileUploader Project),
+                    %w(PersonalFileUploader Snippet),
+                    %w(NamespaceFileUploader Snippet),
+                    %w(FileUploader MergeRequest)].freeze
+
+      def initialize(args, logger)
+        prepare_variables(args, logger)
+      end
+
+      def migrate_to_remote_storage
+        @to_store = ObjectStorage::Store::REMOTE
+
+        uploads.each_batch(of: batch_size, &method(:enqueue_batch))
+      end
+
+      def migrate_to_local_storage
+        @to_store = ObjectStorage::Store::LOCAL
+
+        uploads(ObjectStorage::Store::REMOTE).each_batch(of: batch_size, &method(:enqueue_batch))
+      end
+
+      private
+
+      def batch_size
+        ENV.fetch('MIGRATION_BATCH_SIZE', 200).to_i
+      end
+
+      def prepare_variables(args, logger)
+        @mounted_as     = args.mounted_as&.gsub(':', '')&.to_sym
+        @uploader_class = args.uploader_class.constantize
+        @model_class    = args.model_class.constantize
+        @logger         = logger
+      end
+
+      def enqueue_batch(batch, index)
+        job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch,
+                                                           @model_class,
+                                                           @mounted_as,
+                                                           @to_store)
+        logger.info(message: "[Uploads migration] Enqueued upload migration job", index: index, job_id: job)
+      rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e
+        # continue for the next batch
+        logger.warn(message: "[Uploads migration] Could not enqueue batch", ids: batch.ids, reason: e.message) # rubocop:disable CodeReuse/ActiveRecord
+      end
+
+      # rubocop:disable CodeReuse/ActiveRecord
+      def uploads(store_type = [nil, ObjectStorage::Store::LOCAL])
+        Upload.class_eval { include EachBatch } unless Upload < EachBatch
+
+        Upload
+          .where(store: store_type,
+                 uploader: @uploader_class.to_s,
+                 model_type: @model_class.base_class.sti_name)
+      end
+      # rubocop:enable CodeReuse/ActiveRecord
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake
index 9012e55a70c..0d09fd0a4e3 100644
--- a/lib/tasks/gitlab/artifacts/migrate.rake
+++ b/lib/tasks/gitlab/artifacts/migrate.rake
@@ -6,18 +6,31 @@ namespace :gitlab do
   namespace :artifacts do
     task migrate: :environment do
       logger = Logger.new(STDOUT)
-      logger.info('Starting transfer of artifacts')
+      logger.info('Starting transfer of artifacts to remote storage')
 
-      Ci::Build.joins(:project)
-        .with_artifacts_stored_locally
-        .find_each(batch_size: 10) do |build|
+      helper = Gitlab::Artifacts::MigrationHelper.new
 
-        build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE)
-        build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE)
+      begin
+        helper.migrate_to_remote_storage do |artifact|
+          logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to object storage")
+        end
+      rescue => e
+        logger.error(e.message)
+      end
+    end
+
+    task migrate_to_local: :environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Starting transfer of artifacts to local storage')
+
+      helper = Gitlab::Artifacts::MigrationHelper.new
 
-        logger.info("Transferred artifact ID #{build.id} with size #{build.artifacts_size} to object storage")
+      begin
+        helper.migrate_to_local_storage do |artifact|
+          logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to local storage")
+        end
       rescue => e
-        logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}")
+        logger.error(e.message)
       end
     end
   end
diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake
index 97c15175a23..4142903d9c3 100644
--- a/lib/tasks/gitlab/lfs/migrate.rake
+++ b/lib/tasks/gitlab/lfs/migrate.rake
@@ -17,5 +17,20 @@ namespace :gitlab do
           logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}")
         end
     end
+
+    task migrate_to_local: :environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Starting transfer of LFS files to local storage')
+
+      LfsObject.with_files_stored_remotely
+        .find_each(batch_size: 10) do |lfs_object|
+
+        lfs_object.file.migrate!(LfsObjectUploader::Store::LOCAL)
+
+        logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to local storage")
+      rescue => e
+        logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}")
+      end
+    end
   end
 end
diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake
deleted file mode 100644
index 5e1ec481ece..00000000000
--- a/lib/tasks/gitlab/traces.rake
+++ /dev/null
@@ -1,38 +0,0 @@
-require 'logger'
-require 'resolv-replace'
-
-desc "GitLab | Archive legacy traces to trace artifacts"
-namespace :gitlab do
-  namespace :traces do
-    task archive: :environment do
-      logger = Logger.new(STDOUT)
-      logger.info('Archiving legacy traces')
-
-      Ci::Build.finished.without_archived_trace
-        .order(id: :asc)
-        .find_in_batches(batch_size: 1000) do |jobs|
-        job_ids = jobs.map { |job| [job.id] }
-
-        ArchiveTraceWorker.bulk_perform_async(job_ids)
-
-        logger.info("Scheduled #{job_ids.count} jobs. From #{job_ids.min} to #{job_ids.max}")
-      end
-    end
-
-    task migrate: :environment do
-      logger = Logger.new(STDOUT)
-      logger.info('Starting transfer of job traces')
-
-      Ci::Build.joins(:project)
-        .with_archived_trace_stored_locally
-        .find_each(batch_size: 10) do |build|
-
-        build.job_artifacts_trace.file.migrate!(ObjectStorage::Store::REMOTE)
-
-        logger.info("Transferred job trace of #{build.id} to object storage")
-      rescue => e
-        logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}")
-      end
-    end
-  end
-end
diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake
index 1c93609a006..44536a447c7 100644
--- a/lib/tasks/gitlab/uploads/migrate.rake
+++ b/lib/tasks/gitlab/uploads/migrate.rake
@@ -3,19 +3,7 @@ namespace :gitlab do
     namespace :migrate do
       desc "GitLab | Uploads | Migrate all uploaded files to object storage"
       task all: :environment do
-        categories = [%w(AvatarUploader Project :avatar),
-                      %w(AvatarUploader Group :avatar),
-                      %w(AvatarUploader User :avatar),
-                      %w(AttachmentUploader Note :attachment),
-                      %w(AttachmentUploader Appearance :logo),
-                      %w(AttachmentUploader Appearance :header_logo),
-                      %w(FaviconUploader Appearance :favicon),
-                      %w(FileUploader Project),
-                      %w(PersonalFileUploader Snippet),
-                      %w(NamespaceFileUploader Snippet),
-                      %w(FileUploader MergeRequest)]
-
-        categories.each do |args|
+        Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args|
           Rake::Task["gitlab:uploads:migrate"].invoke(*args)
           Rake::Task["gitlab:uploads:migrate"].reenable
         end
@@ -25,34 +13,23 @@ namespace :gitlab do
     # The following is the actual rake task that migrates uploads of specified
     # category to object storage
     desc 'GitLab | Uploads | Migrate the uploaded files of specified type to object storage'
-    task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args|
-      batch_size     = ENV.fetch('BATCH', 200).to_i
-      @to_store      = ObjectStorage::Store::REMOTE
-      @mounted_as    = args.mounted_as&.gsub(':', '')&.to_sym
-      @uploader_class = args.uploader_class.constantize
-      @model_class    = args.model_class.constantize
-
-      uploads.each_batch(of: batch_size, &method(:enqueue_batch))
+    task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args|
+      Gitlab::Uploads::MigrationHelper.new(args, Logger.new(STDOUT)).migrate_to_remote_storage
     end
 
-    def enqueue_batch(batch, index)
-      job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch,
-                                                         @model_class,
-                                                         @mounted_as,
-                                                         @to_store)
-      puts "Enqueued job ##{index}: #{job}"
-    rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e
-      # continue for the next batch
-      puts "Could not enqueue batch (#{batch.ids}) #{e.message}".color(:red)
+    namespace :migrate_to_local do
+      desc "GitLab | Uploads | Migrate all uploaded files to local storage"
+      task all: :environment do
+        Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args|
+          Rake::Task["gitlab:uploads:migrate_to_local"].invoke(*args)
+          Rake::Task["gitlab:uploads:migrate_to_local"].reenable
+        end
+      end
     end
 
-    def uploads
-      Upload.class_eval { include EachBatch } unless Upload < EachBatch
-
-      Upload
-        .where(store: [nil, ObjectStorage::Store::LOCAL],
-               uploader: @uploader_class.to_s,
-               model_type: @model_class.base_class.sti_name)
+    desc 'GitLab | Uploads | Migrate the uploaded files of specified type to local storage'
+    task :migrate_to_local, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args|
+      Gitlab::Uploads::MigrationHelper.new(args, Logger.new(STDOUT)).migrate_to_local_storage
     end
   end
 end