Merge commit '11c67e7c2f992299ff5918ce67995b73d1e0be6d' into object-storage-ee-to-ce-backport

8 files changed, 97 insertions, 13 deletions

diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb
index e2a1b3dcc41..52e7d346e74 100644
--- a/app/workers/build_finished_worker.rb
+++ b/app/workers/build_finished_worker.rb
@@ -6,6 +6,7 @@ class BuildFinishedWorker
 
   def perform(build_id)
     Ci::Build.find_by(id: build_id).try do |build|
+      BuildTraceSectionsWorker.perform_async(build.id)
       BuildCoverageWorker.new.perform(build.id)
       BuildHooksWorker.new.perform(build.id)
     end
diff --git a/app/workers/build_trace_sections_worker.rb b/app/workers/build_trace_sections_worker.rb
new file mode 100644
index 00000000000..8c57e8f767b
--- /dev/null
+++ b/app/workers/build_trace_sections_worker.rb
@@ -0,0 +1,8 @@
+class BuildTraceSectionsWorker
+  include Sidekiq::Worker
+  include PipelineQueue
+
+  def perform(build_id)
+    Ci::Build.find_by(id: build_id)&.parse_trace_sections!
+  end
+end
diff --git a/app/workers/cluster_provision_worker.rb b/app/workers/cluster_provision_worker.rb
new file mode 100644
index 00000000000..63300b58a25
--- /dev/null
+++ b/app/workers/cluster_provision_worker.rb
@@ -0,0 +1,10 @@
+class ClusterProvisionWorker
+  include Sidekiq::Worker
+  include ClusterQueue
+
+  def perform(cluster_id)
+    Gcp::Cluster.find_by_id(cluster_id).try do |cluster|
+      Ci::ProvisionClusterService.new.execute(cluster)
+    end
+  end
+end
diff --git a/app/workers/concerns/cluster_queue.rb b/app/workers/concerns/cluster_queue.rb
new file mode 100644
index 00000000000..a5074d13220
--- /dev/null
+++ b/app/workers/concerns/cluster_queue.rb
@@ -0,0 +1,10 @@
+##
+# Concern for setting Sidekiq settings for the various Gcp clusters workers.
+#
+module ClusterQueue
+  extend ActiveSupport::Concern
+
+  included do
+    sidekiq_options queue: :gcp_cluster
+  end
+end
diff --git a/app/workers/project_migrate_hashed_storage_worker.rb b/app/workers/project_migrate_hashed_storage_worker.rb
new file mode 100644
index 00000000000..ca276d7801c
--- /dev/null
+++ b/app/workers/project_migrate_hashed_storage_worker.rb
@@ -0,0 +1,11 @@
+class ProjectMigrateHashedStorageWorker
+  include Sidekiq::Worker
+  include DedicatedSidekiqQueue
+
+  def perform(project_id)
+    project = Project.find_by(id: project_id)
+    return if project.nil? || project.pending_delete?
+
+    ::Projects::HashedStorageMigrationService.new(project, logger).execute
+  end
+end
diff --git a/app/workers/storage_migrator_worker.rb b/app/workers/storage_migrator_worker.rb
new file mode 100644
index 00000000000..b48ead799b9
--- /dev/null
+++ b/app/workers/storage_migrator_worker.rb
@@ -0,0 +1,30 @@
+class StorageMigratorWorker
+  include Sidekiq::Worker
+  include DedicatedSidekiqQueue
+
+  BATCH_SIZE = 100
+
+  def perform(start, finish)
+    projects = build_relation(start, finish)
+
+    projects.with_route.find_each(batch_size: BATCH_SIZE) do |project|
+      Rails.logger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..."
+
+      begin
+        project.migrate_to_hashed_storage!
+      rescue => err
+        Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}")
+      end
+    end
+  end
+
+  def build_relation(start, finish)
+    relation = Project
+    table = Project.arel_table
+
+    relation = relation.where(table[:id].gteq(start)) if start
+    relation = relation.where(table[:id].lteq(finish)) if finish
+
+    relation
+  end
+end
diff --git a/app/workers/use_key_worker.rb b/app/workers/use_key_worker.rb
deleted file mode 100644
index c9d382cc5d6..00000000000
--- a/app/workers/use_key_worker.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-class UseKeyWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
-
-  def perform(key_id)
-    key = Key.find(key_id)
-    key.touch(:last_used_at)
-  rescue ActiveRecord::RecordNotFound
-    Rails.logger.error("UseKeyWorker: couldn't find key with ID=#{key_id}, skipping job")
-
-    false
-  end
-end
diff --git a/app/workers/wait_for_cluster_creation_worker.rb b/app/workers/wait_for_cluster_creation_worker.rb
new file mode 100644
index 00000000000..5aa3bbdaa9d
--- /dev/null
+++ b/app/workers/wait_for_cluster_creation_worker.rb
@@ -0,0 +1,27 @@
+class WaitForClusterCreationWorker
+  include Sidekiq::Worker
+  include ClusterQueue
+
+  INITIAL_INTERVAL = 2.minutes
+  EAGER_INTERVAL = 10.seconds
+  TIMEOUT = 20.minutes
+
+  def perform(cluster_id)
+    Gcp::Cluster.find_by_id(cluster_id).try do |cluster|
+      Ci::FetchGcpOperationService.new.execute(cluster) do |operation|
+        case operation.status
+        when 'RUNNING'
+          if TIMEOUT < Time.now.utc - operation.start_time.to_time.utc
+            return cluster.make_errored!("Cluster creation time exceeds timeout; #{TIMEOUT}")
+          end
+
+          WaitForClusterCreationWorker.perform_in(EAGER_INTERVAL, cluster.id)
+        when 'DONE'
+          Ci::FinalizeClusterCreationService.new.execute(cluster)
+        else
+          return cluster.make_errored!("Unexpected operation status; #{operation.status} #{operation.status_message}")
+        end
+      end
+    end
+  end
+end