diff options
author | Kamil Trzciński <ayufan@ayufan.eu> | 2018-02-28 22:35:22 +0300 |
---|---|---|
committer | Kamil Trzciński <ayufan@ayufan.eu> | 2018-02-28 22:35:22 +0300 |
commit | e0401df1214397626e65e58166988fe62715d372 (patch) | |
tree | 087d8ca4a1611aa50a8ac98e66f7d1657ff1f90f /app/workers | |
parent | 2b7b60728426c10ef1188a1073d3630805773a35 (diff) | |
parent | 11c67e7c2f992299ff5918ce67995b73d1e0be6d (diff) |
Merge commit '11c67e7c2f992299ff5918ce67995b73d1e0be6d' into object-storage-ee-to-ce-backport
Diffstat (limited to 'app/workers')
-rw-r--r-- | app/workers/build_finished_worker.rb | 1 | ||||
-rw-r--r-- | app/workers/build_trace_sections_worker.rb | 8 | ||||
-rw-r--r-- | app/workers/cluster_provision_worker.rb | 10 | ||||
-rw-r--r-- | app/workers/concerns/cluster_queue.rb | 10 | ||||
-rw-r--r-- | app/workers/project_migrate_hashed_storage_worker.rb | 11 | ||||
-rw-r--r-- | app/workers/storage_migrator_worker.rb | 30 | ||||
-rw-r--r-- | app/workers/use_key_worker.rb | 13 | ||||
-rw-r--r-- | app/workers/wait_for_cluster_creation_worker.rb | 27 |
8 files changed, 97 insertions, 13 deletions
diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb index e2a1b3dcc41..52e7d346e74 100644 --- a/app/workers/build_finished_worker.rb +++ b/app/workers/build_finished_worker.rb @@ -6,6 +6,7 @@ class BuildFinishedWorker def perform(build_id) Ci::Build.find_by(id: build_id).try do |build| + BuildTraceSectionsWorker.perform_async(build.id) BuildCoverageWorker.new.perform(build.id) BuildHooksWorker.new.perform(build.id) end diff --git a/app/workers/build_trace_sections_worker.rb b/app/workers/build_trace_sections_worker.rb new file mode 100644 index 00000000000..8c57e8f767b --- /dev/null +++ b/app/workers/build_trace_sections_worker.rb @@ -0,0 +1,8 @@ +class BuildTraceSectionsWorker + include Sidekiq::Worker + include PipelineQueue + + def perform(build_id) + Ci::Build.find_by(id: build_id)&.parse_trace_sections! + end +end diff --git a/app/workers/cluster_provision_worker.rb b/app/workers/cluster_provision_worker.rb new file mode 100644 index 00000000000..63300b58a25 --- /dev/null +++ b/app/workers/cluster_provision_worker.rb @@ -0,0 +1,10 @@ +class ClusterProvisionWorker + include Sidekiq::Worker + include ClusterQueue + + def perform(cluster_id) + Gcp::Cluster.find_by_id(cluster_id).try do |cluster| + Ci::ProvisionClusterService.new.execute(cluster) + end + end +end diff --git a/app/workers/concerns/cluster_queue.rb b/app/workers/concerns/cluster_queue.rb new file mode 100644 index 00000000000..a5074d13220 --- /dev/null +++ b/app/workers/concerns/cluster_queue.rb @@ -0,0 +1,10 @@ +## +# Concern for setting Sidekiq settings for the various Gcp clusters workers. +# +module ClusterQueue + extend ActiveSupport::Concern + + included do + sidekiq_options queue: :gcp_cluster + end +end diff --git a/app/workers/project_migrate_hashed_storage_worker.rb b/app/workers/project_migrate_hashed_storage_worker.rb new file mode 100644 index 00000000000..ca276d7801c --- /dev/null +++ b/app/workers/project_migrate_hashed_storage_worker.rb @@ -0,0 +1,11 @@ +class ProjectMigrateHashedStorageWorker + include Sidekiq::Worker + include DedicatedSidekiqQueue + + def perform(project_id) + project = Project.find_by(id: project_id) + return if project.nil? || project.pending_delete? + + ::Projects::HashedStorageMigrationService.new(project, logger).execute + end +end diff --git a/app/workers/storage_migrator_worker.rb b/app/workers/storage_migrator_worker.rb new file mode 100644 index 00000000000..b48ead799b9 --- /dev/null +++ b/app/workers/storage_migrator_worker.rb @@ -0,0 +1,30 @@ +class StorageMigratorWorker + include Sidekiq::Worker + include DedicatedSidekiqQueue + + BATCH_SIZE = 100 + + def perform(start, finish) + projects = build_relation(start, finish) + + projects.with_route.find_each(batch_size: BATCH_SIZE) do |project| + Rails.logger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..." + + begin + project.migrate_to_hashed_storage! + rescue => err + Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") + end + end + end + + def build_relation(start, finish) + relation = Project + table = Project.arel_table + + relation = relation.where(table[:id].gteq(start)) if start + relation = relation.where(table[:id].lteq(finish)) if finish + + relation + end +end diff --git a/app/workers/use_key_worker.rb b/app/workers/use_key_worker.rb deleted file mode 100644 index c9d382cc5d6..00000000000 --- a/app/workers/use_key_worker.rb +++ /dev/null @@ -1,13 +0,0 @@ -class UseKeyWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue - - def perform(key_id) - key = Key.find(key_id) - key.touch(:last_used_at) - rescue ActiveRecord::RecordNotFound - Rails.logger.error("UseKeyWorker: couldn't find key with ID=#{key_id}, skipping job") - - false - end -end diff --git a/app/workers/wait_for_cluster_creation_worker.rb b/app/workers/wait_for_cluster_creation_worker.rb new file mode 100644 index 00000000000..5aa3bbdaa9d --- /dev/null +++ b/app/workers/wait_for_cluster_creation_worker.rb @@ -0,0 +1,27 @@ +class WaitForClusterCreationWorker + include Sidekiq::Worker + include ClusterQueue + + INITIAL_INTERVAL = 2.minutes + EAGER_INTERVAL = 10.seconds + TIMEOUT = 20.minutes + + def perform(cluster_id) + Gcp::Cluster.find_by_id(cluster_id).try do |cluster| + Ci::FetchGcpOperationService.new.execute(cluster) do |operation| + case operation.status + when 'RUNNING' + if TIMEOUT < Time.now.utc - operation.start_time.to_time.utc + return cluster.make_errored!("Cluster creation time exceeds timeout; #{TIMEOUT}") + end + + WaitForClusterCreationWorker.perform_in(EAGER_INTERVAL, cluster.id) + when 'DONE' + Ci::FinalizeClusterCreationService.new.execute(cluster) + else + return cluster.make_errored!("Unexpected operation status; #{operation.status} #{operation.status_message}") + end + end + end + end +end |