1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
# frozen_string_literal: true
namespace :gitlab do
namespace :sidekiq do
def write_yaml(path, banner, object)
File.write(path, banner + YAML.dump(object).gsub(/ *$/m, ''))
end
namespace :migrate_jobs do
desc 'GitLab | Sidekiq | Migrate jobs in the scheduled set to new queue names'
task schedule: :environment do
::Gitlab::SidekiqMigrateJobs
.new(::Gitlab::SidekiqConfig.worker_queue_mappings, logger: Logger.new($stdout))
.migrate_set('schedule')
end
desc 'GitLab | Sidekiq | Migrate jobs in the retry set to new queue names'
task retry: :environment do
::Gitlab::SidekiqMigrateJobs
.new(::Gitlab::SidekiqConfig.worker_queue_mappings, logger: Logger.new($stdout))
.migrate_set('retry')
end
desc 'GitLab | Sidekiq | Migrate jobs in queues outside of routing rules'
task queued: :environment do
::Gitlab::SidekiqMigrateJobs
.new(::Gitlab::SidekiqConfig.worker_queue_mappings, logger: Logger.new($stdout))
.migrate_queues
end
end
task :not_production do
raise 'This task cannot be run in the production environment' if Rails.env.production?
end
namespace :all_queues_yml do
desc 'GitLab | Sidekiq | Generate all_queues.yml based on worker definitions'
task generate: ['gitlab:sidekiq:not_production', :environment] do
banner = <<~BANNER
# This file is generated automatically by
# bin/rake gitlab:sidekiq:all_queues_yml:generate
#
# Do not edit it manually!
BANNER
foss_workers, ee_workers, jh_workers = Gitlab::SidekiqConfig.workers_for_all_queues_yml
write_yaml(Gitlab::SidekiqConfig::FOSS_QUEUE_CONFIG_PATH, banner, foss_workers)
if Gitlab.ee?
write_yaml(Gitlab::SidekiqConfig::EE_QUEUE_CONFIG_PATH, banner, ee_workers)
end
if Gitlab.jh?
write_yaml(Gitlab::SidekiqConfig::JH_QUEUE_CONFIG_PATH, banner, jh_workers)
end
end
desc 'GitLab | Sidekiq | Validate that all_queues.yml matches worker definitions'
task check: ['gitlab:sidekiq:not_production', :environment] do
if Gitlab::SidekiqConfig.all_queues_yml_outdated?
raise <<~MSG
Changes in worker queues found, please update the metadata by running:
bin/rake gitlab:sidekiq:all_queues_yml:generate
Then commit and push the changes from:
- #{Gitlab::SidekiqConfig::FOSS_QUEUE_CONFIG_PATH}
- #{Gitlab::SidekiqConfig::EE_QUEUE_CONFIG_PATH}
#{"- " + Gitlab::SidekiqConfig::JH_QUEUE_CONFIG_PATH if Gitlab.jh?}
MSG
end
end
end
namespace :sidekiq_queues_yml do
desc 'GitLab | Sidekiq | Generate sidekiq_queues.yml based on worker definitions'
task generate: ['gitlab:sidekiq:not_production', :environment] do
banner = <<~BANNER
# This file is generated automatically by
# bin/rake gitlab:sidekiq:sidekiq_queues_yml:generate
#
# Do not edit it manually!
#
# This configuration file should be exclusively used to set queue settings for
# Sidekiq. Any other setting should be specified using the Sidekiq CLI or the
# Sidekiq Ruby API (see config/initializers/sidekiq.rb).
#
# All the queues to process and their weights. Every queue _must_ have a weight
# defined.
#
# The available weights are as follows
#
# 1: low priority
# 2: medium priority
# 3: high priority
# 5: _super_ high priority, this should only be used for _very_ important queues
#
# The formula for calculating the likelihood of a job being popped off a queue
# (given all queues have work to perform) is:
#
# chance = (queue weight / total weight of all queues) * 100
BANNER
queues_and_weights = Gitlab::SidekiqConfig.queues_for_sidekiq_queues_yml
write_yaml(Gitlab::SidekiqConfig::SIDEKIQ_QUEUES_PATH, banner, queues: queues_and_weights)
if Gitlab.jh?
write_yaml(Gitlab::SidekiqConfig::JH_SIDEKIQ_QUEUES_PATH, banner, queues: Gitlab::SidekiqConfig.jh_queues_for_sidekiq_queues_yml)
end
end
desc 'GitLab | Sidekiq | Validate that sidekiq_queues.yml matches worker definitions'
task check: ['gitlab:sidekiq:not_production', :environment] do
if Gitlab::SidekiqConfig.sidekiq_queues_yml_outdated?
raise <<~MSG
Changes in worker queues found, please update the metadata by running:
bin/rake gitlab:sidekiq:sidekiq_queues_yml:generate
Then commit and push the changes from:
- #{Gitlab::SidekiqConfig::SIDEKIQ_QUEUES_PATH}
#{"- " + Gitlab::SidekiqConfig::JH_SIDEKIQ_QUEUES_PATH if Gitlab.jh?}
MSG
end
end
end
namespace :queues do
desc 'GitLab | Sidekiq | Validate all_queues.yml and sidekiq_queues.yml match worker definitions'
task check: ['gitlab:sidekiq:all_queues_yml:check', 'gitlab:sidekiq:sidekiq_queues_yml:check', :environment]
end
end
end
|
