diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-01-22 15:08:40 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-01-22 15:08:40 +0300 |
commit | be3e24ea3c9f497efde85900df298ce9bc42fce8 (patch) | |
tree | fd0de9443253a1b21ca9a2741dc34ba3aef795be /lib | |
parent | 001243986195143c395a9811d8254bbf1b9ebfa1 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/import_export/import_export.yml | 1 | ||||
-rw-r--r-- | lib/gitlab/sidekiq_config.rb | 56 | ||||
-rw-r--r-- | lib/gitlab/sidekiq_config/worker.rb | 51 | ||||
-rw-r--r-- | lib/tasks/gitlab/sidekiq.rake | 48 | ||||
-rw-r--r-- | lib/tasks/lint.rake | 1 |
5 files changed, 145 insertions, 12 deletions
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 2acb79e3e22..afa575241a1 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -178,7 +178,6 @@ excluded_attributes: - :encrypted_secret_token - :encrypted_secret_token_iv - :repository_storage - - :storage_version merge_request_diff: - :external_diff - :stored_externally diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index b246c507e9e..c96212f27a7 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -4,6 +4,22 @@ require 'yaml' module Gitlab module SidekiqConfig + FOSS_QUEUE_CONFIG_PATH = 'app/workers/all_queues.yml' + EE_QUEUE_CONFIG_PATH = 'ee/app/workers/all_queues.yml' + + QUEUE_CONFIG_PATHS = [ + FOSS_QUEUE_CONFIG_PATH, + (EE_QUEUE_CONFIG_PATH if Gitlab.ee?) + ].compact.freeze + + # For queues that don't have explicit workers - default and mailers + DummyWorker = Struct.new(:queue) + + DEFAULT_WORKERS = [ + Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('default'), ee: false), + Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('mailers'), ee: false) + ].freeze + class << self include Gitlab::SidekiqConfig::CliMethods @@ -25,28 +41,46 @@ module Gitlab def workers @workers ||= begin - result = find_workers(Rails.root.join('app', 'workers')) - result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'))) if Gitlab.ee? + result = [] + result.concat(DEFAULT_WORKERS) + result.concat(find_workers(Rails.root.join('app', 'workers'), ee: false)) + + if Gitlab.ee? + result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'), ee: true)) + end + result end end + def workers_for_all_queues_yml + workers.partition(&:ee?).reverse.map(&:sort) + end + + def all_queues_yml_outdated? + foss_workers, ee_workers = workers_for_all_queues_yml + + return true if foss_workers != YAML.safe_load(File.read(FOSS_QUEUE_CONFIG_PATH)) + + Gitlab.ee? && ee_workers != YAML.safe_load(File.read(EE_QUEUE_CONFIG_PATH)) + end + private - def find_workers(root) + def find_workers(root, ee:) concerns = root.join('concerns').to_s - workers = Dir[root.join('**', '*.rb')] + Dir[root.join('**', '*.rb')] .reject { |path| path.start_with?(concerns) } + .map { |path| worker_from_path(path, root) } + .select { |worker| worker < Sidekiq::Worker } + .map { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: ee) } + end - workers.map! do |path| - ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '') - - ns.camelize.constantize - end + def worker_from_path(path, root) + ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '') - # Skip things that aren't workers - workers.select { |w| w < Sidekiq::Worker } + ns.camelize.constantize end end end diff --git a/lib/gitlab/sidekiq_config/worker.rb b/lib/gitlab/sidekiq_config/worker.rb new file mode 100644 index 00000000000..313d9b17b16 --- /dev/null +++ b/lib/gitlab/sidekiq_config/worker.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqConfig + class Worker + include Comparable + + attr_reader :klass + delegate :feature_category_not_owned?, :get_feature_category, + :get_worker_resource_boundary, :latency_sensitive_worker?, + :queue, :worker_has_external_dependencies?, + to: :klass + + def initialize(klass, ee:) + @klass = klass + @ee = ee + end + + def ee? + @ee + end + + def ==(other) + to_yaml == case other + when self.class + other.to_yaml + else + other + end + end + + def <=>(other) + to_sort <=> other.to_sort + end + + # Put namespaced queues first + def to_sort + [queue.include?(':') ? 0 : 1, queue] + end + + # YAML representation + def encode_with(coder) + coder.represent_scalar(nil, to_yaml) + end + + def to_yaml + queue + end + end + end +end diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake new file mode 100644 index 00000000000..f6bb0196236 --- /dev/null +++ b/lib/tasks/gitlab/sidekiq.rake @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +namespace :gitlab do + namespace :sidekiq do + namespace :all_queues_yml do + def write_yaml(path, object) + banner = <<~BANNER + # This file is generated automatically by + # bin/rake gitlab:sidekiq:all_queues_yml:generate + # + # Do not edit it manually! + BANNER + + File.write(path, banner + YAML.dump(object)) + end + + desc 'GitLab | Generate all_queues.yml based on worker definitions' + task generate: :environment do + foss_workers, ee_workers = Gitlab::SidekiqConfig.workers_for_all_queues_yml + + write_yaml(Gitlab::SidekiqConfig::FOSS_QUEUE_CONFIG_PATH, foss_workers) + + if Gitlab.ee? + write_yaml(Gitlab::SidekiqConfig::EE_QUEUE_CONFIG_PATH, ee_workers) + end + end + + desc 'GitLab | Validate that all_queues.yml matches worker definitions' + task check: :environment do + if Gitlab::SidekiqConfig.all_queues_yml_outdated? + raise <<~MSG + Changes in worker queues found, please update the metadata by running: + + bin/rake gitlab:sidekiq:all_queues_yml:generate + + Then commit and push the changes from: + + - #{Gitlab::SidekiqConfig::FOSS_QUEUE_CONFIG_PATH} + - #{Gitlab::SidekiqConfig::EE_QUEUE_CONFIG_PATH} + + MSG + end + end + end + end +end diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake index 9a5693e78a2..42a9c027b6a 100644 --- a/lib/tasks/lint.rake +++ b/lib/tasks/lint.rake @@ -34,6 +34,7 @@ unless Rails.env.production? scss_lint gettext:lint lint:static_verification + gitlab:sidekiq:all_queues_yml:check ] if Gitlab.ee? |