diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-10-21 03:10:26 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-10-21 03:10:26 +0300 |
commit | 3f9ae4674edd5b23ec9cd5b2bbb4a0caae5eec74 (patch) | |
tree | 3d8e65627e0fa03b967f11035b23133df0a860e7 /lib | |
parent | 40e8ba2fc8ac6c3695d7f297ff4143518615a3f9 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/populate.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/database_importers/self_monitoring/project/create_service.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/database_importers/self_monitoring/project/delete_service.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/import_export.yml | 2 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/finder.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/sidekiq_migrate_jobs.rb | 57 | ||||
-rw-r--r-- | lib/tasks/gitlab/sidekiq.rake | 15 |
7 files changed, 66 insertions, 16 deletions
diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 4bec8355732..d1f57ca68e7 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -42,7 +42,7 @@ module Gitlab name = @command.yaml_processor_result.workflow_name - pipeline.build_pipeline_metadata(project: pipeline.project, title: name) + pipeline.build_pipeline_metadata(project: pipeline.project, name: name) end def stage_names diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb index 57d354eb907..be500171bef 100644 --- a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb +++ b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb @@ -98,7 +98,7 @@ module Gitlab if environment.save success(result) else - log_error("Could not create environment for the Self monitoring project. Errors: %{errors}" % { errors: environment.errors.full_messages }) + log_error("Could not create environment for the Self-monitoring project. Errors: %{errors}" % { errors: environment.errors.full_messages }) error(_('Could not create environment')) end end diff --git a/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb b/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb index 998977b4000..d5bed94d735 100644 --- a/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb +++ b/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb @@ -23,7 +23,7 @@ module Gitlab def validate_self_monitoring_project_exists(result) unless project_created? || self_monitoring_project_id.present? - return error(_('Self monitoring project does not exist')) + return error(_('Self-monitoring project does not exist')) end success(result) diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index fb44aaf094e..b2a593397e4 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -585,7 +585,7 @@ included_attributes: - :target_sha pipeline_metadata: - :project_id - - :title + - :name stages: - :name - :status diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb index c8591a81a05..a4964ae0ebc 100644 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -78,7 +78,7 @@ module Gitlab end def predefined_dashboard_services_for(project) - # Only list the self monitoring dashboard on the self monitoring project, + # Only list the self-monitoring dashboard on the self-monitoring project, # since it is the only dashboard (at time of writing) that shows data # about GitLab itself. if project.self_monitoring? diff --git a/lib/gitlab/sidekiq_migrate_jobs.rb b/lib/gitlab/sidekiq_migrate_jobs.rb index 62d62bf82c4..76fa3515253 100644 --- a/lib/gitlab/sidekiq_migrate_jobs.rb +++ b/lib/gitlab/sidekiq_migrate_jobs.rb @@ -4,15 +4,16 @@ module Gitlab class SidekiqMigrateJobs LOG_FREQUENCY = 1_000 - attr_reader :sidekiq_set, :logger + attr_reader :logger, :mappings - def initialize(sidekiq_set, logger: nil) - @sidekiq_set = sidekiq_set + # mappings is a hash of WorkerClassName => target_queue_name + def initialize(mappings, logger: nil) + @mappings = mappings @logger = logger end - # mappings is a hash of WorkerClassName => target_queue_name - def execute(mappings) + # Migrate jobs in SortedSets, i.e. scheduled and retry sets. + def migrate_set(sidekiq_set) source_queues_regex = Regexp.union(mappings.keys) cursor = 0 scanned = 0 @@ -41,7 +42,7 @@ module Gitlab job_hash['queue'] = destination_queue - migrated += migrate_job(job, score, job_hash) + migrated += migrate_job_in_set(sidekiq_set, job, score, job_hash) end end while cursor.to_i != 0 @@ -53,9 +54,45 @@ module Gitlab } end + # Migrates jobs from queues that are outside the mappings + def migrate_queues + routing_rules_queues = mappings.values.uniq + logger&.info("List of queues based on routing rules: #{routing_rules_queues}") + Sidekiq.redis do |conn| + conn.scan_each(match: "queue:*", type: 'list') do |key| + queue_from = key.split(':', 2).last + next if routing_rules_queues.include?(queue_from) + + logger&.info("Migrating #{queue_from} queue") + + migrated = 0 + while queue_length(queue_from) > 0 + begin + if migrated >= 0 && migrated % LOG_FREQUENCY == 0 + logger&.info("Migrating from #{queue_from}. Total: #{queue_length(queue_from)}. Migrated: #{migrated}.") + end + + job = conn.rpop "queue:#{queue_from}" + job_hash = Sidekiq.load_json job + next unless mappings.has_key?(job_hash['class']) + + destination_queue = mappings[job_hash['class']] + job_hash['queue'] = destination_queue + conn.lpush("queue:#{destination_queue}", Sidekiq.dump_json(job_hash)) + migrated += 1 + rescue JSON::ParserError + logger&.error("Unmarshal JSON payload from SidekiqMigrateJobs failed. Job: #{job}") + next + end + end + logger&.info("Finished migrating #{queue_from} queue") + end + end + end + private - def migrate_job(job, score, job_hash) + def migrate_job_in_set(sidekiq_set, job, score, job_hash) Sidekiq.redis do |connection| removed = connection.zrem(sidekiq_set, job) @@ -68,5 +105,11 @@ module Gitlab end end end + + def queue_length(queue_name) + Sidekiq.redis do |conn| + conn.llen("queue:#{queue_name}") + end + end end end diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake index 10492e183c5..dc472305304 100644 --- a/lib/tasks/gitlab/sidekiq.rake +++ b/lib/tasks/gitlab/sidekiq.rake @@ -10,15 +10,22 @@ namespace :gitlab do desc 'GitLab | Sidekiq | Migrate jobs in the scheduled set to new queue names' task schedule: :environment do ::Gitlab::SidekiqMigrateJobs - .new('schedule', logger: Logger.new($stdout)) - .execute(::Gitlab::SidekiqConfig.worker_queue_mappings) + .new(::Gitlab::SidekiqConfig.worker_queue_mappings, logger: Logger.new($stdout) ) + .migrate_set('schedule') end desc 'GitLab | Sidekiq | Migrate jobs in the retry set to new queue names' task retry: :environment do ::Gitlab::SidekiqMigrateJobs - .new('retry', logger: Logger.new($stdout)) - .execute(::Gitlab::SidekiqConfig.worker_queue_mappings) + .new(::Gitlab::SidekiqConfig.worker_queue_mappings, logger: Logger.new($stdout) ) + .migrate_set('retry') + end + + desc 'GitLab | Sidekiq | Migrate jobs in queues outside of routing rules' + task queued: :environment do + ::Gitlab::SidekiqMigrateJobs + .new(::Gitlab::SidekiqConfig.worker_queue_mappings, logger: Logger.new($stdout) ) + .migrate_queues end end |