Add latest changes from gitlab-org/gitlab@master

7 files changed, 66 insertions, 16 deletions

diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb
index 4bec8355732..d1f57ca68e7 100644
--- a/lib/gitlab/ci/pipeline/chain/populate.rb
+++ b/lib/gitlab/ci/pipeline/chain/populate.rb
@@ -42,7 +42,7 @@ module Gitlab
 
             name = @command.yaml_processor_result.workflow_name
 
-            pipeline.build_pipeline_metadata(project: pipeline.project, title: name)
+            pipeline.build_pipeline_metadata(project: pipeline.project, name: name)
           end
 
           def stage_names
diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb
index 57d354eb907..be500171bef 100644
--- a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb
+++ b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb
@@ -98,7 +98,7 @@ module Gitlab
             if environment.save
               success(result)
             else
-              log_error("Could not create environment for the Self monitoring project. Errors: %{errors}" % { errors: environment.errors.full_messages })
+              log_error("Could not create environment for the Self-monitoring project. Errors: %{errors}" % { errors: environment.errors.full_messages })
               error(_('Could not create environment'))
             end
           end
diff --git a/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb b/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb
index 998977b4000..d5bed94d735 100644
--- a/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb
+++ b/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb
@@ -23,7 +23,7 @@ module Gitlab
 
           def validate_self_monitoring_project_exists(result)
             unless project_created? || self_monitoring_project_id.present?
-              return error(_('Self monitoring project does not exist'))
+              return error(_('Self-monitoring project does not exist'))
             end
 
             success(result)
diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml
index fb44aaf094e..b2a593397e4 100644
--- a/lib/gitlab/import_export/project/import_export.yml
+++ b/lib/gitlab/import_export/project/import_export.yml
@@ -585,7 +585,7 @@ included_attributes:
     - :target_sha
   pipeline_metadata:
     - :project_id
-    - :title
+    - :name
   stages:
     - :name
     - :status
diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb
index c8591a81a05..a4964ae0ebc 100644
--- a/lib/gitlab/metrics/dashboard/finder.rb
+++ b/lib/gitlab/metrics/dashboard/finder.rb
@@ -78,7 +78,7 @@ module Gitlab
           end
 
           def predefined_dashboard_services_for(project)
-            # Only list the self monitoring dashboard on the self monitoring project,
+            # Only list the self-monitoring dashboard on the self-monitoring project,
             # since it is the only dashboard (at time of writing) that shows data
             # about GitLab itself.
             if project.self_monitoring?
diff --git a/lib/gitlab/sidekiq_migrate_jobs.rb b/lib/gitlab/sidekiq_migrate_jobs.rb
index 62d62bf82c4..76fa3515253 100644
--- a/lib/gitlab/sidekiq_migrate_jobs.rb
+++ b/lib/gitlab/sidekiq_migrate_jobs.rb
@@ -4,15 +4,16 @@ module Gitlab
   class SidekiqMigrateJobs
     LOG_FREQUENCY = 1_000
 
-    attr_reader :sidekiq_set, :logger
+    attr_reader :logger, :mappings
 
-    def initialize(sidekiq_set, logger: nil)
-      @sidekiq_set = sidekiq_set
+    # mappings is a hash of WorkerClassName => target_queue_name
+    def initialize(mappings, logger: nil)
+      @mappings = mappings
       @logger = logger
     end
 
-    # mappings is a hash of WorkerClassName => target_queue_name
-    def execute(mappings)
+    # Migrate jobs in SortedSets, i.e. scheduled and retry sets.
+    def migrate_set(sidekiq_set)
       source_queues_regex = Regexp.union(mappings.keys)
       cursor = 0
       scanned = 0
@@ -41,7 +42,7 @@ module Gitlab
 
           job_hash['queue'] = destination_queue
 
-          migrated += migrate_job(job, score, job_hash)
+          migrated += migrate_job_in_set(sidekiq_set, job, score, job_hash)
         end
       end while cursor.to_i != 0
 
@@ -53,9 +54,45 @@ module Gitlab
       }
     end
 
+    # Migrates jobs from queues that are outside the mappings
+    def migrate_queues
+      routing_rules_queues = mappings.values.uniq
+      logger&.info("List of queues based on routing rules: #{routing_rules_queues}")
+      Sidekiq.redis do |conn|
+        conn.scan_each(match: "queue:*", type: 'list') do |key|
+          queue_from = key.split(':', 2).last
+          next if routing_rules_queues.include?(queue_from)
+
+          logger&.info("Migrating #{queue_from} queue")
+
+          migrated = 0
+          while queue_length(queue_from) > 0
+            begin
+              if migrated >= 0 && migrated % LOG_FREQUENCY == 0
+                logger&.info("Migrating from #{queue_from}. Total: #{queue_length(queue_from)}. Migrated: #{migrated}.")
+              end
+
+              job = conn.rpop "queue:#{queue_from}"
+              job_hash = Sidekiq.load_json job
+              next unless mappings.has_key?(job_hash['class'])
+
+              destination_queue = mappings[job_hash['class']]
+              job_hash['queue'] = destination_queue
+              conn.lpush("queue:#{destination_queue}", Sidekiq.dump_json(job_hash))
+              migrated += 1
+            rescue JSON::ParserError
+              logger&.error("Unmarshal JSON payload from SidekiqMigrateJobs failed. Job: #{job}")
+              next
+            end
+          end
+          logger&.info("Finished migrating #{queue_from} queue")
+        end
+      end
+    end
+
     private
 
-    def migrate_job(job, score, job_hash)
+    def migrate_job_in_set(sidekiq_set, job, score, job_hash)
       Sidekiq.redis do |connection|
         removed = connection.zrem(sidekiq_set, job)
 
@@ -68,5 +105,11 @@ module Gitlab
         end
       end
     end
+
+    def queue_length(queue_name)
+      Sidekiq.redis do |conn|
+        conn.llen("queue:#{queue_name}")
+      end
+    end
   end
 end
diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake
index 10492e183c5..dc472305304 100644
--- a/lib/tasks/gitlab/sidekiq.rake
+++ b/lib/tasks/gitlab/sidekiq.rake
@@ -10,15 +10,22 @@ namespace :gitlab do
       desc 'GitLab | Sidekiq | Migrate jobs in the scheduled set to new queue names'
       task schedule: :environment do
         ::Gitlab::SidekiqMigrateJobs
-          .new('schedule', logger: Logger.new($stdout))
-          .execute(::Gitlab::SidekiqConfig.worker_queue_mappings)
+          .new(::Gitlab::SidekiqConfig.worker_queue_mappings, logger: Logger.new($stdout) )
+          .migrate_set('schedule')
       end
 
       desc 'GitLab | Sidekiq | Migrate jobs in the retry set to new queue names'
       task retry: :environment do
         ::Gitlab::SidekiqMigrateJobs
-          .new('retry', logger: Logger.new($stdout))
-          .execute(::Gitlab::SidekiqConfig.worker_queue_mappings)
+          .new(::Gitlab::SidekiqConfig.worker_queue_mappings, logger: Logger.new($stdout) )
+          .migrate_set('retry')
+      end
+
+      desc 'GitLab | Sidekiq | Migrate jobs in queues outside of routing rules'
+      task queued: :environment do
+        ::Gitlab::SidekiqMigrateJobs
+          .new(::Gitlab::SidekiqConfig.worker_queue_mappings, logger: Logger.new($stdout) )
+          .migrate_queues
       end
     end