Merge remote-tracking branch 'origin/master' into 40781-os-to-ce

16 files changed, 66 insertions, 23 deletions

diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 9b1656be966..cf3735460b2 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -24,6 +24,7 @@
 - gcp_cluster:cluster_wait_for_app_installation
 - gcp_cluster:wait_for_cluster_creation
 - gcp_cluster:check_gcp_project_billing
+- gcp_cluster:cluster_wait_for_ingress_ip_address
 
 - github_import_advance_stage
 - github_importer:github_import_import_diff_note
@@ -46,12 +47,11 @@
 - pipeline_cache:expire_pipeline_cache
 - pipeline_creation:create_pipeline
 - pipeline_creation:run_pipeline_schedule
+- pipeline_background:archive_trace
 - pipeline_default:build_coverage
 - pipeline_default:build_trace_sections
-- pipeline_default:create_trace_artifact
 - pipeline_default:pipeline_metrics
 - pipeline_default:pipeline_notification
-- pipeline_default:update_head_pipeline_for_merge_request
 - pipeline_hooks:build_hooks
 - pipeline_hooks:pipeline_hooks
 - pipeline_processing:build_finished
@@ -61,6 +61,7 @@
 - pipeline_processing:pipeline_success
 - pipeline_processing:pipeline_update
 - pipeline_processing:stage_update
+- pipeline_processing:update_head_pipeline_for_merge_request
 
 - repository_check:repository_check_clear
 - repository_check:repository_check_single_repository
@@ -88,6 +89,7 @@
 - new_note
 - pages
 - pages_domain_verification
+- plugin
 - post_receive
 - process_commit
 - project_cache
diff --git a/app/workers/archive_trace_worker.rb b/app/workers/archive_trace_worker.rb
new file mode 100644
index 00000000000..dea7425ad88
--- /dev/null
+++ b/app/workers/archive_trace_worker.rb
@@ -0,0 +1,10 @@
+class ArchiveTraceWorker
+  include ApplicationWorker
+  include PipelineBackgroundQueue
+
+  def perform(job_id)
+    Ci::Build.find_by(id: job_id).try do |job|
+      job.trace.archive!
+    end
+  end
+end
diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb
index b5ed8d607b3..46f1ac09915 100644
--- a/app/workers/build_finished_worker.rb
+++ b/app/workers/build_finished_worker.rb
@@ -12,7 +12,7 @@ class BuildFinishedWorker
 
       # We execute that async as this are two indepentent operations that can be executed after TraceSections and Coverage
       BuildHooksWorker.perform_async(build.id)
-      CreateTraceArtifactWorker.perform_async(build.id)
+      ArchiveTraceWorker.perform_async(build.id)
     end
   end
 end
diff --git a/app/workers/cluster_wait_for_ingress_ip_address_worker.rb b/app/workers/cluster_wait_for_ingress_ip_address_worker.rb
new file mode 100644
index 00000000000..8ba5951750c
--- /dev/null
+++ b/app/workers/cluster_wait_for_ingress_ip_address_worker.rb
@@ -0,0 +1,11 @@
+class ClusterWaitForIngressIpAddressWorker
+  include ApplicationWorker
+  include ClusterQueue
+  include ClusterApplications
+
+  def perform(app_name, app_id)
+    find_application(app_name, app_id) do |app|
+      Clusters::Applications::CheckIngressIpAddressService.new(app).execute
+    end
+  end
+end
diff --git a/app/workers/concerns/gitlab/github_import/object_importer.rb b/app/workers/concerns/gitlab/github_import/object_importer.rb
index 9a9fbaad653..100d86e38c8 100644
--- a/app/workers/concerns/gitlab/github_import/object_importer.rb
+++ b/app/workers/concerns/gitlab/github_import/object_importer.rb
@@ -22,7 +22,7 @@ module Gitlab
 
         importer_class.new(object, project, client).execute
 
-        counter.increment(project: project.path_with_namespace)
+        counter.increment(project: project.full_path)
       end
 
       def counter
diff --git a/app/workers/concerns/pipeline_background_queue.rb b/app/workers/concerns/pipeline_background_queue.rb
new file mode 100644
index 00000000000..8bf43de6b26
--- /dev/null
+++ b/app/workers/concerns/pipeline_background_queue.rb
@@ -0,0 +1,10 @@
+##
+# Concern for setting Sidekiq settings for the low priority CI pipeline workers.
+#
+module PipelineBackgroundQueue
+  extend ActiveSupport::Concern
+
+  included do
+    queue_namespace :pipeline_background
+  end
+end
diff --git a/app/workers/create_trace_artifact_worker.rb b/app/workers/create_trace_artifact_worker.rb
deleted file mode 100644
index 11cda58021e..00000000000
--- a/app/workers/create_trace_artifact_worker.rb
+++ /dev/null
@@ -1,10 +0,0 @@
-class CreateTraceArtifactWorker
-  include ApplicationWorker
-  include PipelineQueue
-
-  def perform(job_id)
-    Ci::Build.preload(:project, :user).find_by(id: job_id).try do |job|
-      Ci::CreateTraceArtifactService.new(job.project, job.user).execute(job)
-    end
-  end
-end
diff --git a/app/workers/emails_on_push_worker.rb b/app/workers/emails_on_push_worker.rb
index 21da27973fe..2a4d65b5cb3 100644
--- a/app/workers/emails_on_push_worker.rb
+++ b/app/workers/emails_on_push_worker.rb
@@ -66,7 +66,7 @@ class EmailsOnPushWorker
 
       # These are input errors and won't be corrected even if Sidekiq retries
       rescue Net::SMTPFatalError, Net::SMTPSyntaxError => e
-        logger.info("Failed to send e-mail for project '#{project.name_with_namespace}' to #{recipient}: #{e}")
+        logger.info("Failed to send e-mail for project '#{project.full_name}' to #{recipient}: #{e}")
       end
     end
   ensure
diff --git a/app/workers/git_garbage_collect_worker.rb b/app/workers/git_garbage_collect_worker.rb
index 7ba224d74c8..55fb817ca6e 100644
--- a/app/workers/git_garbage_collect_worker.rb
+++ b/app/workers/git_garbage_collect_worker.rb
@@ -44,6 +44,10 @@ class GitGarbageCollectWorker
 
     # Refresh the branch cache in case garbage collection caused a ref lookup to fail
     flush_ref_caches(project) if task == :gc
+
+    # In case pack files are deleted, release libgit2 cache and open file
+    # descriptors ASAP instead of waiting for Ruby garbage collection
+    project.cleanup
   ensure
     cancel_lease(lease_key, lease_uuid) if lease_key.present? && lease_uuid.present?
   end
diff --git a/app/workers/gitlab/github_import/stage/finish_import_worker.rb b/app/workers/gitlab/github_import/stage/finish_import_worker.rb
index 073d6608082..a779e631516 100644
--- a/app/workers/gitlab/github_import/stage/finish_import_worker.rb
+++ b/app/workers/gitlab/github_import/stage/finish_import_worker.rb
@@ -16,7 +16,7 @@ module Gitlab
 
         def report_import_time(project)
           duration = Time.zone.now - project.created_at
-          path = project.path_with_namespace
+          path = project.full_path
 
           histogram.observe({ project: path }, duration)
           counter.increment
diff --git a/app/workers/pages_worker.rb b/app/workers/pages_worker.rb
index d3b95009364..66a0ff83bef 100644
--- a/app/workers/pages_worker.rb
+++ b/app/workers/pages_worker.rb
@@ -1,7 +1,7 @@
 class PagesWorker
   include ApplicationWorker
 
-  sidekiq_options retry: false
+  sidekiq_options retry: 3
 
   def perform(action, *arg)
     send(action, *arg) # rubocop:disable GitlabSecurity/PublicSend
diff --git a/app/workers/plugin_worker.rb b/app/workers/plugin_worker.rb
new file mode 100644
index 00000000000..bfcc683d99a
--- /dev/null
+++ b/app/workers/plugin_worker.rb
@@ -0,0 +1,15 @@
+class PluginWorker
+  include ApplicationWorker
+
+  sidekiq_options retry: false
+
+  def perform(file_name, data)
+    success, message = Gitlab::Plugin.execute(file_name, data)
+
+    unless success
+      Gitlab::PluginLogger.error("Plugin Error => #{file_name}: #{message}")
+    end
+
+    true
+  end
+end
diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb
index f2b2c4428d3..3909dbf7d7f 100644
--- a/app/workers/post_receive.rb
+++ b/app/workers/post_receive.rb
@@ -55,7 +55,7 @@ class PostReceive
   end
 
   def process_wiki_changes(post_received)
-    # Nothing defined here yet.
+    post_received.project.touch(:last_activity_at, :last_repository_updated_at)
   end
 
   def log(message)
diff --git a/app/workers/process_commit_worker.rb b/app/workers/process_commit_worker.rb
index 5b25d980bdb..201e7f332b4 100644
--- a/app/workers/process_commit_worker.rb
+++ b/app/workers/process_commit_worker.rb
@@ -30,10 +30,9 @@ class ProcessCommitWorker
   end
 
   def process_commit_message(project, commit, user, author, default = false)
-    # this is a GitLab generated commit message, ignore it.
-    return if commit.merged_merge_request?(user)
-
-    closed_issues = default ? commit.closes_issues(user) : []
+    # Ignore closing references from GitLab-generated commit messages.
+    find_closing_issues = default && !commit.merged_merge_request?(user)
+    closed_issues = find_closing_issues ? commit.closes_issues(user) : []
 
     close_issues(project, user, author, commit, closed_issues) if closed_issues.any?
     commit.create_cross_references!(author, closed_issues)
diff --git a/app/workers/remove_expired_members_worker.rb b/app/workers/remove_expired_members_worker.rb
index d80b3b15840..68960f72bf6 100644
--- a/app/workers/remove_expired_members_worker.rb
+++ b/app/workers/remove_expired_members_worker.rb
@@ -5,7 +5,7 @@ class RemoveExpiredMembersWorker
   def perform
     Member.expired.find_each do |member|
       begin
-        Members::AuthorizedDestroyService.new(member).execute
+        Members::DestroyService.new.execute(member, skip_authorization: true)
       rescue => ex
         logger.error("Expired Member ID=#{member.id} cannot be removed - #{ex}")
       end
diff --git a/app/workers/update_head_pipeline_for_merge_request_worker.rb b/app/workers/update_head_pipeline_for_merge_request_worker.rb
index f09d89aa170..76f84ff920f 100644
--- a/app/workers/update_head_pipeline_for_merge_request_worker.rb
+++ b/app/workers/update_head_pipeline_for_merge_request_worker.rb
@@ -2,6 +2,8 @@ class UpdateHeadPipelineForMergeRequestWorker
   include ApplicationWorker
   include PipelineQueue
 
+  queue_namespace :pipeline_processing
+
   def perform(merge_request_id)
     merge_request = MergeRequest.find(merge_request_id)
     pipeline = Ci::Pipeline.where(project: merge_request.source_project, ref: merge_request.source_branch).last