From 264bf229277caf1c1eaca4e83921ca1b305d5401 Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Wed, 3 May 2017 17:20:12 +0200
Subject: add propagate service worker and updated spec and controller

---
 app/controllers/admin/services_controller.rb    |  2 +-
 app/models/service.rb                           | 10 +++++
 app/workers/propagate_project_service_worker.rb | 49 +++++++++++++++++++++++++
 3 files changed, 60 insertions(+), 1 deletion(-)
 create mode 100644 app/workers/propagate_project_service_worker.rb

(limited to 'app')

diff --git a/app/controllers/admin/services_controller.rb b/app/controllers/admin/services_controller.rb
index 37a1a23178e..2b6f335cb2b 100644
--- a/app/controllers/admin/services_controller.rb
+++ b/app/controllers/admin/services_controller.rb
@@ -15,7 +15,7 @@ class Admin::ServicesController < Admin::ApplicationController
   end
 
   def update
-    if service.update_attributes(service_params[:service])
+    if service.update_and_propagate(service_params[:service])
       redirect_to admin_application_settings_services_path,
         notice: 'Application settings saved successfully'
     else
diff --git a/app/models/service.rb b/app/models/service.rb
index c71a7d169ec..dea22fd96a7 100644
--- a/app/models/service.rb
+++ b/app/models/service.rb
@@ -254,6 +254,16 @@ class Service < ActiveRecord::Base
     service
   end
 
+  def update_and_propagate(service_params)
+    return false unless update_attributes(service_params)
+
+    if service_params[:active] == 1
+      PropagateProjectServiceWorker.perform_async(service_params[:id])
+    end
+
+    true
+  end
+
   private
 
   def cache_project_has_external_issue_tracker
diff --git a/app/workers/propagate_project_service_worker.rb b/app/workers/propagate_project_service_worker.rb
new file mode 100644
index 00000000000..53551770968
--- /dev/null
+++ b/app/workers/propagate_project_service_worker.rb
@@ -0,0 +1,49 @@
+# Worker for updating any project specific caches.
+class PropagateProjectServiceWorker
+  include Sidekiq::Worker
+  include DedicatedSidekiqQueue
+
+  LEASE_TIMEOUT = 30.minutes.to_i
+
+  def perform(template_id)
+    template = Service.find_by(id: template_id)
+
+    return unless template&.active
+    return unless try_obtain_lease_for(template.id)
+
+    Rails.logger.info("Propagating services for template #{template.id}")
+
+    project_ids_for_template(template) do |project_id|
+      Service.build_from_template(project_id, template).save!
+    end
+  end
+
+  private
+
+  def project_ids_for_template(template)
+    limit = 100
+    offset = 0
+
+    loop do
+      batch = project_ids_batch(limit, offset, template.type)
+
+      batch.each { |project_id| yield(project_id) }
+
+      break if batch.count < limit
+
+      offset += limit
+    end
+  end
+
+  def project_ids_batch(limit, offset, template_type)
+    Project.joins('LEFT JOIN services ON services.project_id = projects.id').
+      where('services.type != ? OR services.id IS NULL', template_type).
+      limit(limit).offset(offset).pluck(:id)
+  end
+
+  def try_obtain_lease_for(template_id)
+    Gitlab::ExclusiveLease.
+      new("propagate_project_service_worker:#{template_id}", timeout: LEASE_TIMEOUT).
+      try_obtain
+  end
+end
-- 
cgit v1.2.3


From f81cf84035213002ce7931af6c3ffa917fe7fcbd Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Thu, 4 May 2017 12:13:33 +0200
Subject: refactor worker into service

---
 app/services/projects/propagate_service.rb      | 47 +++++++++++++++++++++++++
 app/workers/propagate_project_service_worker.rb | 34 +++---------------
 2 files changed, 51 insertions(+), 30 deletions(-)
 create mode 100644 app/services/projects/propagate_service.rb

(limited to 'app')

diff --git a/app/services/projects/propagate_service.rb b/app/services/projects/propagate_service.rb
new file mode 100644
index 00000000000..3c05dcce07c
--- /dev/null
+++ b/app/services/projects/propagate_service.rb
@@ -0,0 +1,47 @@
+module Projects
+  class PropagateService
+    BATCH_SIZE = 100
+
+    def self.propagate!(*args)
+      new(*args).propagate!
+    end
+
+    def initialize(template)
+      @template = template
+    end
+
+    def propagate!
+      return unless @template&.active
+
+      Rails.logger.info("Propagating services for template #{@template.id}")
+
+      propagate_projects_with_template
+    end
+
+    private
+
+    def propagate_projects_with_template
+      offset = 0
+
+      loop do
+        batch = project_ids_batch(offset)
+
+        batch.each { |project_id| create_from_template(project_id) }
+
+        break if batch.count < BATCH_SIZE
+
+        offset += BATCH_SIZE
+      end
+    end
+
+    def create_from_template(project_id)
+      Service.build_from_template(project_id, @template).save!
+    end
+
+    def project_ids_batch(offset)
+      Project.joins('LEFT JOIN services ON services.project_id = projects.id').
+        where('services.type != ? OR services.id IS NULL', @template.type).
+        limit(BATCH_SIZE).offset(offset).pluck(:id)
+    end
+  end
+end
diff --git a/app/workers/propagate_project_service_worker.rb b/app/workers/propagate_project_service_worker.rb
index 53551770968..ab2b7738f9a 100644
--- a/app/workers/propagate_project_service_worker.rb
+++ b/app/workers/propagate_project_service_worker.rb
@@ -3,44 +3,18 @@ class PropagateProjectServiceWorker
   include Sidekiq::Worker
   include DedicatedSidekiqQueue
 
+  sidekiq_options retry: 3
+
   LEASE_TIMEOUT = 30.minutes.to_i
 
   def perform(template_id)
-    template = Service.find_by(id: template_id)
-
-    return unless template&.active
-    return unless try_obtain_lease_for(template.id)
-
-    Rails.logger.info("Propagating services for template #{template.id}")
+    return unless try_obtain_lease_for(template_id)
 
-    project_ids_for_template(template) do |project_id|
-      Service.build_from_template(project_id, template).save!
-    end
+    Projects::PropagateService.propagate!(Service.find_by(id: template_id))
   end
 
   private
 
-  def project_ids_for_template(template)
-    limit = 100
-    offset = 0
-
-    loop do
-      batch = project_ids_batch(limit, offset, template.type)
-
-      batch.each { |project_id| yield(project_id) }
-
-      break if batch.count < limit
-
-      offset += limit
-    end
-  end
-
-  def project_ids_batch(limit, offset, template_type)
-    Project.joins('LEFT JOIN services ON services.project_id = projects.id').
-      where('services.type != ? OR services.id IS NULL', template_type).
-      limit(limit).offset(offset).pluck(:id)
-  end
-
   def try_obtain_lease_for(template_id)
     Gitlab::ExclusiveLease.
       new("propagate_project_service_worker:#{template_id}", timeout: LEASE_TIMEOUT).
-- 
cgit v1.2.3


From 3d807dc81b27c6366390b2355e40a5c65bbf02c2 Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Thu, 4 May 2017 12:21:35 +0200
Subject: update lease timeout

---
 app/workers/propagate_project_service_worker.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app')

diff --git a/app/workers/propagate_project_service_worker.rb b/app/workers/propagate_project_service_worker.rb
index ab2b7738f9a..6cc3fe84635 100644
--- a/app/workers/propagate_project_service_worker.rb
+++ b/app/workers/propagate_project_service_worker.rb
@@ -5,7 +5,7 @@ class PropagateProjectServiceWorker
 
   sidekiq_options retry: 3
 
-  LEASE_TIMEOUT = 30.minutes.to_i
+  LEASE_TIMEOUT = 4.hours.to_i
 
   def perform(template_id)
     return unless try_obtain_lease_for(template_id)
-- 
cgit v1.2.3


From 3bff8da8c1e3223e81bccd5343902b840f005fcf Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Thu, 4 May 2017 12:37:54 +0200
Subject: fix service spec

---
 app/models/service.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app')

diff --git a/app/models/service.rb b/app/models/service.rb
index dea22fd96a7..18c046aff54 100644
--- a/app/models/service.rb
+++ b/app/models/service.rb
@@ -257,7 +257,7 @@ class Service < ActiveRecord::Base
   def update_and_propagate(service_params)
     return false unless update_attributes(service_params)
 
-    if service_params[:active] == 1
+    if service_params[:active]
       PropagateProjectServiceWorker.perform_async(service_params[:id])
     end
 
-- 
cgit v1.2.3


From b871564383cbade7fff312b8f045cee6c871f1e0 Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Thu, 4 May 2017 12:46:03 +0200
Subject: fix service spec

---
 app/models/service.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app')

diff --git a/app/models/service.rb b/app/models/service.rb
index 18c046aff54..f8534387703 100644
--- a/app/models/service.rb
+++ b/app/models/service.rb
@@ -258,7 +258,7 @@ class Service < ActiveRecord::Base
     return false unless update_attributes(service_params)
 
     if service_params[:active]
-      PropagateProjectServiceWorker.perform_async(service_params[:id])
+      PropagateProjectServiceWorker.perform_async(id)
     end
 
     true
-- 
cgit v1.2.3


From cf002738e766f977bdb0e857759f548a5c65c9bd Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Thu, 4 May 2017 18:11:28 +0200
Subject: refactor a few things based on feedback

---
 app/controllers/admin/services_controller.rb    |  4 +++-
 app/models/service.rb                           | 10 ----------
 app/services/projects/propagate_service.rb      |  8 ++++----
 app/workers/propagate_project_service_worker.rb |  2 +-
 4 files changed, 8 insertions(+), 16 deletions(-)

(limited to 'app')

diff --git a/app/controllers/admin/services_controller.rb b/app/controllers/admin/services_controller.rb
index 2b6f335cb2b..e335fbfffed 100644
--- a/app/controllers/admin/services_controller.rb
+++ b/app/controllers/admin/services_controller.rb
@@ -15,7 +15,9 @@ class Admin::ServicesController < Admin::ApplicationController
   end
 
   def update
-    if service.update_and_propagate(service_params[:service])
+    if service.update_attributes(service_params[:service])
+      PropagateProjectServiceWorker.perform_async(service.id) if  service.active?
+
       redirect_to admin_application_settings_services_path,
         notice: 'Application settings saved successfully'
     else
diff --git a/app/models/service.rb b/app/models/service.rb
index f8534387703..c71a7d169ec 100644
--- a/app/models/service.rb
+++ b/app/models/service.rb
@@ -254,16 +254,6 @@ class Service < ActiveRecord::Base
     service
   end
 
-  def update_and_propagate(service_params)
-    return false unless update_attributes(service_params)
-
-    if service_params[:active]
-      PropagateProjectServiceWorker.perform_async(id)
-    end
-
-    true
-  end
-
   private
 
   def cache_project_has_external_issue_tracker
diff --git a/app/services/projects/propagate_service.rb b/app/services/projects/propagate_service.rb
index 3c05dcce07c..6e24a67d8b0 100644
--- a/app/services/projects/propagate_service.rb
+++ b/app/services/projects/propagate_service.rb
@@ -2,15 +2,15 @@ module Projects
   class PropagateService
     BATCH_SIZE = 100
 
-    def self.propagate!(*args)
-      new(*args).propagate!
+    def self.propagate(*args)
+      new(*args).propagate
     end
 
     def initialize(template)
       @template = template
     end
 
-    def propagate!
+    def propagate
       return unless @template&.active
 
       Rails.logger.info("Propagating services for template #{@template.id}")
@@ -28,7 +28,7 @@ module Projects
 
         batch.each { |project_id| create_from_template(project_id) }
 
-        break if batch.count < BATCH_SIZE
+        break if batch.size < BATCH_SIZE
 
         offset += BATCH_SIZE
       end
diff --git a/app/workers/propagate_project_service_worker.rb b/app/workers/propagate_project_service_worker.rb
index 6cc3fe84635..5eabe4eaecd 100644
--- a/app/workers/propagate_project_service_worker.rb
+++ b/app/workers/propagate_project_service_worker.rb
@@ -10,7 +10,7 @@ class PropagateProjectServiceWorker
   def perform(template_id)
     return unless try_obtain_lease_for(template_id)
 
-    Projects::PropagateService.propagate!(Service.find_by(id: template_id))
+    Projects::PropagateService.propagate(Service.find_by(id: template_id))
   end
 
   private
-- 
cgit v1.2.3


From 1fe8b7f646603239f530b1a18427f4f5bc0e2060 Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Fri, 5 May 2017 09:40:44 +0200
Subject: refactor propagate service to use batch inserts and subquery instead
 of left join

---
 app/services/projects/propagate_service.rb | 32 ++++++++++++++++++++++++------
 1 file changed, 26 insertions(+), 6 deletions(-)

(limited to 'app')

diff --git a/app/services/projects/propagate_service.rb b/app/services/projects/propagate_service.rb
index 6e24a67d8b0..b067fc2cd64 100644
--- a/app/services/projects/propagate_service.rb
+++ b/app/services/projects/propagate_service.rb
@@ -26,7 +26,7 @@ module Projects
       loop do
         batch = project_ids_batch(offset)
 
-        batch.each { |project_id| create_from_template(project_id) }
+        bulk_create_from_template(batch)
 
         break if batch.size < BATCH_SIZE
 
@@ -34,14 +34,34 @@ module Projects
       end
     end
 
-    def create_from_template(project_id)
-      Service.build_from_template(project_id, @template).save!
+    def bulk_create_from_template(batch)
+      service_hash_list = batch.map do |project_id|
+        service_hash.merge('project_id' => project_id)
+      end
+
+      Project.transaction do
+        Service.create!(service_hash_list)
+      end
     end
 
     def project_ids_batch(offset)
-      Project.joins('LEFT JOIN services ON services.project_id = projects.id').
-        where('services.type != ? OR services.id IS NULL', @template.type).
-        limit(BATCH_SIZE).offset(offset).pluck(:id)
+      Project.connection.execute(
+        <<-SQL
+          SELECT id
+          FROM projects
+          WHERE NOT EXISTS (
+            SELECT true
+            FROM services
+            WHERE services.project_id = projects.id
+            AND services.type = '#{@template.type}'
+          )
+          LIMIT #{BATCH_SIZE} OFFSET #{offset}
+      SQL
+      ).to_a.flatten
+    end
+
+    def service_hash
+      @service_hash ||= @template.as_json(methods: :type).except('id', 'template')
     end
   end
 end
-- 
cgit v1.2.3


From adcff298f8f3041faa29b75ee3711fb4ce1cbb69 Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Fri, 5 May 2017 10:43:56 +0200
Subject: fixed all issues - not doing bulk create.

---
 app/services/projects/propagate_service.rb | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

(limited to 'app')

diff --git a/app/services/projects/propagate_service.rb b/app/services/projects/propagate_service.rb
index b067fc2cd64..716a6209537 100644
--- a/app/services/projects/propagate_service.rb
+++ b/app/services/projects/propagate_service.rb
@@ -21,16 +21,12 @@ module Projects
     private
 
     def propagate_projects_with_template
-      offset = 0
-
       loop do
-        batch = project_ids_batch(offset)
+        batch = project_ids_batch
 
         bulk_create_from_template(batch)
 
         break if batch.size < BATCH_SIZE
-
-        offset += BATCH_SIZE
       end
     end
 
@@ -44,7 +40,7 @@ module Projects
       end
     end
 
-    def project_ids_batch(offset)
+    def project_ids_batch
       Project.connection.execute(
         <<-SQL
           SELECT id
@@ -55,7 +51,7 @@ module Projects
             WHERE services.project_id = projects.id
             AND services.type = '#{@template.type}'
           )
-          LIMIT #{BATCH_SIZE} OFFSET #{offset}
+          LIMIT #{BATCH_SIZE}
       SQL
       ).to_a.flatten
     end
-- 
cgit v1.2.3


From 9ec39568c5284f5a3a17a342d12f87befb6cfb4c Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Fri, 5 May 2017 10:51:25 +0200
Subject: use select_values

---
 app/services/projects/propagate_service.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'app')

diff --git a/app/services/projects/propagate_service.rb b/app/services/projects/propagate_service.rb
index 716a6209537..f952b5108bb 100644
--- a/app/services/projects/propagate_service.rb
+++ b/app/services/projects/propagate_service.rb
@@ -41,7 +41,7 @@ module Projects
     end
 
     def project_ids_batch
-      Project.connection.execute(
+      Project.connection.select_values(
         <<-SQL
           SELECT id
           FROM projects
@@ -53,7 +53,7 @@ module Projects
           )
           LIMIT #{BATCH_SIZE}
       SQL
-      ).to_a.flatten
+      )
     end
 
     def service_hash
-- 
cgit v1.2.3


From 606584c115d5f7a22f3b5c7e0ac6803b96fe999e Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Fri, 5 May 2017 14:43:22 +0200
Subject: bulk insert FTW - This would introduce more complexity, but should be
 faster

---
 app/services/projects/propagate_service.rb | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

(limited to 'app')

diff --git a/app/services/projects/propagate_service.rb b/app/services/projects/propagate_service.rb
index f952b5108bb..c420f24fe02 100644
--- a/app/services/projects/propagate_service.rb
+++ b/app/services/projects/propagate_service.rb
@@ -24,20 +24,23 @@ module Projects
       loop do
         batch = project_ids_batch
 
-        bulk_create_from_template(batch)
+        bulk_create_from_template(batch) unless batch.empty?
 
         break if batch.size < BATCH_SIZE
       end
     end
 
     def bulk_create_from_template(batch)
-      service_hash_list = batch.map do |project_id|
-        service_hash.merge('project_id' => project_id)
+      service_list = batch.map do |project_id|
+        service_hash.merge('project_id' => project_id).values
       end
 
-      Project.transaction do
-        Service.create!(service_hash_list)
-      end
+      # Project.transaction do
+      #   Service.create!(service_hash_list)
+      # end
+      Gitlab::SQL::BulkInsert.new(service_hash.keys + ['project_id'],
+                                  service_list,
+                                  'services').execute
     end
 
     def project_ids_batch
@@ -57,7 +60,17 @@ module Projects
     end
 
     def service_hash
-      @service_hash ||= @template.as_json(methods: :type).except('id', 'template')
+      @service_hash ||=
+        begin
+          template_hash = @template.as_json(methods: :type).except('id', 'template', 'project_id')
+
+          template_hash.each_with_object({}) do |(key, value), service_hash|
+            value = value.is_a?(Hash) ? value.to_json : value
+            key = Gitlab::Database.postgresql? ? "\"#{key}\"" : "`#{key}`"
+
+            service_hash[key] = ActiveRecord::Base.sanitize(value)
+          end
+        end
     end
   end
 end
-- 
cgit v1.2.3


From ce418036c763219df8239632f71ef0e9782be7ea Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Fri, 5 May 2017 16:16:02 +0200
Subject: add callbacks in bulk

---
 app/services/projects/propagate_service.rb | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

(limited to 'app')

diff --git a/app/services/projects/propagate_service.rb b/app/services/projects/propagate_service.rb
index c420f24fe02..f4fae478609 100644
--- a/app/services/projects/propagate_service.rb
+++ b/app/services/projects/propagate_service.rb
@@ -35,12 +35,12 @@ module Projects
         service_hash.merge('project_id' => project_id).values
       end
 
-      # Project.transaction do
-      #   Service.create!(service_hash_list)
-      # end
-      Gitlab::SQL::BulkInsert.new(service_hash.keys + ['project_id'],
-                                  service_list,
-                                  'services').execute
+      Project.transaction do
+        Gitlab::SQL::BulkInsert.new(service_hash.keys + ['project_id'],
+                                    service_list,
+                                    'services').execute
+        run_callbacks(batch)
+      end
     end
 
     def project_ids_batch
@@ -72,5 +72,23 @@ module Projects
           end
         end
     end
+
+    def run_callbacks(batch)
+      if active_external_issue_tracker?
+        Project.where(id: batch).update_all(has_external_issue_tracker: true)
+      end
+
+      if active_external_wiki?
+        Project.where(id: batch).update_all(has_external_wiki: true)
+      end
+    end
+
+    def active_external_issue_tracker?
+      @template['category'] == 'issue_tracker' && @template['active'] && !@template['default']
+    end
+
+    def active_external_wiki?
+      @template['type'] == 'ExternalWikiService' && @template['active']
+    end
   end
 end
-- 
cgit v1.2.3


From 6ecf16b8f70335e1e6868b7c918cd031f5eb2f8d Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Fri, 5 May 2017 18:01:33 +0200
Subject: refactor code based on feedback

---
 app/controllers/admin/services_controller.rb       |   2 +-
 app/services/projects/propagate_service.rb         |  94 -------------------
 .../projects/propagate_service_template.rb         | 103 +++++++++++++++++++++
 app/workers/propagate_project_service_worker.rb    |  23 -----
 app/workers/propagate_service_template_worker.rb   |  23 +++++
 5 files changed, 127 insertions(+), 118 deletions(-)
 delete mode 100644 app/services/projects/propagate_service.rb
 create mode 100644 app/services/projects/propagate_service_template.rb
 delete mode 100644 app/workers/propagate_project_service_worker.rb
 create mode 100644 app/workers/propagate_service_template_worker.rb

(limited to 'app')

diff --git a/app/controllers/admin/services_controller.rb b/app/controllers/admin/services_controller.rb
index e335fbfffed..a40ce3c2418 100644
--- a/app/controllers/admin/services_controller.rb
+++ b/app/controllers/admin/services_controller.rb
@@ -16,7 +16,7 @@ class Admin::ServicesController < Admin::ApplicationController
 
   def update
     if service.update_attributes(service_params[:service])
-      PropagateProjectServiceWorker.perform_async(service.id) if  service.active?
+      PropagateServiceTemplateWorker.perform_async(service.id) if  service.active?
 
       redirect_to admin_application_settings_services_path,
         notice: 'Application settings saved successfully'
diff --git a/app/services/projects/propagate_service.rb b/app/services/projects/propagate_service.rb
deleted file mode 100644
index f4fae478609..00000000000
--- a/app/services/projects/propagate_service.rb
+++ /dev/null
@@ -1,94 +0,0 @@
-module Projects
-  class PropagateService
-    BATCH_SIZE = 100
-
-    def self.propagate(*args)
-      new(*args).propagate
-    end
-
-    def initialize(template)
-      @template = template
-    end
-
-    def propagate
-      return unless @template&.active
-
-      Rails.logger.info("Propagating services for template #{@template.id}")
-
-      propagate_projects_with_template
-    end
-
-    private
-
-    def propagate_projects_with_template
-      loop do
-        batch = project_ids_batch
-
-        bulk_create_from_template(batch) unless batch.empty?
-
-        break if batch.size < BATCH_SIZE
-      end
-    end
-
-    def bulk_create_from_template(batch)
-      service_list = batch.map do |project_id|
-        service_hash.merge('project_id' => project_id).values
-      end
-
-      Project.transaction do
-        Gitlab::SQL::BulkInsert.new(service_hash.keys + ['project_id'],
-                                    service_list,
-                                    'services').execute
-        run_callbacks(batch)
-      end
-    end
-
-    def project_ids_batch
-      Project.connection.select_values(
-        <<-SQL
-          SELECT id
-          FROM projects
-          WHERE NOT EXISTS (
-            SELECT true
-            FROM services
-            WHERE services.project_id = projects.id
-            AND services.type = '#{@template.type}'
-          )
-          LIMIT #{BATCH_SIZE}
-      SQL
-      )
-    end
-
-    def service_hash
-      @service_hash ||=
-        begin
-          template_hash = @template.as_json(methods: :type).except('id', 'template', 'project_id')
-
-          template_hash.each_with_object({}) do |(key, value), service_hash|
-            value = value.is_a?(Hash) ? value.to_json : value
-            key = Gitlab::Database.postgresql? ? "\"#{key}\"" : "`#{key}`"
-
-            service_hash[key] = ActiveRecord::Base.sanitize(value)
-          end
-        end
-    end
-
-    def run_callbacks(batch)
-      if active_external_issue_tracker?
-        Project.where(id: batch).update_all(has_external_issue_tracker: true)
-      end
-
-      if active_external_wiki?
-        Project.where(id: batch).update_all(has_external_wiki: true)
-      end
-    end
-
-    def active_external_issue_tracker?
-      @template['category'] == 'issue_tracker' && @template['active'] && !@template['default']
-    end
-
-    def active_external_wiki?
-      @template['type'] == 'ExternalWikiService' && @template['active']
-    end
-  end
-end
diff --git a/app/services/projects/propagate_service_template.rb b/app/services/projects/propagate_service_template.rb
new file mode 100644
index 00000000000..32ad68673ac
--- /dev/null
+++ b/app/services/projects/propagate_service_template.rb
@@ -0,0 +1,103 @@
+module Projects
+  class PropagateServiceTemplate
+    BATCH_SIZE = 100
+
+    def self.propagate(*args)
+      new(*args).propagate
+    end
+
+    def initialize(template)
+      @template = template
+    end
+
+    def propagate
+      return unless @template&.active
+
+      Rails.logger.info("Propagating services for template #{@template.id}")
+
+      propagate_projects_with_template
+    end
+
+    private
+
+    def propagate_projects_with_template
+      loop do
+        batch = project_ids_batch
+
+        bulk_create_from_template(batch) unless batch.empty?
+
+        break if batch.size < BATCH_SIZE
+      end
+    end
+
+    def bulk_create_from_template(batch)
+      service_list = batch.map do |project_id|
+        service_hash.merge('project_id' => project_id).values
+      end
+
+      Project.transaction do
+        bulk_insert_services(service_hash.keys + ['project_id'], service_list)
+        run_callbacks(batch)
+      end
+    end
+
+    def project_ids_batch
+      Project.connection.select_values(
+        <<-SQL
+          SELECT id
+          FROM projects
+          WHERE NOT EXISTS (
+            SELECT true
+            FROM services
+            WHERE services.project_id = projects.id
+            AND services.type = '#{@template.type}'
+          )
+          AND projects.pending_delete = false
+          AND projects.archived = false
+          LIMIT #{BATCH_SIZE}
+      SQL
+      )
+    end
+
+    def bulk_insert_services(columns, values_array)
+      ActiveRecord::Base.connection.execute(
+        <<-SQL.strip_heredoc
+          INSERT INTO services (#{columns.join(', ')})
+          VALUES #{values_array.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')}
+      SQL
+      )
+    end
+
+    def service_hash
+      @service_hash ||=
+        begin
+          template_hash = @template.as_json(methods: :type).except('id', 'template', 'project_id')
+
+          template_hash.each_with_object({}) do |(key, value), service_hash|
+            value = value.is_a?(Hash) ? value.to_json : value
+            key = Gitlab::Database.postgresql? ? "\"#{key}\"" : "`#{key}`"
+
+            service_hash[key] = ActiveRecord::Base.sanitize(value)
+          end
+        end
+    end
+
+    def run_callbacks(batch)
+      if active_external_issue_tracker?
+        Project.where(id: batch).update_all(has_external_issue_tracker: true)
+      end
+
+      if active_external_wiki?
+        Project.where(id: batch).update_all(has_external_wiki: true)
+      end
+    end
+
+    def active_external_issue_tracker?
+      @template.category == :issue_tracker && !@template.default
+    end
+
+    def active_external_wiki?
+      @template.type == 'ExternalWikiService'
+    end
+  end
+end
diff --git a/app/workers/propagate_project_service_worker.rb b/app/workers/propagate_project_service_worker.rb
deleted file mode 100644
index 5eabe4eaecd..00000000000
--- a/app/workers/propagate_project_service_worker.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-# Worker for updating any project specific caches.
-class PropagateProjectServiceWorker
-  include Sidekiq::Worker
-  include DedicatedSidekiqQueue
-
-  sidekiq_options retry: 3
-
-  LEASE_TIMEOUT = 4.hours.to_i
-
-  def perform(template_id)
-    return unless try_obtain_lease_for(template_id)
-
-    Projects::PropagateService.propagate(Service.find_by(id: template_id))
-  end
-
-  private
-
-  def try_obtain_lease_for(template_id)
-    Gitlab::ExclusiveLease.
-      new("propagate_project_service_worker:#{template_id}", timeout: LEASE_TIMEOUT).
-      try_obtain
-  end
-end
diff --git a/app/workers/propagate_service_template_worker.rb b/app/workers/propagate_service_template_worker.rb
new file mode 100644
index 00000000000..f1fc7ccb955
--- /dev/null
+++ b/app/workers/propagate_service_template_worker.rb
@@ -0,0 +1,23 @@
+# Worker for updating any project specific caches.
+class PropagateServiceTemplateWorker
+  include Sidekiq::Worker
+  include DedicatedSidekiqQueue
+
+  sidekiq_options retry: 3
+
+  LEASE_TIMEOUT = 4.hours.to_i
+
+  def perform(template_id)
+    return unless try_obtain_lease_for(template_id)
+
+    Projects::PropagateServiceTemplate.propagate(Service.find_by(id: template_id))
+  end
+
+  private
+
+  def try_obtain_lease_for(template_id)
+    Gitlab::ExclusiveLease.
+      new("propagate_service_template_worker:#{template_id}", timeout: LEASE_TIMEOUT).
+      try_obtain
+  end
+end
-- 
cgit v1.2.3


From f15466bd5bd2ce5390e392785d7c750c176acbec Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Fri, 5 May 2017 18:18:39 +0200
Subject: refactor code based on feedback

---
 app/controllers/admin/services_controller.rb        | 2 +-
 app/services/projects/propagate_service_template.rb | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'app')

diff --git a/app/controllers/admin/services_controller.rb b/app/controllers/admin/services_controller.rb
index a40ce3c2418..4c3d336b3af 100644
--- a/app/controllers/admin/services_controller.rb
+++ b/app/controllers/admin/services_controller.rb
@@ -16,7 +16,7 @@ class Admin::ServicesController < Admin::ApplicationController
 
   def update
     if service.update_attributes(service_params[:service])
-      PropagateServiceTemplateWorker.perform_async(service.id) if  service.active?
+      PropagateServiceTemplateWorker.perform_async(service.id) if service.active?
 
       redirect_to admin_application_settings_services_path,
         notice: 'Application settings saved successfully'
diff --git a/app/services/projects/propagate_service_template.rb b/app/services/projects/propagate_service_template.rb
index 32ad68673ac..2999e1af385 100644
--- a/app/services/projects/propagate_service_template.rb
+++ b/app/services/projects/propagate_service_template.rb
@@ -11,7 +11,7 @@ module Projects
     end
 
     def propagate
-      return unless @template&.active
+      return unless @template&.active?
 
       Rails.logger.info("Propagating services for template #{@template.id}")
 
-- 
cgit v1.2.3


From 856a511b4804a0b78294a29bbba86ac111d960f8 Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Fri, 5 May 2017 18:57:52 +0200
Subject: refactor code based on feedback

---
 app/services/projects/propagate_service_template.rb | 12 ++++++------
 app/workers/propagate_service_template_worker.rb    |  2 --
 2 files changed, 6 insertions(+), 8 deletions(-)

(limited to 'app')

diff --git a/app/services/projects/propagate_service_template.rb b/app/services/projects/propagate_service_template.rb
index 2999e1af385..a8ef2108492 100644
--- a/app/services/projects/propagate_service_template.rb
+++ b/app/services/projects/propagate_service_template.rb
@@ -11,7 +11,7 @@ module Projects
     end
 
     def propagate
-      return unless @template&.active?
+      return unless @template.active?
 
       Rails.logger.info("Propagating services for template #{@template.id}")
 
@@ -32,11 +32,11 @@ module Projects
 
     def bulk_create_from_template(batch)
       service_list = batch.map do |project_id|
-        service_hash.merge('project_id' => project_id).values
+        service_hash.values << project_id
       end
 
       Project.transaction do
-        bulk_insert_services(service_hash.keys + ['project_id'], service_list)
+        bulk_insert_services(service_hash.keys << 'project_id', service_list)
         run_callbacks(batch)
       end
     end
@@ -75,9 +75,9 @@ module Projects
 
           template_hash.each_with_object({}) do |(key, value), service_hash|
             value = value.is_a?(Hash) ? value.to_json : value
-            key = Gitlab::Database.postgresql? ? "\"#{key}\"" : "`#{key}`"
 
-            service_hash[key] = ActiveRecord::Base.sanitize(value)
+            service_hash[ActiveRecord::Base.connection.quote_column_name(key)] =
+              ActiveRecord::Base.sanitize(value)
           end
         end
     end
@@ -93,7 +93,7 @@ module Projects
     end
 
     def active_external_issue_tracker?
-      @template.category == :issue_tracker && !@template.default
+      @template.issue_tracker? && !@template.default
     end
 
     def active_external_wiki?
diff --git a/app/workers/propagate_service_template_worker.rb b/app/workers/propagate_service_template_worker.rb
index f1fc7ccb955..5ce0e0405d0 100644
--- a/app/workers/propagate_service_template_worker.rb
+++ b/app/workers/propagate_service_template_worker.rb
@@ -3,8 +3,6 @@ class PropagateServiceTemplateWorker
   include Sidekiq::Worker
   include DedicatedSidekiqQueue
 
-  sidekiq_options retry: 3
-
   LEASE_TIMEOUT = 4.hours.to_i
 
   def perform(template_id)
-- 
cgit v1.2.3