diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-05-20 21:08:00 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-05-20 21:08:00 +0300 |
commit | f781b0b69368ea3181cf892305c60a22886c0d7e (patch) | |
tree | 0737d7313d4e5760e3addcec0b0f40c474008dcf /app | |
parent | 1d5ae049f089db097048fa896105ad75fe859596 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'app')
-rw-r--r-- | app/controllers/concerns/integrations_actions.rb | 2 | ||||
-rw-r--r-- | app/models/data_list.rb | 25 | ||||
-rw-r--r-- | app/models/service.rb | 8 | ||||
-rw-r--r-- | app/models/service_list.rb | 27 | ||||
-rw-r--r-- | app/services/admin/propagate_integration_service.rb | 144 | ||||
-rw-r--r-- | app/services/projects/propagate_service_template.rb | 16 | ||||
-rw-r--r-- | app/workers/all_queues.yml | 7 | ||||
-rw-r--r-- | app/workers/propagate_integration_worker.rb | 16 |
8 files changed, 236 insertions, 9 deletions
diff --git a/app/controllers/concerns/integrations_actions.rb b/app/controllers/concerns/integrations_actions.rb index ff283f9bb62..b3ad89f3227 100644 --- a/app/controllers/concerns/integrations_actions.rb +++ b/app/controllers/concerns/integrations_actions.rb @@ -16,10 +16,12 @@ module IntegrationsActions def update saved = integration.update(service_params[:service]) + overwrite = ActiveRecord::Type::Boolean.new.cast(params[:overwrite]) respond_to do |format| format.html do if saved + PropagateIntegrationWorker.perform_async(integration.id, overwrite) redirect_to scoped_edit_integration_path(integration), notice: success_message else render 'shared/integrations/edit' diff --git a/app/models/data_list.rb b/app/models/data_list.rb new file mode 100644 index 00000000000..12011cb17f7 --- /dev/null +++ b/app/models/data_list.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class DataList + def initialize(batch, data_fields_hash, klass) + @batch = batch + @data_fields_hash = data_fields_hash + @klass = klass + end + + def to_array + [klass, columns, values] + end + + private + + attr_reader :batch, :data_fields_hash, :klass + + def columns + data_fields_hash.keys << 'service_id' + end + + def values + batch.map { |row| data_fields_hash.values << row['id'] } + end +end diff --git a/app/models/service.rb b/app/models/service.rb index 396c0c530ab..a2c23947932 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -134,6 +134,14 @@ class Service < ApplicationRecord %w(active) end + def to_service_hash + as_json(methods: :type, except: %w[id template instance project_id]) + end + + def to_data_fields_hash + data_fields.as_json(only: data_fields.class.column_names).except('id', 'service_id') + end + def test_data(project, user) Gitlab::DataBuilder::Push.build_sample(project, user) end diff --git a/app/models/service_list.rb b/app/models/service_list.rb new file mode 100644 index 00000000000..fa3760f0c56 --- /dev/null +++ b/app/models/service_list.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class ServiceList + def initialize(batch, service_hash, extra_hash = {}) + @batch = batch + @service_hash = service_hash + @extra_hash = extra_hash + end + + def to_array + [Service, columns, values] + end + + private + + attr_reader :batch, :service_hash, :extra_hash + + def columns + (service_hash.keys << 'project_id') + extra_hash.keys + end + + def values + batch.map do |project_id| + (service_hash.values << project_id) + extra_hash.values + end + end +end diff --git a/app/services/admin/propagate_integration_service.rb b/app/services/admin/propagate_integration_service.rb new file mode 100644 index 00000000000..0a3c61816f8 --- /dev/null +++ b/app/services/admin/propagate_integration_service.rb @@ -0,0 +1,144 @@ +# frozen_string_literal: true + +module Admin + class PropagateIntegrationService + BATCH_SIZE = 100 + + delegate :data_fields_present?, to: :integration + + def self.propagate(integration:, overwrite:) + new(integration, overwrite).propagate + end + + def initialize(integration, overwrite) + @integration = integration + @overwrite = overwrite + end + + def propagate + if overwrite + update_integration_for_all_projects + else + update_integration_for_inherited_projects + end + + create_integration_for_projects_without_integration + end + + private + + attr_reader :integration, :overwrite + + # rubocop: disable Cop/InBatches + # rubocop: disable CodeReuse/ActiveRecord + def update_integration_for_inherited_projects + Service.where(type: integration.type, inherit_from_id: integration.id).in_batches(of: BATCH_SIZE) do |batch| + bulk_update_from_integration(batch) + end + end + + def update_integration_for_all_projects + Service.where(type: integration.type).in_batches(of: BATCH_SIZE) do |batch| + bulk_update_from_integration(batch) + end + end + # rubocop: enable Cop/InBatches + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord + def bulk_update_from_integration(batch) + # Retrieving the IDs instantiates the ActiveRecord relation (batch) + # into concrete models, otherwise update_all will clear the relation. + # https://stackoverflow.com/q/34811646/462015 + batch_ids = batch.pluck(:id) + + Service.transaction do + batch.update_all(service_hash) + + if data_fields_present? + integration.data_fields.class.where(service_id: batch_ids).update_all(data_fields_hash) + end + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def create_integration_for_projects_without_integration + loop do + batch = Project.uncached { project_ids_without_integration } + + bulk_create_from_integration(batch) unless batch.empty? + + break if batch.size < BATCH_SIZE + end + end + + def bulk_create_from_integration(batch) + service_list = ServiceList.new(batch, service_hash, { 'inherit_from_id' => integration.id }).to_array + + Project.transaction do + results = bulk_insert(*service_list) + + if data_fields_present? + data_list = DataList.new(results, data_fields_hash, integration.data_fields.class).to_array + + bulk_insert(*data_list) + end + + run_callbacks(batch) + end + end + + def bulk_insert(klass, columns, values_array) + items_to_insert = values_array.map { |array| Hash[columns.zip(array)] } + + klass.insert_all(items_to_insert, returning: [:id]) + end + + # rubocop: disable CodeReuse/ActiveRecord + def run_callbacks(batch) + if active_external_issue_tracker? + Project.where(id: batch).update_all(has_external_issue_tracker: true) + end + + if active_external_wiki? + Project.where(id: batch).update_all(has_external_wiki: true) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def active_external_issue_tracker? + integration.issue_tracker? && !integration.default + end + + def active_external_wiki? + integration.type == 'ExternalWikiService' + end + + def project_ids_without_integration + Project.connection.select_values( + <<-SQL + SELECT id + FROM projects + WHERE NOT EXISTS ( + SELECT true + FROM services + WHERE services.project_id = projects.id + AND services.type = #{ActiveRecord::Base.connection.quote(integration.type)} + ) + AND projects.pending_delete = false + AND projects.archived = false + LIMIT #{BATCH_SIZE} + SQL + ) + end + + def service_hash + @service_hash ||= integration.to_service_hash + .tap { |json| json['inherit_from_id'] = integration.id } + end + + def data_fields_hash + @data_fields_hash ||= integration.to_data_fields_hash + end + end +end diff --git a/app/services/projects/propagate_service_template.rb b/app/services/projects/propagate_service_template.rb index 0483c951f1e..ecca9715940 100644 --- a/app/services/projects/propagate_service_template.rb +++ b/app/services/projects/propagate_service_template.rb @@ -35,17 +35,15 @@ module Projects end def bulk_create_from_template(batch) - service_list = batch.map do |project_id| - service_hash.values << project_id - end + service_list = ServiceList.new(batch, service_hash).to_array Project.transaction do - results = bulk_insert(Service, service_hash.keys << 'project_id', service_list) + results = bulk_insert(*service_list) if data_fields_present? - data_list = results.map { |row| data_hash.values << row['id'] } + data_list = DataList.new(results, data_fields_hash, template.data_fields.class).to_array - bulk_insert(template.data_fields.class, data_hash.keys << 'service_id', data_list) + bulk_insert(*data_list) end run_callbacks(batch) @@ -77,11 +75,11 @@ module Projects end def service_hash - @service_hash ||= template.as_json(methods: :type, except: %w[id template project_id]) + @service_hash ||= template.to_service_hash end - def data_hash - @data_hash ||= template.data_fields.as_json(only: template.data_fields.class.column_names).except('id', 'service_id') + def data_fields_hash + @data_fields_hash ||= template.to_data_fields_hash end # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 1f9a53d64d9..1454ededc04 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -1291,6 +1291,13 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true +- :name: propagate_integration + :feature_category: :integrations + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true - :name: propagate_service_template :feature_category: :source_code_management :has_external_dependencies: diff --git a/app/workers/propagate_integration_worker.rb b/app/workers/propagate_integration_worker.rb new file mode 100644 index 00000000000..cbab38465bc --- /dev/null +++ b/app/workers/propagate_integration_worker.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class PropagateIntegrationWorker + include ApplicationWorker + + feature_category :integrations + + idempotent! + + def perform(integration_id, overwrite) + Admin::PropagateIntegrationService.propagate( + integration: Service.find(integration_id), + overwrite: overwrite + ) + end +end |