Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to 'lib/bulk_imports/pipeline/runner.rb')
-rw-r--r--lib/bulk_imports/pipeline/runner.rb61
1 files changed, 44 insertions, 17 deletions
diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb
index cf94b500612..04038e50399 100644
--- a/lib/bulk_imports/pipeline/runner.rb
+++ b/lib/bulk_imports/pipeline/runner.rb
@@ -6,34 +6,61 @@ module BulkImports
extend ActiveSupport::Concern
included do
- attr_reader :extractors, :transformers, :loaders
+ private
- def initialize
- @extractors = self.class.extractors.map(&method(:instantiate))
- @transformers = self.class.transformers.map(&method(:instantiate))
- @loaders = self.class.loaders.map(&method(:instantiate))
+ def extractors
+ @extractors ||= self.class.extractors.map(&method(:instantiate))
+ end
- super
+ def transformers
+ @transformers ||= self.class.transformers.map(&method(:instantiate))
end
- def run(context)
- extractors.each do |extractor|
- extractor.extract(context).each do |entry|
- transformers.each do |transformer|
- entry = transformer.transform(context, entry)
- end
+ def loaders
+ @loaders ||= self.class.loaders.map(&method(:instantiate))
+ end
- loaders.each do |loader|
- loader.load(context, entry)
- end
- end
- end
+ def pipeline_name
+ @pipeline ||= self.class.name
end
def instantiate(class_config)
class_config[:klass].new(class_config[:options])
end
end
+
+ def run(context)
+ info(context, message: "Pipeline started", pipeline: pipeline_name)
+
+ extractors.each do |extractor|
+ extractor.extract(context).each do |entry|
+ info(context, extractor: extractor.class.name)
+
+ transformers.each do |transformer|
+ info(context, transformer: transformer.class.name)
+ entry = transformer.transform(context, entry)
+ end
+
+ loaders.each do |loader|
+ info(context, loader: loader.class.name)
+ loader.load(context, entry)
+ end
+ end
+ end
+ end
+
+ private # rubocop:disable Lint/UselessAccessModifier
+
+ def info(context, extra = {})
+ logger.info({
+ entity: context.entity.id,
+ entity_type: context.entity.source_type
+ }.merge(extra))
+ end
+
+ def logger
+ @logger ||= Gitlab::Import::Logger.build
+ end
end
end
end