diff options
Diffstat (limited to 'lib/bulk_imports/pipeline/runner.rb')
-rw-r--r-- | lib/bulk_imports/pipeline/runner.rb | 61 |
1 files changed, 44 insertions, 17 deletions
diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index cf94b500612..04038e50399 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -6,34 +6,61 @@ module BulkImports extend ActiveSupport::Concern included do - attr_reader :extractors, :transformers, :loaders + private - def initialize - @extractors = self.class.extractors.map(&method(:instantiate)) - @transformers = self.class.transformers.map(&method(:instantiate)) - @loaders = self.class.loaders.map(&method(:instantiate)) + def extractors + @extractors ||= self.class.extractors.map(&method(:instantiate)) + end - super + def transformers + @transformers ||= self.class.transformers.map(&method(:instantiate)) end - def run(context) - extractors.each do |extractor| - extractor.extract(context).each do |entry| - transformers.each do |transformer| - entry = transformer.transform(context, entry) - end + def loaders + @loaders ||= self.class.loaders.map(&method(:instantiate)) + end - loaders.each do |loader| - loader.load(context, entry) - end - end - end + def pipeline_name + @pipeline ||= self.class.name end def instantiate(class_config) class_config[:klass].new(class_config[:options]) end end + + def run(context) + info(context, message: "Pipeline started", pipeline: pipeline_name) + + extractors.each do |extractor| + extractor.extract(context).each do |entry| + info(context, extractor: extractor.class.name) + + transformers.each do |transformer| + info(context, transformer: transformer.class.name) + entry = transformer.transform(context, entry) + end + + loaders.each do |loader| + info(context, loader: loader.class.name) + loader.load(context, entry) + end + end + end + end + + private # rubocop:disable Lint/UselessAccessModifier + + def info(context, extra = {}) + logger.info({ + entity: context.entity.id, + entity_type: context.entity.source_type + }.merge(extra)) + end + + def logger + @logger ||= Gitlab::Import::Logger.build + end end end end |