diff options
Diffstat (limited to 'lib/bulk_imports/pipeline/runner.rb')
| -rw-r--r-- | lib/bulk_imports/pipeline/runner.rb | 27 |
1 files changed, 18 insertions, 9 deletions
diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index 588d2c87209..b756fba3bee 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -14,19 +14,24 @@ module BulkImports extracted_data = extracted_data_from - extracted_data&.each do |entry| - transformers.each do |transformer| - entry = run_pipeline_step(:transformer, transformer.class.name) do - transformer.transform(context, entry) + if extracted_data + extracted_data.each do |entry| + transformers.each do |transformer| + entry = run_pipeline_step(:transformer, transformer.class.name) do + transformer.transform(context, entry) + end end - end - run_pipeline_step(:loader, loader.class.name) do - loader.load(context, entry) + run_pipeline_step(:loader, loader.class.name) do + loader.load(context, entry) + end end - end - if extracted_data && respond_to?(:after_run) + tracker.update!( + has_next_page: extracted_data.has_next_page?, + next_page: extracted_data.next_page + ) + run_pipeline_step(:after_run) do after_run(extracted_data) end @@ -65,6 +70,10 @@ module BulkImports end end + def after_run(extracted_data) + run if extracted_data.has_next_page? + end + def mark_as_failed warn(message: 'Pipeline failed') |