diff options
Diffstat (limited to 'app/models/bulk_imports')
-rw-r--r-- | app/models/bulk_imports/batch_tracker.rb | 46 | ||||
-rw-r--r-- | app/models/bulk_imports/entity.rb | 23 | ||||
-rw-r--r-- | app/models/bulk_imports/export.rb | 1 | ||||
-rw-r--r-- | app/models/bulk_imports/export_batch.rb | 33 | ||||
-rw-r--r-- | app/models/bulk_imports/export_upload.rb | 1 | ||||
-rw-r--r-- | app/models/bulk_imports/file_transfer.rb | 4 | ||||
-rw-r--r-- | app/models/bulk_imports/file_transfer/base_config.rb | 3 | ||||
-rw-r--r-- | app/models/bulk_imports/tracker.rb | 3 |
8 files changed, 103 insertions, 11 deletions
diff --git a/app/models/bulk_imports/batch_tracker.rb b/app/models/bulk_imports/batch_tracker.rb new file mode 100644 index 00000000000..df1fab89ee6 --- /dev/null +++ b/app/models/bulk_imports/batch_tracker.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module BulkImports + class BatchTracker < ApplicationRecord + self.table_name = 'bulk_import_batch_trackers' + + belongs_to :tracker, class_name: 'BulkImports::Tracker' + + validates :batch_number, presence: true, uniqueness: { scope: :tracker_id } + + state_machine :status, initial: :created do + state :created, value: 0 + state :started, value: 1 + state :finished, value: 2 + state :timeout, value: 3 + state :failed, value: -1 + state :skipped, value: -2 + + event :start do + transition created: :started + end + + event :retry do + transition started: :created + end + + event :finish do + transition started: :finished + transition failed: :failed + transition skipped: :skipped + end + + event :skip do + transition any => :skipped + end + + event :fail_op do + transition any => :failed + end + + event :cleanup_stale do + transition [:created, :started] => :timeout + end + end + end +end diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index 6fc24c77f1d..ae2d3758110 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -26,10 +26,11 @@ class BulkImports::Entity < ApplicationRecord belongs_to :parent, class_name: 'BulkImports::Entity', optional: true belongs_to :project, optional: true - belongs_to :group, foreign_key: :namespace_id, optional: true + belongs_to :group, foreign_key: :namespace_id, optional: true, inverse_of: :bulk_import_entities has_many :trackers, class_name: 'BulkImports::Tracker', + inverse_of: :entity, foreign_key: :bulk_import_entity_id has_many :failures, @@ -104,6 +105,12 @@ class BulkImports::Entity < ApplicationRecord transition created: :timeout transition started: :timeout end + + # rubocop:disable Style/SymbolProc + after_transition any => [:finished, :failed, :timeout] do |entity| + entity.update_has_failures + end + # rubocop:enable Style/SymbolProc end def self.all_human_statuses @@ -185,6 +192,13 @@ class BulkImports::Entity < ApplicationRecord default_project_visibility end + def update_has_failures + return if has_failures + return unless failures.any? + + update!(has_failures: true) + end + private def validate_parent_is_a_group @@ -194,13 +208,6 @@ class BulkImports::Entity < ApplicationRecord end def validate_imported_entity_type - if project_entity? && !BulkImports::Features.project_migration_enabled?(destination_namespace) - errors.add( - :base, - s_('BulkImport|invalid entity source type') - ) - end - if group.present? && project_entity? errors.add( :group, diff --git a/app/models/bulk_imports/export.rb b/app/models/bulk_imports/export.rb index 8d4d31ee92d..1ea317a100a 100644 --- a/app/models/bulk_imports/export.rb +++ b/app/models/bulk_imports/export.rb @@ -14,6 +14,7 @@ module BulkImports belongs_to :group, optional: true has_one :upload, class_name: 'BulkImports::ExportUpload' + has_many :batches, class_name: 'BulkImports::ExportBatch' validates :project, presence: true, unless: :group validates :group, presence: true, unless: :project diff --git a/app/models/bulk_imports/export_batch.rb b/app/models/bulk_imports/export_batch.rb new file mode 100644 index 00000000000..9d34dae12d0 --- /dev/null +++ b/app/models/bulk_imports/export_batch.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module BulkImports + class ExportBatch < ApplicationRecord + self.table_name = 'bulk_import_export_batches' + + BATCH_SIZE = 1000 + + belongs_to :export, class_name: 'BulkImports::Export' + has_one :upload, class_name: 'BulkImports::ExportUpload', foreign_key: :batch_id, inverse_of: :batch + + validates :batch_number, presence: true, uniqueness: { scope: :export_id } + + state_machine :status, initial: :started do + state :started, value: 0 + state :finished, value: 1 + state :failed, value: -1 + + event :start do + transition any => :started + end + + event :finish do + transition started: :finished + transition failed: :failed + end + + event :fail_op do + transition any => :failed + end + end + end +end diff --git a/app/models/bulk_imports/export_upload.rb b/app/models/bulk_imports/export_upload.rb index 4304032b28c..00f8e8f1304 100644 --- a/app/models/bulk_imports/export_upload.rb +++ b/app/models/bulk_imports/export_upload.rb @@ -7,6 +7,7 @@ module BulkImports self.table_name = 'bulk_import_export_uploads' belongs_to :export, class_name: 'BulkImports::Export' + belongs_to :batch, class_name: 'BulkImports::ExportBatch', optional: true mount_uploader :export_file, ExportUploader diff --git a/app/models/bulk_imports/file_transfer.rb b/app/models/bulk_imports/file_transfer.rb index 5be954b98da..c6af4e0c833 100644 --- a/app/models/bulk_imports/file_transfer.rb +++ b/app/models/bulk_imports/file_transfer.rb @@ -9,9 +9,9 @@ module BulkImports def config_for(portable) case portable when ::Project - FileTransfer::ProjectConfig.new(portable) + ::BulkImports::FileTransfer::ProjectConfig.new(portable) when ::Group - FileTransfer::GroupConfig.new(portable) + ::BulkImports::FileTransfer::GroupConfig.new(portable) else raise(UnsupportedObjectType, "Unsupported object type: #{portable.class}") end diff --git a/app/models/bulk_imports/file_transfer/base_config.rb b/app/models/bulk_imports/file_transfer/base_config.rb index 036d511bc59..67c4e7400b3 100644 --- a/app/models/bulk_imports/file_transfer/base_config.rb +++ b/app/models/bulk_imports/file_transfer/base_config.rb @@ -51,7 +51,8 @@ module BulkImports end def portable_relations_tree - @portable_relations_tree ||= attributes_finder.find_relations_tree(portable_class_sym).deep_stringify_keys + @portable_relations_tree ||= attributes_finder + .find_relations_tree(portable_class_sym, include_import_only_tree: true).deep_stringify_keys end private diff --git a/app/models/bulk_imports/tracker.rb b/app/models/bulk_imports/tracker.rb index b04ef1cb7ae..55502721a76 100644 --- a/app/models/bulk_imports/tracker.rb +++ b/app/models/bulk_imports/tracker.rb @@ -7,9 +7,12 @@ class BulkImports::Tracker < ApplicationRecord belongs_to :entity, class_name: 'BulkImports::Entity', + inverse_of: :trackers, foreign_key: :bulk_import_entity_id, optional: false + has_many :batches, class_name: 'BulkImports::BatchTracker', inverse_of: :tracker + validates :relation, presence: true, uniqueness: { scope: :bulk_import_entity_id } |