diff options
Diffstat (limited to 'app/workers/concerns/gitlab')
8 files changed, 212 insertions, 35 deletions
diff --git a/app/workers/concerns/gitlab/bitbucket_import/object_importer.rb b/app/workers/concerns/gitlab/bitbucket_import/object_importer.rb new file mode 100644 index 00000000000..26e6e2675ed --- /dev/null +++ b/app/workers/concerns/gitlab/bitbucket_import/object_importer.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + # ObjectImporter defines the base behaviour for every Sidekiq worker that + # imports a single resource such as a note or pull request. + module ObjectImporter + extend ActiveSupport::Concern + + included do + include ApplicationWorker + + data_consistency :always + + feature_category :importers + + worker_has_external_dependencies! + + sidekiq_retries_exhausted do |msg| + args = msg['args'] + jid = msg['jid'] + + # If a job is being exhausted we still want to notify the + # Gitlab::Import::AdvanceStageWorker to prevent the entire import from getting stuck + key = args.last + JobWaiter.notify(key, jid) if args.length == 3 && key && key.is_a?(String) + end + end + + def perform(project_id, hash, notify_key) + project = Project.find_by_id(project_id) + + return unless project + + if project.import_state&.canceled? + info(project.id, message: 'project import canceled') + return + end + + import(project, hash) + ensure + notify_waiter(notify_key) + end + + private + + # project - An instance of `Project` to import the data into. + # hash - A Hash containing the details of the object to import. + def import(project, hash) + info(project.id, message: 'importer started') + + importer_class.new(project, hash).execute + + info(project.id, message: 'importer finished') + rescue ActiveRecord::RecordInvalid => e + # We do not raise exception to prevent job retry + track_exception(project, e) + rescue StandardError => e + track_and_raise_exception(project, e) + end + + def notify_waiter(key) + JobWaiter.notify(key, jid) + end + + # Returns the class to use for importing the object. + def importer_class + raise NotImplementedError + end + + def info(project_id, extra = {}) + Logger.info(log_attributes(project_id, extra)) + end + + def log_attributes(project_id, extra = {}) + extra.merge( + project_id: project_id, + importer: importer_class.name + ) + end + + def track_exception(project, exception, fail_import: false) + Gitlab::Import::ImportFailureService.track( + project_id: project.id, + error_source: importer_class.name, + exception: exception, + fail_import: fail_import + ) + end + + def track_and_raise_exception(project, exception, fail_import: false) + track_exception(project, exception, fail_import: fail_import) + + raise(exception) + end + end + end +end diff --git a/app/workers/concerns/gitlab/bitbucket_import/stage_methods.rb b/app/workers/concerns/gitlab/bitbucket_import/stage_methods.rb new file mode 100644 index 00000000000..2885cc29532 --- /dev/null +++ b/app/workers/concerns/gitlab/bitbucket_import/stage_methods.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module StageMethods + extend ActiveSupport::Concern + + included do + include ApplicationWorker + + worker_has_external_dependencies! + + feature_category :importers + + data_consistency :always + + sidekiq_options dead: false, retry: 3 + + sidekiq_retries_exhausted do |msg, e| + Gitlab::Import::ImportFailureService.track( + project_id: msg['args'][0], + exception: e, + fail_import: true + ) + end + end + + # project_id - The ID of the GitLab project to import the data into. + def perform(project_id) + info(project_id, message: 'starting stage') + + project = find_project(project_id) + + return unless project + + import(project) + + info(project_id, message: 'stage finished') + rescue StandardError => e + Gitlab::Import::ImportFailureService.track( + project_id: project_id, + exception: e, + error_source: self.class.name, + fail_import: abort_on_failure + ) + + raise(e) + end + + def find_project(id) + # If the project has been marked as failed we want to bail out + # automatically. + # rubocop: disable CodeReuse/ActiveRecord + Project.joins_import_state.where(import_state: { status: :started }).find_by_id(id) + # rubocop: enable CodeReuse/ActiveRecord + end + + def abort_on_failure + false + end + + private + + def info(project_id, extra = {}) + Logger.info(log_attributes(project_id, extra)) + end + + def log_attributes(project_id, extra = {}) + extra.merge( + project_id: project_id, + import_stage: self.class.name + ) + end + end + end +end diff --git a/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb b/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb index b209719479b..1090d82c922 100644 --- a/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb +++ b/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb @@ -23,7 +23,7 @@ module Gitlab # If a job is being exhausted we still want to notify the # Gitlab::Import::AdvanceStageWorker to prevent the entire import from getting stuck if args.length == 3 && (key = args.last) && key.is_a?(String) - JobWaiter.notify(key, jid) + JobWaiter.notify(key, jid, ttl: Gitlab::Import::JOB_WAITER_TTL) end end end @@ -61,7 +61,7 @@ module Gitlab end def notify_waiter(key) - JobWaiter.notify(key, jid) + JobWaiter.notify(key, jid, ttl: Gitlab::Import::JOB_WAITER_TTL) end # Returns the class to use for importing the object. diff --git a/app/workers/concerns/gitlab/github_import/object_importer.rb b/app/workers/concerns/gitlab/github_import/object_importer.rb index 6cb9bd34969..e190ced5073 100644 --- a/app/workers/concerns/gitlab/github_import/object_importer.rb +++ b/app/workers/concerns/gitlab/github_import/object_importer.rb @@ -27,7 +27,7 @@ module Gitlab # If a job is being exhausted we still want to notify the # Gitlab::Import::AdvanceStageWorker to prevent the entire import from getting stuck if args.length == 3 && (key = args.last) && key.is_a?(String) - JobWaiter.notify(key, jid) + JobWaiter.notify(key, jid, ttl: Gitlab::Import::JOB_WAITER_TTL) end end end @@ -38,7 +38,7 @@ module Gitlab # client - An instance of `Gitlab::GithubImport::Client` # hash - A Hash containing the details of the object to import. def import(project, client, hash) - unless project.import_state&.in_progress? + if project.import_state&.completed? info( project.id, message: 'Project import is no longer running. Stopping worker.', diff --git a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb index b40914770b5..f6feb6d1598 100644 --- a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb +++ b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb @@ -39,7 +39,7 @@ module Gitlab end def notify_waiter(key = nil) - JobWaiter.notify(key, jid) if key + JobWaiter.notify(key, jid, ttl: Gitlab::Import::JOB_WAITER_TTL) if key end def reschedule_job(project, client, hash, notify_key) diff --git a/app/workers/concerns/gitlab/github_import/stage_methods.rb b/app/workers/concerns/gitlab/github_import/stage_methods.rb index a5287fcfbe2..75db5589415 100644 --- a/app/workers/concerns/gitlab/github_import/stage_methods.rb +++ b/app/workers/concerns/gitlab/github_import/stage_methods.rb @@ -9,7 +9,7 @@ module Gitlab return unless (project = find_project(project_id)) - unless project.import_state&.in_progress? + if project.import_state&.completed? info( project_id, message: 'Project import is no longer running. Stopping worker.', diff --git a/app/workers/concerns/gitlab/import/notify_upon_death.rb b/app/workers/concerns/gitlab/import/notify_upon_death.rb new file mode 100644 index 00000000000..ae726a673c2 --- /dev/null +++ b/app/workers/concerns/gitlab/import/notify_upon_death.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +# NotifyUponDeath can be included into a worker class if it should +# notify any JobWaiter instances upon being moved to the Sidekiq dead queue. +# +# Note that this will only notify the waiter upon graceful termination, a +# SIGKILL will still result in the waiter _not_ being notified. +# +# Workers including this module must have jobs passed where the last +# argument is the key to notify, as a String. +module Gitlab + module Import + module NotifyUponDeath + extend ActiveSupport::Concern + + included do + # If a job is being exhausted we still want to notify the + # Gitlab::Import::AdvanceStageWorker. This prevents the entire import from getting stuck + # just because 1 job threw too many errors. + sidekiq_retries_exhausted do |job| + args = job['args'] + jid = job['jid'] + key = args.last + + next unless args.length == 3 && key.is_a?(String) + + JobWaiter.notify(key, jid, ttl: Gitlab::Import::JOB_WAITER_TTL) + end + end + end + end +end diff --git a/app/workers/concerns/gitlab/notify_upon_death.rb b/app/workers/concerns/gitlab/notify_upon_death.rb deleted file mode 100644 index 66dc6270637..00000000000 --- a/app/workers/concerns/gitlab/notify_upon_death.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - # NotifyUponDeath can be included into a worker class if it should - # notify any JobWaiter instances upon being moved to the Sidekiq dead queue. - # - # Note that this will only notify the waiter upon graceful termination, a - # SIGKILL will still result in the waiter _not_ being notified. - # - # Workers including this module must have jobs passed where the last - # argument is the key to notify, as a String. - module NotifyUponDeath - extend ActiveSupport::Concern - - included do - # If a job is being exhausted we still want to notify the - # Gitlab::Import::AdvanceStageWorker. This prevents the entire import from getting stuck - # just because 1 job threw too many errors. - sidekiq_retries_exhausted do |job| - args = job['args'] - jid = job['jid'] - - if args.length == 3 && (key = args.last) && key.is_a?(String) - JobWaiter.notify(key, jid) - end - end - end - end -end |