diff options
Diffstat (limited to 'app/workers/concerns/gitlab')
5 files changed, 20 insertions, 13 deletions
diff --git a/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb b/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb index 1090d82c922..fbcb5d81c8a 100644 --- a/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb +++ b/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb @@ -7,6 +7,8 @@ module Gitlab module ObjectImporter extend ActiveSupport::Concern + FAILED_IMPORT_STATES = %w[canceled failed].freeze + included do include ApplicationWorker @@ -33,8 +35,10 @@ module Gitlab return unless project - if project.import_state&.canceled? - info(project.id, message: 'project import canceled') + import_state = project.import_status + + if FAILED_IMPORT_STATES.include?(import_state) + info(project.id, message: "project import #{import_state}") return end diff --git a/app/workers/concerns/gitlab/github_import/object_importer.rb b/app/workers/concerns/gitlab/github_import/object_importer.rb index fcc7a96fa2b..15156e1deef 100644 --- a/app/workers/concerns/gitlab/github_import/object_importer.rb +++ b/app/workers/concerns/gitlab/github_import/object_importer.rb @@ -16,6 +16,7 @@ module Gitlab feature_category :importers worker_has_external_dependencies! + sidekiq_options retry: 5 sidekiq_retries_exhausted do |msg| args = msg['args'] jid = msg['jid'] @@ -57,12 +58,7 @@ module Gitlab end info(project.id, message: 'importer finished') - rescue NoMethodError => e - # This exception will be more useful in development when a new - # Representation is created but the developer forgot to add a - # `#github_identifiers` method. - track_and_raise_exception(project, e, fail_import: true) - rescue ActiveRecord::RecordInvalid, NotRetriableError => e + rescue ActiveRecord::RecordInvalid, NotRetriableError, NoMethodError => e # We do not raise exception to prevent job retry track_exception(project, e) rescue StandardError => e diff --git a/app/workers/concerns/gitlab/github_import/queue.rb b/app/workers/concerns/gitlab/github_import/queue.rb index 7cc23dd7c0b..5aabc74a3d5 100644 --- a/app/workers/concerns/gitlab/github_import/queue.rb +++ b/app/workers/concerns/gitlab/github_import/queue.rb @@ -14,7 +14,7 @@ module Gitlab # the dead queue. This does mean some resources may not be imported, but # this is better than a project being stuck in the "import" state # forever. - sidekiq_options dead: false, retry: 5 + sidekiq_options dead: false end end end diff --git a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb index 316d30d94da..e2808f45821 100644 --- a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb +++ b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb @@ -8,6 +8,8 @@ module Gitlab extend ActiveSupport::Concern include JobDelayCalculator + attr_reader :project + ENQUEUED_JOB_COUNT = 'github-importer/enqueued_job_count/%{project}/%{collection}' included do @@ -17,8 +19,10 @@ module Gitlab # project_id - The ID of the GitLab project to import the note into. # hash - A Hash containing the details of the GitHub object to import. # notify_key - The Redis key to notify upon completion, if any. + def perform(project_id, hash, notify_key = nil) - project = Project.find_by_id(project_id) + @project = Project.find_by_id(project_id) # rubocop:disable Gitlab/ModuleWithInstanceVariables -- GitHub Import + # uses modules everywhere. Too big to refactor. return notify_waiter(notify_key) unless project diff --git a/app/workers/concerns/gitlab/github_import/stage_methods.rb b/app/workers/concerns/gitlab/github_import/stage_methods.rb index 5c63c667a03..5f6812ab84f 100644 --- a/app/workers/concerns/gitlab/github_import/stage_methods.rb +++ b/app/workers/concerns/gitlab/github_import/stage_methods.rb @@ -9,6 +9,11 @@ module Gitlab included do include ApplicationWorker + include GithubImport::Queue + + sidekiq_options retry: 6 + + sidekiq_options status_expiration: Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION sidekiq_retries_exhausted do |msg, e| Gitlab::Import::ImportFailureService.track( @@ -37,8 +42,6 @@ module Gitlab # - Continue their loop from where it left off: # https://gitlab.com/gitlab-org/gitlab/-/blob/024235ec/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb#L15 def resumes_work_when_interrupted! - return unless Feature.enabled?(:github_importer_raise_max_interruptions) - sidekiq_options max_retries_after_interruption: MAX_RETRIES_AFTER_INTERRUPTION end end @@ -79,7 +82,7 @@ module Gitlab # client - An instance of Gitlab::GithubImport::Client. # project - An instance of Project. def try_import(client, project) - project.import_state.refresh_jid_expiration + RefreshImportJidWorker.perform_in_the_future(project.id, jid) import(client, project) rescue RateLimitError |