diff options
Diffstat (limited to 'app/workers/concerns')
7 files changed, 92 insertions, 15 deletions
diff --git a/app/workers/concerns/click_house_worker.rb b/app/workers/concerns/click_house_worker.rb new file mode 100644 index 00000000000..6399796f6df --- /dev/null +++ b/app/workers/concerns/click_house_worker.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module ClickHouseWorker + extend ActiveSupport::Concern + + class_methods do + def register_click_house_worker? + click_house_worker_attrs.present? + end + + def click_house_worker_attrs + get_class_attribute(:click_house_worker_attrs) + end + + def click_house_migration_lock(ttl) + raise ArgumentError unless ttl.is_a?(ActiveSupport::Duration) + + set_class_attribute( + :click_house_worker_attrs, + (click_house_worker_attrs || {}).merge(migration_lock_ttl: ttl) + ) + end + end + + included do + click_house_migration_lock(ClickHouse::MigrationSupport::ExclusiveLock::DEFAULT_CLICKHOUSE_WORKER_TTL) + + pause_control :click_house_migration + end +end diff --git a/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb b/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb index 1090d82c922..fbcb5d81c8a 100644 --- a/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb +++ b/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb @@ -7,6 +7,8 @@ module Gitlab module ObjectImporter extend ActiveSupport::Concern + FAILED_IMPORT_STATES = %w[canceled failed].freeze + included do include ApplicationWorker @@ -33,8 +35,10 @@ module Gitlab return unless project - if project.import_state&.canceled? - info(project.id, message: 'project import canceled') + import_state = project.import_status + + if FAILED_IMPORT_STATES.include?(import_state) + info(project.id, message: "project import #{import_state}") return end diff --git a/app/workers/concerns/gitlab/github_import/object_importer.rb b/app/workers/concerns/gitlab/github_import/object_importer.rb index fcc7a96fa2b..15156e1deef 100644 --- a/app/workers/concerns/gitlab/github_import/object_importer.rb +++ b/app/workers/concerns/gitlab/github_import/object_importer.rb @@ -16,6 +16,7 @@ module Gitlab feature_category :importers worker_has_external_dependencies! + sidekiq_options retry: 5 sidekiq_retries_exhausted do |msg| args = msg['args'] jid = msg['jid'] @@ -57,12 +58,7 @@ module Gitlab end info(project.id, message: 'importer finished') - rescue NoMethodError => e - # This exception will be more useful in development when a new - # Representation is created but the developer forgot to add a - # `#github_identifiers` method. - track_and_raise_exception(project, e, fail_import: true) - rescue ActiveRecord::RecordInvalid, NotRetriableError => e + rescue ActiveRecord::RecordInvalid, NotRetriableError, NoMethodError => e # We do not raise exception to prevent job retry track_exception(project, e) rescue StandardError => e diff --git a/app/workers/concerns/gitlab/github_import/queue.rb b/app/workers/concerns/gitlab/github_import/queue.rb index 7cc23dd7c0b..5aabc74a3d5 100644 --- a/app/workers/concerns/gitlab/github_import/queue.rb +++ b/app/workers/concerns/gitlab/github_import/queue.rb @@ -14,7 +14,7 @@ module Gitlab # the dead queue. This does mean some resources may not be imported, but # this is better than a project being stuck in the "import" state # forever. - sidekiq_options dead: false, retry: 5 + sidekiq_options dead: false end end end diff --git a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb index 316d30d94da..e2808f45821 100644 --- a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb +++ b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb @@ -8,6 +8,8 @@ module Gitlab extend ActiveSupport::Concern include JobDelayCalculator + attr_reader :project + ENQUEUED_JOB_COUNT = 'github-importer/enqueued_job_count/%{project}/%{collection}' included do @@ -17,8 +19,10 @@ module Gitlab # project_id - The ID of the GitLab project to import the note into. # hash - A Hash containing the details of the GitHub object to import. # notify_key - The Redis key to notify upon completion, if any. + def perform(project_id, hash, notify_key = nil) - project = Project.find_by_id(project_id) + @project = Project.find_by_id(project_id) # rubocop:disable Gitlab/ModuleWithInstanceVariables -- GitHub Import + # uses modules everywhere. Too big to refactor. return notify_waiter(notify_key) unless project diff --git a/app/workers/concerns/gitlab/github_import/stage_methods.rb b/app/workers/concerns/gitlab/github_import/stage_methods.rb index 5c63c667a03..5f6812ab84f 100644 --- a/app/workers/concerns/gitlab/github_import/stage_methods.rb +++ b/app/workers/concerns/gitlab/github_import/stage_methods.rb @@ -9,6 +9,11 @@ module Gitlab included do include ApplicationWorker + include GithubImport::Queue + + sidekiq_options retry: 6 + + sidekiq_options status_expiration: Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION sidekiq_retries_exhausted do |msg, e| Gitlab::Import::ImportFailureService.track( @@ -37,8 +42,6 @@ module Gitlab # - Continue their loop from where it left off: # https://gitlab.com/gitlab-org/gitlab/-/blob/024235ec/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb#L15 def resumes_work_when_interrupted! - return unless Feature.enabled?(:github_importer_raise_max_interruptions) - sidekiq_options max_retries_after_interruption: MAX_RETRIES_AFTER_INTERRUPTION end end @@ -79,7 +82,7 @@ module Gitlab # client - An instance of Gitlab::GithubImport::Client. # project - An instance of Project. def try_import(client, project) - project.import_state.refresh_jid_expiration + RefreshImportJidWorker.perform_in_the_future(project.id, jid) import(client, project) rescue RateLimitError diff --git a/app/workers/concerns/update_repository_storage_worker.rb b/app/workers/concerns/update_repository_storage_worker.rb index 01744d1e57d..fd437ebc158 100644 --- a/app/workers/concerns/update_repository_storage_worker.rb +++ b/app/workers/concerns/update_repository_storage_worker.rb @@ -11,7 +11,19 @@ module UpdateRepositoryStorageWorker urgency :throttled end - def perform(container_id, new_repository_storage_key, repository_storage_move_id = nil) + LEASE_TIMEOUT = 30.minutes.to_i + + # `container_id` and `new_repository_storage_key` arguments have been deprecated. + # `repository_storage_move_id` is now a mandatory argument. + # We are using *args for backwards compatability. Previously defined as: + # perform(container_id, new_repository_storage_key, repository_storage_move_id = nil) + def perform(*args) + if args.length == 1 + repository_storage_move_id = args[0] + else + container_id, new_repository_storage_key, repository_storage_move_id = *args + end + repository_storage_move = if repository_storage_move_id find_repository_storage_move(repository_storage_move_id) @@ -24,7 +36,35 @@ module UpdateRepositoryStorageWorker ) end - update_repository_storage(repository_storage_move) + container_id ||= repository_storage_move.container_id + + # Use exclusive lock to prevent multiple storage migrations at the same time + # + # Note: instead of using a randomly generated `uuid`, we provide a worker jid value. + # That will allow to track a worker that requested a lease. + lease_key = [self.class.name.underscore, container_id].join(':') + exclusive_lease = Gitlab::ExclusiveLease.new(lease_key, uuid: jid, timeout: LEASE_TIMEOUT) + lease = exclusive_lease.try_obtain + + if lease + begin + update_repository_storage(repository_storage_move) + ensure + exclusive_lease.cancel + end + else + # If there is an ungoing storage migration, then the current one should be marked as failed + repository_storage_move.do_fail! + + # A special case + # Sidekiq can receive an interrupt signal during the processing. + # It kills existing workers and reschedules their jobs using the same jid. + # But it can cause a situation when the migration is only half complete (see https://gitlab.com/gitlab-org/gitlab/-/issues/429049#note_1635650597) + # + # Here we detect this case and release the lock. + uuid = Gitlab::ExclusiveLease.get_uuid(lease_key) + exclusive_lease.cancel if uuid == jid + end end private |