diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-11-14 11:41:52 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-11-14 11:41:52 +0300 |
commit | 585826cb22ecea5998a2c2a4675735c94bdeedac (patch) | |
tree | 5b05f0b30d33cef48963609e8a18a4dff260eab3 /app/workers/concerns | |
parent | df221d036e5d0c6c0ee4d55b9c97f481ee05dee8 (diff) |
Add latest changes from gitlab-org/gitlab@16-6-stable-eev16.6.0-rc42
Diffstat (limited to 'app/workers/concerns')
3 files changed, 30 insertions, 4 deletions
diff --git a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb index f6feb6d1598..316d30d94da 100644 --- a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb +++ b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb @@ -52,8 +52,7 @@ module Gitlab job_delay = client.rate_limit_resets_in + calculate_job_delay(enqueued_job_counter) - self.class - .perform_in(job_delay, project.id, hash, notify_key) + self.class.perform_in(job_delay, project.id, hash.deep_stringify_keys, notify_key.to_s) end end end diff --git a/app/workers/concerns/gitlab/github_import/stage_methods.rb b/app/workers/concerns/gitlab/github_import/stage_methods.rb index 80013ff3cd9..5c63c667a03 100644 --- a/app/workers/concerns/gitlab/github_import/stage_methods.rb +++ b/app/workers/concerns/gitlab/github_import/stage_methods.rb @@ -5,6 +5,8 @@ module Gitlab module StageMethods extend ActiveSupport::Concern + MAX_RETRIES_AFTER_INTERRUPTION = 20 + included do include ApplicationWorker @@ -18,6 +20,29 @@ module Gitlab end end + class_methods do + # We can increase the number of times a GitHubImport::Stage worker is retried + # after being interrupted if the importer it executes can restart exactly + # from where it left off. + # + # It is not safe to call this method if the importer loops over its data from + # the beginning when restarted, even if it skips data that is already imported + # inside the loop, as there is a possibility the importer will never reach + # the end of the loop. + # + # Examples of stage workers that call this method are ones that execute services that: + # + # - Continue paging an endpoint from where it left off: + # https://gitlab.com/gitlab-org/gitlab/-/blob/487521cc/lib/gitlab/github_import/parallel_scheduling.rb#L114-117 + # - Continue their loop from where it left off: + # https://gitlab.com/gitlab-org/gitlab/-/blob/024235ec/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb#L15 + def resumes_work_when_interrupted! + return unless Feature.enabled?(:github_importer_raise_max_interruptions) + + sidekiq_options max_retries_after_interruption: MAX_RETRIES_AFTER_INTERRUPTION + end + end + # project_id - The ID of the GitLab project to import the data into. def perform(project_id) info(project_id, message: 'starting stage') @@ -54,6 +79,8 @@ module Gitlab # client - An instance of Gitlab::GithubImport::Client. # project - An instance of Project. def try_import(client, project) + project.import_state.refresh_jid_expiration + import(client, project) rescue RateLimitError self.class.perform_in(client.rate_limit_resets_in, project.id) diff --git a/app/workers/concerns/worker_attributes.rb b/app/workers/concerns/worker_attributes.rb index cb09aaf1a6a..28c82a5a38e 100644 --- a/app/workers/concerns/worker_attributes.rb +++ b/app/workers/concerns/worker_attributes.rb @@ -201,10 +201,10 @@ module WorkerAttributes !!get_class_attribute(:big_payload) end - def defer_on_database_health_signal(gitlab_schema, tables = [], delay_by = DEFAULT_DEFER_DELAY) + def defer_on_database_health_signal(gitlab_schema, tables = [], delay_by = DEFAULT_DEFER_DELAY, &block) set_class_attribute( :database_health_check_attrs, - { gitlab_schema: gitlab_schema, tables: tables, delay_by: delay_by } + { gitlab_schema: gitlab_schema, tables: tables, delay_by: delay_by, block: block } ) end |