diff options
Diffstat (limited to 'app/workers/concerns/gitlab')
5 files changed, 171 insertions, 0 deletions
diff --git a/app/workers/concerns/gitlab/github_import/notify_upon_death.rb b/app/workers/concerns/gitlab/github_import/notify_upon_death.rb new file mode 100644 index 00000000000..3d7120665b6 --- /dev/null +++ b/app/workers/concerns/gitlab/github_import/notify_upon_death.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # NotifyUponDeath can be included into a GitHub worker class if it should + # notify any JobWaiter instances upon being moved to the Sidekiq dead queue. + # + # Note that this will only notify the waiter upon graceful termination, a + # SIGKILL will still result in the waiter _not_ being notified. + # + # Workers including this module must have jobs passed where the last + # argument is the key to notify, as a String. + module NotifyUponDeath + extend ActiveSupport::Concern + + included do + # If a job is being exhausted we still want to notify the + # AdvanceStageWorker. This prevents the entire import from getting stuck + # just because 1 job threw too many errors. + sidekiq_retries_exhausted do |job| + args = job['args'] + jid = job['jid'] + + if args.length == 3 && (key = args.last) && key.is_a?(String) + JobWaiter.notify(key, jid) + end + end + end + end + end +end diff --git a/app/workers/concerns/gitlab/github_import/object_importer.rb b/app/workers/concerns/gitlab/github_import/object_importer.rb new file mode 100644 index 00000000000..67e36c811de --- /dev/null +++ b/app/workers/concerns/gitlab/github_import/object_importer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # ObjectImporter defines the base behaviour for every Sidekiq worker that + # imports a single resource such as a note or pull request. + module ObjectImporter + extend ActiveSupport::Concern + + included do + include Sidekiq::Worker + include GithubImport::Queue + include ReschedulingMethods + include NotifyUponDeath + end + + # project - An instance of `Project` to import the data into. + # client - An instance of `Gitlab::GithubImport::Client` + # hash - A Hash containing the details of the object to import. + def import(project, client, hash) + object = representation_class.from_json_hash(hash) + + importer_class.new(object, project, client).execute + + counter.increment(project: project.path_with_namespace) + end + + def counter + @counter ||= Gitlab::Metrics.counter(counter_name, counter_description) + end + + # Returns the representation class to use for the object. This class must + # define the class method `from_json_hash`. + def representation_class + raise NotImplementedError + end + + # Returns the class to use for importing the object. + def importer_class + raise NotImplementedError + end + + # Returns the name (as a Symbol) of the Prometheus counter. + def counter_name + raise NotImplementedError + end + + # Returns the description (as a String) of the Prometheus counter. + def counter_description + raise NotImplementedError + end + end + end +end diff --git a/app/workers/concerns/gitlab/github_import/queue.rb b/app/workers/concerns/gitlab/github_import/queue.rb new file mode 100644 index 00000000000..a2bee361b86 --- /dev/null +++ b/app/workers/concerns/gitlab/github_import/queue.rb @@ -0,0 +1,16 @@ +module Gitlab + module GithubImport + module Queue + extend ActiveSupport::Concern + + included do + # If a job produces an error it may block a stage from advancing + # forever. To prevent this from happening we prevent jobs from going to + # the dead queue. This does mean some resources may not be imported, but + # this is better than a project being stuck in the "import" state + # forever. + sidekiq_options queue: 'github_importer', dead: false, retry: 5 + end + end + end +end diff --git a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb new file mode 100644 index 00000000000..692ca6b7f42 --- /dev/null +++ b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # Module that provides methods shared by the various workers used for + # importing GitHub projects. + module ReschedulingMethods + # project_id - The ID of the GitLab project to import the note into. + # hash - A Hash containing the details of the GitHub object to imoprt. + # notify_key - The Redis key to notify upon completion, if any. + def perform(project_id, hash, notify_key = nil) + project = Project.find_by(id: project_id) + + return notify_waiter(notify_key) unless project + + client = GithubImport.new_client_for(project, parallel: true) + + if try_import(project, client, hash) + notify_waiter(notify_key) + else + # In the event of hitting the rate limit we want to reschedule the job + # so its retried after our rate limit has been reset. + self.class + .perform_in(client.rate_limit_resets_in, project.id, hash, notify_key) + end + end + + def try_import(*args) + import(*args) + true + rescue RateLimitError + false + end + + def notify_waiter(key = nil) + JobWaiter.notify(key, jid) if key + end + end + end +end diff --git a/app/workers/concerns/gitlab/github_import/stage_methods.rb b/app/workers/concerns/gitlab/github_import/stage_methods.rb new file mode 100644 index 00000000000..147c8c8d683 --- /dev/null +++ b/app/workers/concerns/gitlab/github_import/stage_methods.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module StageMethods + # project_id - The ID of the GitLab project to import the data into. + def perform(project_id) + return unless (project = find_project(project_id)) + + client = GithubImport.new_client_for(project) + + try_import(client, project) + end + + # client - An instance of Gitlab::GithubImport::Client. + # project - An instance of Project. + def try_import(client, project) + import(client, project) + rescue RateLimitError + self.class.perform_in(client.rate_limit_resets_in, project.id) + end + + def find_project(id) + # If the project has been marked as failed we want to bail out + # automatically. + Project.import_started.find_by(id: id) + end + end + end +end |