diff options
Diffstat (limited to 'lib/gitlab/bitbucket_import')
| -rw-r--r-- | lib/gitlab/bitbucket_import/importer.rb | 6 | ||||
| -rw-r--r-- | lib/gitlab/bitbucket_import/importers/pull_request_importer.rb | 88 | ||||
| -rw-r--r-- | lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb | 47 | ||||
| -rw-r--r-- | lib/gitlab/bitbucket_import/importers/repository_importer.rb | 78 | ||||
| -rw-r--r-- | lib/gitlab/bitbucket_import/loggable.rb | 41 | ||||
| -rw-r--r-- | lib/gitlab/bitbucket_import/logger.rb | 11 | ||||
| -rw-r--r-- | lib/gitlab/bitbucket_import/parallel_importer.rb | 37 | ||||
| -rw-r--r-- | lib/gitlab/bitbucket_import/parallel_scheduling.rb | 93 | ||||
| -rw-r--r-- | lib/gitlab/bitbucket_import/user_finder.rb | 46 |
9 files changed, 446 insertions, 1 deletions
diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index e785ce558db..7f228c19b6b 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -210,7 +210,11 @@ module Gitlab source_branch_sha = pull_request.source_branch_sha target_branch_sha = pull_request.target_branch_sha - source_branch_sha = project.repository.commit(source_branch_sha)&.sha || source_branch_sha + + source_sha_from_commit_sha = project.repository.commit(source_branch_sha)&.sha + source_sha_from_merge_sha = project.repository.commit(pull_request.merge_commit_sha)&.sha + + source_branch_sha = source_sha_from_commit_sha || source_sha_from_merge_sha || source_branch_sha target_branch_sha = project.repository.commit(target_branch_sha)&.sha || target_branch_sha merge_request = project.merge_requests.create!( diff --git a/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb new file mode 100644 index 00000000000..d76e08e1039 --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class PullRequestImporter + include Loggable + + def initialize(project, hash) + @project = project + @formatter = Gitlab::ImportFormatter.new + @user_finder = UserFinder.new(project) + @object = hash.with_indifferent_access + end + + def execute + log_info(import_stage: 'import_pull_request', message: 'starting', iid: object[:iid]) + + description = '' + description += author_line + description += object[:description] if object[:description] + + attributes = { + iid: object[:iid], + title: object[:title], + description: description, + source_project_id: project.id, + source_branch: Gitlab::Git.ref_name(object[:source_branch_name]), + source_branch_sha: source_branch_sha, + target_project_id: project.id, + target_branch: Gitlab::Git.ref_name(object[:target_branch_name]), + target_branch_sha: object[:target_branch_sha], + state_id: MergeRequest.available_states[object[:state]], + author_id: author_id, + created_at: object[:created_at], + updated_at: object[:updated_at] + } + + creator = Gitlab::Import::MergeRequestCreator.new(project) + + merge_request = creator.execute(attributes) + + if merge_request + merge_request.assignee_ids = [author_id] + merge_request.reviewer_ids = reviewers + merge_request.save! + end + + log_info(import_stage: 'import_pull_request', message: 'finished', iid: object[:iid]) + rescue StandardError => e + Gitlab::Import::ImportFailureService.track(project_id: project.id, exception: e) + end + + private + + attr_reader :object, :project, :formatter, :user_finder + + def author_line + return '' if find_user_id + + formatter.author_line(object[:author]) + end + + def find_user_id + user_finder.find_user_id(object[:author]) + end + + def author_id + user_finder.gitlab_user_id(project, object[:author]) + end + + def reviewers + return [] unless object[:reviewers].present? + + object[:reviewers].filter_map do |reviewer| + user_finder.find_user_id(reviewer) + end + end + + def source_branch_sha + project.repository.commit(object[:source_branch_sha])&.sha || + project.repository.commit(object[:merge_commit_sha])&.sha || + object[:source_branch_sha] + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb new file mode 100644 index 00000000000..1c7ce7f2f3a --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class PullRequestsImporter + include ParallelScheduling + + def execute + log_info(import_stage: 'import_pull_requests', message: 'importing pull requests') + + pull_requests = client.pull_requests(project.import_source) + + pull_requests.each do |pull_request| + job_waiter.jobs_remaining += 1 + + next if already_enqueued?(pull_request) + + job_delay = calculate_job_delay(job_waiter.jobs_remaining) + + sidekiq_worker_class.perform_in(job_delay, project.id, pull_request.to_hash, job_waiter.key) + + mark_as_enqueued(pull_request) + end + + job_waiter + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + private + + def sidekiq_worker_class + ImportPullRequestWorker + end + + def collection_method + :pull_requests + end + + def id_for_already_enqueued_cache(object) + object.iid + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/repository_importer.rb b/lib/gitlab/bitbucket_import/importers/repository_importer.rb new file mode 100644 index 00000000000..7b0362b6ec6 --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/repository_importer.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class RepositoryImporter + include Loggable + + def initialize(project) + @project = project + end + + def execute + log_info(import_stage: 'import_repository', message: 'starting import') + + if project.empty_repo? + project.repository.import_repository(project.import_url) + project.repository.fetch_as_mirror(project.import_url, refmap: refmap) + + validate_repository_size! + + update_clone_time + end + + import_wiki + + log_info(import_stage: 'import_repository', message: 'finished import') + + true + rescue ::Gitlab::Git::CommandError => e + Gitlab::ErrorTracking.log_exception( + e, import_stage: 'import_repository', message: 'failed import', error: e.message + ) + + # Expire cache to prevent scenarios such as: + # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true + # 2. Retried import, repo is broken or not imported but +exists?+ still returns true + project.repository.expire_content_cache if project.repository_exists? + + raise + end + + private + + attr_reader :project + + def refmap + # We omit :heads and :tags since these are fetched in the import_repository + ['+refs/pull-requests/*/to:refs/merge-requests/*/head'] + end + + def import_wiki + return if project.wiki.repository_exists? + + project.wiki.repository.import_repository(wiki.import_url) + rescue StandardError => e + Gitlab::ErrorTracking.log_exception( + e, import_stage: 'import_repository', message: 'failed to import wiki', error: e.message + ) + end + + def wiki + WikiFormatter.new(project) + end + + def update_clone_time + project.touch(:last_repository_updated_at) + end + + def validate_repository_size! + # Defined in EE + end + end + end + end +end + +Gitlab::BitbucketImport::Importers::RepositoryImporter.prepend_mod diff --git a/lib/gitlab/bitbucket_import/loggable.rb b/lib/gitlab/bitbucket_import/loggable.rb new file mode 100644 index 00000000000..eda3cc96d4d --- /dev/null +++ b/lib/gitlab/bitbucket_import/loggable.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Loggable + def log_debug(messages) + logger.debug(log_data(messages)) + end + + def log_info(messages) + logger.info(log_data(messages)) + end + + def log_warn(messages) + logger.warn(log_data(messages)) + end + + def log_error(messages) + logger.error(log_data(messages)) + end + + private + + def logger + Gitlab::BitbucketImport::Logger + end + + def log_data(messages) + messages.merge(log_base_data) + end + + def log_base_data + { + class: self.class.name, + project_id: project.id, + project_path: project.full_path + } + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/logger.rb b/lib/gitlab/bitbucket_import/logger.rb new file mode 100644 index 00000000000..1f4d175f8a3 --- /dev/null +++ b/lib/gitlab/bitbucket_import/logger.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + class Logger < ::Gitlab::Import::Logger + def default_attributes + super.merge(import_type: :bitbucket) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/parallel_importer.rb b/lib/gitlab/bitbucket_import/parallel_importer.rb new file mode 100644 index 00000000000..1563261fa4a --- /dev/null +++ b/lib/gitlab/bitbucket_import/parallel_importer.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + class ParallelImporter + def self.async? + true + end + + def self.imports_repository? + true + end + + def self.track_start_import(project) + Gitlab::Import::Metrics.new(:bitbucket_importer, project).track_start_import + end + + def initialize(project) + @project = project + end + + def execute + Gitlab::Import::SetAsyncJid.set_jid(project.import_state) + + Stage::ImportRepositoryWorker + .with_status + .perform_async(project.id) + + true + end + + private + + attr_reader :project + end + end +end diff --git a/lib/gitlab/bitbucket_import/parallel_scheduling.rb b/lib/gitlab/bitbucket_import/parallel_scheduling.rb new file mode 100644 index 00000000000..f4df9a35526 --- /dev/null +++ b/lib/gitlab/bitbucket_import/parallel_scheduling.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module ParallelScheduling + include Loggable + + attr_reader :project, :already_enqueued_cache_key, :job_waiter_cache_key + + # The base cache key to use for tracking already enqueued objects. + ALREADY_ENQUEUED_CACHE_KEY = + 'bitbucket-importer/already-enqueued/%{project}/%{collection}' + + # The base cache key to use for storing job waiter key + JOB_WAITER_CACHE_KEY = + 'bitbucket-importer/job-waiter/%{project}/%{collection}' + + BATCH_SIZE = 100 + + # project - An instance of `Project`. + def initialize(project) + @project = project + + @already_enqueued_cache_key = + format(ALREADY_ENQUEUED_CACHE_KEY, project: project.id, collection: collection_method) + @job_waiter_cache_key = + format(JOB_WAITER_CACHE_KEY, project: project.id, collection: collection_method) + end + + private + + def client + @client ||= Bitbucket::Client.new(project.import_data.credentials) + end + + # Returns the ID to use for the cache used for checking if an object has + # already been enqueued or not. + # + # object - The object we may want to import. + def id_for_already_enqueued_cache(object) + raise NotImplementedError + end + + # The Sidekiq worker class used for scheduling the importing of objects in + # parallel. + def sidekiq_worker_class + raise NotImplementedError + end + + # The name of the method to call to retrieve the data to import. + def collection_method + raise NotImplementedError + end + + def job_waiter + @job_waiter ||= begin + key = Gitlab::Cache::Import::Caching.read(job_waiter_cache_key) + key ||= Gitlab::Cache::Import::Caching.write(job_waiter_cache_key, JobWaiter.generate_key) + + JobWaiter.new(0, key) + end + end + + def already_enqueued?(object) + id = id_for_already_enqueued_cache(object) + + Gitlab::Cache::Import::Caching.set_includes?(already_enqueued_cache_key, id) + end + + # Marks the given object as "already enqueued". + def mark_as_enqueued(object) + id = id_for_already_enqueued_cache(object) + + Gitlab::Cache::Import::Caching.set_add(already_enqueued_cache_key, id) + end + + def calculate_job_delay(job_index) + multiplier = (job_index / BATCH_SIZE) + + (multiplier * 1.minute) + 1.second + end + + def track_import_failure!(project, exception:, **args) + Gitlab::Import::ImportFailureService.track( + project_id: project.id, + error_source: self.class.name, + exception: exception, + **args + ) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/user_finder.rb b/lib/gitlab/bitbucket_import/user_finder.rb new file mode 100644 index 00000000000..70ed94351d5 --- /dev/null +++ b/lib/gitlab/bitbucket_import/user_finder.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + class UserFinder + USER_ID_FOR_AUTHOR_CACHE_KEY = 'bitbucket-importer/user-finder/%{project_id}/%{author}' + CACHE_USER_ID_NOT_FOUND = -1 + + attr_reader :project + + def initialize(project) + @project = project + end + + def find_user_id(author) + return unless author + + cache_key = build_cache_key(author) + cached_id = cache.read_integer(cache_key) + + return if cached_id == CACHE_USER_ID_NOT_FOUND + return cached_id if cached_id + + id = User.by_provider_and_extern_uid(:bitbucket, author).select(:id).first&.id + + cache.write(cache_key, id || CACHE_USER_ID_NOT_FOUND) + + id + end + + def gitlab_user_id(project, username) + find_user_id(username) || project.creator_id + end + + private + + def cache + Cache::Import::Caching + end + + def build_cache_key(author) + format(USER_ID_FOR_AUTHOR_CACHE_KEY, project_id: project.id, author: author) + end + end + end +end |