diff options
Diffstat (limited to 'lib/bulk_imports')
| -rw-r--r-- | lib/bulk_imports/common/extractors/ndjson_extractor.rb | 24 | ||||
| -rw-r--r-- | lib/bulk_imports/common/pipelines/milestones_pipeline.rb (renamed from lib/bulk_imports/groups/pipelines/milestones_pipeline.rb) | 2 | ||||
| -rw-r--r-- | lib/bulk_imports/common/pipelines/uploads_pipeline.rb | 52 | ||||
| -rw-r--r-- | lib/bulk_imports/common/pipelines/wiki_pipeline.rb | 42 | ||||
| -rw-r--r-- | lib/bulk_imports/groups/graphql/get_milestones_query.rb | 56 | ||||
| -rw-r--r-- | lib/bulk_imports/groups/loaders/group_loader.rb | 27 | ||||
| -rw-r--r-- | lib/bulk_imports/groups/stage.rb | 2 | ||||
| -rw-r--r-- | lib/bulk_imports/ndjson_pipeline.rb | 9 | ||||
| -rw-r--r-- | lib/bulk_imports/projects/pipelines/external_pull_requests_pipeline.rb | 15 | ||||
| -rw-r--r-- | lib/bulk_imports/projects/pipelines/merge_requests_pipeline.rb | 19 | ||||
| -rw-r--r-- | lib/bulk_imports/projects/pipelines/protected_branches_pipeline.rb | 15 | ||||
| -rw-r--r-- | lib/bulk_imports/projects/pipelines/repository_pipeline.rb | 10 | ||||
| -rw-r--r-- | lib/bulk_imports/projects/stage.rb | 26 |
13 files changed, 215 insertions, 84 deletions
diff --git a/lib/bulk_imports/common/extractors/ndjson_extractor.rb b/lib/bulk_imports/common/extractors/ndjson_extractor.rb index 6b4acd45ea9..ecd7c08bd25 100644 --- a/lib/bulk_imports/common/extractors/ndjson_extractor.rb +++ b/lib/bulk_imports/common/extractors/ndjson_extractor.rb @@ -7,10 +7,6 @@ module BulkImports include Gitlab::ImportExport::CommandLineUtil include Gitlab::Utils::StrongMemoize - FILE_SIZE_LIMIT = 5.gigabytes - ALLOWED_CONTENT_TYPES = %w(application/gzip application/octet-stream).freeze - EXPORT_DOWNLOAD_URL_PATH = "/%{resource}/%{full_path}/export_relations/download?relation=%{relation}" - def initialize(relation:) @relation = relation @tmp_dir = Dir.mktmpdir @@ -39,33 +35,19 @@ module BulkImports def download_service(tmp_dir, context) @download_service ||= BulkImports::FileDownloadService.new( configuration: context.configuration, - relative_url: relative_resource_url(context), + relative_url: context.entity.relation_download_url_path(relation), dir: tmp_dir, - filename: filename, - file_size_limit: FILE_SIZE_LIMIT, - allowed_content_types: ALLOWED_CONTENT_TYPES + filename: filename ) end def decompression_service(tmp_dir) - @decompression_service ||= BulkImports::FileDecompressionService.new( - dir: tmp_dir, - filename: filename - ) + @decompression_service ||= BulkImports::FileDecompressionService.new(dir: tmp_dir, filename: filename) end def ndjson_reader(tmp_dir) @ndjson_reader ||= Gitlab::ImportExport::Json::NdjsonReader.new(tmp_dir) end - - def relative_resource_url(context) - strong_memoize(:relative_resource_url) do - resource = context.entity.pluralized_name - encoded_full_path = context.entity.encoded_source_full_path - - EXPORT_DOWNLOAD_URL_PATH % { resource: resource, full_path: encoded_full_path, relation: relation } - end - end end end end diff --git a/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb b/lib/bulk_imports/common/pipelines/milestones_pipeline.rb index b2bd14952e7..aea2a04c1c7 100644 --- a/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/milestones_pipeline.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module BulkImports - module Groups + module Common module Pipelines class MilestonesPipeline include NdjsonPipeline diff --git a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb new file mode 100644 index 00000000000..15e126e1646 --- /dev/null +++ b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Pipelines + class UploadsPipeline + include Pipeline + include Gitlab::ImportExport::CommandLineUtil + + FILENAME = 'uploads.tar.gz' + + def extract(context) + download_service(tmp_dir, context).execute + untar_zxf(archive: File.join(tmp_dir, FILENAME), dir: tmp_dir) + upload_file_paths = Dir.glob(File.join(tmp_dir, '**', '*')) + + BulkImports::Pipeline::ExtractedData.new(data: upload_file_paths) + end + + def load(context, file_path) + dynamic_path = FileUploader.extract_dynamic_path(file_path) + + return unless dynamic_path + return if File.directory?(file_path) + + named_captures = dynamic_path.named_captures.symbolize_keys + + UploadService.new(context.portable, File.open(file_path, 'r'), FileUploader, **named_captures).execute + end + + def after_run(_) + FileUtils.remove_entry(tmp_dir) + end + + private + + def download_service(tmp_dir, context) + BulkImports::FileDownloadService.new( + configuration: context.configuration, + relative_url: context.entity.relation_download_url_path('uploads'), + dir: tmp_dir, + filename: FILENAME + ) + end + + def tmp_dir + @tmp_dir ||= Dir.mktmpdir('bulk_imports') + end + end + end + end +end diff --git a/lib/bulk_imports/common/pipelines/wiki_pipeline.rb b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb new file mode 100644 index 00000000000..ccab0b979b2 --- /dev/null +++ b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Pipelines + class WikiPipeline + include Pipeline + + def extract(*) + BulkImports::Pipeline::ExtractedData.new(data: { url: url_from_parent_path(context.entity.source_full_path) }) + end + + def transform(_, data) + data&.slice(:url) + end + + def load(context, data) + return unless context.portable.wiki + + url = data[:url].sub("://", "://oauth2:#{context.configuration.access_token}@") + + Gitlab::UrlBlocker.validate!(url, allow_local_network: allow_local_requests?, allow_localhost: allow_local_requests?) + + context.portable.wiki.ensure_repository + context.portable.wiki.repository.fetch_as_mirror(url) + end + + private + + def url_from_parent_path(parent_path) + wiki_path = parent_path + ".wiki.git" + root = context.configuration.url + Gitlab::Utils.append_path(root, wiki_path) + end + + def allow_local_requests? + Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? + end + end + end + end +end diff --git a/lib/bulk_imports/groups/graphql/get_milestones_query.rb b/lib/bulk_imports/groups/graphql/get_milestones_query.rb deleted file mode 100644 index 5dd5b31cf0e..00000000000 --- a/lib/bulk_imports/groups/graphql/get_milestones_query.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Groups - module Graphql - module GetMilestonesQuery - extend self - - def to_s - <<-'GRAPHQL' - query ($full_path: ID!, $cursor: String, $per_page: Int) { - group(fullPath: $full_path) { - milestones(first: $per_page, after: $cursor, includeDescendants: false) { - page_info: pageInfo { - next_page: endCursor - has_next_page: hasNextPage - } - nodes { - iid - title - description - state - start_date: startDate - due_date: dueDate - created_at: createdAt - updated_at: updatedAt - } - } - } - } - GRAPHQL - end - - def variables(context) - { - full_path: context.entity.source_full_path, - cursor: context.tracker.next_page, - per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE - } - end - - def base_path - %w[data group milestones] - end - - def data_path - base_path << 'nodes' - end - - def page_info_path - base_path << 'page_info' - end - end - end - end -end diff --git a/lib/bulk_imports/groups/loaders/group_loader.rb b/lib/bulk_imports/groups/loaders/group_loader.rb index a631685c2ad..5f5307123a5 100644 --- a/lib/bulk_imports/groups/loaders/group_loader.rb +++ b/lib/bulk_imports/groups/loaders/group_loader.rb @@ -4,10 +4,21 @@ module BulkImports module Groups module Loaders class GroupLoader + GroupCreationError = Class.new(StandardError) + def load(context, data) - return unless user_can_create_group?(context.current_user, data) + path = data['path'] + current_user = context.current_user + destination_namespace = context.entity.destination_namespace + + raise(GroupCreationError, 'Path is missing') unless path.present? + raise(GroupCreationError, 'Destination is not a group') if user_namespace_destination?(destination_namespace) + raise(GroupCreationError, 'User not allowed to create group') unless user_can_create_group?(current_user, data) + raise(GroupCreationError, 'Group exists') if group_exists?(destination_namespace, path) + + group = ::Groups::CreateService.new(current_user, data).execute - group = ::Groups::CreateService.new(context.current_user, data).execute + raise(GroupCreationError, group.errors.full_messages.to_sentence) if group.errors.any? context.entity.update!(group: group) @@ -25,6 +36,18 @@ module BulkImports Ability.allowed?(current_user, :create_group) end end + + def group_exists?(destination_namespace, path) + full_path = destination_namespace.present? ? File.join(destination_namespace, path) : path + + Group.find_by_full_path(full_path).present? + end + + def user_namespace_destination?(destination_namespace) + return false unless destination_namespace.present? + + Namespace.find_by_full_path(destination_namespace)&.user_namespace? + end end end end diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb index a1869b4cb0e..241dd428dd5 100644 --- a/lib/bulk_imports/groups/stage.rb +++ b/lib/bulk_imports/groups/stage.rb @@ -28,7 +28,7 @@ module BulkImports stage: 1 }, milestones: { - pipeline: BulkImports::Groups::Pipelines::MilestonesPipeline, + pipeline: BulkImports::Common::Pipelines::MilestonesPipeline, stage: 1 }, badges: { diff --git a/lib/bulk_imports/ndjson_pipeline.rb b/lib/bulk_imports/ndjson_pipeline.rb index f01ce22a46d..6cc29d63919 100644 --- a/lib/bulk_imports/ndjson_pipeline.rb +++ b/lib/bulk_imports/ndjson_pipeline.rb @@ -13,7 +13,7 @@ module BulkImports relation_hash, relation_index = data relation_definition = import_export_config.top_relation_tree(relation) - deep_transform_relation!(relation_hash, relation, relation_definition) do |key, hash| + relation_object = deep_transform_relation!(relation_hash, relation, relation_definition) do |key, hash| relation_factory.create( relation_index: relation_index, relation_sym: key.to_sym, @@ -25,6 +25,9 @@ module BulkImports excluded_keys: import_export_config.relation_excluded_keys(key) ) end + + relation_object.assign_attributes(portable_class_sym => portable) + relation_object end def load(_, object) @@ -94,6 +97,10 @@ module BulkImports def members_mapper @members_mapper ||= BulkImports::UsersMapper.new(context: context) end + + def portable_class_sym + portable.class.to_s.downcase.to_sym + end end end end diff --git a/lib/bulk_imports/projects/pipelines/external_pull_requests_pipeline.rb b/lib/bulk_imports/projects/pipelines/external_pull_requests_pipeline.rb new file mode 100644 index 00000000000..1f720596c8f --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/external_pull_requests_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ExternalPullRequestsPipeline + include NdjsonPipeline + + relation_name 'external_pull_requests' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/merge_requests_pipeline.rb b/lib/bulk_imports/projects/pipelines/merge_requests_pipeline.rb new file mode 100644 index 00000000000..264bda6e654 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/merge_requests_pipeline.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class MergeRequestsPipeline + include NdjsonPipeline + + relation_name 'merge_requests' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + + def after_run(_) + context.portable.merge_requests.set_latest_merge_request_diff_ids! + end + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/protected_branches_pipeline.rb b/lib/bulk_imports/projects/pipelines/protected_branches_pipeline.rb new file mode 100644 index 00000000000..a570143227d --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/protected_branches_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ProtectedBranchesPipeline + include NdjsonPipeline + + relation_name 'protected_branches' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/repository_pipeline.rb b/lib/bulk_imports/projects/pipelines/repository_pipeline.rb index 86e696f87a4..6bbd4d0688b 100644 --- a/lib/bulk_imports/projects/pipelines/repository_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/repository_pipeline.rb @@ -17,10 +17,18 @@ module BulkImports def load(context, data) url = data['httpUrlToRepo'] url = url.sub("://", "://oauth2:#{context.configuration.access_token}@") + project = context.portable Gitlab::UrlBlocker.validate!(url, allow_local_network: allow_local_requests?, allow_localhost: allow_local_requests?) - context.portable.repository.import_repository(url) + project.ensure_repository + project.repository.fetch_as_mirror(url) + end + + # The initial fetch can bring in lots of loose refs and objects. + # Running a `git gc` will make importing merge requests faster. + def after_run(_) + ::Repositories::HousekeepingService.new(context.portable, :gc).execute end private diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb index 3ada0f406ca..9ccc9efff1d 100644 --- a/lib/bulk_imports/projects/stage.rb +++ b/lib/bulk_imports/projects/stage.rb @@ -19,6 +19,10 @@ module BulkImports pipeline: BulkImports::Common::Pipelines::LabelsPipeline, stage: 2 }, + milestones: { + pipeline: BulkImports::Common::Pipelines::MilestonesPipeline, + stage: 2 + }, issues: { pipeline: BulkImports::Projects::Pipelines::IssuesPipeline, stage: 3 @@ -27,9 +31,29 @@ module BulkImports pipeline: BulkImports::Common::Pipelines::BoardsPipeline, stage: 4 }, + merge_requests: { + pipeline: BulkImports::Projects::Pipelines::MergeRequestsPipeline, + stage: 4 + }, + external_pull_requests: { + pipeline: BulkImports::Projects::Pipelines::ExternalPullRequestsPipeline, + stage: 4 + }, + protected_branches: { + pipeline: BulkImports::Projects::Pipelines::ProtectedBranchesPipeline, + stage: 4 + }, + wiki: { + pipeline: BulkImports::Common::Pipelines::WikiPipeline, + stage: 5 + }, + uploads: { + pipeline: BulkImports::Common::Pipelines::UploadsPipeline, + stage: 5 + }, finisher: { pipeline: BulkImports::Common::Pipelines::EntityFinisher, - stage: 5 + stage: 6 } } end |