diff options
author | Shinya Maeda <shinya@gitlab.com> | 2018-08-03 08:17:22 +0300 |
---|---|---|
committer | Shinya Maeda <shinya@gitlab.com> | 2018-08-03 08:17:22 +0300 |
commit | 1f53cf7cf0cb53b5d69ab141fa9020356e62027e (patch) | |
tree | fac87bd3fcf2900116bbc2777eb6f347f50922f6 /lib/gitlab | |
parent | d867081df185b873667d9eec1184ac92efc8973e (diff) | |
parent | 5f664759b57ef1c0fcfb7e95dfbff6c080a7a72f (diff) |
Merge branch 'master-ce' into artifact-format-v2-with-parser
Diffstat (limited to 'lib/gitlab')
-rw-r--r-- | lib/gitlab/bare_repository_import/importer.rb | 36 | ||||
-rw-r--r-- | lib/gitlab/bare_repository_import/repository.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/cleanup/project_uploads.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/git/repository.rb | 13 | ||||
-rw-r--r-- | lib/gitlab/import_export/command_line_util.rb | 15 | ||||
-rw-r--r-- | lib/gitlab/import_export/file_importer.rb | 24 | ||||
-rw-r--r-- | lib/gitlab/import_export/importer.rb | 12 | ||||
-rw-r--r-- | lib/gitlab/import_export/uploads_manager.rb | 5 | ||||
-rw-r--r-- | lib/gitlab/kubernetes/helm.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/template_helper.rb | 14 |
10 files changed, 84 insertions, 41 deletions
diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 4ca5a78e068..04aa6aab771 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -26,6 +26,12 @@ module Gitlab end end + # This is called from within a rake task only used by Admins, so allow writing + # to STDOUT + def self.log(message) + puts message # rubocop:disable Rails/Output + end + attr_reader :user, :project_name, :bare_repo delegate :log, to: :class @@ -59,11 +65,10 @@ module Gitlab import_type: 'bare_repository', namespace_id: group&.id).execute - if project.persisted? && mv_repo(project) + if project.persisted? && mv_repositories(project) log " * Created #{project.name} (#{project_full_path})".color(:green) project.write_repository_config - Gitlab::Git::Repository.create_hooks(project.repository.path_to_repo, Gitlab.config.gitlab_shell.hooks_path) ProjectCacheWorker.perform_async(project.id) else @@ -74,12 +79,11 @@ module Gitlab project end - def mv_repo(project) - storage_path = storage_path_for_shard(project.repository_storage) - FileUtils.mv(repo_path, project.repository.path_to_repo) + def mv_repositories(project) + mv_repo(bare_repo.repo_path, project.repository) if bare_repo.wiki_exists? - FileUtils.mv(wiki_path, File.join(storage_path, project.disk_path + '.wiki.git')) + mv_repo(bare_repo.wiki_path, project.wiki.repository) end true @@ -89,6 +93,11 @@ module Gitlab false end + def mv_repo(path, repository) + repository.create_from_bundle(bundle(path)) + FileUtils.rm_rf(path) + end + def storage_path_for_shard(shard) Gitlab.config.repositories.storages[shard].legacy_disk_path end @@ -101,10 +110,17 @@ module Gitlab Groups::NestedCreateService.new(user, group_path: group_path).execute end - # This is called from within a rake task only used by Admins, so allow writing - # to STDOUT - def self.log(message) - puts message # rubocop:disable Rails/Output + def bundle(repo_path) + # TODO: we could save some time and disk space by using + # `git bundle create - --all` and streaming the bundle directly to + # Gitaly, rather than writing it on disk first + bundle_path = "#{repo_path}.bundle" + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all) + output, status = Gitlab::Popen.popen(cmd) + + raise output unless status.zero? + + bundle_path end end end diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb index fe267248275..c0c666dfb7b 100644 --- a/lib/gitlab/bare_repository_import/repository.rb +++ b/lib/gitlab/bare_repository_import/repository.rb @@ -1,5 +1,3 @@ -# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/953 -# module Gitlab module BareRepositoryImport class Repository diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb index b88e00311d5..f55ab535efe 100644 --- a/lib/gitlab/cleanup/project_uploads.rb +++ b/lib/gitlab/cleanup/project_uploads.rb @@ -40,7 +40,7 @@ module Gitlab # Accepts a path in the form of "#{hex_secret}/#{filename}" def find_correct_path(upload_path) upload = Upload.find_by(uploader: 'FileUploader', path: upload_path) - return unless upload && upload.local? + return unless upload && upload.local? && upload.model upload.absolute_path rescue => e diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index eb02c7ac8e1..73151e4a4c5 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -39,19 +39,6 @@ module Gitlab ChecksumError = Class.new(StandardError) class << self - # Unlike `new`, `create` takes the repository path - def create(repo_path, bare: true, symlink_hooks_to: nil) - FileUtils.mkdir_p(repo_path, mode: 0770) - - # Equivalent to `git --git-path=#{repo_path} init [--bare]` - repo = Rugged::Repository.init_at(repo_path, bare) - repo.close - - create_hooks(repo_path, symlink_hooks_to) if symlink_hooks_to.present? - - true - end - def create_hooks(repo_path, global_hooks_path) local_hooks_path = File.join(repo_path, 'hooks') real_local_hooks_path = :not_found diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 2f163db936b..3adc44f8044 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -18,6 +18,21 @@ module Gitlab private + def download_or_copy_upload(uploader, upload_path) + if uploader.upload.local? + copy_files(uploader.path, upload_path) + else + download(uploader.url, upload_path) + end + end + + def download(url, upload_path) + File.open(upload_path, 'w') do |file| + # Download (stream) file from the uploader's location + IO.copy_stream(URI.parse(url).open, file) + end + end + def tar_with_options(archive:, dir:, options:) execute(%W(tar -#{options} #{archive} -C #{dir} .)) end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 4c411f4847e..7fd66b4e244 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -10,15 +10,18 @@ module Gitlab new(*args).import end - def initialize(archive_file:, shared:) + def initialize(project:, archive_file:, shared:) + @project = project @archive_file = archive_file @shared = shared end def import mkdir_p(@shared.export_path) + mkdir_p(@shared.archive_path) - remove_symlinks! + remove_symlinks + copy_archive wait_for_archived_file do decompress_archive @@ -27,7 +30,8 @@ module Gitlab @shared.error(e) false ensure - remove_symlinks! + remove_import_file + remove_symlinks end private @@ -51,7 +55,15 @@ module Gitlab result end - def remove_symlinks! + def copy_archive + return if @archive_file + + @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) + + download_or_copy_upload(@project.import_export_upload.import_file, @archive_file) + end + + def remove_symlinks extracted_files.each do |path| FileUtils.rm(path) if File.lstat(path).symlink? end @@ -59,6 +71,10 @@ module Gitlab true end + def remove_import_file + FileUtils.rm_rf(@archive_file) + end + def extracted_files Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| IGNORED_FILENAMES.include?(File.basename(f)) } end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 63cab07324a..4e179f63d8c 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -35,7 +35,8 @@ module Gitlab end def import_file - Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, + Gitlab::ImportExport::FileImporter.import(project: @project, + archive_file: @archive_file, shared: @shared) end @@ -91,7 +92,14 @@ module Gitlab end def remove_import_file - FileUtils.rm_rf(@archive_file) + return unless Gitlab::ImportExport.object_storage? + + upload = @project.import_export_upload + + return unless upload&.import_file&.file + + upload.remove_import_file! + upload.save! end def overwrite_project diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index 1110149712d..07875ebb56a 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -91,10 +91,7 @@ module Gitlab mkdir_p(File.join(uploads_export_path, secret)) - File.open(upload_path, 'w') do |file| - # Download (stream) file from the uploader's location - IO.copy_stream(URI.parse(upload.file.url).open, file) - end + download_or_copy_upload(upload, upload_path) end end end diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 0f0588b8b23..530ccf88053 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -1,7 +1,7 @@ module Gitlab module Kubernetes module Helm - HELM_VERSION = '2.7.0'.freeze + HELM_VERSION = '2.7.2'.freeze NAMESPACE = 'gitlab-managed-apps'.freeze end end diff --git a/lib/gitlab/template_helper.rb b/lib/gitlab/template_helper.rb index 3b8e45e0688..3918abaab45 100644 --- a/lib/gitlab/template_helper.rb +++ b/lib/gitlab/template_helper.rb @@ -2,11 +2,17 @@ module Gitlab module TemplateHelper include Gitlab::Utils::StrongMemoize - def prepare_template_environment(file_path) - return unless file_path.present? + def prepare_template_environment(file) + return unless file&.path.present? - FileUtils.mkdir_p(File.dirname(import_upload_path)) - FileUtils.copy_entry(file_path, import_upload_path) + if Gitlab::ImportExport.object_storage? + params[:import_export_upload] = ImportExportUpload.new(import_file: file) + else + FileUtils.mkdir_p(File.dirname(import_upload_path)) + FileUtils.copy_entry(file.path, import_upload_path) + + params[:import_source] = import_upload_path + end end def import_upload_path |