diff options
Diffstat (limited to 'app/controllers/import')
| -rw-r--r-- | app/controllers/import/base_controller.rb | 76 | ||||
| -rw-r--r-- | app/controllers/import/bitbucket_controller.rb | 53 | ||||
| -rw-r--r-- | app/controllers/import/bitbucket_server_controller.rb | 77 | ||||
| -rw-r--r-- | app/controllers/import/fogbugz_controller.rb | 33 | ||||
| -rw-r--r-- | app/controllers/import/github_controller.rb | 2 | ||||
| -rw-r--r-- | app/controllers/import/gitlab_controller.rb | 29 | ||||
| -rw-r--r-- | app/controllers/import/gitlab_groups_controller.rb | 65 | ||||
| -rw-r--r-- | app/controllers/import/gitlab_projects_controller.rb | 27 |
8 files changed, 317 insertions, 45 deletions
diff --git a/app/controllers/import/base_controller.rb b/app/controllers/import/base_controller.rb index 04919a4b9d0..afdea4f7c9d 100644 --- a/app/controllers/import/base_controller.rb +++ b/app/controllers/import/base_controller.rb @@ -1,10 +1,86 @@ # frozen_string_literal: true class Import::BaseController < ApplicationController + include ActionView::Helpers::SanitizeHelper + before_action :import_rate_limit, only: [:create] + def status + respond_to do |format| + format.json do + render json: { imported_projects: serialized_imported_projects, + provider_repos: serialized_provider_repos, + incompatible_repos: serialized_incompatible_repos, + namespaces: serialized_namespaces } + end + format.html + end + end + + def realtime_changes + Gitlab::PollingInterval.set_header(response, interval: 3_000) + + render json: already_added_projects.to_json(only: [:id], methods: [:import_status]) + end + + protected + + def importable_repos + raise NotImplementedError + end + + def incompatible_repos + [] + end + + def provider_name + raise NotImplementedError + end + + def provider_url + raise NotImplementedError + end + private + def filter_attribute + :name + end + + def sanitized_filter_param + @filter ||= sanitize(params[:filter]) + end + + def filtered(collection) + return collection unless sanitized_filter_param + + collection.select { |item| item[filter_attribute].include?(sanitized_filter_param) } + end + + def serialized_provider_repos + Import::ProviderRepoSerializer.new(current_user: current_user).represent(importable_repos, provider: provider_name, provider_url: provider_url) + end + + def serialized_incompatible_repos + Import::ProviderRepoSerializer.new(current_user: current_user).represent(incompatible_repos, provider: provider_name, provider_url: provider_url) + end + + def serialized_imported_projects + ProjectSerializer.new.represent(already_added_projects, serializer: :import, provider_url: provider_url) + end + + def already_added_projects + @already_added_projects ||= filtered(find_already_added_projects(provider_name)) + end + + def serialized_namespaces + NamespaceSerializer.new.represent(namespaces) + end + + def namespaces + current_user.manageable_groups_with_routes + end + # rubocop: disable CodeReuse/ActiveRecord def find_already_added_projects(import_type) current_user.created_projects.where(import_type: import_type).with_import_state diff --git a/app/controllers/import/bitbucket_controller.rb b/app/controllers/import/bitbucket_controller.rb index c37e799de62..4886aeb5e3f 100644 --- a/app/controllers/import/bitbucket_controller.rb +++ b/app/controllers/import/bitbucket_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Import::BitbucketController < Import::BaseController + extend ::Gitlab::Utils::Override + include ActionView::Helpers::SanitizeHelper before_action :verify_bitbucket_import_enabled @@ -10,7 +12,7 @@ class Import::BitbucketController < Import::BaseController rescue_from Bitbucket::Error::Unauthorized, with: :bitbucket_unauthorized def callback - response = client.auth_code.get_token(params[:code], redirect_uri: users_import_bitbucket_callback_url) + response = oauth_client.auth_code.get_token(params[:code], redirect_uri: users_import_bitbucket_callback_url) session[:bitbucket_token] = response.token session[:bitbucket_expires_at] = response.expires_at @@ -22,9 +24,10 @@ class Import::BitbucketController < Import::BaseController # rubocop: disable CodeReuse/ActiveRecord def status + return super if Feature.enabled?(:new_import_ui) + bitbucket_client = Bitbucket::Client.new(credentials) repos = bitbucket_client.repos(filter: sanitized_filter_param) - @repos, @incompatible_repos = repos.partition { |repo| repo.valid? } @already_added_projects = find_already_added_projects('bitbucket') @@ -38,6 +41,10 @@ class Import::BitbucketController < Import::BaseController render json: find_jobs('bitbucket') end + def realtime_changes + super + end + def create bitbucket_client = Bitbucket::Client.new(credentials) @@ -59,7 +66,7 @@ class Import::BitbucketController < Import::BaseController project = Gitlab::BitbucketImport::ProjectCreator.new(repo, project_name, target_namespace, current_user, credentials).execute if project.persisted? - render json: ProjectSerializer.new.represent(project) + render json: ProjectSerializer.new.represent(project, serializer: :import) else render json: { errors: project_save_error(project) }, status: :unprocessable_entity end @@ -68,16 +75,50 @@ class Import::BitbucketController < Import::BaseController end end + protected + + # rubocop: disable CodeReuse/ActiveRecord + override :importable_repos + def importable_repos + already_added_projects_names = already_added_projects.map(&:import_source) + + bitbucket_repos.reject { |repo| already_added_projects_names.include?(repo.full_name) || !repo.valid? } + end + # rubocop: enable CodeReuse/ActiveRecord + + override :incompatible_repos + def incompatible_repos + bitbucket_repos.reject { |repo| repo.valid? } + end + + override :provider_name + def provider_name + :bitbucket + end + + override :provider_url + def provider_url + provider.url + end + private - def client - @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options) + def oauth_client + @oauth_client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options) end def provider Gitlab::Auth::OAuth::Provider.config_for('bitbucket') end + def client + @client ||= Bitbucket::Client.new(credentials) + end + + def bitbucket_repos + @bitbucket_repos ||= client.repos(filter: sanitized_filter_param).to_a + end + def options OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys end @@ -91,7 +132,7 @@ class Import::BitbucketController < Import::BaseController end def go_to_bitbucket_for_permissions - redirect_to client.auth_code.authorize_url(redirect_uri: users_import_bitbucket_callback_url) + redirect_to oauth_client.auth_code.authorize_url(redirect_uri: users_import_bitbucket_callback_url) end def bitbucket_unauthorized diff --git a/app/controllers/import/bitbucket_server_controller.rb b/app/controllers/import/bitbucket_server_controller.rb index 5fb7b5dccc5..9aa8110257d 100644 --- a/app/controllers/import/bitbucket_server_controller.rb +++ b/app/controllers/import/bitbucket_server_controller.rb @@ -1,12 +1,16 @@ # frozen_string_literal: true class Import::BitbucketServerController < Import::BaseController + extend ::Gitlab::Utils::Override + include ActionView::Helpers::SanitizeHelper before_action :verify_bitbucket_server_import_enabled before_action :bitbucket_auth, except: [:new, :configure] before_action :validate_import_params, only: [:create] + rescue_from BitbucketServer::Connection::ConnectionError, with: :bitbucket_connection_error + # As a basic sanity check to prevent URL injection, restrict project # repository input and repository slugs to allowed characters. For Bitbucket: # @@ -24,7 +28,7 @@ class Import::BitbucketServerController < Import::BaseController end def create - repo = bitbucket_client.repo(@project_key, @repo_slug) + repo = client.repo(@project_key, @repo_slug) unless repo return render json: { errors: _("Project %{project_repo} could not be found") % { project_repo: "#{@project_key}/#{@repo_slug}" } }, status: :unprocessable_entity @@ -38,15 +42,13 @@ class Import::BitbucketServerController < Import::BaseController project = Gitlab::BitbucketServerImport::ProjectCreator.new(@project_key, @repo_slug, repo, project_name, target_namespace, current_user, credentials).execute if project.persisted? - render json: ProjectSerializer.new.represent(project) + render json: ProjectSerializer.new.represent(project, serializer: :import) else render json: { errors: project_save_error(project) }, status: :unprocessable_entity end else render json: { errors: _('This namespace has already been taken! Please choose another one.') }, status: :unprocessable_entity end - rescue BitbucketServer::Connection::ConnectionError => error - render json: { errors: _("Unable to connect to server: %{error}") % { error: error } }, status: :unprocessable_entity end def configure @@ -59,7 +61,9 @@ class Import::BitbucketServerController < Import::BaseController # rubocop: disable CodeReuse/ActiveRecord def status - @collection = bitbucket_client.repos(page_offset: page_offset, limit: limit_per_page, filter: sanitized_filter_param) + return super if Feature.enabled?(:new_import_ui) + + @collection = client.repos(page_offset: page_offset, limit: limit_per_page, filter: sanitized_filter_param) @repos, @incompatible_repos = @collection.partition { |repo| repo.valid? } # Use the import URL to filter beyond what BaseService#find_already_added_projects @@ -67,10 +71,6 @@ class Import::BitbucketServerController < Import::BaseController already_added_projects_names = @already_added_projects.pluck(:import_source) @repos.reject! { |repo| already_added_projects_names.include?(repo.browse_url) } - rescue BitbucketServer::Connection::ConnectionError => error - flash[:alert] = _("Unable to connect to server: %{error}") % { error: error } - clear_session_data - redirect_to new_import_bitbucket_server_path end # rubocop: enable CodeReuse/ActiveRecord @@ -78,6 +78,38 @@ class Import::BitbucketServerController < Import::BaseController render json: find_jobs('bitbucket_server') end + def realtime_changes + super + end + + protected + + # rubocop: disable CodeReuse/ActiveRecord + override :importable_repos + def importable_repos + # Use the import URL to filter beyond what BaseService#find_already_added_projects + already_added_projects = filter_added_projects('bitbucket_server', bitbucket_repos.map(&:browse_url)) + already_added_projects_names = already_added_projects.map(&:import_source) + + bitbucket_repos.reject { |repo| already_added_projects_names.include?(repo.browse_url) || !repo.valid? } + end + # rubocop: enable CodeReuse/ActiveRecord + + override :incompatible_repos + def incompatible_repos + bitbucket_repos.reject { |repo| repo.valid? } + end + + override :provider_name + def provider_name + :bitbucket_server + end + + override :provider_url + def provider_url + session[bitbucket_server_url_key] + end + private # rubocop: disable CodeReuse/ActiveRecord @@ -86,8 +118,12 @@ class Import::BitbucketServerController < Import::BaseController end # rubocop: enable CodeReuse/ActiveRecord - def bitbucket_client - @bitbucket_client ||= BitbucketServer::Client.new(credentials) + def client + @client ||= BitbucketServer::Client.new(credentials) + end + + def bitbucket_repos + @bitbucket_repos ||= client.repos(page_offset: page_offset, limit: limit_per_page, filter: sanitized_filter_param).to_a end def validate_import_params @@ -153,4 +189,23 @@ class Import::BitbucketServerController < Import::BaseController def sanitized_filter_param sanitize(params[:filter]) end + + def bitbucket_connection_error(error) + flash[:alert] = _("Unable to connect to server: %{error}") % { error: error } + clear_session_data + + respond_to do |format| + format.json do + render json: { + error: { + message: _("Unable to connect to server: %{error}") % { error: error }, + redirect: new_import_bitbucket_server_path + } + }, status: :unprocessable_entity + end + format.html do + redirect_to new_import_bitbucket_server_path + end + end + end end diff --git a/app/controllers/import/fogbugz_controller.rb b/app/controllers/import/fogbugz_controller.rb index 4fb6efde7ff..91779a5d6cc 100644 --- a/app/controllers/import/fogbugz_controller.rb +++ b/app/controllers/import/fogbugz_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Import::FogbugzController < Import::BaseController + extend ::Gitlab::Utils::Override + before_action :verify_fogbugz_import_enabled before_action :user_map, only: [:new_user_map, :create_user_map] before_action :verify_blocked_uri, only: :callback @@ -48,6 +50,8 @@ class Import::FogbugzController < Import::BaseController return redirect_to new_import_fogbugz_path end + return super if Feature.enabled?(:new_import_ui) + @repos = client.repos @already_added_projects = find_already_added_projects('fogbugz') @@ -57,6 +61,10 @@ class Import::FogbugzController < Import::BaseController end # rubocop: enable CodeReuse/ActiveRecord + def realtime_changes + super + end + def jobs render json: find_jobs('fogbugz') end @@ -69,12 +77,35 @@ class Import::FogbugzController < Import::BaseController project = Gitlab::FogbugzImport::ProjectCreator.new(repo, fb_session, current_user.namespace, current_user, umap).execute if project.persisted? - render json: ProjectSerializer.new.represent(project) + render json: ProjectSerializer.new.represent(project, serializer: :import) else render json: { errors: project_save_error(project) }, status: :unprocessable_entity end end + protected + + # rubocop: disable CodeReuse/ActiveRecord + override :importable_repos + def importable_repos + repos = client.repos + + already_added_projects_names = already_added_projects.map(&:import_source) + + repos.reject { |repo| already_added_projects_names.include? repo.name } + end + # rubocop: enable CodeReuse/ActiveRecord + + override :provider_name + def provider_name + :fogbugz + end + + override :provider_url + def provider_url + session[:fogbugz_uri] + end + private def client diff --git a/app/controllers/import/github_controller.rb b/app/controllers/import/github_controller.rb index 4e8ceae75bd..097edcd6075 100644 --- a/app/controllers/import/github_controller.rb +++ b/app/controllers/import/github_controller.rb @@ -76,7 +76,7 @@ class Import::GithubController < Import::BaseController def serialized_provider_repos repos = client_repos.reject { |repo| already_added_project_names.include? repo.full_name } - ProviderRepoSerializer.new(current_user: current_user).represent(repos, provider: provider, provider_url: provider_url) + Import::ProviderRepoSerializer.new(current_user: current_user).represent(repos, provider: provider, provider_url: provider_url) end def serialized_namespaces diff --git a/app/controllers/import/gitlab_controller.rb b/app/controllers/import/gitlab_controller.rb index 5ec8e9e6fc5..a95a67e208c 100644 --- a/app/controllers/import/gitlab_controller.rb +++ b/app/controllers/import/gitlab_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Import::GitlabController < Import::BaseController + extend ::Gitlab::Utils::Override + MAX_PROJECT_PAGES = 15 PER_PAGE_PROJECTS = 100 @@ -16,6 +18,8 @@ class Import::GitlabController < Import::BaseController # rubocop: disable CodeReuse/ActiveRecord def status + return super if Feature.enabled?(:new_import_ui) + @repos = client.projects(starting_page: 1, page_limit: MAX_PROJECT_PAGES, per_page: PER_PAGE_PROJECTS) @already_added_projects = find_already_added_projects('gitlab') @@ -37,7 +41,7 @@ class Import::GitlabController < Import::BaseController project = Gitlab::GitlabImport::ProjectCreator.new(repo, target_namespace, current_user, access_params).execute if project.persisted? - render json: ProjectSerializer.new.represent(project) + render json: ProjectSerializer.new.represent(project, serializer: :import) else render json: { errors: project_save_error(project) }, status: :unprocessable_entity end @@ -46,6 +50,29 @@ class Import::GitlabController < Import::BaseController end end + protected + + # rubocop: disable CodeReuse/ActiveRecord + override :importable_repos + def importable_repos + repos = client.projects(starting_page: 1, page_limit: MAX_PROJECT_PAGES, per_page: PER_PAGE_PROJECTS) + + already_added_projects_names = already_added_projects.map(&:import_source) + + repos.reject { |repo| already_added_projects_names.include? repo["path_with_namespace"] } + end + # rubocop: enable CodeReuse/ActiveRecord + + override :provider_name + def provider_name + :gitlab + end + + override :provider_url + def provider_url + 'https://gitlab.com' + end + private def client diff --git a/app/controllers/import/gitlab_groups_controller.rb b/app/controllers/import/gitlab_groups_controller.rb new file mode 100644 index 00000000000..330af68385e --- /dev/null +++ b/app/controllers/import/gitlab_groups_controller.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +class Import::GitlabGroupsController < ApplicationController + include WorkhorseImportExportUpload + + before_action :ensure_group_import_enabled + before_action :import_rate_limit, only: %i[create] + + def create + unless file_is_valid?(group_params[:file]) + return redirect_back_or_default(options: { alert: s_('GroupImport|Unable to process group import file') }) + end + + group_data = group_params.except(:file).merge( + visibility_level: closest_allowed_visibility_level, + import_export_upload: ImportExportUpload.new(import_file: group_params[:file]) + ) + + group = ::Groups::CreateService.new(current_user, group_data).execute + + if group.persisted? + if Groups::ImportExport::ImportService.new(group: group, user: current_user).async_execute + redirect_to( + group_path(group), + notice: s_("GroupImport|Group '%{group_name}' is being imported.") % { group_name: group.name } + ) + else + redirect_to group_path(group), alert: _("Group import could not be scheduled") + end + else + redirect_back_or_default( + options: { alert: s_("GroupImport|Group could not be imported: %{errors}") % { errors: group.errors.full_messages.to_sentence } } + ) + end + end + + private + + def group_params + params.permit(:path, :name, :parent_id, :file) + end + + def closest_allowed_visibility_level + if group_params[:parent_id].present? + parent_group = Group.find(group_params[:parent_id]) + + Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level) + else + Gitlab::VisibilityLevel::PRIVATE + end + end + + def ensure_group_import_enabled + render_404 unless Feature.enabled?(:group_import_export, @group, default_enabled: true) + end + + def import_rate_limit + if Gitlab::ApplicationRateLimiter.throttled?(:group_import, scope: current_user) + Gitlab::ApplicationRateLimiter.log_request(request, :group_import_request_limit, current_user) + + flash[:alert] = _('This endpoint has been requested too many times. Try again later.') + redirect_to new_group_path + end + end +end diff --git a/app/controllers/import/gitlab_projects_controller.rb b/app/controllers/import/gitlab_projects_controller.rb index 6a3715a4675..39d053347f0 100644 --- a/app/controllers/import/gitlab_projects_controller.rb +++ b/app/controllers/import/gitlab_projects_controller.rb @@ -1,14 +1,11 @@ # frozen_string_literal: true class Import::GitlabProjectsController < Import::BaseController - include WorkhorseRequest + include WorkhorseImportExportUpload before_action :whitelist_query_limiting, only: [:create] before_action :verify_gitlab_project_import_enabled - skip_before_action :verify_authenticity_token, only: [:authorize] - before_action :verify_workhorse_api!, only: [:authorize] - def new @namespace = Namespace.find(project_params[:namespace_id]) return render_404 unless current_user.can?(:create_projects, @namespace) @@ -17,7 +14,7 @@ class Import::GitlabProjectsController < Import::BaseController end def create - unless file_is_valid? + unless file_is_valid?(project_params[:file]) return redirect_back_or_default(options: { alert: _("You need to upload a GitLab project export archive (ending in .gz).") }) end @@ -33,28 +30,8 @@ class Import::GitlabProjectsController < Import::BaseController end end - def authorize - set_workhorse_internal_api_content_type - - authorized = ImportExportUploader.workhorse_authorize( - has_length: false, - maximum_size: Gitlab::CurrentSettings.max_attachment_size.megabytes.to_i) - - render json: authorized - rescue SocketError - render json: _("Error uploading file"), status: :internal_server_error - end - private - def file_is_valid? - return false unless project_params[:file].is_a?(::UploadedFile) - - filename = project_params[:file].original_filename - - ImportExportUploader::EXTENSION_WHITELIST.include?(File.extname(filename).delete('.')) - end - def verify_gitlab_project_import_enabled render_404 unless gitlab_project_import_enabled? end |