diff options
author | Heinrich Lee Yu <heinrich@gitlab.com> | 2018-12-06 03:51:30 +0300 |
---|---|---|
committer | Heinrich Lee Yu <hleeyu@gmail.com> | 2019-01-07 06:16:58 +0300 |
commit | 3c026971149c95f076b8c50a52ddbfed139d5b20 (patch) | |
tree | 6bf0e1e8e28968887156dd524dc37e498229a9e6 /app | |
parent | 876ab436fabf2f44e2a6912262f980256b7c9736 (diff) |
Import CSV Backend
Process CSV uploads async using a worker then email results
Diffstat (limited to 'app')
-rw-r--r-- | app/controllers/projects/issues_controller.rb | 21 | ||||
-rw-r--r-- | app/mailers/emails/issues.rb | 11 | ||||
-rw-r--r-- | app/mailers/previews/notify_preview.rb | 4 | ||||
-rw-r--r-- | app/policies/project_policy.rb | 1 | ||||
-rw-r--r-- | app/services/issues/import_csv_service.rb | 65 | ||||
-rw-r--r-- | app/services/upload_service.rb | 8 | ||||
-rw-r--r-- | app/views/notify/import_issues_csv_email.html.haml | 18 | ||||
-rw-r--r-- | app/views/notify/import_issues_csv_email.text.erb | 11 | ||||
-rw-r--r-- | app/views/projects/issues/_nav_btns.html.haml | 2 | ||||
-rw-r--r-- | app/views/projects/issues/import_csv/_modal.html.haml | 4 | ||||
-rw-r--r-- | app/workers/all_queues.yml | 1 | ||||
-rw-r--r-- | app/workers/import_issues_csv_worker.rb | 16 |
12 files changed, 150 insertions, 12 deletions
diff --git a/app/controllers/projects/issues_controller.rb b/app/controllers/projects/issues_controller.rb index ae48a02f623..f88eb9e0322 100644 --- a/app/controllers/projects/issues_controller.rb +++ b/app/controllers/projects/issues_controller.rb @@ -155,11 +155,11 @@ class Projects::IssuesController < Projects::ApplicationController def can_create_branch can_create = current_user && can?(current_user, :push_code, @project) && - issue.can_be_worked_on? + @issue.can_be_worked_on? respond_to do |format| format.json do - render json: { can_create_branch: can_create, suggested_branch_name: issue.suggested_branch_name } + render json: { can_create_branch: can_create, suggested_branch_name: @issue.suggested_branch_name } end end end @@ -176,10 +176,19 @@ class Projects::IssuesController < Projects::ApplicationController end def import_csv - redirect_to( - project_issues_path(project), - notice: _("Your issues are being imported. Once finished, you'll get a confirmation email.") - ) + return render_404 unless Feature.enabled?(:issues_import_csv) && can?(current_user, :import_issues, project) + + service = UploadService.new(project, params[:file]) + + if service.execute + ImportIssuesCsvWorker.perform_async(current_user.id, project.id, service.uploader.upload.id) + + flash[:notice] = _("Your issues are being imported. Once finished, you'll get a confirmation email.") + else + flash[:alert] = _("File upload error.") + end + + redirect_to project_issues_path(project) end protected diff --git a/app/mailers/emails/issues.rb b/app/mailers/emails/issues.rb index 370e6d2f90b..654ae211310 100644 --- a/app/mailers/emails/issues.rb +++ b/app/mailers/emails/issues.rb @@ -77,6 +77,17 @@ module Emails mail_answer_thread(issue, issue_thread_options(updated_by_user.id, recipient.id, reason)) end + def import_issues_csv_email(user_id, project_id, results) + @user = User.find(user_id) + @project = Project.find(project_id) + @results = results + + mail(to: @user.notification_email, subject: subject('Imported issues')) do |format| + format.html { render layout: 'mailer' } + format.text { render layout: 'mailer' } + end + end + private def setup_issue_mail(issue_id, recipient_id) diff --git a/app/mailers/previews/notify_preview.rb b/app/mailers/previews/notify_preview.rb index 2ac4610967d..80e0a17c312 100644 --- a/app/mailers/previews/notify_preview.rb +++ b/app/mailers/previews/notify_preview.rb @@ -76,6 +76,10 @@ class NotifyPreview < ActionMailer::Preview Notify.changed_milestone_issue_email(user.id, issue.id, milestone, user.id) end + def import_issues_csv_email + Notify.import_issues_csv_email(user, project, { success: 3, errors: [5, 6, 7], valid_file: true }) + end + def closed_merge_request_email Notify.closed_merge_request_email(user.id, issue.id, user.id).message end diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb index 3146f26bed5..0026a88b972 100644 --- a/app/policies/project_policy.rb +++ b/app/policies/project_policy.rb @@ -223,6 +223,7 @@ class ProjectPolicy < BasePolicy rule { ~request_access_enabled }.prevent :request_access rule { can?(:developer_access) }.policy do + enable :import_issues enable :admin_merge_request enable :admin_milestone enable :update_merge_request diff --git a/app/services/issues/import_csv_service.rb b/app/services/issues/import_csv_service.rb new file mode 100644 index 00000000000..411000a7066 --- /dev/null +++ b/app/services/issues/import_csv_service.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Issues + class ImportCsvService + def initialize(user, project, upload) + @user = user + @project = project + @uploader = upload.build_uploader + @results = { success: 0, errors: [], valid_file: true } + end + + def execute + # Cache remote file locally for processing + @uploader.cache_stored_file! unless @uploader.file_storage? + + process_csv + email_results_to_user + + cleanup_cache unless @uploader.file_storage? + + @results + end + + private + + def process_csv + CSV.foreach(@uploader.file.path, col_sep: detect_col_sep, headers: true).with_index(2) do |row, line_no| + issue = Issues::CreateService.new(@project, @user, title: row[0], description: row[1]).execute + + if issue.persisted? + @results[:success] += 1 + else + @results[:errors].push(line_no) + end + end + rescue ArgumentError, CSV::MalformedCSVError + @results[:valid_file] = false + end + + def email_results_to_user + Notify.import_issues_csv_email(@user.id, @project.id, @results).deliver_now + end + + def detect_col_sep + header = File.open(@uploader.file.path, &:readline) + + if header.include?(",") + "," + elsif header.include?(";") + ";" + elsif header.include?("\t") + "\t" + else + raise CSV::MalformedCSVError + end + end + + def cleanup_cache + cached_file_path = @uploader.file.cache_path + + File.delete(cached_file_path) + Dir.delete(File.dirname(cached_file_path)) + end + end +end diff --git a/app/services/upload_service.rb b/app/services/upload_service.rb index 39909ee4f82..47903eb48c3 100644 --- a/app/services/upload_service.rb +++ b/app/services/upload_service.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class UploadService + attr_accessor :uploader + def initialize(model, file, uploader_class = FileUploader, **uploader_context) @model, @file, @uploader_class, @uploader_context = model, file, uploader_class, uploader_context end @@ -8,10 +10,10 @@ class UploadService def execute return nil unless @file && @file.size <= max_attachment_size - uploader = @uploader_class.new(@model, nil, @uploader_context) - uploader.store!(@file) + @uploader = @uploader_class.new(@model, nil, @uploader_context) + @uploader.store!(@file) - uploader.to_h + @uploader.to_h end private diff --git a/app/views/notify/import_issues_csv_email.html.haml b/app/views/notify/import_issues_csv_email.html.haml new file mode 100644 index 00000000000..a6a797a5fb7 --- /dev/null +++ b/app/views/notify/import_issues_csv_email.html.haml @@ -0,0 +1,18 @@ +- text_style = 'font-size:16px; text-align:center; line-height:30px;' + +%p{ style: text_style } + Your CSV import for project + %a{ href: project_url(@project), style: "color:#3777b0; text-decoration:none;" } + = @project.full_name + has been completed. + +%p{ style: text_style } + #{pluralize(@results[:success], 'issue')} imported successfully. + +- if @results[:errors].present? + %p{ style: text_style } + Errors found on line #{'number'.pluralize(@results[:errors].size)}: #{@results[:errors].join(', ')}. + +- unless @results[:valid_file] + %p{ style: text_style } + Error parsing CSV file. diff --git a/app/views/notify/import_issues_csv_email.text.erb b/app/views/notify/import_issues_csv_email.text.erb new file mode 100644 index 00000000000..54a1762c4ec --- /dev/null +++ b/app/views/notify/import_issues_csv_email.text.erb @@ -0,0 +1,11 @@ +Your CSV import for project <%= @project.full_name %> (<%= project_url(@project) %>) has been completed. + +<%= pluralize(@results[:success], 'issue') %> imported successfully. + +<% if @results[:errors].present? %> +Errors found on line <%= 'number'.pluralize(@results[:errors].size) %>: <%= @results[:errors].join(', ') %>. +<% end %> + +<% unless @results[:valid_file] %> +Error parsing CSV file. +<% end %> diff --git a/app/views/projects/issues/_nav_btns.html.haml b/app/views/projects/issues/_nav_btns.html.haml index 4a4b8a9fcad..fd6559e37ba 100644 --- a/app/views/projects/issues/_nav_btns.html.haml +++ b/app/views/projects/issues/_nav_btns.html.haml @@ -1,5 +1,5 @@ - show_feed_buttons = local_assigns.fetch(:show_feed_buttons, true) -- show_import_button = local_assigns.fetch(:show_import_button, true) && Feature.enabled?(:issues_import_csv) +- show_import_button = local_assigns.fetch(:show_import_button, true) && Feature.enabled?(:issues_import_csv) && can?(current_user, :import_issues, @project) - show_export_button = local_assigns.fetch(:show_export_button, true) .nav-controls.issues-nav-controls diff --git a/app/views/projects/issues/import_csv/_modal.html.haml b/app/views/projects/issues/import_csv/_modal.html.haml index e8576a2f17d..03a6c79ff74 100644 --- a/app/views/projects/issues/import_csv/_modal.html.haml +++ b/app/views/projects/issues/import_csv/_modal.html.haml @@ -13,9 +13,9 @@ %p = _("Your issues will be imported in the background. Once finished, you'll get a confirmation email.") .form-group - = label_tag :file, _('Upload CSV File'), class: 'label-bold' + = label_tag :file, _('Upload CSV file'), class: 'label-bold' %div - = file_field_tag :file, accept: '.csv', required: true + = file_field_tag :file, accept: 'text/csv', required: true %p.text-secondary = _('It must have a header row and at least two columns: the first column is the issue title and the second column is the issue description. The separator is automatically detected.') %p.text-secondary diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index d3cf21db335..223ddc80c88 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -140,3 +140,4 @@ - detect_repository_languages - repository_cleanup - delete_stored_files +- import_issues_csv diff --git a/app/workers/import_issues_csv_worker.rb b/app/workers/import_issues_csv_worker.rb new file mode 100644 index 00000000000..5d8c02a0605 --- /dev/null +++ b/app/workers/import_issues_csv_worker.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class ImportIssuesCsvWorker + include ApplicationWorker + + def perform(current_user_id, project_id, upload_id) + @user = User.find(current_user_id) + @project = Project.find(project_id) + @upload = Upload.find(upload_id) + + importer = Issues::ImportCsvService.new(@user, @project, @upload) + importer.execute + + @upload.destroy + end +end |