diff options
author | Douwe Maan <douwe@gitlab.com> | 2015-04-15 09:53:42 +0300 |
---|---|---|
committer | Douwe Maan <douwe@gitlab.com> | 2015-04-15 09:53:42 +0300 |
commit | 63b21945d2a6d53c7baa65bffcac33ab5fa8bb91 (patch) | |
tree | 250754a33d53845d5f720d97a7648fd9ebe97eb9 | |
parent | 61c79418c46105ecedfca3701b7e4ac9ba6ba601 (diff) | |
parent | 9d00bb08962e26b0c284bcfc5e7ed52cf141d5c6 (diff) |
Merge branch 'google-code-import' into 'master'
Import projects from Google Code.
Resolves #1257.
Issue import logic almost entirely taken from https://gitlab.com/o9000/google-code-to-gitlab ( :hearts: @o9000).
### To do
- [x] List projects from Google Takeout file
- [x] Import Git repository
- [x] Import issues
- [x] Link to correct attachment URL (https://code.google.com/p/support-tools/issues/detail?id=50)
- [x] Handle deleted attachments
- [x] Handle blockedOn attribute
- [x] Add directions on how to get data from Google Takeout
### Import instructions
![Screen_Shot_2015-04-03_at_16.02.21](https://gitlab.com/gitlab-org/gitlab-ce/uploads/4a87038035fbad9441cf613b8cdcc690/Screen_Shot_2015-04-03_at_16.02.21.png)
### Imported issue
![Screen_Shot_2015-04-03_at_16.25.49](https://gitlab.com/gitlab-org/gitlab-ce/uploads/b2c2eaf5ecfcf57b3c48511eb5d26db4/Screen_Shot_2015-04-03_at_16.25.49.png)
See merge request !471
22 files changed, 1382 insertions, 16 deletions
diff --git a/app/assets/stylesheets/base/mixins.scss b/app/assets/stylesheets/base/mixins.scss index ccba65e3fd5..216f25cdcd5 100644 --- a/app/assets/stylesheets/base/mixins.scss +++ b/app/assets/stylesheets/base/mixins.scss @@ -119,6 +119,22 @@ li { line-height: 1.5; } + + a[href*="/uploads/"], a[href*="storage.googleapis.com/google-code-attachments/"] { + &:before { + margin-right: 4px; + + font: normal normal normal 14px/1 FontAwesome; + font-size: inherit; + text-rendering: auto; + -webkit-font-smoothing: antialiased; + content: "\f0c6"; + } + + &:hover:before { + text-decoration: none; + } + } } @mixin str-truncated($max_width: 82%) { diff --git a/app/assets/stylesheets/pages/notes.scss b/app/assets/stylesheets/pages/notes.scss index d66093bc2e5..facd7e19314 100644 --- a/app/assets/stylesheets/pages/notes.scss +++ b/app/assets/stylesheets/pages/notes.scss @@ -62,20 +62,8 @@ ul.notes { word-wrap: break-word; @include md-typography; - a[href*="/uploads/"] { - &:before { - margin-right: 4px; - - font: normal normal normal 14px/1 FontAwesome; - font-size: inherit; - text-rendering: auto; - -webkit-font-smoothing: antialiased; - content: "\f0c6"; - } - - &:hover:before { - text-decoration: none; - } + hr { + margin: 10px 0; } } } diff --git a/app/controllers/import/google_code_controller.rb b/app/controllers/import/google_code_controller.rb new file mode 100644 index 00000000000..fb4ef987367 --- /dev/null +++ b/app/controllers/import/google_code_controller.rb @@ -0,0 +1,111 @@ +class Import::GoogleCodeController < Import::BaseController + before_filter :user_map, only: [:new_user_map, :create_user_map] + + def new + + end + + def callback + dump_file = params[:dump_file] + + unless dump_file.respond_to?(:read) + return redirect_to :back, alert: "You need to upload a Google Takeout archive." + end + + begin + dump = JSON.parse(dump_file.read) + rescue + return redirect_to :back, alert: "The uploaded file is not a valid Google Takeout archive." + end + + client = Gitlab::GoogleCodeImport::Client.new(dump) + unless client.valid? + return redirect_to :back, alert: "The uploaded file is not a valid Google Takeout archive." + end + + session[:google_code_dump] = dump + + if params[:create_user_map] == "1" + redirect_to new_user_map_import_google_code_path + else + redirect_to status_import_google_code_path + end + end + + def new_user_map + + end + + def create_user_map + user_map_json = params[:user_map] + user_map_json = "{}" if user_map_json.blank? + + begin + user_map = JSON.parse(user_map_json) + rescue + flash.now[:alert] = "The entered user map is not a valid JSON user map." + + render "new_user_map" and return + end + + unless user_map.is_a?(Hash) && user_map.all? { |k, v| k.is_a?(String) && v.is_a?(String) } + flash.now[:alert] = "The entered user map is not a valid JSON user map." + + render "new_user_map" and return + end + + session[:google_code_user_map] = user_map + + flash[:notice] = "The user map has been saved. Continue by selecting the projects you want to import." + + redirect_to status_import_google_code_path + end + + def status + unless client.valid? + return redirect_to new_import_google_path + end + + @repos = client.repos + + @already_added_projects = current_user.created_projects.where(import_type: "google_code") + already_added_projects_names = @already_added_projects.pluck(:import_source) + + @repos.reject! { |repo| already_added_projects_names.include? repo.name } + end + + def jobs + jobs = current_user.created_projects.where(import_type: "google_code").to_json(only: [:id, :import_status]) + render json: jobs + end + + def create + @repo_id = params[:repo_id] + repo = client.repo(@repo_id) + @target_namespace = current_user.namespace + @project_name = repo.name + + namespace = @target_namespace + + user_map = session[:google_code_user_map] + + @project = Gitlab::GoogleCodeImport::ProjectCreator.new(repo, namespace, current_user, user_map).execute + end + + private + + def client + @client ||= Gitlab::GoogleCodeImport::Client.new(session[:google_code_dump]) + end + + def user_map + @user_map ||= begin + user_map = client.user_map + + stored_user_map = session[:google_code_user_map] + user_map.update(stored_user_map) if stored_user_map + + Hash[user_map.sort] + end + end +end diff --git a/app/models/project.rb b/app/models/project.rb index dcbafd76475..c4f50e02ddd 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -27,6 +27,7 @@ # import_type :string(255) # import_source :string(255) # avatar :string(255) +# import_data :text # require 'carrierwave/orm/activerecord' @@ -50,6 +51,8 @@ class Project < ActiveRecord::Base default_value_for :wall_enabled, false default_value_for :snippets_enabled, gitlab_config_features.snippets + serialize :import_data, JSON + # set last_activity_at to the same as created_at after_create :set_last_activity_at def set_last_activity_at @@ -185,6 +188,7 @@ class Project < ActiveRecord::Base state :failed after_transition any => :started, do: :add_import_job + after_transition any => :finished, do: :clear_import_data end class << self @@ -262,6 +266,11 @@ class Project < ActiveRecord::Base RepositoryImportWorker.perform_in(2.seconds, id) end + def clear_import_data + self.import_data = nil + self.save + end + def import? import_url.present? end diff --git a/app/views/import/google_code/new.html.haml b/app/views/import/google_code/new.html.haml new file mode 100644 index 00000000000..ce78fec205f --- /dev/null +++ b/app/views/import/google_code/new.html.haml @@ -0,0 +1,60 @@ +%h3.page-title + %i.fa.fa-google + Import projects from Google Code +%hr + += form_tag callback_import_google_code_path, class: 'form-horizontal', multipart: true do + %p + Follow the steps below to export your Google Code project data. + In the next step, you'll be able to select the projects you want to import. + %ol + %li + %p + Go to + #{link_to "Google Takeout", "https://www.google.com/settings/takeout", target: "_blank"}. + %li + %p + Make sure you're logged into the account that owns the projects you'd like to import. + %li + %p + Click the <strong>Select none</strong> button on the right, since we only need "Google Code Project Hosting". + %li + %p + Scroll down to <strong>Google Code Project Hosting</strong> and enable the switch on the right. + %li + %p + Choose <strong>Next</strong> at the bottom of the page. + %li + %p + Leave the "File type" and "Delivery method" options on their default values. + %li + %p + Choose <strong>Create archive</strong> and wait for archiving to complete. + %li + %p + Click the <strong>Download</strong> button and wait for downloading to complete. + %li + %p + Find the downloaded ZIP file and decompress it. + %li + %p + Find the newly extracted <code>Takeout/Google Code Project Hosting/GoogleCodeProjectHosting.json</code> file. + %li + %p + Upload <code>GoogleCodeProjectHosting.json</code> here: + %p + %input{type: "file", name: "dump_file", id: "dump_file"} + %li + %p + Do you want to customize how Google Code email addresses and usernames are imported into GitLab? + %p + = label_tag :create_user_map_0 do + = radio_button_tag :create_user_map, 0, true + No, directly import the existing email addresses and usernames. + %p + = label_tag :create_user_map_1 do + = radio_button_tag :create_user_map, 1, false + Yes, let me map Google Code users to full names or GitLab users. + %li + %p + = submit_tag 'Continue to the next step', class: "btn btn-create" diff --git a/app/views/import/google_code/new_user_map.html.haml b/app/views/import/google_code/new_user_map.html.haml new file mode 100644 index 00000000000..d55fcfc97a8 --- /dev/null +++ b/app/views/import/google_code/new_user_map.html.haml @@ -0,0 +1,20 @@ +%h3.page-title + %i.fa.fa-google + Import projects from Google Code +%hr + += form_tag create_user_map_import_google_code_path, class: 'form-horizontal' do + %p + Customize how Google Code email addresses and usernames are imported into GitLab. + In the next step, you'll be able to select the projects you want to import. + %p + The user map is a JSON document mapping Google Code users (as keys) to the way they will be imported into GitLab (as values). By default the username is masked to ensure users' privacy. + %p + To map a Google Code user to a full name or GitLab user, simply replace the value, e.g. <code>"johnsmith@gmail.com": "John Smith"</code> or <code>"johnsmith@gmail.com": "@johnsmith"</code>. Be sure to preserve the surrounding double quotes and other punctuation. + + .form-group + .col-sm-12 + = text_area_tag :user_map, JSON.pretty_generate(@user_map), class: 'form-control', rows: 15 + + .form-actions + = submit_tag 'Continue to the next step', class: "btn btn-create" diff --git a/app/views/import/google_code/status.html.haml b/app/views/import/google_code/status.html.haml new file mode 100644 index 00000000000..2013b8c03c6 --- /dev/null +++ b/app/views/import/google_code/status.html.haml @@ -0,0 +1,49 @@ +%h3.page-title + %i.fa.fa-google + Import projects from Google Code + +%p.light + Select projects you want to import. +%p.light + Optionally, you can + = link_to "customize", new_user_map_import_google_code_path + how Google Code email addresses and usernames are imported into GitLab. +%hr +%p + = button_tag 'Import all projects', class: "btn btn-success js-import-all" + +%table.table.import-jobs + %thead + %tr + %th From Google Code + %th To GitLab + %th Status + %tbody + - @already_added_projects.each do |project| + %tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"} + %td + = link_to project.import_source, "https://code.google.com/p/#{project.import_source}", target: "_blank" + %td + %strong= link_to project.path_with_namespace, [project.namespace.becomes(Namespace), project] + %td.job-status + - if project.import_status == 'finished' + %span + %i.fa.fa-check + done + - elsif project.import_status == 'started' + %i.fa.fa-spinner.fa-spin + started + - else + = project.human_import_status_name + + - @repos.each do |repo| + %tr{id: "repo_#{repo.id}"} + %td + = link_to repo.name, "https://code.google.com/p/#{repo.name}", target: "_blank" + %td.import-target + = "#{current_user.username}/#{repo.name}" + %td.import-actions.job-status + = button_tag "Import", class: "btn js-add-to-import" + +:coffeescript + new ImporterStatus("#{jobs_import_google_code_path}", "#{import_google_code_path}") diff --git a/app/views/projects/new.html.haml b/app/views/projects/new.html.haml index 69909a8554e..a06c85b4251 100644 --- a/app/views/projects/new.html.haml +++ b/app/views/projects/new.html.haml @@ -62,6 +62,10 @@ %i.icon-gitorious.icon-gitorious-small Gitorious.org + = link_to new_import_google_code_path, class: 'btn' do + %i.fa.fa-google + Google Code + = link_to "#", class: 'btn js-toggle-button' do %i.fa.fa-git %span Any repo by URL diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 437640d2305..e6a50afedb1 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -18,6 +18,8 @@ class RepositoryImportWorker Gitlab::GitlabImport::Importer.new(project).execute elsif project.import_type == 'bitbucket' Gitlab::BitbucketImport::Importer.new(project).execute + elsif project.import_type == 'google_code' + Gitlab::GoogleCodeImport::Importer.new(project).execute else true end diff --git a/config/routes.rb b/config/routes.rb index 63eb7d0fe85..5d176590d70 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -81,6 +81,15 @@ Gitlab::Application.routes.draw do get :callback get :jobs end + + resource :google_code, only: [:create, :new], controller: :google_code do + get :status + post :callback + get :jobs + + get :new_user_map, path: :user_map + post :create_user_map, path: :user_map + end end # diff --git a/db/migrate/20150327150017_add_import_data_to_project.rb b/db/migrate/20150327150017_add_import_data_to_project.rb new file mode 100644 index 00000000000..12c00339eec --- /dev/null +++ b/db/migrate/20150327150017_add_import_data_to_project.rb @@ -0,0 +1,5 @@ +class AddImportDataToProject < ActiveRecord::Migration + def change + add_column :projects, :import_data, :text + end +end diff --git a/db/schema.rb b/db/schema.rb index 7f464d7f973..a54fb4b7098 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -343,6 +343,7 @@ ActiveRecord::Schema.define(version: 20150413192223) do t.integer "star_count", default: 0, null: false t.string "import_type" t.string "import_source" + t.text "import_data" end add_index "projects", ["created_at", "id"], name: "index_projects_on_created_at_and_id", using: :btree diff --git a/lib/gitlab/google_code_import/client.rb b/lib/gitlab/google_code_import/client.rb new file mode 100644 index 00000000000..02f31e45f88 --- /dev/null +++ b/lib/gitlab/google_code_import/client.rb @@ -0,0 +1,48 @@ +module Gitlab + module GoogleCodeImport + class Client + attr_reader :raw_data + + def self.mask_email(author) + parts = author.split("@", 2) + parts[0] = "#{parts[0][0...-3]}..." + parts.join("@") + end + + def initialize(raw_data) + @raw_data = raw_data + end + + def valid? + raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.has_key?("projects") + end + + def repos + @repos ||= raw_data["projects"].map { |raw_repo| GoogleCodeImport::Repository.new(raw_repo) }.select(&:git?) + end + + def repo(id) + repos.find { |repo| repo.id == id } + end + + def user_map + user_map = Hash.new { |hash, user| hash[user] = self.class.mask_email(user) } + + repos.each do |repo| + next unless repo.valid? && repo.issues + + repo.issues.each do |raw_issue| + # Touching is enough to add the entry and masked email. + user_map[raw_issue["author"]["name"]] + + raw_issue["comments"]["items"].each do |raw_comment| + user_map[raw_comment["author"]["name"]] + end + end + end + + Hash[user_map.sort] + end + end + end +end diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb new file mode 100644 index 00000000000..777472cf3c5 --- /dev/null +++ b/lib/gitlab/google_code_import/importer.rb @@ -0,0 +1,365 @@ +module Gitlab + module GoogleCodeImport + class Importer + attr_reader :project, :repo + + def initialize(project) + @project = project + @repo = GoogleCodeImport::Repository.new(project.import_data["repo"]) + + @closed_statuses = [] + @known_labels = Set.new + end + + def execute + return true unless repo.valid? + + import_status_labels + + import_labels + + import_issues + + true + end + + private + + def user_map + @user_map ||= begin + user_map = Hash.new { |hash, user| hash[user] = Client.mask_email(user) } + + stored_user_map = project.import_data["user_map"] + user_map.update(stored_user_map) if stored_user_map + + user_map + end + end + + def import_status_labels + repo.raw_data["issuesConfig"]["statuses"].each do |status| + closed = !status["meansOpen"] + @closed_statuses << status["status"] if closed + + name = nice_status_name(status["status"]) + create_label(name) + @known_labels << name + end + end + + def import_labels + repo.raw_data["issuesConfig"]["labels"].each do |label| + name = nice_label_name(label["label"]) + create_label(name) + @known_labels << name + end + end + + def import_issues + return unless repo.issues + + last_id = 0 + + deleted_issues = [] + + repo.issues.each do |raw_issue| + while raw_issue["id"] > last_id + 1 + last_id += 1 + + issue = project.issues.create!( + title: "Deleted issue", + description: "*This issue has been deleted*", + author_id: project.creator_id, + state: "closed" + ) + deleted_issues << issue + end + last_id = raw_issue["id"] + + author = user_map[raw_issue["author"]["name"]] + date = DateTime.parse(raw_issue["published"]).to_formatted_s(:long) + + comments = raw_issue["comments"]["items"] + issue_comment = comments.shift + + content = format_content(issue_comment["content"]) + attachments = format_attachments(raw_issue["id"], 0, issue_comment["attachments"]) + + body = format_issue_body(author, date, content, attachments) + + labels = [] + raw_issue["labels"].each do |label| + name = nice_label_name(label) + labels << name + + unless @known_labels.include?(name) + create_label(name) + @known_labels << name + end + end + labels << nice_status_name(raw_issue["status"]) + + assignee_id = nil + if raw_issue.has_key?("owner") + username = user_map[raw_issue["owner"]["name"]] + + if username.start_with?("@") + username = username[1..-1] + + if user = User.find_by(username: username) + assignee_id = user.id + end + end + end + + issue = project.issues.create!( + title: raw_issue["title"], + description: body, + author_id: project.creator_id, + assignee_id: assignee_id, + state: raw_issue["state"] == "closed" ? "closed" : "opened" + ) + issue.add_labels_by_names(labels) + + import_issue_comments(issue, comments) + end + + deleted_issues.each(&:destroy!) + end + + def import_issue_comments(issue, comments) + comments.each do |raw_comment| + next if raw_comment.has_key?("deletedBy") + + content = format_content(raw_comment["content"]) + updates = format_updates(raw_comment["updates"]) + attachments = format_attachments(issue.iid, raw_comment["id"], raw_comment["attachments"]) + + next if content.blank? && updates.blank? && attachments.blank? + + author = user_map[raw_comment["author"]["name"]] + date = DateTime.parse(raw_comment["published"]).to_formatted_s(:long) + + body = format_issue_comment_body( + raw_comment["id"], + author, + date, + content, + updates, + attachments + ) + + issue.notes.create!( + project_id: project.id, + author_id: project.creator_id, + note: body + ) + end + end + + def nice_label_color(name) + case name + when /\AComponent:/ + "#fff39e" + when /\AOpSys:/ + "#e2e2e2" + when /\AMilestone:/ + "#fee3ff" + + when *@closed_statuses.map { |s| nice_status_name(s) } + "#cfcfcf" + when "Status: New" + "#428bca" + when "Status: Accepted" + "#5cb85c" + when "Status: Started" + "#8e44ad" + + when "Priority: Critical" + "#ffcfcf" + when "Priority: High" + "#deffcf" + when "Priority: Medium" + "#fff5cc" + when "Priority: Low" + "#cfe9ff" + + when "Type: Defect" + "#d9534f" + when "Type: Enhancement" + "#44ad8e" + when "Type: Task" + "#4b6dd0" + when "Type: Review" + "#8e44ad" + when "Type: Other" + "#7f8c8d" + else + "#e2e2e2" + end + end + + def nice_label_name(name) + name.sub("-", ": ") + end + + def nice_status_name(name) + "Status: #{name}" + end + + def linkify_issues(s) + s.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2') + end + + def escape_for_markdown(s) + s = s.gsub("*", "\\*") + s = s.gsub("#", "\\#") + s = s.gsub("`", "\\`") + s = s.gsub(":", "\\:") + s = s.gsub("-", "\\-") + s = s.gsub("+", "\\+") + s = s.gsub("_", "\\_") + s = s.gsub("(", "\\(") + s = s.gsub(")", "\\)") + s = s.gsub("[", "\\[") + s = s.gsub("]", "\\]") + s = s.gsub("<", "\\<") + s = s.gsub(">", "\\>") + s = s.gsub("\r", "") + s = s.gsub("\n", " \n") + s + end + + def create_label(name) + color = nice_label_color(name) + project.labels.create!(name: name, color: color) + end + + def format_content(raw_content) + linkify_issues(escape_for_markdown(raw_content)) + end + + def format_updates(raw_updates) + updates = [] + + if raw_updates.has_key?("status") + updates << "*Status: #{raw_updates["status"]}*" + end + + if raw_updates.has_key?("owner") + updates << "*Owner: #{user_map[raw_updates["owner"]]}*" + end + + if raw_updates.has_key?("cc") + cc = raw_updates["cc"].map do |l| + deleted = l.start_with?("-") + l = l[1..-1] if deleted + l = user_map[l] + l = "~~#{l}~~" if deleted + l + end + + updates << "*Cc: #{cc.join(", ")}*" + end + + if raw_updates.has_key?("labels") + labels = raw_updates["labels"].map do |l| + deleted = l.start_with?("-") + l = l[1..-1] if deleted + l = nice_label_name(l) + l = "~~#{l}~~" if deleted + l + end + + updates << "*Labels: #{labels.join(", ")}*" + end + + if raw_updates.has_key?("mergedInto") + updates << "*Merged into: ##{raw_updates["mergedInto"]}*" + end + + if raw_updates.has_key?("blockedOn") + blocked_ons = raw_updates["blockedOn"].map do |raw_blocked_on| + name, id = raw_blocked_on.split(":", 2) + if name == project.import_source + "##{id}" + else + "#{project.namespace.path}/#{name}##{id}" + end + end + updates << "*Blocked on: #{blocked_ons.join(", ")}*" + end + + if raw_updates.has_key?("blocking") + blockings = raw_updates["blocking"].map do |raw_blocked_on| + name, id = raw_blocked_on.split(":", 2) + if name == project.import_source + "##{id}" + else + "#{project.namespace.path}/#{name}##{id}" + end + end + updates << "*Blocking: #{blockings.join(", ")}*" + end + + updates + end + + def format_attachments(issue_id, comment_id, raw_attachments) + return [] unless raw_attachments + + raw_attachments.map do |attachment| + next if attachment["isDeleted"] + + filename = attachment["fileName"] + link = "https://storage.googleapis.com/google-code-attachments/#{@repo.name}/issue-#{issue_id}/comment-#{comment_id}/#{filename}" + + text = "[#{filename}](#{link})" + text = "!#{text}" if filename =~ /\.(png|jpg|jpeg|gif|bmp|tiff)\z/ + text + end.compact + end + + def format_issue_comment_body(id, author, date, content, updates, attachments) + body = [] + body << "*Comment #{id} by #{author} on #{date}*" + body << "---" + + if content.blank? + content = "*(No comment has been entered for this change)*" + end + body << content + + if updates.any? + body << "---" + body += updates + end + + if attachments.any? + body << "---" + body += attachments + end + + body.join("\n\n") + end + + def format_issue_body(author, date, content, attachments) + body = [] + body << "*By #{author} on #{date}*" + body << "---" + + if content.blank? + content = "*(No description has been entered for this issue)*" + end + body << content + + if attachments.any? + body << "---" + body += attachments + end + + body.join("\n\n") + end + end + end +end diff --git a/lib/gitlab/google_code_import/project_creator.rb b/lib/gitlab/google_code_import/project_creator.rb new file mode 100644 index 00000000000..7ac4387d79d --- /dev/null +++ b/lib/gitlab/google_code_import/project_creator.rb @@ -0,0 +1,46 @@ +module Gitlab + module GoogleCodeImport + class ProjectCreator + attr_reader :repo, :namespace, :current_user, :user_map + + def initialize(repo, namespace, current_user, user_map = nil) + @repo = repo + @namespace = namespace + @current_user = current_user + @user_map = user_map + end + + def execute + import_data = { + "repo" => repo.raw_data, + "user_map" => user_map + } + + @project = Project.new( + name: repo.name, + path: repo.name, + description: repo.summary, + namespace: namespace, + creator: current_user, + visibility_level: Gitlab::VisibilityLevel::PUBLIC, + import_type: "google_code", + import_source: repo.name, + import_url: repo.import_url, + import_data: import_data + ) + + if @project.save! + @project.reload + + if @project.import_failed? + @project.import_retry + else + @project.import_start + end + end + + @project + end + end + end +end diff --git a/lib/gitlab/google_code_import/repository.rb b/lib/gitlab/google_code_import/repository.rb new file mode 100644 index 00000000000..ad33fc2cad2 --- /dev/null +++ b/lib/gitlab/google_code_import/repository.rb @@ -0,0 +1,43 @@ +module Gitlab + module GoogleCodeImport + class Repository + attr_accessor :raw_data + + def initialize(raw_data) + @raw_data = raw_data + end + + def valid? + raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#project" + end + + def id + raw_data["externalId"] + end + + def name + raw_data["name"] + end + + def summary + raw_data["summary"] + end + + def description + raw_data["description"] + end + + def git? + raw_data["versionControlSystem"] == "git" + end + + def import_url + raw_data["repositoryUrls"].first + end + + def issues + raw_data["issues"] && raw_data["issues"]["items"] + end + end + end +end diff --git a/spec/controllers/import/google_code_controller_spec.rb b/spec/controllers/import/google_code_controller_spec.rb new file mode 100644 index 00000000000..037cddb4600 --- /dev/null +++ b/spec/controllers/import/google_code_controller_spec.rb @@ -0,0 +1,47 @@ +require 'spec_helper' + +describe Import::GoogleCodeController do + let(:user) { create(:user) } + let(:dump_file) { fixture_file_upload(Rails.root + 'spec/fixtures/GoogleCodeProjectHosting.json', 'application/json') } + + before do + sign_in(user) + end + + describe "POST callback" do + it "stores Google Takeout dump list in session" do + post :callback, dump_file: dump_file + + expect(session[:google_code_dump]).to be_a(Hash) + expect(session[:google_code_dump]["kind"]).to eq("projecthosting#user") + expect(session[:google_code_dump]).to have_key("projects") + end + end + + describe "GET status" do + before do + @repo = OpenStruct.new(name: 'vim') + controller.stub_chain(:client, :valid?).and_return(true) + end + + it "assigns variables" do + @project = create(:project, import_type: 'google_code', creator_id: user.id) + controller.stub_chain(:client, :repos).and_return([@repo]) + + get :status + + expect(assigns(:already_added_projects)).to eq([@project]) + expect(assigns(:repos)).to eq([@repo]) + end + + it "does not show already added project" do + @project = create(:project, import_type: 'google_code', creator_id: user.id, import_source: 'vim') + controller.stub_chain(:client, :repos).and_return([@repo]) + + get :status + + expect(assigns(:already_added_projects)).to eq([@project]) + expect(assigns(:repos)).to eq([]) + end + end +end diff --git a/spec/fixtures/GoogleCodeProjectHosting.json b/spec/fixtures/GoogleCodeProjectHosting.json new file mode 100644 index 00000000000..d05e77271ae --- /dev/null +++ b/spec/fixtures/GoogleCodeProjectHosting.json @@ -0,0 +1,407 @@ +{ + "kind" : "projecthosting#user", + "id" : "@WRRVSlFXARlCVgB6", + "projects" : [ { + "kind" : "projecthosting#project", + "name" : "pmn", + "externalId" : "pmn", + "htmlLink" : "/p/pmn/", + "summary" : "Shows an icon in the system tray when you have new emails", + "description" : "IMAP client that shows an icon in the system tray when you have new emails.", + "labels" : [ "Mail" ], + "versionControlSystem" : "svn", + "repositoryUrls" : [ "https://pmn.googlecode.com/svn/" ], + "issuesConfig" : { + "kind" : "projecthosting#projectIssueConfig", + "statuses" : [ { + "status" : "New", + "meansOpen" : true, + "description" : "Issue has not had initial review yet" + }, { + "status" : "Accepted", + "meansOpen" : true, + "description" : "Problem reproduced / Need acknowledged" + }, { + "status" : "Started", + "meansOpen" : true, + "description" : "Work on this issue has begun" + }, { + "status" : "Fixed", + "meansOpen" : false, + "description" : "Developer made source code changes, QA should verify" + }, { + "status" : "Verified", + "meansOpen" : false, + "description" : "QA has verified that the fix worked" + }, { + "status" : "Invalid", + "meansOpen" : false, + "description" : "This was not a valid issue report" + }, { + "status" : "Duplicate", + "meansOpen" : false, + "description" : "This report duplicates an existing issue" + }, { + "status" : "WontFix", + "meansOpen" : false, + "description" : "We decided to not take action on this issue" + }, { + "status" : "Done", + "meansOpen" : false, + "description" : "The requested non-coding task was completed" + } ], + "labels" : [ { + "label" : "Type-Defect", + "description" : "Report of a software defect" + }, { + "label" : "Type-Enhancement", + "description" : "Request for enhancement" + }, { + "label" : "Type-Task", + "description" : "Work item that doesn't change the code or docs" + }, { + "label" : "Type-Review", + "description" : "Request for a source code review" + }, { + "label" : "Type-Other", + "description" : "Some other kind of issue" + }, { + "label" : "Priority-Critical", + "description" : "Must resolve in the specified milestone" + }, { + "label" : "Priority-High", + "description" : "Strongly want to resolve in the specified milestone" + }, { + "label" : "Priority-Medium", + "description" : "Normal priority" + }, { + "label" : "Priority-Low", + "description" : "Might slip to later milestone" + }, { + "label" : "OpSys-All", + "description" : "Affects all operating systems" + }, { + "label" : "OpSys-Windows", + "description" : "Affects Windows users" + }, { + "label" : "OpSys-Linux", + "description" : "Affects Linux users" + }, { + "label" : "OpSys-OSX", + "description" : "Affects Mac OS X users" + }, { + "label" : "Milestone-Release1.0", + "description" : "All essential functionality working" + }, { + "label" : "Component-UI", + "description" : "Issue relates to program UI" + }, { + "label" : "Component-Logic", + "description" : "Issue relates to application logic" + }, { + "label" : "Component-Persistence", + "description" : "Issue relates to data storage components" + }, { + "label" : "Component-Scripts", + "description" : "Utility and installation scripts" + }, { + "label" : "Component-Docs", + "description" : "Issue relates to end-user documentation" + }, { + "label" : "Security", + "description" : "Security risk to users" + }, { + "label" : "Performance", + "description" : "Performance issue" + }, { + "label" : "Usability", + "description" : "Affects program usability" + }, { + "label" : "Maintainability", + "description" : "Hinders future changes" + } ], + "prompts" : [ { + "name" : "Defect report from user", + "title" : "Enter one-line summary", + "description" : "What steps will reproduce the problem?\n1. \n2. \n3. \n\nWhat is the expected output? What do you see instead?\n\n\nWhat version of the product are you using? On what operating system?\n\n\nPlease provide any additional information below.\n", + "titleMustBeEdited" : true, + "status" : "New", + "labels" : [ "Type-Defect", "Priority-Medium" ] + }, { + "name" : "Defect report from developer", + "title" : "Enter one-line summary", + "description" : "What steps will reproduce the problem?\n1. \n2. \n3. \n\nWhat is the expected output? What do you see instead?\n\n\nPlease use labels and text to provide additional information.\n", + "titleMustBeEdited" : true, + "status" : "Accepted", + "labels" : [ "Type-Defect", "Priority-Medium" ], + "membersOnly" : true + }, { + "name" : "Review request", + "title" : "Code review request", + "description" : "Branch name:\n\nPurpose of code changes on this branch:\n\n\nWhen reviewing my code changes, please focus on:\n\n\nAfter the review, I'll merge this branch into:\n/trunk\n", + "status" : "New", + "labels" : [ "Type-Review", "Priority-Medium" ], + "membersOnly" : true, + "defaultToMember" : false + } ], + "defaultPromptForMembers" : 1, + "defaultPromptForNonMembers" : 0 + }, + "role" : "owner", + "members" : [ { + "kind" : "projecthosting#issuePerson", + "name" : "mrovi9000", + "htmlLink" : "https://code.google.com/u/106736353629303906862/" + } ], + "issues" : { + "kind" : "projecthosting#issueList", + "totalResults" : 0, + "items" : [ ] + } + }, { + "kind" : "projecthosting#project", + "name" : "tint2", + "externalId" : "tint2", + "htmlLink" : "/p/tint2/", + "summary" : "tint2 is a lightweight panel/taskbar.", + "description" : "tint2 is a simple _*panel/taskbar*_ unintrusive and light (memory / cpu / aestetic). <br>We follow freedesktop specifications.\r\n \r\n=== 0.11 features ===\r\n * panel with taskbar, systray, clock and battery status\r\n * easy to customize : color/transparency on font, icon, border and background\r\n * pager like capability : send task from one workspace to another, switch workspace\r\n * multi-monitor capability : one panel per monitor, show task from current monitor\r\n * customize mouse event\r\n * window manager's menu\r\n * tooltip\r\n * autohide\r\n * clock timezones\r\n * real & fake transparency with autodetection of composite manager\r\n * panel's theme switcher 'tint2conf' \r\n\r\n=== Other project ===\r\n * Lightweight volume control http://softwarebakery.com/maato/volumeicon.html\r\n * Lightweight calendar http://code.google.com/p/gsimplecal/\r\n * Graphical config tool http://code.google.com/p/tintwizard/\r\n * Command line theme switcher http://github.com/dbbolton/scripts/blob/master/tint2theme\r\n\r\n\r\n=== Snapshot SVN ===\r\n\r\nhttp://img252.imageshack.us/img252/1433/wallpaper2td.jpg\r\n\r\n\r\n", + "labels" : [ "taskbar", "panel", "lightweight", "desktop", "openbox", "pager", "tint2" ], + "versionControlSystem" : "git", + "repositoryUrls" : [ "https://tint2.googlecode.com/git/" ], + "issuesConfig" : { + "kind" : "projecthosting#projectIssueConfig", + "defaultColumns" : [ "ID", "Status", "Type", "Milestone", "Priority", "Component", "Owner", "Summary", "Modified", "Stars" ], + "defaultSorting" : [ "-ID" ], + "statuses" : [ { + "status" : "New", + "meansOpen" : true, + "description" : "Issue has not had initial review yet" + }, { + "status" : "NeedInfo", + "meansOpen" : true, + "description" : "More information is needed before deciding what action should be taken" + }, { + "status" : "Accepted", + "meansOpen" : true, + "description" : "A Defect that a developer has reproduced or an Enhancement that a developer has committed to addressing" + }, { + "status" : "Wishlist", + "meansOpen" : true, + "description" : "An Enhancement which is valid, but no developers have committed to addressing" + }, { + "status" : "Started", + "meansOpen" : true, + "description" : "Work on this issue has begun" + }, { + "status" : "Fixed", + "meansOpen" : false, + "description" : "Work has completed" + }, { + "status" : "Invalid", + "meansOpen" : false, + "description" : "This was not a valid issue report" + }, { + "status" : "Duplicate", + "meansOpen" : false, + "description" : "This report duplicates an existing issue" + }, { + "status" : "WontFix", + "meansOpen" : false, + "description" : "We decided to not take action on this issue" + }, { + "status" : "Incomplete", + "meansOpen" : false, + "description" : "Not enough information and no activity for a long period of time" + } ], + "labels" : [ { + "label" : "Type-Defect", + "description" : "Report of a software defect" + }, { + "label" : "Type-Enhancement", + "description" : "Request for enhancement" + }, { + "label" : "Type-Task", + "description" : "Work item that does not change the code" + }, { + "label" : "Type-Review", + "description" : "Request for a source code review" + }, { + "label" : "Type-Other", + "description" : "Some other kind of issue" + }, { + "label" : "Milestone-0.12", + "description" : "Fix should be included in release 0.12" + }, { + "label" : "Priority-Critical", + "description" : "Must resolve in the specified milestone" + }, { + "label" : "Priority-High", + "description" : "Strongly want to resolve in the specified milestone" + }, { + "label" : "Priority-Medium", + "description" : "Normal priority" + }, { + "label" : "Priority-Low", + "description" : "Might slip to later milestone" + }, { + "label" : "OpSys-All", + "description" : "Affects all operating systems" + }, { + "label" : "OpSys-Windows", + "description" : "Affects Windows users" + }, { + "label" : "OpSys-Linux", + "description" : "Affects Linux users" + }, { + "label" : "OpSys-OSX", + "description" : "Affects Mac OS X users" + }, { + "label" : "Security", + "description" : "Security risk to users" + }, { + "label" : "Performance", + "description" : "Performance issue" + }, { + "label" : "Usability", + "description" : "Affects program usability" + }, { + "label" : "Maintainability", + "description" : "Hinders future changes" + }, { + "label" : "Component-Panel", + "description" : "Issue relates to the panel (e.g. positioning, hiding, transparency)" + }, { + "label" : "Component-Taskbar", + "description" : "Issue relates to the taskbar (e.g. tasks, multiple desktops)" + }, { + "label" : "Component-Battery", + "description" : "Issue relates to the battery" + }, { + "label" : "Component-Systray", + "description" : "Issue relates to the system tray" + }, { + "label" : "Component-Clock", + "description" : "Issue relates to the clock" + }, { + "label" : "Component-Launcher", + "description" : "Issue relates to the launcher" + }, { + "label" : "Component-Tint2conf", + "description" : "Issue relates to the configuration GUI (tint2conf)" + }, { + "label" : "Component-Docs", + "description" : "Issue relates to end-user documentation" + }, { + "label" : "Component-New", + "description" : "Issue describes a new component proposal" + } ], + "prompts" : [ { + "name" : "Defect report from user", + "title" : "Enter one-line summary", + "description" : "What steps will reproduce the problem?\n1.\n2.\n3.\n\nWhat is the expected output? What do you see instead?\n\n\nWhat version of the product are you using? On what operating system?\n\n\nWhich window manager (e.g. openbox, xfwm, metacity, mutter, kwin) or\nwhich desktop environment (e.g. Gnome 2, Gnome 3, LXDE, XFCE, KDE)\nare you using?\n\n\nPlease provide any additional information below. It might be helpful\nto attach your tint2rc file (usually located at ~/.config/tint2/tint2rc).", + "titleMustBeEdited" : true, + "status" : "New", + "labels" : [ "Priority-Medium" ], + "defaultToMember" : true + }, { + "name" : "Defect report from developer", + "title" : "Enter one-line summary", + "description" : "What steps will reproduce the problem?\n1.\n2.\n3.\n\nWhat is the expected output? What do you see instead?\n\n\nPlease use labels and text to provide additional information.", + "titleMustBeEdited" : true, + "status" : "Accepted", + "labels" : [ "Type-Defect", "Priority-Medium" ], + "membersOnly" : true, + "defaultToMember" : true + }, { + "name" : "Review request", + "title" : "Code review request", + "description" : "Purpose of code changes on this branch:\n\n\nWhen reviewing my code changes, please focus on:\n\n\nAfter the review, I'll merge this branch into:\n/trunk", + "status" : "New", + "labels" : [ "Type-Review", "Priority-Medium" ], + "membersOnly" : true, + "defaultToMember" : true + } ], + "defaultPromptForMembers" : 1, + "defaultPromptForNonMembers" : 0, + "usersCanSetLabels" : false + }, + "role" : "owner", + "issues" : { + "kind" : "projecthosting#issueList", + "totalResults" : 473, + "items" : [ { + "kind" : "projecthosting#issue", + "id" : 169, + "title" : "Scrolling through tasks", + "summary" : "Scrolling through tasks", + "stars" : 1, + "starred" : false, + "status" : "Fixed", + "state" : "closed", + "labels" : [ "Type-Enhancement", "Priority-Medium" ], + "author" : { + "kind" : "projecthosting#issuePerson", + "name" : "schattenpr...", + "htmlLink" : "https://code.google.com/u/106498139506637530000/" + }, + "owner" : { + "kind" : "projecthosting#issuePerson", + "name" : "thilo...", + "htmlLink" : "https://code.google.com/u/104224918623172014000/" + }, + "updated" : "2009-11-18T05:14:58.000Z", + "published" : "2009-11-18T00:20:19.000Z", + "closed" : "2009-11-18T05:14:58.000Z", + "projectId" : "tint2", + "canComment" : true, + "canEdit" : true, + "comments" : { + "kind" : "projecthosting#issueCommentList", + "totalResults" : 2, + "items" : [ { + "id" : 0, + "kind" : "projecthosting#issueComment", + "author" : { + "kind" : "projecthosting#issuePerson", + "name" : "schattenpr...", + "htmlLink" : "https://code.google.com/u/10649813950663753000/" + }, + "content" : "I like to scroll through the tasks with my scrollwheel (like in fluxbox). \r\n\r\nPatch is attached that adds two new mouse-actions (next_task+prev_task) \r\nthat can be used for exactly that purpose. \r\n\r\nall the best!", + "published" : "2009-11-18T00:20:19.000Z", + "updates" : { + "kind" : "projecthosting#issueCommentUpdate" + }, + "canDelete" : true, + "attachments" : [ { + "attachmentId" : "8901002890399325565", + "fileName" : "tint2_task_scrolling.diff", + "fileSize" : 3059, + "mimetype" : "text/x-c++; charset=us-ascii" + }, { + "attachmentId" : "000", + "fileName" : "screenshot.png", + "fileSize" : 0, + "mimetype" : "image/png" + } ] + }, { + "id" : 1, + "kind" : "projecthosting#issueComment", + "author" : { + "kind" : "projecthosting#issuePerson", + "name" : "thilo...", + "htmlLink" : "https://code.google.com/u/104224918623172014000/" + }, + "content" : "applied, thanks.\r\n", + "published" : "2009-11-18T05:14:58.000Z", + "updates" : { + "kind" : "projecthosting#issueCommentUpdate", + "status" : "Fixed", + "labels" : [ "-Type-Defect", "Type-Enhancement" ] + }, + "canDelete" : true + } ] + } + } ] + } + } ] +} diff --git a/spec/lib/gitlab/gitorious_import/project_creator.rb b/spec/lib/gitlab/gitorious_import/project_creator_spec.rb index cf2318bb3a2..90271ea15e3 100644 --- a/spec/lib/gitlab/gitorious_import/project_creator.rb +++ b/spec/lib/gitlab/gitorious_import/project_creator_spec.rb @@ -9,8 +9,7 @@ describe Gitlab::GitoriousImport::ProjectCreator do allow_any_instance_of(Project).to receive(:add_import_job) project_creator = Gitlab::GitoriousImport::ProjectCreator.new(repo, namespace, user) - project_creator.execute - project = Project.last + project = project_creator.execute expect(project.name).to eq("Bar Baz Qux") expect(project.path).to eq("bar-baz-qux") diff --git a/spec/lib/gitlab/google_code_import/client_spec.rb b/spec/lib/gitlab/google_code_import/client_spec.rb new file mode 100644 index 00000000000..d2bf871daa8 --- /dev/null +++ b/spec/lib/gitlab/google_code_import/client_spec.rb @@ -0,0 +1,34 @@ +require "spec_helper" + +describe Gitlab::GoogleCodeImport::Client do + let(:raw_data) { JSON.parse(File.read(Rails.root.join("spec/fixtures/GoogleCodeProjectHosting.json"))) } + subject { described_class.new(raw_data) } + + describe "#valid?" do + context "when the data is valid" do + it "returns true" do + expect(subject).to be_valid + end + end + + context "when the data is invalid" do + let(:raw_data) { "No clue" } + + it "returns true" do + expect(subject).to_not be_valid + end + end + end + + describe "#repos" do + it "returns only Git repositories" do + expect(subject.repos.length).to eq(1) + end + end + + describe "#repo" do + it "returns the referenced repository" do + expect(subject.repo("tint2").name).to eq("tint2") + end + end +end diff --git a/spec/lib/gitlab/google_code_import/importer_spec.rb b/spec/lib/gitlab/google_code_import/importer_spec.rb new file mode 100644 index 00000000000..107ba49962a --- /dev/null +++ b/spec/lib/gitlab/google_code_import/importer_spec.rb @@ -0,0 +1,80 @@ +require "spec_helper" + +describe Gitlab::GoogleCodeImport::Importer do + let(:mapped_user) { create(:user, username: "thilo123") } + let(:raw_data) { JSON.parse(File.read(Rails.root.join("spec/fixtures/GoogleCodeProjectHosting.json"))) } + let(:client) { Gitlab::GoogleCodeImport::Client.new(raw_data) } + let(:import_data) { + { + "repo" => client.repo("tint2").raw_data, + "user_map" => { + "thilo..." => "@#{mapped_user.username}" + } + } + } + let(:project) { create(:project, import_data: import_data) } + subject { described_class.new(project) } + + describe "#execute" do + + it "imports status labels" do + subject.execute + + %w(New NeedInfo Accepted Wishlist Started Fixed Invalid Duplicate WontFix Incomplete).each do |status| + expect(project.labels.find_by(name: "Status: #{status}")).to_not be_nil + end + end + + it "imports labels" do + subject.execute + + %w( + Type-Defect Type-Enhancement Type-Task Type-Review Type-Other Milestone-0.12 Priority-Critical + Priority-High Priority-Medium Priority-Low OpSys-All OpSys-Windows OpSys-Linux OpSys-OSX Security + Performance Usability Maintainability Component-Panel Component-Taskbar Component-Battery + Component-Systray Component-Clock Component-Launcher Component-Tint2conf Component-Docs Component-New + ).each do |label| + label.sub!("-", ": ") + expect(project.labels.find_by(name: label)).to_not be_nil + end + end + + it "imports issues" do + subject.execute + + issue = project.issues.first + expect(issue).to_not be_nil + expect(issue.iid).to eq(169) + expect(issue.author).to eq(project.creator) + expect(issue.assignee).to eq(mapped_user) + expect(issue.state).to eq("closed") + expect(issue.label_names).to include("Priority: Medium") + expect(issue.label_names).to include("Status: Fixed") + expect(issue.label_names).to include("Type: Enhancement") + expect(issue.title).to eq("Scrolling through tasks") + expect(issue.state).to eq("closed") + expect(issue.description).to include("schattenpr...") + expect(issue.description).to include("November 18, 2009 00:20") + expect(issue.description).to include('I like to scroll through the tasks with my scrollwheel \(like in fluxbox\).') + expect(issue.description).to include('Patch is attached that adds two new mouse\-actions \(next\_taskprev\_task\)') + expect(issue.description).to include('that can be used for exactly that purpose.') + expect(issue.description).to include('all the best!') + expect(issue.description).to include('[tint2_task_scrolling.diff](https://storage.googleapis.com/google-code-attachments/tint2/issue-169/comment-0/tint2_task_scrolling.diff)') + expect(issue.description).to include('![screenshot.png](https://storage.googleapis.com/google-code-attachments/tint2/issue-169/comment-0/screenshot.png)') + end + + it "imports issue comments" do + subject.execute + + note = project.issues.first.notes.first + expect(note).to_not be_nil + expect(note.note).to include("Comment 1") + expect(note.note).to include("@#{mapped_user.username}") + expect(note.note).to include("November 18, 2009 05:14") + expect(note.note).to include("applied, thanks.") + expect(note.note).to include("Status: Fixed") + expect(note.note).to include("~~Type: Defect~~") + expect(note.note).to include("Type: Enhancement") + end + end +end diff --git a/spec/lib/gitlab/google_code_import/project_creator_spec.rb b/spec/lib/gitlab/google_code_import/project_creator_spec.rb new file mode 100644 index 00000000000..6cdd4dbbfb1 --- /dev/null +++ b/spec/lib/gitlab/google_code_import/project_creator_spec.rb @@ -0,0 +1,23 @@ +require 'spec_helper' + +describe Gitlab::GoogleCodeImport::ProjectCreator do + let(:user) { create(:user) } + let(:repo) { + Gitlab::GoogleCodeImport::Repository.new( + "name" => 'vim', + "summary" => 'VI Improved', + "repositoryUrls" => [ "https://vim.googlecode.com/git/" ] + ) + } + let(:namespace) { create(:namespace) } + + it 'creates project' do + allow_any_instance_of(Project).to receive(:add_import_job) + + project_creator = Gitlab::GoogleCodeImport::ProjectCreator.new(repo, namespace, user) + project = project_creator.execute + + expect(project.import_url).to eq("https://vim.googlecode.com/git/") + expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::PUBLIC) + end +end |