diff options
author | Tiago Botelho <tiagonbotelho@hotmail.com> | 2018-05-02 16:35:04 +0300 |
---|---|---|
committer | Tiago Botelho <tiagonbotelho@hotmail.com> | 2018-05-04 18:33:26 +0300 |
commit | bddbcaefc2389e4c61763472cecbea150f10cd75 (patch) | |
tree | 38ba599beed53505dfca09fce7a890bf3b125bb3 /lib | |
parent | 7603beffc916d06039cac63b223d8e6234b5d666 (diff) |
Backports every CE related change from ee-44542 to CE
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/entities.rb | 1 | ||||
-rw-r--r-- | lib/gitlab/background_migration/populate_import_state.rb | 39 | ||||
-rw-r--r-- | lib/gitlab/background_migration/rollback_import_state_data.rb | 40 | ||||
-rw-r--r-- | lib/gitlab/github_import/parallel_importer.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/legacy_github_import/importer.rb | 3 |
5 files changed, 84 insertions, 2 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 086071161b7..a9bab5c56cf 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -136,6 +136,7 @@ module API def self.preload_relation(projects_relation, options = {}) projects_relation.preload(:project_feature, :route) + .preload(:import_state) .preload(namespace: [:route, :owner], tags: :taggings) end diff --git a/lib/gitlab/background_migration/populate_import_state.rb b/lib/gitlab/background_migration/populate_import_state.rb new file mode 100644 index 00000000000..695a2a713c5 --- /dev/null +++ b/lib/gitlab/background_migration/populate_import_state.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This background migration creates all the records on the + # import state table for projects that are considered imports or forks + class PopulateImportState + def perform(start_id, end_id) + move_attributes_data_to_import_state(start_id, end_id) + rescue ActiveRecord::RecordNotUnique + retry + end + + def move_attributes_data_to_import_state(start_id, end_id) + Rails.logger.info("#{self.class.name} - Moving import attributes data to project mirror data table: #{start_id} - #{end_id}") + + ActiveRecord::Base.connection.execute <<~SQL + INSERT INTO project_mirror_data (project_id, status, jid, last_error) + SELECT id, import_status, import_jid, import_error + FROM projects + WHERE projects.import_status != 'none' + AND projects.id BETWEEN #{start_id} AND #{end_id} + AND NOT EXISTS ( + SELECT id + FROM project_mirror_data + WHERE project_id = projects.id + ) + SQL + + ActiveRecord::Base.connection.execute <<~SQL + UPDATE projects + SET import_status = 'none' + WHERE import_status != 'none' + AND id BETWEEN #{start_id} AND #{end_id} + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/rollback_import_state_data.rb b/lib/gitlab/background_migration/rollback_import_state_data.rb new file mode 100644 index 00000000000..a7c986747d8 --- /dev/null +++ b/lib/gitlab/background_migration/rollback_import_state_data.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This background migration migrates all the data of import_state + # back to the projects table for projects that are considered imports or forks + class RollbackImportStateData + def perform(start_id, end_id) + move_attributes_data_to_project(start_id, end_id) + end + + def move_attributes_data_to_project(start_id, end_id) + Rails.logger.info("#{self.class.name} - Moving import attributes data to projects table: #{start_id} - #{end_id}") + + if Gitlab::Database.mysql? + ActiveRecord::Base.connection.execute <<~SQL + UPDATE projects, project_mirror_data + SET + projects.import_status = project_mirror_data.status, + projects.import_jid = project_mirror_data.jid, + projects.import_error = project_mirror_data.last_error + WHERE project_mirror_data.project_id = projects.id + AND project_mirror_data.id BETWEEN #{start_id} AND #{end_id} + SQL + else + ActiveRecord::Base.connection.execute <<~SQL + UPDATE projects + SET + import_status = project_mirror_data.status, + import_jid = project_mirror_data.jid, + import_error = project_mirror_data.last_error + FROM project_mirror_data + WHERE project_mirror_data.project_id = projects.id + AND project_mirror_data.id BETWEEN #{start_id} AND #{end_id} + SQL + end + end + end + end +end diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb index 6da11e6ef08..b02b123c98e 100644 --- a/lib/gitlab/github_import/parallel_importer.rb +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -32,7 +32,8 @@ module Gitlab Gitlab::SidekiqStatus .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) - project.update_column(:import_jid, jid) + project.ensure_import_state + project.import_state&.update_column(:jid, jid) Stage::ImportRepositoryWorker .perform_async(project.id) diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 7edd0ad2033..b04d678cf98 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -78,7 +78,8 @@ module Gitlab def handle_errors return unless errors.any? - project.update_column(:import_error, { + project.ensure_import_state + project.import_state&.update_column(:last_error, { message: 'The remote data could not be fully imported.', errors: errors }.to_json) |