diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-09-29 03:09:59 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-09-29 03:09:59 +0300 |
commit | a111182345afb4625cf044ca829f946c7fc3d8a6 (patch) | |
tree | 934b134682850661ef81134e72faf6a2fc7917ba /app | |
parent | e994d22d1d0b24e6920a6cf7202601c8318ec0a2 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'app')
-rw-r--r-- | app/assets/javascripts/boards/index.js | 13 | ||||
-rw-r--r-- | app/graphql/resolvers/concerns/looks_ahead.rb | 4 | ||||
-rw-r--r-- | app/models/design_management/design.rb | 10 | ||||
-rw-r--r-- | app/models/design_management/design_collection.rb | 1 | ||||
-rw-r--r-- | app/services/design_management/copy_design_collection.rb | 6 | ||||
-rw-r--r-- | app/services/design_management/copy_design_collection/copy_service.rb | 306 | ||||
-rw-r--r-- | app/services/design_management/copy_design_collection/queue_service.rb | 42 | ||||
-rw-r--r-- | app/services/design_management/design_service.rb | 1 | ||||
-rw-r--r-- | app/services/design_management/runs_design_actions.rb | 11 | ||||
-rw-r--r-- | app/services/design_management/save_designs_service.rb | 8 | ||||
-rw-r--r-- | app/services/issues/move_service.rb | 25 | ||||
-rw-r--r-- | app/services/projects/update_service.rb | 4 | ||||
-rw-r--r-- | app/workers/all_queues.yml | 8 | ||||
-rw-r--r-- | app/workers/design_management/copy_design_collection_worker.rb | 26 | ||||
-rw-r--r-- | app/workers/design_management/new_version_worker.rb | 4 |
15 files changed, 448 insertions, 21 deletions
diff --git a/app/assets/javascripts/boards/index.js b/app/assets/javascripts/boards/index.js index cbd3280f0d3..df6792023ed 100644 --- a/app/assets/javascripts/boards/index.js +++ b/app/assets/javascripts/boards/index.js @@ -132,6 +132,7 @@ export default () => { eventHub.$on('clearDetailIssue', this.clearDetailIssue); sidebarEventHub.$on('toggleSubscription', this.toggleSubscription); eventHub.$on('performSearch', this.performSearch); + eventHub.$on('initialBoardLoad', this.initialBoardLoad); }, beforeDestroy() { eventHub.$off('updateTokens', this.updateTokens); @@ -139,6 +140,7 @@ export default () => { eventHub.$off('clearDetailIssue', this.clearDetailIssue); sidebarEventHub.$off('toggleSubscription', this.toggleSubscription); eventHub.$off('performSearch', this.performSearch); + eventHub.$off('initialBoardLoad', this.initialBoardLoad); }, mounted() { this.filterManager = new FilteredSearchBoards(boardsStore.filter, true, boardsStore.cantEdit); @@ -149,6 +151,12 @@ export default () => { boardsStore.disabled = this.disabled; if (!gon.features.graphqlBoardLists) { + this.initialBoardLoad(); + } + }, + methods: { + ...mapActions(['setInitialBoardData', 'setFilters', 'fetchEpicsSwimlanes', 'resetIssues']), + initialBoardLoad() { boardsStore .all() .then(res => res.data) @@ -161,10 +169,7 @@ export default () => { .catch(() => { Flash(__('An error occurred while fetching the board lists. Please try again.')); }); - } - }, - methods: { - ...mapActions(['setInitialBoardData', 'setFilters', 'fetchEpicsSwimlanes', 'resetIssues']), + }, updateTokens() { this.filterManager.updateTokens(); }, diff --git a/app/graphql/resolvers/concerns/looks_ahead.rb b/app/graphql/resolvers/concerns/looks_ahead.rb index e7230287e13..61f23920ebb 100644 --- a/app/graphql/resolvers/concerns/looks_ahead.rb +++ b/app/graphql/resolvers/concerns/looks_ahead.rb @@ -3,8 +3,6 @@ module LooksAhead extend ActiveSupport::Concern - FEATURE_FLAG = :graphql_lookahead_support - included do attr_accessor :lookahead end @@ -16,8 +14,6 @@ module LooksAhead end def apply_lookahead(query) - return query unless Feature.enabled?(FEATURE_FLAG) - selection = node_selection includes = preloads.each.flat_map do |name, requirements| diff --git a/app/models/design_management/design.rb b/app/models/design_management/design.rb index 57bb250829d..62e4bd6cebc 100644 --- a/app/models/design_management/design.rb +++ b/app/models/design_management/design.rb @@ -167,6 +167,10 @@ module DesignManagement end end + def self.build_full_path(issue, design) + File.join(DesignManagement.designs_directory, "issue-#{issue.iid}", design.filename) + end + def to_ability_name 'design' end @@ -180,7 +184,7 @@ module DesignManagement end def full_path - @full_path ||= File.join(DesignManagement.designs_directory, "issue-#{issue.iid}", filename) + @full_path ||= self.class.build_full_path(issue, self) end def diff_refs @@ -224,6 +228,10 @@ module DesignManagement !interloper.exists? end + def notes_with_associations + notes.includes(:author) + end + private def head_version diff --git a/app/models/design_management/design_collection.rb b/app/models/design_management/design_collection.rb index c48b36588c9..6deba14a6ba 100644 --- a/app/models/design_management/design_collection.rb +++ b/app/models/design_management/design_collection.rb @@ -5,6 +5,7 @@ module DesignManagement attr_reader :issue delegate :designs, :project, to: :issue + delegate :empty?, to: :designs state_machine :copy_state, initial: :ready, namespace: :copy do after_transition any => any, do: :update_stored_copy_state! diff --git a/app/services/design_management/copy_design_collection.rb b/app/services/design_management/copy_design_collection.rb new file mode 100644 index 00000000000..66cf6112062 --- /dev/null +++ b/app/services/design_management/copy_design_collection.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +module DesignManagement + module CopyDesignCollection + end +end diff --git a/app/services/design_management/copy_design_collection/copy_service.rb b/app/services/design_management/copy_design_collection/copy_service.rb new file mode 100644 index 00000000000..7484c43a428 --- /dev/null +++ b/app/services/design_management/copy_design_collection/copy_service.rb @@ -0,0 +1,306 @@ +# frozen_string_literal: true + +# Service to copy a DesignCollection from one Issue to another. +# Copies the DesignCollection's Designs, Versions, and Notes on Designs. +module DesignManagement + module CopyDesignCollection + class CopyService < DesignService + # rubocop: disable CodeReuse/ActiveRecord + def initialize(project, user, params = {}) + super + + @target_issue = params.fetch(:target_issue) + @target_project = @target_issue.project + @target_repository = @target_project.design_repository + @target_design_collection = @target_issue.design_collection + @temporary_branch = "CopyDesignCollectionService_#{SecureRandom.hex}" + + @designs = DesignManagement::Design.unscoped.where(issue: issue).order(:id).load + @versions = DesignManagement::Version.unscoped.where(issue: issue).order(:id).includes(:designs).load + + @sha_attribute = Gitlab::Database::ShaAttribute.new + @shas = [] + @event_enum_map = DesignManagement::DesignAction::EVENT_FOR_GITALY_ACTION.invert + end + # rubocop: enable CodeReuse/ActiveRecord + + def execute + return error('User cannot copy design collection to issue') unless user_can_copy? + return error('Target design collection must first be queued') unless target_design_collection.copy_in_progress? + return error('Design collection has no designs') if designs.empty? + return error('Target design collection already has designs') unless target_design_collection.empty? + + with_temporary_branch do + copy_commits! + + ActiveRecord::Base.transaction do + design_ids = copy_designs! + version_ids = copy_versions! + copy_actions!(design_ids, version_ids) + link_lfs_files! + copy_notes!(design_ids) + finalize! + end + end + + ServiceResponse.success + rescue => error + log_exception(error) + + target_design_collection.error_copy! + + error('Designs were unable to be copied successfully') + end + + private + + attr_reader :designs, :event_enum_map, :sha_attribute, :shas, :temporary_branch, + :target_design_collection, :target_issue, :target_repository, + :target_project, :versions + + alias_method :merge_branch, :target_branch + + def log_exception(exception) + payload = { + issue_id: issue.id, + project_id: project.id, + target_issue_id: target_issue.id, + target_project: target_project.id + } + + Gitlab::ErrorTracking.track_exception(exception, payload) + end + + def error(message) + ServiceResponse.error(message: message) + end + + def user_can_copy? + current_user.can?(:read_design, design_collection) && + current_user.can?(:admin_issue, target_issue) + end + + def with_temporary_branch(&block) + target_repository.create_if_not_exists + + create_master_branch! if target_repository.empty? + create_temporary_branch! + + yield + ensure + remove_temporary_branch! + end + + # A project that does not have any designs will have a blank design + # repository. To create a temporary branch from `master` we need + # create `master` first by adding a file to it. + def create_master_branch! + target_repository.create_file( + current_user, + ".CopyDesignCollectionService_#{Time.now.to_i}", + '.gitlab', + message: "Commit to create #{merge_branch} branch in CopyDesignCollectionService", + branch_name: merge_branch + ) + end + + def create_temporary_branch! + target_repository.add_branch( + current_user, + temporary_branch, + target_repository.root_ref + ) + end + + def remove_temporary_branch! + return unless target_repository.branch_exists?(temporary_branch) + + target_repository.rm_branch(current_user, temporary_branch) + end + + # Merge the temporary branch containing the commits to `master` + # and update the state of the target_design_collection. + def finalize! + source_sha = shas.last + + target_repository.raw.merge( + current_user, + source_sha, + merge_branch, + 'CopyDesignCollectionService finalize merge' + ) { nil } + + target_design_collection.end_copy! + end + + # rubocop: disable CodeReuse/ActiveRecord + def copy_commits! + # Execute another query to include actions and their designs + DesignManagement::Version.unscoped.where(id: versions).order(:id).includes(actions: :design).find_each(batch_size: 100) do |version| + gitaly_actions = version.actions.map do |action| + design = action.design + # Map the raw Action#event enum value to a Gitaly "action" for the + # `Repository#multi_action` call. + gitaly_action_name = @event_enum_map[action.event_before_type_cast] + # `content` will be the LfsPointer file and not the design file, + # and can be nil for deletions. + content = blobs.dig(version.sha, design.filename)&.data + file_path = DesignManagement::Design.build_full_path(target_issue, design) + + { + action: gitaly_action_name, + file_path: file_path, + content: content + }.compact + end + + sha = target_repository.multi_action( + current_user, + branch_name: temporary_branch, + message: commit_message(version), + actions: gitaly_actions + ) + + shas << sha + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def copy_designs! + design_attributes = attributes_config[:design_attributes] + + new_rows = designs.map do |design| + design.attributes.slice(*design_attributes).merge( + issue_id: target_issue.id, + project_id: target_project.id + ) + end + + # TODO Replace `Gitlab::Database.bulk_insert` with `BulkInsertSafe` + # once https://gitlab.com/gitlab-org/gitlab/-/issues/247718 is fixed. + ::Gitlab::Database.bulk_insert( # rubocop:disable Gitlab/BulkInsert + DesignManagement::Design.table_name, + new_rows, + return_ids: true + ) + end + + def copy_versions! + version_attributes = attributes_config[:version_attributes] + # `shas` are the list of Git commits made during the Git copy phase, + # and will be ordered 1:1 with old versions + shas_enum = shas.to_enum + + new_rows = versions.map do |version| + version.attributes.slice(*version_attributes).merge( + issue_id: target_issue.id, + sha: sha_attribute.serialize(shas_enum.next) + ) + end + + # TODO Replace `Gitlab::Database.bulk_insert` with `BulkInsertSafe` + # once https://gitlab.com/gitlab-org/gitlab/-/issues/247718 is fixed. + ::Gitlab::Database.bulk_insert( # rubocop:disable Gitlab/BulkInsert + DesignManagement::Version.table_name, + new_rows, + return_ids: true + ) + end + + # rubocop: disable CodeReuse/ActiveRecord + def copy_actions!(new_design_ids, new_version_ids) + # Create a map of <Old design id> => <New design id> + design_id_map = new_design_ids.each_with_index.to_h do |design_id, i| + [designs[i].id, design_id] + end + + # Create a map of <Old version id> => <New version id> + version_id_map = new_version_ids.each_with_index.to_h do |version_id, i| + [versions[i].id, version_id] + end + + actions = DesignManagement::Action.unscoped.select(:design_id, :version_id, :event).where(design: designs, version: versions) + + new_rows = actions.map do |action| + { + design_id: design_id_map[action.design_id], + version_id: version_id_map[action.version_id], + event: action.event_before_type_cast + } + end + + # We cannot use `BulkInsertSafe` because of the uploader mounted in `Action`. + ::Gitlab::Database.bulk_insert( # rubocop:disable Gitlab/BulkInsert + DesignManagement::Action.table_name, + new_rows + ) + end + # rubocop: enable CodeReuse/ActiveRecord + + def commit_message(version) + "Copy commit #{version.sha} from issue #{issue.to_reference(full: true)}" + end + + # rubocop: disable CodeReuse/ActiveRecord + def copy_notes!(design_ids) + new_designs = DesignManagement::Design.unscoped.find(design_ids) + + # Execute another query to filter only designs with notes + DesignManagement::Design.unscoped.where(id: designs).joins(:notes).find_each(batch_size: 100) do |old_design| + new_design = new_designs.find { |d| d.filename == old_design.filename } + + Notes::CopyService.new(current_user, old_design, new_design).execute + end + end + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord + def link_lfs_files! + oids = blobs.values.flat_map(&:values).map(&:lfs_oid) + repository_type = LfsObjectsProject.repository_types[:design] + + new_rows = LfsObject.where(oid: oids).map do |lfs_object| + { + project_id: target_project.id, + lfs_object_id: lfs_object.id, + repository_type: repository_type + } + end + + # We cannot use `BulkInsertSafe` due to the LfsObjectsProject#update_project_statistics + # callback that fires after_commit. + ::Gitlab::Database.bulk_insert( # rubocop:disable Gitlab/BulkInsert + LfsObjectsProject.table_name, + new_rows, + on_conflict: :do_nothing # Upsert + ) + end + # rubocop: enable CodeReuse/ActiveRecord + + # Blob data is used to find the oids for LfsObjects and to copy to Git. + # Blobs are reasonably small in memory, as their data are LFS Pointer files. + # + # Returns all blobs for the designs as a Hash of `{ Blob#commit_id => { Design#filename => Blob } }` + def blobs + @blobs ||= begin + items = versions.flat_map { |v| v.designs.map { |d| [v.sha, DesignManagement::Design.build_full_path(issue, d)] } } + + repository.blobs_at(items).each_with_object({}) do |blob, h| + design = designs.find { |d| DesignManagement::Design.build_full_path(issue, d) == blob.path } + + h[blob.commit_id] ||= {} + h[blob.commit_id][design.filename] = blob + end + end + end + + def attributes_config + @attributes_config ||= YAML.load_file(attributes_config_file).symbolize_keys + end + + def attributes_config_file + Rails.root.join('lib/gitlab/design_management/copy_design_collection_model_attributes.yml') + end + end + end +end diff --git a/app/services/design_management/copy_design_collection/queue_service.rb b/app/services/design_management/copy_design_collection/queue_service.rb new file mode 100644 index 00000000000..f76917dbe47 --- /dev/null +++ b/app/services/design_management/copy_design_collection/queue_service.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +# Service for setting the initial copy_state on the target DesignCollection +# and queuing a CopyDesignCollectionWorker. +module DesignManagement + module CopyDesignCollection + class QueueService + def initialize(current_user, issue, target_issue) + @current_user = current_user + @issue = issue + @target_issue = target_issue + @target_design_collection = target_issue.design_collection + end + + def execute + return error('User cannot copy designs to issue') unless user_can_copy? + return error('Target design collection copy state must be `ready`') unless target_design_collection.can_start_copy? + + target_design_collection.start_copy! + + DesignManagement::CopyDesignCollectionWorker.perform_async(current_user.id, issue.id, target_issue.id) + + ServiceResponse.success + end + + private + + delegate :design_collection, to: :issue + + attr_reader :current_user, :issue, :target_design_collection, :target_issue + + def error(message) + ServiceResponse.error(message: message) + end + + def user_can_copy? + current_user.can?(:read_design, issue) && + current_user.can?(:admin_issue, target_issue) + end + end + end +end diff --git a/app/services/design_management/design_service.rb b/app/services/design_management/design_service.rb index 54e53609646..5aa2a2f73bc 100644 --- a/app/services/design_management/design_service.rb +++ b/app/services/design_management/design_service.rb @@ -19,6 +19,7 @@ module DesignManagement def collection issue.design_collection end + alias_method :design_collection, :collection def repository collection.repository diff --git a/app/services/design_management/runs_design_actions.rb b/app/services/design_management/runs_design_actions.rb index 4bd6bb45658..ee6aa9286d3 100644 --- a/app/services/design_management/runs_design_actions.rb +++ b/app/services/design_management/runs_design_actions.rb @@ -4,14 +4,15 @@ module DesignManagement module RunsDesignActions NoActions = Class.new(StandardError) - # this concern requires the following methods to be implemented: + # This concern requires the following methods to be implemented: # current_user, target_branch, repository, commit_message # # Before calling `run_actions`, you should ensure the repository exists, by # calling `repository.create_if_not_exists`. # # @raise [NoActions] if actions are empty - def run_actions(actions) + # @return [DesignManagement::Version] + def run_actions(actions, skip_system_notes: false) raise NoActions if actions.empty? sha = repository.multi_action(current_user, @@ -21,14 +22,14 @@ module DesignManagement ::DesignManagement::Version .create_for_designs(actions, sha, current_user) - .tap { |version| post_process(version) } + .tap { |version| post_process(version, skip_system_notes) } end private - def post_process(version) + def post_process(version, skip_system_notes) version.run_after_commit_or_now do - ::DesignManagement::NewVersionWorker.perform_async(id) + ::DesignManagement::NewVersionWorker.perform_async(id, skip_system_notes) end end end diff --git a/app/services/design_management/save_designs_service.rb b/app/services/design_management/save_designs_service.rb index f3ac5b74ca5..8dcc678e87e 100644 --- a/app/services/design_management/save_designs_service.rb +++ b/app/services/design_management/save_designs_service.rb @@ -17,11 +17,14 @@ module DesignManagement return error("Not allowed!") unless can_create_designs? return error("Only #{MAX_FILES} files are allowed simultaneously") if files.size > MAX_FILES return error("Duplicate filenames are not allowed!") if files.map(&:original_filename).uniq.length != files.length + return error("Design copy is in progress") if design_collection.copy_in_progress? uploaded_designs, version = upload_designs! skipped_designs = designs - uploaded_designs create_events + design_collection.reset_copy! + success({ designs: uploaded_designs, version: version, skipped_designs: skipped_designs }) rescue ::ActiveRecord::RecordInvalid => e error(e.message) @@ -35,7 +38,10 @@ module DesignManagement ::DesignManagement::Version.with_lock(project.id, repository) do actions = build_actions - [actions.map(&:design), actions.presence && run_actions(actions)] + [ + actions.map(&:design), + actions.presence && run_actions(actions) + ] end end diff --git a/app/services/issues/move_service.rb b/app/services/issues/move_service.rb index 60e0d1eec3d..e8747b9d6d8 100644 --- a/app/services/issues/move_service.rb +++ b/app/services/issues/move_service.rb @@ -23,11 +23,15 @@ module Issues # to receive service desk emails on the new moved issue. update_service_desk_sent_notifications + queue_copy_designs + new_entity end private + attr_reader :target_project + def update_service_desk_sent_notifications return unless original_entity.from_service_desk? @@ -46,7 +50,7 @@ module Issues new_params = { id: nil, iid: nil, - project: @target_project, + project: target_project, author: original_entity.author, assignee_ids: original_entity.assignee_ids } @@ -58,6 +62,23 @@ module Issues CreateService.new(@target_project, @current_user, new_params).execute(skip_system_notes: true) end + def queue_copy_designs + return unless copy_designs_enabled? && original_entity.designs.present? + + response = DesignManagement::CopyDesignCollection::QueueService.new( + current_user, + original_entity, + new_entity + ).execute + + log_error(response.message) if response.error? + end + + def copy_designs_enabled? + Feature.enabled?(:design_management_copy_designs, old_project) && + Feature.enabled?(:design_management_copy_designs, target_project) + end + def mark_as_moved original_entity.update(moved_to: new_entity) end @@ -75,7 +96,7 @@ module Issues end def add_note_from - SystemNoteService.noteable_moved(new_entity, @target_project, + SystemNoteService.noteable_moved(new_entity, target_project, original_entity, current_user, direction: :from) end diff --git a/app/services/projects/update_service.rb b/app/services/projects/update_service.rb index bb430811497..d44f5e637f1 100644 --- a/app/services/projects/update_service.rb +++ b/app/services/projects/update_service.rb @@ -135,8 +135,8 @@ module Projects end def ensure_wiki_exists - ProjectWiki.new(project, project.owner).wiki - rescue Wiki::CouldNotCreateWikiError + return if project.create_wiki + log_error("Could not create wiki for #{project.full_name}") Gitlab::Metrics.counter(:wiki_can_not_be_created_total, 'Counts the times we failed to create a wiki').increment end diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index d7d9762b45c..6a22844038e 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -1332,6 +1332,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: design_management_copy_design_collection + :feature_category: :design_management + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: design_management_new_version :feature_category: :design_management :has_external_dependencies: diff --git a/app/workers/design_management/copy_design_collection_worker.rb b/app/workers/design_management/copy_design_collection_worker.rb new file mode 100644 index 00000000000..0a6e23fe9da --- /dev/null +++ b/app/workers/design_management/copy_design_collection_worker.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module DesignManagement + class CopyDesignCollectionWorker + include ApplicationWorker + + feature_category :design_management + idempotent! + urgency :low + + def perform(user_id, issue_id, target_issue_id) + user = User.find(user_id) + issue = Issue.find(issue_id) + target_issue = Issue.find(target_issue_id) + + response = DesignManagement::CopyDesignCollection::CopyService.new( + target_issue.project, + user, + issue: issue, + target_issue: target_issue + ).execute + + Gitlab::AppLogger.warn(response.message) if response.error? + end + end +end diff --git a/app/workers/design_management/new_version_worker.rb b/app/workers/design_management/new_version_worker.rb index 3634dcbcebd..4fbf2067be4 100644 --- a/app/workers/design_management/new_version_worker.rb +++ b/app/workers/design_management/new_version_worker.rb @@ -9,10 +9,10 @@ module DesignManagement # `GenerateImageVersionsService` resizing designs worker_resource_boundary :memory - def perform(version_id) + def perform(version_id, skip_system_notes = false) version = DesignManagement::Version.find(version_id) - add_system_note(version) + add_system_note(version) unless skip_system_notes generate_image_versions(version) rescue ActiveRecord::RecordNotFound => e Sidekiq.logger.warn(e) |