Add latest changes from gitlab-org/gitlab@master

14 files changed, 552 insertions, 14 deletions

diff --git a/app/services/design_management/delete_designs_service.rb b/app/services/design_management/delete_designs_service.rb
new file mode 100644
index 00000000000..e69f07db5bf
--- /dev/null
+++ b/app/services/design_management/delete_designs_service.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module DesignManagement
+  class DeleteDesignsService < DesignService
+    include RunsDesignActions
+    include OnSuccessCallbacks
+
+    def initialize(project, user, params = {})
+      super
+
+      @designs = params.fetch(:designs)
+    end
+
+    def execute
+      return error('Forbidden!') unless can_delete_designs?
+
+      version = delete_designs!
+
+      success(version: version)
+    end
+
+    def commit_message
+      n = designs.size
+
+      <<~MSG
+      Removed #{n} #{'designs'.pluralize(n)}
+
+      #{formatted_file_list}
+      MSG
+    end
+
+    private
+
+    attr_reader :designs
+
+    def delete_designs!
+      DesignManagement::Version.with_lock(project.id, repository) do
+        run_actions(build_actions)
+      end
+    end
+
+    def can_delete_designs?
+      Ability.allowed?(current_user, :destroy_design, issue)
+    end
+
+    def build_actions
+      designs.map { |d| design_action(d) }
+    end
+
+    def design_action(design)
+      on_success { counter.count(:delete) }
+
+      DesignManagement::DesignAction.new(design, :delete)
+    end
+
+    def counter
+      ::Gitlab::UsageDataCounters::DesignsCounter
+    end
+
+    def formatted_file_list
+      designs.map { |design| "- #{design.full_path}" }.join("\n")
+    end
+  end
+end
+
+DesignManagement::DeleteDesignsService.prepend_if_ee('EE::DesignManagement::DeleteDesignsService')
diff --git a/app/services/design_management/design_service.rb b/app/services/design_management/design_service.rb
new file mode 100644
index 00000000000..54e53609646
--- /dev/null
+++ b/app/services/design_management/design_service.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module DesignManagement
+  class DesignService < ::BaseService
+    def initialize(project, user, params = {})
+      super
+
+      @issue = params.fetch(:issue)
+    end
+
+    # Accessors common to all subclasses:
+
+    attr_reader :issue
+
+    def target_branch
+      repository.root_ref || "master"
+    end
+
+    def collection
+      issue.design_collection
+    end
+
+    def repository
+      collection.repository
+    end
+
+    def project
+      issue.project
+    end
+  end
+end
diff --git a/app/services/design_management/generate_image_versions_service.rb b/app/services/design_management/generate_image_versions_service.rb
new file mode 100644
index 00000000000..213aac164ff
--- /dev/null
+++ b/app/services/design_management/generate_image_versions_service.rb
@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module DesignManagement
+  # This service generates smaller image versions for `DesignManagement::Design`
+  # records within a given `DesignManagement::Version`.
+  class GenerateImageVersionsService < DesignService
+    # We limit processing to only designs with file sizes that don't
+    # exceed `MAX_DESIGN_SIZE`.
+    #
+    # Note, we may be able to remove checking this limit, if when we come to
+    # implement a file size limit for designs, there are no designs that
+    # exceed 40MB on GitLab.com
+    #
+    # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22860#note_281780387
+    MAX_DESIGN_SIZE = 40.megabytes.freeze
+
+    def initialize(version)
+      super(version.project, version.author, issue: version.issue)
+
+      @version = version
+    end
+
+    def execute
+      # rubocop: disable CodeReuse/ActiveRecord
+      version.actions.includes(:design).each do |action|
+        generate_image(action)
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      success(version: version)
+    end
+
+    private
+
+    attr_reader :version
+
+    def generate_image(action)
+      raw_file = get_raw_file(action)
+
+      unless raw_file
+        log_error("No design file found for Action: #{action.id}")
+        return
+      end
+
+      # Skip attempting to process images that would be rejected by CarrierWave.
+      return unless DesignManagement::DesignV432x230Uploader::MIME_TYPE_WHITELIST.include?(raw_file.content_type)
+
+      # Store and process the file
+      action.image_v432x230.store!(raw_file)
+      action.save!
+    rescue CarrierWave::UploadError => e
+      Gitlab::ErrorTracking.track_exception(e, project_id: project.id, design_id: action.design_id, version_id: action.version_id)
+      log_error(e.message)
+    end
+
+    # Returns the `CarrierWave::SanitizedFile` of the original design file
+    def get_raw_file(action)
+      raw_files_by_path[action.design.full_path]
+    end
+
+    # Returns the `Carrierwave:SanitizedFile` instances for all of the original
+    # design files, mapping to { design.filename => `Carrierwave::SanitizedFile` }.
+    #
+    # As design files are stored in Git LFS, the only way to retrieve their original
+    # files is to first fetch the LFS pointer file data from the Git design repository.
+    # The LFS pointer file data contains an "OID" that lets us retrieve `LfsObject`
+    # records, which have an Uploader (`LfsObjectUploader`) for the original design file.
+    def raw_files_by_path
+      @raw_files_by_path ||= begin
+        LfsObject.for_oids(blobs_by_oid.keys).each_with_object({}) do |lfs_object, h|
+          blob = blobs_by_oid[lfs_object.oid]
+          file = lfs_object.file.file
+          # The `CarrierWave::SanitizedFile` is loaded without knowing the `content_type`
+          # of the file, due to the file not having an extension.
+          #
+          # Set the content_type from the `Blob`.
+          file.content_type = blob.content_type
+          h[blob.path] = file
+        end
+      end
+    end
+
+    # Returns the `Blob`s that correspond to the design files in the repository.
+    #
+    # All design `Blob`s are LFS Pointer files, and are therefore small amounts
+    # of data to load.
+    #
+    # `Blob`s whose size are above a certain threshold: `MAX_DESIGN_SIZE`
+    # are filtered out.
+    def blobs_by_oid
+      @blobs ||= begin
+        items = version.designs.map { |design| [version.sha, design.full_path] }
+        blobs = repository.blobs_at(items)
+        blobs.reject! { |blob| blob.lfs_size > MAX_DESIGN_SIZE }
+        blobs.index_by(&:lfs_oid)
+      end
+    end
+  end
+end
diff --git a/app/services/design_management/on_success_callbacks.rb b/app/services/design_management/on_success_callbacks.rb
new file mode 100644
index 00000000000..be55890a02d
--- /dev/null
+++ b/app/services/design_management/on_success_callbacks.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module DesignManagement
+  module OnSuccessCallbacks
+    def on_success(&block)
+      success_callbacks.push(block)
+    end
+
+    def success(*_)
+      while cb = success_callbacks.pop
+        cb.call
+      end
+
+      super
+    end
+
+    private
+
+    def success_callbacks
+      @success_callbacks ||= []
+    end
+  end
+end
diff --git a/app/services/design_management/runs_design_actions.rb b/app/services/design_management/runs_design_actions.rb
new file mode 100644
index 00000000000..4bd6bb45658
--- /dev/null
+++ b/app/services/design_management/runs_design_actions.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module DesignManagement
+  module RunsDesignActions
+    NoActions = Class.new(StandardError)
+
+    # this concern requires the following methods to be implemented:
+    #   current_user, target_branch, repository, commit_message
+    #
+    # Before calling `run_actions`, you should ensure the repository exists, by
+    # calling `repository.create_if_not_exists`.
+    #
+    # @raise [NoActions] if actions are empty
+    def run_actions(actions)
+      raise NoActions if actions.empty?
+
+      sha = repository.multi_action(current_user,
+                                    branch_name: target_branch,
+                                    message: commit_message,
+                                    actions: actions.map(&:gitaly_action))
+
+      ::DesignManagement::Version
+        .create_for_designs(actions, sha, current_user)
+        .tap { |version| post_process(version) }
+    end
+
+    private
+
+    def post_process(version)
+      version.run_after_commit_or_now do
+        ::DesignManagement::NewVersionWorker.perform_async(id)
+      end
+    end
+  end
+end
diff --git a/app/services/design_management/save_designs_service.rb b/app/services/design_management/save_designs_service.rb
new file mode 100644
index 00000000000..a09c19bc885
--- /dev/null
+++ b/app/services/design_management/save_designs_service.rb
@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module DesignManagement
+  class SaveDesignsService < DesignService
+    include RunsDesignActions
+    include OnSuccessCallbacks
+
+    MAX_FILES = 10
+
+    def initialize(project, user, params = {})
+      super
+
+      @files = params.fetch(:files)
+    end
+
+    def execute
+      return error("Not allowed!") unless can_create_designs?
+      return error("Only #{MAX_FILES} files are allowed simultaneously") if files.size > MAX_FILES
+
+      uploaded_designs, version = upload_designs!
+      skipped_designs = designs - uploaded_designs
+
+      success({ designs: uploaded_designs, version: version, skipped_designs: skipped_designs })
+    rescue ::ActiveRecord::RecordInvalid => e
+      error(e.message)
+    end
+
+    private
+
+    attr_reader :files
+
+    def upload_designs!
+      ::DesignManagement::Version.with_lock(project.id, repository) do
+        actions = build_actions
+
+        [actions.map(&:design), actions.presence && run_actions(actions)]
+      end
+    end
+
+    # Returns `Design` instances that correspond with `files`.
+    # New `Design`s will be created where a file name does not match
+    # an existing `Design`
+    def designs
+      @designs ||= files.map do |file|
+        collection.find_or_create_design!(filename: file.original_filename)
+      end
+    end
+
+    def build_actions
+      files.zip(designs).flat_map do |(file, design)|
+        Array.wrap(build_design_action(file, design))
+      end
+    end
+
+    def build_design_action(file, design)
+      content = file_content(file, design.full_path)
+      return if design_unchanged?(design, content)
+
+      action = new_file?(design) ? :create : :update
+      on_success { ::Gitlab::UsageDataCounters::DesignsCounter.count(action) }
+
+      DesignManagement::DesignAction.new(design, action, content)
+    end
+
+    # Returns true if the design file is the same as its latest version
+    def design_unchanged?(design, content)
+      content == existing_blobs[design]&.data
+    end
+
+    def commit_message
+      <<~MSG
+      Updated #{files.size} #{'designs'.pluralize(files.size)}
+
+      #{formatted_file_list}
+      MSG
+    end
+
+    def formatted_file_list
+      filenames.map { |name| "- #{name}" }.join("\n")
+    end
+
+    def filenames
+      @filenames ||= files.map(&:original_filename)
+    end
+
+    def can_create_designs?
+      Ability.allowed?(current_user, :create_design, issue)
+    end
+
+    def new_file?(design)
+      !existing_blobs[design]
+    end
+
+    def file_content(file, full_path)
+      transformer = ::Lfs::FileTransformer.new(project, repository, target_branch)
+      transformer.new_file(full_path, file.to_io).content
+    end
+
+    # Returns the latest blobs for the designs as a Hash of `{ Design => Blob }`
+    def existing_blobs
+      @existing_blobs ||= begin
+        items = designs.map { |d| ['HEAD', d.full_path] }
+
+        repository.blobs_at(items).each_with_object({}) do |blob, h|
+          design = designs.find { |d| d.full_path == blob.path }
+
+          h[design] = blob
+        end
+      end
+    end
+  end
+end
+
+DesignManagement::SaveDesignsService.prepend_if_ee('EE::DesignManagement::SaveDesignsService')
diff --git a/app/services/lfs/file_transformer.rb b/app/services/lfs/file_transformer.rb
index 88f59b820a4..69d33e1c873 100644
--- a/app/services/lfs/file_transformer.rb
+++ b/app/services/lfs/file_transformer.rb
@@ -5,8 +5,7 @@ module Lfs
   #        return a transformed result with `content` and `encoding` to commit.
   #
   #        The `repository` passed to the initializer can be a Repository or
-  #        a DesignManagement::Repository (an EE-specific class that inherits
-  #        from Repository).
+  #        class that inherits from Repository.
   #
   #        The `repository_type` property will be one of the types named in
   #        `Gitlab::GlRepository.types`, and is recorded on the `LfsObjectsProject`
diff --git a/app/services/notes/post_process_service.rb b/app/services/notes/post_process_service.rb
index 53b3b57f4af..bc86118a150 100644
--- a/app/services/notes/post_process_service.rb
+++ b/app/services/notes/post_process_service.rb
@@ -16,10 +16,18 @@ module Notes
         return if @note.for_personal_snippet?
 
         @note.create_cross_references!
+        ::SystemNoteService.design_discussion_added(@note) if create_design_discussion_system_note?
+
         execute_note_hooks
       end
     end
 
+    private
+
+    def create_design_discussion_system_note?
+      @note && @note.for_design? && @note.start_of_discussion?
+    end
+
     def hook_data
       Gitlab::DataBuilder::Note.build(@note, @note.author)
     end
diff --git a/app/services/projects/hashed_storage/base_repository_service.rb b/app/services/projects/hashed_storage/base_repository_service.rb
index d81aa4de9f1..065bf8725be 100644
--- a/app/services/projects/hashed_storage/base_repository_service.rb
+++ b/app/services/projects/hashed_storage/base_repository_service.rb
@@ -8,13 +8,15 @@ module Projects
     class BaseRepositoryService < BaseService
       include Gitlab::ShellAdapter
 
-      attr_reader :old_disk_path, :new_disk_path, :old_storage_version, :logger, :move_wiki
+      attr_reader :old_disk_path, :new_disk_path, :old_storage_version,
+                  :logger, :move_wiki, :move_design
 
       def initialize(project:, old_disk_path:, logger: nil)
         @project = project
         @logger = logger || Gitlab::AppLogger
         @old_disk_path = old_disk_path
         @move_wiki = has_wiki?
+        @move_design = has_design?
       end
 
       protected
@@ -23,6 +25,10 @@ module Projects
         gitlab_shell.repository_exists?(project.repository_storage, "#{old_wiki_disk_path}.git")
       end
 
+      def has_design?
+        gitlab_shell.repository_exists?(project.repository_storage, "#{old_design_disk_path}.git")
+      end
+
       def move_repository(from_name, to_name)
         from_exists = gitlab_shell.repository_exists?(project.repository_storage, "#{from_name}.git")
         to_exists = gitlab_shell.repository_exists?(project.repository_storage, "#{to_name}.git")
@@ -58,12 +64,18 @@ module Projects
           project.clear_memoization(:wiki)
         end
 
+        if move_design
+          result &&= move_repository(old_design_disk_path, new_design_disk_path)
+          project.clear_memoization(:design_repository)
+        end
+
         result
       end
 
       def rollback_folder_move
         move_repository(new_disk_path, old_disk_path)
         move_repository(new_wiki_disk_path, old_wiki_disk_path)
+        move_repository(new_design_disk_path, old_design_disk_path) if move_design
       end
 
       def try_to_set_repository_read_only!
@@ -87,8 +99,18 @@ module Projects
       def new_wiki_disk_path
         @new_wiki_disk_path ||= "#{new_disk_path}#{wiki_path_suffix}"
       end
+
+      def design_path_suffix
+        @design_path_suffix ||= ::Gitlab::GlRepository::DESIGN.path_suffix
+      end
+
+      def old_design_disk_path
+        @old_design_disk_path ||= "#{old_disk_path}#{design_path_suffix}"
+      end
+
+      def new_design_disk_path
+        @new_design_disk_path ||= "#{new_disk_path}#{design_path_suffix}"
+      end
     end
   end
 end
-
-Projects::HashedStorage::BaseRepositoryService.prepend_if_ee('EE::Projects::HashedStorage::BaseRepositoryService')
diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb
index 309eab59463..60e5b7e2639 100644
--- a/app/services/projects/transfer_service.rb
+++ b/app/services/projects/transfer_service.rb
@@ -135,7 +135,8 @@ module Projects
       return if project.hashed_storage?(:repository)
 
       move_repo_folder(@new_path, @old_path)
-      move_repo_folder("#{@new_path}.wiki", "#{@old_path}.wiki")
+      move_repo_folder(new_wiki_repo_path, old_wiki_repo_path)
+      move_repo_folder(new_design_repo_path, old_design_repo_path)
     end
 
     def move_repo_folder(from_name, to_name)
@@ -157,8 +158,9 @@ module Projects
       # Disk path is changed; we need to ensure we reload it
       project.reload_repository!
 
-      # Move wiki repo also if present
-      move_repo_folder("#{@old_path}.wiki", "#{@new_path}.wiki")
+      # Move wiki and design repos also if present
+      move_repo_folder(old_wiki_repo_path, new_wiki_repo_path)
+      move_repo_folder(old_design_repo_path, new_design_repo_path)
     end
 
     def move_project_uploads(project)
@@ -170,6 +172,22 @@ module Projects
         @new_namespace.full_path
       )
     end
+
+    def old_wiki_repo_path
+      "#{old_path}#{::Gitlab::GlRepository::WIKI.path_suffix}"
+    end
+
+    def new_wiki_repo_path
+      "#{new_path}#{::Gitlab::GlRepository::WIKI.path_suffix}"
+    end
+
+    def old_design_repo_path
+      "#{old_path}#{::Gitlab::GlRepository::DESIGN.path_suffix}"
+    end
+
+    def new_design_repo_path
+      "#{new_path}#{::Gitlab::GlRepository::DESIGN.path_suffix}"
+    end
   end
 end
 
diff --git a/app/services/projects/update_repository_storage_service.rb b/app/services/projects/update_repository_storage_service.rb
index 21cd84117d7..0082b7bfed5 100644
--- a/app/services/projects/update_repository_storage_service.rb
+++ b/app/services/projects/update_repository_storage_service.rb
@@ -58,6 +58,10 @@ module Projects
       if project.wiki.repository_exists?
         mirror_repository(type: Gitlab::GlRepository::WIKI)
       end
+
+      if project.design_repository.exists?
+        mirror_repository(type: ::Gitlab::GlRepository::DESIGN)
+      end
     end
 
     def mirror_repository(type: Gitlab::GlRepository::PROJECT)
@@ -106,6 +110,13 @@ module Projects
                                           wiki.disk_path,
                                           "#{new_project_path}.wiki")
         end
+
+        if design_repository.exists?
+          GitlabShellWorker.perform_async(:mv_repository,
+                                          old_repository_storage,
+                                          design_repository.disk_path,
+                                          "#{new_project_path}.design")
+        end
       end
     end
 
@@ -140,5 +151,3 @@ module Projects
     end
   end
 end
-
-Projects::UpdateRepositoryStorageService.prepend_if_ee('EE::Projects::UpdateRepositoryStorageService')
diff --git a/app/services/system_note_service.rb b/app/services/system_note_service.rb
index 1b9f5971f73..6bf04c55415 100644
--- a/app/services/system_note_service.rb
+++ b/app/services/system_note_service.rb
@@ -245,6 +245,34 @@ module SystemNoteService
   def auto_resolve_prometheus_alert(noteable, project, author)
     ::SystemNotes::IssuablesService.new(noteable: noteable, project: project, author: author).auto_resolve_prometheus_alert
   end
+
+  # Parameters:
+  #   - version [DesignManagement::Version]
+  #
+  # Example Note text:
+  #
+  #   "added [1 designs](link-to-version)"
+  #   "changed [2 designs](link-to-version)"
+  #
+  # Returns [Array<Note>]: the created Note objects
+  def design_version_added(version)
+    ::SystemNotes::DesignManagementService.new(noteable: version.issue, project: version.issue.project, author: version.author).design_version_added(version)
+  end
+
+  # Called when a new discussion is created on a design
+  #
+  # discussion_note - DiscussionNote
+  #
+  # Example Note text:
+  #
+  #   "started a discussion on screen.png"
+  #
+  # Returns the created Note object
+  def design_discussion_added(discussion_note)
+    design = discussion_note.noteable
+
+    ::SystemNotes::DesignManagementService.new(noteable: design.issue, project: design.project, author: discussion_note.author).design_discussion_added(discussion_note)
+  end
 end
 
 SystemNoteService.prepend_if_ee('EE::SystemNoteService')
diff --git a/app/services/system_notes/design_management_service.rb b/app/services/system_notes/design_management_service.rb
new file mode 100644
index 00000000000..a773877e25b
--- /dev/null
+++ b/app/services/system_notes/design_management_service.rb
@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module SystemNotes
+  class DesignManagementService < ::SystemNotes::BaseService
+    include ActionView::RecordIdentifier
+
+    # Parameters:
+    #   - version [DesignManagement::Version]
+    #
+    # Example Note text:
+    #
+    #   "added [1 designs](link-to-version)"
+    #   "changed [2 designs](link-to-version)"
+    #
+    # Returns [Array<Note>]: the created Note objects
+    def design_version_added(version)
+      events = DesignManagement::Action.events
+      link_href = designs_path(version: version.id)
+
+      version.designs_by_event.map do |(event_name, designs)|
+        note_data = self.class.design_event_note_data(events[event_name])
+        icon_name = note_data[:icon]
+        n = designs.size
+
+        body = "%s [%d %s](%s)" % [note_data[:past_tense], n, 'design'.pluralize(n), link_href]
+
+        create_note(NoteSummary.new(noteable, project, author, body, action: icon_name))
+      end
+    end
+
+    # Called when a new discussion is created on a design
+    #
+    # discussion_note - DiscussionNote
+    #
+    # Example Note text:
+    #
+    #   "started a discussion on screen.png"
+    #
+    # Returns the created Note object
+    def design_discussion_added(discussion_note)
+      design = discussion_note.noteable
+
+      body = _('started a discussion on %{design_link}') % {
+        design_link: '[%s](%s)' % [
+          design.filename,
+          designs_path(vueroute: design.filename, anchor: dom_id(discussion_note))
+        ]
+      }
+
+      action = :designs_discussion_added
+
+      create_note(NoteSummary.new(noteable, project, author, body, action: action))
+    end
+
+    # Take one of the `DesignManagement::Action.events` and
+    # return:
+    #   * an English past-tense verb.
+    #   * the name of an icon used in renderin a system note
+    #
+    # We do not currently internationalize our system notes,
+    # instead we just produce English-language descriptions.
+    # See: https://gitlab.com/gitlab-org/gitlab/issues/30408
+    # See: https://gitlab.com/gitlab-org/gitlab/issues/14056
+    def self.design_event_note_data(event)
+      case event
+      when DesignManagement::Action.events[:creation]
+        { icon: 'designs_added', past_tense: 'added' }
+      when DesignManagement::Action.events[:modification]
+        { icon: 'designs_modified', past_tense: 'updated' }
+      when DesignManagement::Action.events[:deletion]
+        { icon: 'designs_removed', past_tense: 'removed' }
+      else
+        raise "Unknown event: #{event}"
+      end
+    end
+
+    private
+
+    def designs_path(params = {})
+      url_helpers.designs_project_issue_path(project, noteable, params)
+    end
+  end
+end
diff --git a/app/services/wikis/create_attachment_service.rb b/app/services/wikis/create_attachment_service.rb
index 6ef6cbc3c12..82179459345 100644
--- a/app/services/wikis/create_attachment_service.rb
+++ b/app/services/wikis/create_attachment_service.rb
@@ -5,12 +5,15 @@ module Wikis
     ATTACHMENT_PATH = 'uploads'
     MAX_FILENAME_LENGTH = 255
 
-    delegate :wiki, to: :project
+    attr_reader :container
+
+    delegate :wiki, to: :container
     delegate :repository, to: :wiki
 
-    def initialize(*args)
-      super
+    def initialize(container:, current_user: nil, params: {})
+      super(nil, current_user, params)
 
+      @container = container
       @file_name = clean_file_name(params[:file_name])
       @file_path = File.join(ATTACHMENT_PATH, SecureRandom.hex, @file_name) if @file_name
       @commit_message ||= "Upload attachment #{@file_name}"
@@ -51,7 +54,7 @@ module Wikis
     end
 
     def validate_permissions!
-      unless can?(current_user, :create_wiki, project)
+      unless can?(current_user, :create_wiki, container)
         raise_error('You are not allowed to push to the wiki')
       end
     end