diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-05-20 17:34:42 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-05-20 17:34:42 +0300 |
commit | 9f46488805e86b1bc341ea1620b866016c2ce5ed (patch) | |
tree | f9748c7e287041e37d6da49e0a29c9511dc34768 /app/models/design_management | |
parent | dfc92d081ea0332d69c8aca2f0e745cb48ae5e6d (diff) |
Add latest changes from gitlab-org/gitlab@13-0-stable-ee
Diffstat (limited to 'app/models/design_management')
-rw-r--r-- | app/models/design_management/action.rb | 44 | ||||
-rw-r--r-- | app/models/design_management/design.rb | 266 | ||||
-rw-r--r-- | app/models/design_management/design_action.rb | 64 | ||||
-rw-r--r-- | app/models/design_management/design_at_version.rb | 119 | ||||
-rw-r--r-- | app/models/design_management/design_collection.rb | 30 | ||||
-rw-r--r-- | app/models/design_management/repository.rb | 51 | ||||
-rw-r--r-- | app/models/design_management/version.rb | 144 |
7 files changed, 718 insertions, 0 deletions
diff --git a/app/models/design_management/action.rb b/app/models/design_management/action.rb new file mode 100644 index 00000000000..ecd7973a523 --- /dev/null +++ b/app/models/design_management/action.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require_dependency 'design_management' + +module DesignManagement + class Action < ApplicationRecord + include WithUploads + + self.table_name = "#{DesignManagement.table_name_prefix}designs_versions" + + mount_uploader :image_v432x230, DesignManagement::DesignV432x230Uploader + + belongs_to :design, class_name: "DesignManagement::Design", inverse_of: :actions + belongs_to :version, class_name: "DesignManagement::Version", inverse_of: :actions + + enum event: { creation: 0, modification: 1, deletion: 2 } + + # we assume sequential ordering. + scope :ordered, -> { order(version_id: :asc) } + + # For each design, only select the most recent action + scope :most_recent, -> do + selection = Arel.sql("DISTINCT ON (#{table_name}.design_id) #{table_name}.*") + + order(arel_table[:design_id].asc, arel_table[:version_id].desc).select(selection) + end + + # Find all records created before or at the given version, or all if nil + scope :up_to_version, ->(version = nil) do + case version + when nil + all + when DesignManagement::Version + where(arel_table[:version_id].lteq(version.id)) + when ::Gitlab::Git::COMMIT_ID + versions = DesignManagement::Version.arel_table + subquery = versions.project(versions[:id]).where(versions[:sha].eq(version)) + where(arel_table[:version_id].lteq(subquery)) + else + raise ArgumentError, "Expected a DesignManagement::Version, got #{version}" + end + end + end +end diff --git a/app/models/design_management/design.rb b/app/models/design_management/design.rb new file mode 100644 index 00000000000..e9b69eab7a7 --- /dev/null +++ b/app/models/design_management/design.rb @@ -0,0 +1,266 @@ +# frozen_string_literal: true + +module DesignManagement + class Design < ApplicationRecord + include Importable + include Noteable + include Gitlab::FileTypeDetection + include Gitlab::Utils::StrongMemoize + include Referable + include Mentionable + include WhereComposite + + belongs_to :project, inverse_of: :designs + belongs_to :issue + + has_many :actions + has_many :versions, through: :actions, class_name: 'DesignManagement::Version', inverse_of: :designs + # This is a polymorphic association, so we can't count on FK's to delete the + # data + has_many :notes, as: :noteable, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + has_many :user_mentions, class_name: 'DesignUserMention', dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + + validates :project, :filename, presence: true + validates :issue, presence: true, unless: :importing? + validates :filename, uniqueness: { scope: :issue_id } + validate :validate_file_is_image + + alias_attribute :title, :filename + + # Pre-fetching scope to include the data necessary to construct a + # reference using `to_reference`. + scope :for_reference, -> { includes(issue: [{ project: [:route, :namespace] }]) } + + # A design can be uniquely identified by issue_id and filename + # Takes one or more sets of composite IDs of the form: + # `{issue_id: Integer, filename: String}`. + # + # @see WhereComposite::where_composite + # + # e.g: + # + # by_issue_id_and_filename(issue_id: 1, filename: 'homescreen.jpg') + # by_issue_id_and_filename([]) # returns ActiveRecord::NullRelation + # by_issue_id_and_filename([ + # { issue_id: 1, filename: 'homescreen.jpg' }, + # { issue_id: 2, filename: 'homescreen.jpg' }, + # { issue_id: 1, filename: 'menu.png' } + # ]) + # + scope :by_issue_id_and_filename, ->(composites) do + where_composite(%i[issue_id filename], composites) + end + + # Find designs visible at the given version + # + # @param version [nil, DesignManagement::Version]: + # the version at which the designs must be visible + # Passing `nil` is the same as passing the most current version + # + # Restricts to designs + # - created at least *before* the given version + # - not deleted as of the given version. + # + # As a query, we ascertain this by finding the last event prior to + # (or equal to) the cut-off, and seeing whether that version was a deletion. + scope :visible_at_version, -> (version) do + deletion = ::DesignManagement::Action.events[:deletion] + designs = arel_table + actions = ::DesignManagement::Action + .most_recent.up_to_version(version) + .arel.as('most_recent_actions') + + join = designs.join(actions) + .on(actions[:design_id].eq(designs[:id])) + + joins(join.join_sources).where(actions[:event].not_eq(deletion)).order(:id) + end + + scope :with_filename, -> (filenames) { where(filename: filenames) } + scope :on_issue, ->(issue) { where(issue_id: issue) } + + # Scope called by our REST API to avoid N+1 problems + scope :with_api_entity_associations, -> { preload(:issue) } + + # A design is current if the most recent event is not a deletion + scope :current, -> { visible_at_version(nil) } + + def status + if new_design? + :new + elsif deleted? + :deleted + else + :current + end + end + + def deleted? + most_recent_action&.deletion? + end + + # A design is visible_in? a version if: + # * it was created before that version + # * the most recent action before the version was not a deletion + def visible_in?(version) + map = strong_memoize(:visible_in) do + Hash.new do |h, k| + h[k] = self.class.visible_at_version(k).where(id: id).exists? + end + end + + map[version] + end + + def most_recent_action + strong_memoize(:most_recent_action) { actions.ordered.last } + end + + # A reference for a design is the issue reference, indexed by the filename + # with an optional infix when full. + # + # e.g. + # #123[homescreen.png] + # other-project#72[sidebar.jpg] + # #38/designs[transition.gif] + # #12["filename with [] in it.jpg"] + def to_reference(from = nil, full: false) + infix = full ? '/designs' : '' + totally_simple = %r{ \A #{self.class.simple_file_name} \z }x + safe_name = if totally_simple.match?(filename) + filename + elsif filename =~ /[<>]/ + %Q{base64:#{Base64.strict_encode64(filename)}} + else + escaped = filename.gsub(%r{[\\"]}) { |x| "\\#{x}" } + %Q{"#{escaped}"} + end + + "#{issue.to_reference(from, full: full)}#{infix}[#{safe_name}]" + end + + def self.reference_pattern + @reference_pattern ||= begin + # Filenames can be escaped with double quotes to name filenames + # that include square brackets, or other special characters + %r{ + #{Issue.reference_pattern} + (\/designs)? + \[ + (?<design> #{simple_file_name} | #{quoted_file_name} | #{base_64_encoded_name}) + \] + }x + end + end + + def self.simple_file_name + %r{ + (?<simple_file_name> + ( \w | [_:,'-] | \. | \s )+ + \. + \w+ + ) + }x + end + + def self.base_64_encoded_name + %r{ + base64: + (?<base_64_encoded_name> + [A-Za-z0-9+\n]+ + =? + ) + }x + end + + def self.quoted_file_name + %r{ + " + (?<escaped_filename> + (\\ \\ | \\ " | [^"\\])+ + ) + " + }x + end + + def self.link_reference_pattern + @link_reference_pattern ||= begin + exts = SAFE_IMAGE_EXT + DANGEROUS_IMAGE_EXT + path_segment = %r{issues/#{Gitlab::Regex.issue}/designs} + filename_pattern = %r{(?<simple_file_name>[a-z0-9_=-]+\.(#{exts.join('|')}))}i + + super(path_segment, filename_pattern) + end + end + + def to_ability_name + 'design' + end + + def description + '' + end + + def new_design? + strong_memoize(:new_design) { actions.none? } + end + + def full_path + @full_path ||= File.join(DesignManagement.designs_directory, "issue-#{issue.iid}", filename) + end + + def diff_refs + strong_memoize(:diff_refs) { head_version&.diff_refs } + end + + def clear_version_cache + [versions, actions].each(&:reset) + %i[new_design diff_refs head_sha visible_in most_recent_action].each do |key| + clear_memoization(key) + end + end + + def repository + project.design_repository + end + + def user_notes_count + user_notes_count_service.count + end + + def after_note_changed(note) + user_notes_count_service.delete_cache unless note.system? + end + alias_method :after_note_created, :after_note_changed + alias_method :after_note_destroyed, :after_note_changed + + private + + def head_version + strong_memoize(:head_sha) { versions.ordered.first } + end + + def allow_dangerous_images? + Feature.enabled?(:design_management_allow_dangerous_images, project) + end + + def valid_file_extensions + allow_dangerous_images? ? (SAFE_IMAGE_EXT + DANGEROUS_IMAGE_EXT) : SAFE_IMAGE_EXT + end + + def validate_file_is_image + unless image? || (dangerous_image? && allow_dangerous_images?) + message = _('does not have a supported extension. Only %{extension_list} are supported') % { + extension_list: valid_file_extensions.to_sentence + } + errors.add(:filename, message) + end + end + + def user_notes_count_service + strong_memoize(:user_notes_count_service) do + ::DesignManagement::DesignUserNotesCountService.new(self) # rubocop: disable CodeReuse/ServiceClass + end + end + end +end diff --git a/app/models/design_management/design_action.rb b/app/models/design_management/design_action.rb new file mode 100644 index 00000000000..22baa916296 --- /dev/null +++ b/app/models/design_management/design_action.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module DesignManagement + # Parameter object which is a tuple of the database record and the + # last gitaly call made to change it. This serves to perform the + # logical mapping from git action to database representation. + class DesignAction + include ActiveModel::Validations + + EVENT_FOR_GITALY_ACTION = { + create: DesignManagement::Action.events[:creation], + update: DesignManagement::Action.events[:modification], + delete: DesignManagement::Action.events[:deletion] + }.freeze + + attr_reader :design, :action, :content + + delegate :issue_id, to: :design + + validates :design, presence: true + validates :action, presence: true, inclusion: { in: EVENT_FOR_GITALY_ACTION.keys } + validates :content, + absence: { if: :forbids_content?, + message: 'this action forbids content' }, + presence: { if: :needs_content?, + message: 'this action needs content' } + + # Parameters: + # - design [DesignManagement::Design]: the design that was changed + # - action [Symbol]: the action that gitaly performed + def initialize(design, action, content = nil) + @design, @action, @content = design, action, content + validate! + end + + def row_attrs(version) + { design_id: design.id, version_id: version.id, event: event } + end + + def gitaly_action + { action: action, file_path: design.full_path, content: content }.compact + end + + # This action has been performed - do any post-creation actions + # such as clearing method caches. + def performed + design.clear_version_cache + end + + private + + def needs_content? + action != :delete + end + + def forbids_content? + action == :delete + end + + def event + EVENT_FOR_GITALY_ACTION[action] + end + end +end diff --git a/app/models/design_management/design_at_version.rb b/app/models/design_management/design_at_version.rb new file mode 100644 index 00000000000..b4cafb93c2c --- /dev/null +++ b/app/models/design_management/design_at_version.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +# Tuple of design and version +# * has a composite ID, with lazy_find +module DesignManagement + class DesignAtVersion + include ActiveModel::Validations + include GlobalID::Identification + include Gitlab::Utils::StrongMemoize + + attr_reader :version + attr_reader :design + + validates :version, presence: true + validates :design, presence: true + + validate :design_and_version_belong_to_the_same_issue + validate :design_and_version_have_issue_id + + def initialize(design: nil, version: nil) + @design, @version = design, version + end + + def self.instantiate(attrs) + new(attrs).tap { |obj| obj.validate! } + end + + # The ID, needed by GraphQL types and as part of the Lazy-fetch + # protocol, includes information about both the design and the version. + # + # The particular format is not interesting, and should be treated as opaque + # by all callers. + def id + "#{design.id}.#{version.id}" + end + + def ==(other) + return false unless other && self.class == other.class + + other.id == id + end + + alias_method :eql?, :== + + def self.lazy_find(id) + BatchLoader.for(id).batch do |ids, callback| + find(ids).each do |record| + callback.call(record.id, record) + end + end + end + + def self.find(ids) + pairs = ids.map { |id| id.split('.').map(&:to_i) } + + design_ids = pairs.map(&:first).uniq + version_ids = pairs.map(&:second).uniq + + designs = ::DesignManagement::Design + .where(id: design_ids) + .index_by(&:id) + + versions = ::DesignManagement::Version + .where(id: version_ids) + .index_by(&:id) + + pairs.map do |(design_id, version_id)| + design = designs[design_id] + version = versions[version_id] + + obj = new(design: design, version: version) + + obj if obj.valid? + end.compact + end + + def status + if not_created_yet? + :not_created_yet + elsif deleted? + :deleted + else + :current + end + end + + def deleted? + action&.deletion? + end + + def not_created_yet? + action.nil? + end + + private + + def action + strong_memoize(:most_recent_action) do + ::DesignManagement::Action + .most_recent.up_to_version(version) + .find_by(design: design) + end + end + + def design_and_version_belong_to_the_same_issue + id_a, id_b = [design, version].map { |obj| obj&.issue_id } + + return if id_a == id_b + + errors.add(:issue, 'must be the same on design and version') + end + + def design_and_version_have_issue_id + return if [design, version].all? { |obj| obj.try(:issue_id).present? } + + errors.add(:issue, 'must be present on both design and version') + end + end +end diff --git a/app/models/design_management/design_collection.rb b/app/models/design_management/design_collection.rb new file mode 100644 index 00000000000..18d1541e9c7 --- /dev/null +++ b/app/models/design_management/design_collection.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module DesignManagement + class DesignCollection + attr_reader :issue + + delegate :designs, :project, to: :issue + + def initialize(issue) + @issue = issue + end + + def find_or_create_design!(filename:) + designs.find { |design| design.filename == filename } || + designs.safe_find_or_create_by!(project: project, filename: filename) + end + + def versions + @versions ||= DesignManagement::Version.for_designs(designs) + end + + def repository + project.design_repository + end + + def designs_by_filename(filenames) + designs.current.where(filename: filenames) + end + end +end diff --git a/app/models/design_management/repository.rb b/app/models/design_management/repository.rb new file mode 100644 index 00000000000..985d6317d5d --- /dev/null +++ b/app/models/design_management/repository.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module DesignManagement + class Repository < ::Repository + extend ::Gitlab::Utils::Override + + # We define static git attributes for the design repository as this + # repository is entirely GitLab-managed rather than user-facing. + # + # Enable all uploaded files to be stored in LFS. + MANAGED_GIT_ATTRIBUTES = <<~GA.freeze + /#{DesignManagement.designs_directory}/* filter=lfs diff=lfs merge=lfs -text + GA + + def initialize(project) + full_path = project.full_path + Gitlab::GlRepository::DESIGN.path_suffix + disk_path = project.disk_path + Gitlab::GlRepository::DESIGN.path_suffix + + super(full_path, project, shard: project.repository_storage, disk_path: disk_path, repo_type: Gitlab::GlRepository::DESIGN) + end + + # Override of a method called on Repository instances but sent via + # method_missing to Gitlab::Git::Repository where it is defined + def info_attributes + @info_attributes ||= Gitlab::Git::AttributesParser.new(MANAGED_GIT_ATTRIBUTES) + end + + # Override of a method called on Repository instances but sent via + # method_missing to Gitlab::Git::Repository where it is defined + def attributes(path) + info_attributes.attributes(path) + end + + # Override of a method called on Repository instances but sent via + # method_missing to Gitlab::Git::Repository where it is defined + def gitattribute(path, name) + attributes(path)[name] + end + + # Override of a method called on Repository instances but sent via + # method_missing to Gitlab::Git::Repository where it is defined + def attributes_at(_ref = nil) + info_attributes + end + + override :copy_gitattributes + def copy_gitattributes(_ref = nil) + true + end + end +end diff --git a/app/models/design_management/version.rb b/app/models/design_management/version.rb new file mode 100644 index 00000000000..6be98fe3d44 --- /dev/null +++ b/app/models/design_management/version.rb @@ -0,0 +1,144 @@ +# frozen_string_literal: true + +module DesignManagement + class Version < ApplicationRecord + include Importable + include ShaAttribute + include AfterCommitQueue + include Gitlab::Utils::StrongMemoize + extend Gitlab::ExclusiveLeaseHelpers + + NotSameIssue = Class.new(StandardError) + + class CouldNotCreateVersion < StandardError + attr_reader :sha, :issue_id, :actions + + def initialize(sha, issue_id, actions) + @sha, @issue_id, @actions = sha, issue_id, actions + end + + def message + "could not create version from commit: #{sha}" + end + + def sentry_extra_data + { + sha: sha, + issue_id: issue_id, + design_ids: actions.map { |a| a.design.id } + } + end + end + + belongs_to :issue + belongs_to :author, class_name: 'User' + has_many :actions + has_many :designs, + through: :actions, + class_name: "DesignManagement::Design", + source: :design, + inverse_of: :versions + + validates :designs, presence: true, unless: :importing? + validates :sha, presence: true + validates :sha, uniqueness: { case_sensitive: false, scope: :issue_id } + validates :author, presence: true + # We are not validating the issue object as it incurs an extra query to fetch + # the record from the DB. Instead, we rely on the foreign key constraint to + # ensure referential integrity. + validates :issue_id, presence: true, unless: :importing? + + sha_attribute :sha + + delegate :project, to: :issue + + scope :for_designs, -> (designs) do + where(id: ::DesignManagement::Action.where(design_id: designs).select(:version_id)).distinct + end + scope :earlier_or_equal_to, -> (version) { where("(#{table_name}.id) <= ?", version) } # rubocop:disable GitlabSecurity/SqlInjection + scope :ordered, -> { order(id: :desc) } + scope :for_issue, -> (issue) { where(issue: issue) } + scope :by_sha, -> (sha) { where(sha: sha) } + + # This is the one true way to create a Version. + # + # This method means you can avoid the paradox of versions being invalid without + # designs, and not being able to add designs without a saved version. Also this + # method inserts designs in bulk, rather than one by one. + # + # Before calling this method, callers must guard against concurrent + # modification by obtaining the lock on the design repository. See: + # `DesignManagement::Version.with_lock`. + # + # Parameters: + # - design_actions [DesignManagement::DesignAction]: + # the actions that have been performed in the repository. + # - sha [String]: + # the SHA of the commit that performed them + # - author [User]: + # the user who performed the commit + # returns [DesignManagement::Version] + def self.create_for_designs(design_actions, sha, author) + issue_id, not_uniq = design_actions.map(&:issue_id).compact.uniq + raise NotSameIssue, 'All designs must belong to the same issue!' if not_uniq + + transaction do + version = new(sha: sha, issue_id: issue_id, author: author) + version.save(validate: false) # We need it to have an ID. Validate later when designs are present + + rows = design_actions.map { |action| action.row_attrs(version) } + + Gitlab::Database.bulk_insert(::DesignManagement::Action.table_name, rows) + version.designs.reset + version.validate! + design_actions.each(&:performed) + + version + end + rescue + raise CouldNotCreateVersion.new(sha, issue_id, design_actions) + end + + CREATION_TTL = 5.seconds + RETRY_DELAY = ->(num) { 0.2.seconds * num**2 } + + def self.with_lock(project_id, repository, &block) + key = "with_lock:#{name}:{#{project_id}}" + + in_lock(key, ttl: CREATION_TTL, retries: 5, sleep_sec: RETRY_DELAY) do |_retried| + repository.create_if_not_exists + yield + end + end + + def designs_by_event + actions + .includes(:design) + .group_by(&:event) + .transform_values { |group| group.map(&:design) } + end + + def author + super || (commit_author if persisted?) + end + + def diff_refs + strong_memoize(:diff_refs) { commit&.diff_refs } + end + + def reset + %i[diff_refs commit].each { |k| clear_memoization(k) } + super + end + + private + + def commit_author + commit&.author + end + + def commit + strong_memoize(:commit) { issue.project.design_repository.commit(sha) } + end + end +end |