diff options
Diffstat (limited to 'app/models/concerns')
-rw-r--r-- | app/models/concerns/awareness.rb | 41 | ||||
-rw-r--r-- | app/models/concerns/cache_markdown_field.rb | 9 | ||||
-rw-r--r-- | app/models/concerns/ci/artifactable.rb | 2 | ||||
-rw-r--r-- | app/models/concerns/ci/bulk_insertable_tags.rb | 24 | ||||
-rw-r--r-- | app/models/concerns/ci/has_status.rb | 6 | ||||
-rw-r--r-- | app/models/concerns/each_batch.rb | 61 | ||||
-rw-r--r-- | app/models/concerns/enums/ci/commit_status.rb | 5 | ||||
-rw-r--r-- | app/models/concerns/integrations/has_issue_tracker_fields.rb | 18 | ||||
-rw-r--r-- | app/models/concerns/integrations/slack_mattermost_notifier.rb | 2 | ||||
-rw-r--r-- | app/models/concerns/loose_index_scan.rb | 67 | ||||
-rw-r--r-- | app/models/concerns/milestoneable.rb | 2 | ||||
-rw-r--r-- | app/models/concerns/notification_branch_selection.rb | 16 | ||||
-rw-r--r-- | app/models/concerns/packages/fips.rb | 11 | ||||
-rw-r--r-- | app/models/concerns/participable.rb | 23 | ||||
-rw-r--r-- | app/models/concerns/require_email_verification.rb | 52 | ||||
-rw-r--r-- | app/models/concerns/vulnerability_finding_helpers.rb | 37 |
16 files changed, 352 insertions, 24 deletions
diff --git a/app/models/concerns/awareness.rb b/app/models/concerns/awareness.rb new file mode 100644 index 00000000000..da87d87e838 --- /dev/null +++ b/app/models/concerns/awareness.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Awareness + extend ActiveSupport::Concern + + KEY_NAMESPACE = "gitlab:awareness" + private_constant :KEY_NAMESPACE + + def join(session) + session.join(self) + + nil + end + + def leave(session) + session.leave(self) + + nil + end + + def session_ids + with_redis do |redis| + redis + .smembers(user_sessions_key) + # converts session ids from (internal) integer to hex presentation + .map { |key| key.to_i.to_s(16) } + end + end + + private + + def user_sessions_key + "#{KEY_NAMESPACE}:user:#{id}:sessions" + end + + def with_redis + Gitlab::Redis::SharedState.with do |redis| + yield redis if block_given? + end + end +end diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index 99dbe464a7c..9ee0fd1db1d 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -172,7 +172,7 @@ module CacheMarkdownField refs = all_references(self.author) references = {} - references[:mentioned_users_ids] = refs.mentioned_user_ids.presence + references[:mentioned_users_ids] = mentioned_filtered_user_ids_for(refs) references[:mentioned_groups_ids] = refs.mentioned_group_ids.presence references[:mentioned_projects_ids] = refs.mentioned_project_ids.presence @@ -185,6 +185,13 @@ module CacheMarkdownField true end + # Overriden on objects that needs to filter + # mentioned users that cannot read them, for example, + # guest users that are referenced on a confidential note. + def mentioned_filtered_user_ids_for(refs) + refs.mentioned_user_ids.presence + end + def mentionable_attributes_changed?(changes = saved_changes) return false unless is_a?(Mentionable) diff --git a/app/models/concerns/ci/artifactable.rb b/app/models/concerns/ci/artifactable.rb index 78340cf967b..fb4ea4206f4 100644 --- a/app/models/concerns/ci/artifactable.rb +++ b/app/models/concerns/ci/artifactable.rb @@ -30,6 +30,8 @@ module Ci raise NotSupportedAdapterError, 'This file format requires a dedicated adapter' end + ::Gitlab::ApplicationContext.push(artifact: file.model) + file.open do |stream| file_format_adapter_class.new(stream).each_blob(&blk) end diff --git a/app/models/concerns/ci/bulk_insertable_tags.rb b/app/models/concerns/ci/bulk_insertable_tags.rb new file mode 100644 index 00000000000..453b3b3fbc9 --- /dev/null +++ b/app/models/concerns/ci/bulk_insertable_tags.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Ci + module BulkInsertableTags + extend ActiveSupport::Concern + + BULK_INSERT_TAG_THREAD_KEY = 'ci_bulk_insert_tags' + + class << self + def with_bulk_insert_tags + previous = Thread.current[BULK_INSERT_TAG_THREAD_KEY] + Thread.current[BULK_INSERT_TAG_THREAD_KEY] = true + yield + ensure + Thread.current[BULK_INSERT_TAG_THREAD_KEY] = previous + end + end + + # overrides save_tags from acts-as-taggable + def save_tags + super unless Thread.current[BULK_INSERT_TAG_THREAD_KEY] + end + end +end diff --git a/app/models/concerns/ci/has_status.rb b/app/models/concerns/ci/has_status.rb index cca66c3ec94..721cb14201f 100644 --- a/app/models/concerns/ci/has_status.rb +++ b/app/models/concerns/ci/has_status.rb @@ -23,11 +23,9 @@ module Ci UnknownStatusError = Class.new(StandardError) class_methods do - # The parameter `project` is only used for the feature flag check, and will be removed with - # https://gitlab.com/gitlab-org/gitlab/-/issues/321972 - def composite_status(project: nil) + def composite_status Gitlab::Ci::Status::Composite - .new(all, with_allow_failure: columns_hash.key?('allow_failure'), project: project) + .new(all, with_allow_failure: columns_hash.key?('allow_failure')) .status end diff --git a/app/models/concerns/each_batch.rb b/app/models/concerns/each_batch.rb index 443e1ab53b4..dbc0887dc97 100644 --- a/app/models/concerns/each_batch.rb +++ b/app/models/concerns/each_batch.rb @@ -2,6 +2,7 @@ module EachBatch extend ActiveSupport::Concern + include LooseIndexScan class_methods do # Iterates over the rows in a relation in batches, similar to Rails' @@ -100,5 +101,65 @@ module EachBatch break unless stop end end + + # Iterates over the rows in a relation in batches by skipping duplicated values in the column. + # Example: counting the number of distinct authors in `issues` + # + # - Table size: 100_000 + # - Column: author_id + # - Distinct author_ids in the table: 1000 + # + # The query will read maximum 1000 rows if we have index coverage on user_id. + # + # > count = 0 + # > Issue.distinct_each_batch(column: 'author_id', of: 1000) { |r| count += r.count(:author_id) } + def distinct_each_batch(column:, order: :asc, of: 1000) + start = except(:select) + .select(column) + .reorder(column => order) + + start = start.take + + return unless start + + start_id = start[column] + arel_table = self.arel_table + arel_column = arel_table[column.to_s] + + 1.step do |index| + stop = loose_index_scan(column: column, order: order) do |cte_query, inner_query| + if order == :asc + [cte_query.where(arel_column.gteq(start_id)), inner_query] + else + [cte_query.where(arel_column.lteq(start_id)), inner_query] + end + end.offset(of).take + + if stop + stop_id = stop[column] + + relation = loose_index_scan(column: column, order: order) do |cte_query, inner_query| + if order == :asc + [cte_query.where(arel_column.gteq(start_id)), inner_query.where(arel_column.lt(stop_id))] + else + [cte_query.where(arel_column.lteq(start_id)), inner_query.where(arel_column.gt(stop_id))] + end + end + start_id = stop_id + else + relation = loose_index_scan(column: column, order: order) do |cte_query, inner_query| + if order == :asc + [cte_query.where(arel_column.gteq(start_id)), inner_query] + else + [cte_query.where(arel_column.lteq(start_id)), inner_query] + end + end + end + + unscoped { yield relation, index } + + break unless stop + end + end end end diff --git a/app/models/concerns/enums/ci/commit_status.rb b/app/models/concerns/enums/ci/commit_status.rb index 445277a7a7c..ecb120d8013 100644 --- a/app/models/concerns/enums/ci/commit_status.rb +++ b/app/models/concerns/enums/ci/commit_status.rb @@ -29,9 +29,12 @@ module Enums builds_disabled: 20, environment_creation_failure: 21, deployment_rejected: 22, + protected_environment_failure: 1_000, insufficient_bridge_permissions: 1_001, downstream_bridge_project_not_found: 1_002, invalid_bridge_trigger: 1_003, + upstream_bridge_project_not_found: 1_004, + insufficient_upstream_permissions: 1_005, bridge_pipeline_is_child_pipeline: 1_006, # not used anymore, but cannot be deleted because of old data downstream_pipeline_creation_failed: 1_007, secrets_provider_not_found: 1_008, @@ -42,5 +45,3 @@ module Enums end end end - -Enums::Ci::CommitStatus.prepend_mod_with('Enums::Ci::CommitStatus') diff --git a/app/models/concerns/integrations/has_issue_tracker_fields.rb b/app/models/concerns/integrations/has_issue_tracker_fields.rb index b1def38d019..57f8e21c5a6 100644 --- a/app/models/concerns/integrations/has_issue_tracker_fields.rb +++ b/app/models/concerns/integrations/has_issue_tracker_fields.rb @@ -5,26 +5,32 @@ module Integrations extend ActiveSupport::Concern included do + self.field_storage = :data_fields + field :project_url, required: true, - storage: :data_fields, title: -> { _('Project URL') }, - help: -> { s_('IssueTracker|The URL to the project in the external issue tracker.') } + help: -> do + s_('IssueTracker|The URL to the project in the external issue tracker.') + end field :issues_url, required: true, - storage: :data_fields, title: -> { s_('IssueTracker|Issue URL') }, help: -> do - format s_('IssueTracker|The URL to view an issue in the external issue tracker. Must contain %{colon_id}.'), + ERB::Util.html_escape( + s_('IssueTracker|The URL to view an issue in the external issue tracker. Must contain %{colon_id}.') + ) % { colon_id: '<code>:id</code>'.html_safe + } end field :new_issue_url, required: true, - storage: :data_fields, title: -> { s_('IssueTracker|New issue URL') }, - help: -> { s_('IssueTracker|The URL to create an issue in the external issue tracker.') } + help: -> do + s_('IssueTracker|The URL to create an issue in the external issue tracker.') + end end end end diff --git a/app/models/concerns/integrations/slack_mattermost_notifier.rb b/app/models/concerns/integrations/slack_mattermost_notifier.rb index 3bdaa852ddf..142e62bb501 100644 --- a/app/models/concerns/integrations/slack_mattermost_notifier.rb +++ b/app/models/concerns/integrations/slack_mattermost_notifier.rb @@ -34,7 +34,7 @@ module Integrations class HTTPClient def self.post(uri, params = {}) params.delete(:http_options) # these are internal to the client and we do not want them - Gitlab::HTTP.post(uri, body: params, use_read_total_timeout: true) + Gitlab::HTTP.post(uri, body: params) end end end diff --git a/app/models/concerns/loose_index_scan.rb b/app/models/concerns/loose_index_scan.rb new file mode 100644 index 00000000000..5d37a30171a --- /dev/null +++ b/app/models/concerns/loose_index_scan.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module LooseIndexScan + extend ActiveSupport::Concern + + class_methods do + # Builds a recursive query to read distinct values from a column. + # + # Example 1: collect all distinct author ids for the `issues` table + # + # Bad: The DB reads all issues, sorts and dedups them in memory + # + # > Issue.select(:author_id).distinct.map(&:author_id) + # + # Good: Use loose index scan (skip index scan) + # + # > Issue.loose_index_scan(column: :author_id).map(&:author_id) + # + # Example 2: List of users for the DONE todos selector. Select all users who created a todo. + # + # Bad: Loads all DONE todos for the given user and extracts the author_ids + # + # > User.where(id: Todo.where(user_id: 4156052).done.select(:author_id)) + # + # Good: Loads distinct author_ids from todos and then loads users + # + # > distinct_authors = Todo.where(user_id: 4156052).done.loose_index_scan(column: :author_id).select(:author_id) + # > User.where(id: distinct_authors) + def loose_index_scan(column:, order: :asc) + arel_table = self.arel_table + arel_column = arel_table[column.to_s] + + cte = Gitlab::SQL::RecursiveCTE.new(:loose_index_scan_cte, union_args: { remove_order: false }) + + cte_query = except(:select) + .select(column) + .order(column => order) + .limit(1) + + inner_query = except(:select) + + cte_query, inner_query = yield([cte_query, inner_query]) if block_given? + cte << cte_query + + inner_query = if order == :asc + inner_query.where(arel_column.gt(cte.table[column.to_s])) + else + inner_query.where(arel_column.lt(cte.table[column.to_s])) + end + + inner_query = inner_query.order(column => order) + .select(column) + .limit(1) + + cte << cte.table + .project(Arel::Nodes::Grouping.new(Arel.sql(inner_query.to_sql)).as(column.to_s)) + + unscoped do + select(column) + .with + .recursive(cte.to_arel) + .from(cte.alias_to(arel_table)) + .where(arel_column.not_eq(nil)) # filtering out the last NULL value + end + end + end +end diff --git a/app/models/concerns/milestoneable.rb b/app/models/concerns/milestoneable.rb index 12041b103f6..14c54d99ef3 100644 --- a/app/models/concerns/milestoneable.rb +++ b/app/models/concerns/milestoneable.rb @@ -16,7 +16,7 @@ module Milestoneable scope :any_milestone, -> { where.not(milestone_id: nil) } scope :with_milestone, ->(title) { left_joins_milestones.where(milestones: { title: title }) } - scope :without_particular_milestone, ->(title) { left_outer_joins(:milestone).where("milestones.title != ? OR milestone_id IS NULL", title) } + scope :without_particular_milestones, ->(titles) { left_outer_joins(:milestone).where("milestones.title NOT IN (?) OR milestone_id IS NULL", titles) } scope :any_release, -> { joins_milestone_releases } scope :with_release, -> (tag, project_id) { joins_milestone_releases.where( milestones: { releases: { tag: tag, project_id: project_id } } ) } scope :without_particular_release, -> (tag, project_id) { joins_milestone_releases.where.not(milestones: { releases: { tag: tag, project_id: project_id } }) } diff --git a/app/models/concerns/notification_branch_selection.rb b/app/models/concerns/notification_branch_selection.rb index 18ec996c3df..f2df7579a65 100644 --- a/app/models/concerns/notification_branch_selection.rb +++ b/app/models/concerns/notification_branch_selection.rb @@ -6,13 +6,15 @@ module NotificationBranchSelection extend ActiveSupport::Concern - def branch_choices - [ - [_('All branches'), 'all'].freeze, - [_('Default branch'), 'default'].freeze, - [_('Protected branches'), 'protected'].freeze, - [_('Default branch and protected branches'), 'default_and_protected'].freeze - ].freeze + class_methods do + def branch_choices + [ + [_('All branches'), 'all'].freeze, + [_('Default branch'), 'default'].freeze, + [_('Protected branches'), 'protected'].freeze, + [_('Default branch and protected branches'), 'default_and_protected'].freeze + ].freeze + end end def notify_for_branch?(data) diff --git a/app/models/concerns/packages/fips.rb b/app/models/concerns/packages/fips.rb new file mode 100644 index 00000000000..b8589cdc991 --- /dev/null +++ b/app/models/concerns/packages/fips.rb @@ -0,0 +1,11 @@ +# rubocop:disable Naming/FileName +# frozen_string_literal: true + +module Packages + module FIPS + extend ActiveSupport::Concern + + DisabledError = Class.new(StandardError) + end +end +# rubocop:enable Naming/FileName diff --git a/app/models/concerns/participable.rb b/app/models/concerns/participable.rb index 20743ebcb52..f59b5d1ecc8 100644 --- a/app/models/concerns/participable.rb +++ b/app/models/concerns/participable.rb @@ -92,7 +92,13 @@ module Participable end def raw_participants(current_user = nil, verify_access: false) - ext = Gitlab::ReferenceExtractor.new(project, current_user) + extractor = Gitlab::ReferenceExtractor.new(project, current_user) + + # Used to extract references from confidential notes. + # Referenced users that cannot read confidential notes are + # later removed from participants array. + internal_notes_extractor = Gitlab::ReferenceExtractor.new(project, current_user) + participants = Set.new process = [self] @@ -107,6 +113,8 @@ module Participable source.class.participant_attrs.each do |attr| if attr.respond_to?(:call) + ext = use_internal_notes_extractor_for?(source) ? internal_notes_extractor : extractor + source.instance_exec(current_user, ext, &attr) else process << source.__send__(attr) # rubocop:disable GitlabSecurity/PublicSend @@ -121,7 +129,18 @@ module Participable end end - participants.merge(ext.users) + participants.merge(users_that_can_read_internal_notes(internal_notes_extractor)) + participants.merge(extractor.users) + end + + def use_internal_notes_extractor_for?(source) + source.is_a?(Note) && source.confidential? + end + + def users_that_can_read_internal_notes(extractor) + return [] unless self.is_a?(Noteable) && self.try(:resource_parent) + + Ability.users_that_can_read_internal_notes(extractor.users, self.resource_parent) end def source_visible_to_user?(source, user) diff --git a/app/models/concerns/require_email_verification.rb b/app/models/concerns/require_email_verification.rb new file mode 100644 index 00000000000..cf6a31e6ebd --- /dev/null +++ b/app/models/concerns/require_email_verification.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +# == Require Email Verification module +# +# Contains functionality to handle email verification +module RequireEmailVerification + extend ActiveSupport::Concern + include Gitlab::Utils::StrongMemoize + + # This value is twice the amount we want it to be, because due to a bug in the devise-two-factor + # gem every failed login attempt increments the value of failed_attempts by 2 instead of 1. + # See: https://github.com/tinfoil/devise-two-factor/issues/127 + MAXIMUM_ATTEMPTS = 3 * 2 + UNLOCK_IN = 24.hours + + included do + # Virtual attribute for the email verification token form + attr_accessor :verification_token + end + + # When overridden, do not send Devise unlock instructions when locking access. + def lock_access!(opts = {}) + return super unless override_devise_lockable? + + super({ send_instructions: false }) + end + + protected + + # We cannot override the class methods `maximum_attempts` and `unlock_in`, because we want to + # check for 2FA being enabled on the instance. So instead override the Devise Lockable methods + # where those values are used. + def attempts_exceeded? + return super unless override_devise_lockable? + + failed_attempts >= MAXIMUM_ATTEMPTS + end + + def lock_expired? + return super unless override_devise_lockable? + + locked_at && locked_at < UNLOCK_IN.ago + end + + private + + def override_devise_lockable? + strong_memoize(:override_devise_lockable) do + Feature.enabled?(:require_email_verification, self) && !two_factor_enabled? + end + end +end diff --git a/app/models/concerns/vulnerability_finding_helpers.rb b/app/models/concerns/vulnerability_finding_helpers.rb index 7f96b3901f1..4cf36f83857 100644 --- a/app/models/concerns/vulnerability_finding_helpers.rb +++ b/app/models/concerns/vulnerability_finding_helpers.rb @@ -42,4 +42,41 @@ module VulnerabilityFindingHelpers ) end end + + def build_vulnerability_finding(security_finding) + report_finding = report_finding_for(security_finding) + return Vulnerabilities::Finding.new unless report_finding + + finding_data = report_finding.to_hash.except(:compare_key, :identifiers, :location, :scanner, :links, :signatures, + :flags, :evidence) + identifiers = report_finding.identifiers.map do |identifier| + Vulnerabilities::Identifier.new(identifier.to_hash) + end + signatures = report_finding.signatures.map do |signature| + Vulnerabilities::FindingSignature.new(signature.to_hash) + end + evidence = Vulnerabilities::Finding::Evidence.new(data: report_finding.evidence.data) if report_finding.evidence + + Vulnerabilities::Finding.new(finding_data).tap do |finding| + finding.location_fingerprint = report_finding.location.fingerprint + finding.vulnerability = vulnerability_for(security_finding.uuid) + finding.project = project + finding.sha = pipeline.sha + finding.scanner = security_finding.scanner + finding.finding_evidence = evidence + + if calculate_false_positive? + finding.vulnerability_flags = report_finding.flags.map do |flag| + Vulnerabilities::Flag.new(flag) + end + end + + finding.identifiers = identifiers + finding.signatures = signatures + end + end + + def calculate_false_positive? + project.licensed_feature_available?(:sast_fp_reduction) + end end |