diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-09-19 04:45:44 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-09-19 04:45:44 +0300 |
commit | 85dc423f7090da0a52c73eb66faf22ddb20efff9 (patch) | |
tree | 9160f299afd8c80c038f08e1545be119f5e3f1e1 /app/models/concerns | |
parent | 15c2c8c66dbe422588e5411eee7e68f1fa440bb8 (diff) |
Add latest changes from gitlab-org/gitlab@13-4-stable-ee
Diffstat (limited to 'app/models/concerns')
25 files changed, 682 insertions, 377 deletions
diff --git a/app/models/concerns/admin_changed_password_notifier.rb b/app/models/concerns/admin_changed_password_notifier.rb new file mode 100644 index 00000000000..f6c2abc7e0f --- /dev/null +++ b/app/models/concerns/admin_changed_password_notifier.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module AdminChangedPasswordNotifier + # This module is responsible for triggering the `Password changed by administrator` emails + # when a GitLab administrator changes the password of another user. + + # Usage + # These emails are disabled by default and are never trigerred after updating the password, unless + # explicitly specified. + + # To explicitly trigger this email, the `send_only_admin_changed_your_password_notification!` + # method should be called, so like: + + # user = User.find_by(email: 'hello@example.com') + # user.send_only_admin_changed_your_password_notification! + # user.password = user.password_confirmation = 'new_password' + # user.save! + + # The `send_only_admin_changed_your_password_notification` has 2 responsibilities. + # It prevents triggering Devise's default `Password changed` email. + # It trigggers the `Password changed by administrator` email. + + # It is important to skip sending the default Devise email when sending out `Password changed by administrator` + # email because we should not be sending 2 emails for the same event, + # hence the only public API made available from this module is `send_only_admin_changed_your_password_notification!` + + # There is no public API made available to send the `Password changed by administrator` email, + # *without* skipping the default `Password changed` email, to prevent the problem mentioned above. + + extend ActiveSupport::Concern + + included do + after_update :send_admin_changed_your_password_notification, if: :send_admin_changed_your_password_notification? + end + + def initialize(*args, &block) + @allow_admin_changed_your_password_notification = false # These emails are off by default + super + end + + def send_only_admin_changed_your_password_notification! + skip_password_change_notification! # skip sending the default Devise 'password changed' notification + allow_admin_changed_your_password_notification! + end + + private + + def send_admin_changed_your_password_notification + send_devise_notification(:password_change_by_admin) + end + + def allow_admin_changed_your_password_notification! + @allow_admin_changed_your_password_notification = true # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + + def send_admin_changed_your_password_notification? + self.class.send_password_change_notification && saved_change_to_encrypted_password? && + @allow_admin_changed_your_password_notification # rubocop:disable Gitlab/ModuleWithInstanceVariables + end +end diff --git a/app/models/concerns/bulk_member_access_load.rb b/app/models/concerns/bulk_member_access_load.rb index 041ed3755e0..f44ad474cd5 100644 --- a/app/models/concerns/bulk_member_access_load.rb +++ b/app/models/concerns/bulk_member_access_load.rb @@ -22,7 +22,7 @@ module BulkMemberAccessLoad end # Look up only the IDs we need - resource_ids = resource_ids - access.keys + resource_ids -= access.keys return access if resource_ids.empty? diff --git a/app/models/concerns/checksummable.rb b/app/models/concerns/checksummable.rb index 1f76eb87aa5..d6d17bfc604 100644 --- a/app/models/concerns/checksummable.rb +++ b/app/models/concerns/checksummable.rb @@ -3,9 +3,13 @@ module Checksummable extend ActiveSupport::Concern + def crc32(data) + Zlib.crc32(data) + end + class_methods do def hexdigest(path) - Digest::SHA256.file(path).hexdigest + ::Digest::SHA256.file(path).hexdigest end end end diff --git a/app/models/concerns/ci/artifactable.rb b/app/models/concerns/ci/artifactable.rb index 54fb9021f2f..24df86dbc3c 100644 --- a/app/models/concerns/ci/artifactable.rb +++ b/app/models/concerns/ci/artifactable.rb @@ -4,6 +4,8 @@ module Ci module Artifactable extend ActiveSupport::Concern + NotSupportedAdapterError = Class.new(StandardError) + FILE_FORMAT_ADAPTERS = { gzip: Gitlab::Ci::Build::Artifacts::Adapters::GzipStream, raw: Gitlab::Ci::Build::Artifacts::Adapters::RawStream @@ -15,6 +17,24 @@ module Ci zip: 2, gzip: 3 }, _suffix: true + + scope :expired, -> (limit) { where('expire_at < ?', Time.current).limit(limit) } + end + + def each_blob(&blk) + unless file_format_adapter_class + raise NotSupportedAdapterError, 'This file format requires a dedicated adapter' + end + + file.open do |stream| + file_format_adapter_class.new(stream).each_blob(&blk) + end + end + + private + + def file_format_adapter_class + FILE_FORMAT_ADAPTERS[file_format.to_sym] end end end diff --git a/app/models/concerns/discussion_on_diff.rb b/app/models/concerns/discussion_on_diff.rb index 8542c48f366..40891073738 100644 --- a/app/models/concerns/discussion_on_diff.rb +++ b/app/models/concerns/discussion_on_diff.rb @@ -13,14 +13,12 @@ module DiscussionOnDiff :diff_line, :active?, :created_at_diff?, - to: :first_note delegate :file_path, :blob, :highlighted_diff_lines, :diff_lines, - to: :diff_file, allow_nil: true end diff --git a/app/models/concerns/enums/ci/pipeline.rb b/app/models/concerns/enums/ci/pipeline.rb new file mode 100644 index 00000000000..f1bc43a12d8 --- /dev/null +++ b/app/models/concerns/enums/ci/pipeline.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Enums + module Ci + module Pipeline + # Returns the `Hash` to use for creating the `failure_reason` enum for + # `Ci::Pipeline`. + def self.failure_reasons + { + unknown_failure: 0, + config_error: 1, + external_validation_failure: 2 + } + end + + # Returns the `Hash` to use for creating the `sources` enum for + # `Ci::Pipeline`. + def self.sources + { + unknown: nil, + push: 1, + web: 2, + trigger: 3, + schedule: 4, + api: 5, + external: 6, + # TODO: Rename `pipeline` to `cross_project_pipeline` in 13.0 + # https://gitlab.com/gitlab-org/gitlab/issues/195991 + pipeline: 7, + chat: 8, + webide: 9, + merge_request_event: 10, + external_pull_request_event: 11, + parent_pipeline: 12, + ondemand_dast_scan: 13 + } + end + + # Dangling sources are those events that generate pipelines for which + # we don't want to directly affect the ref CI status. + # - when a webide pipeline fails it does not change the ref CI status to failed + # - when a child pipeline (from parent_pipeline source) fails it affects its + # parent pipeline. It's up to the parent to affect the ref CI status + # - when an ondemand_dast_scan pipeline runs it is for testing purpose and should + # not affect the ref CI status. + def self.dangling_sources + sources.slice(:webide, :parent_pipeline, :ondemand_dast_scan) + end + + # CI sources are those pipeline events that affect the CI status of the ref + # they run for. By definition it excludes dangling pipelines. + def self.ci_sources + sources.except(*dangling_sources.keys) + end + + # Returns the `Hash` to use for creating the `config_sources` enum for + # `Ci::Pipeline`. + def self.config_sources + { + unknown_source: nil, + repository_source: 1, + auto_devops_source: 2, + webide_source: 3, + remote_source: 4, + external_project_source: 5, + bridge_source: 6, + parameter_source: 7 + } + end + end + end +end + +Enums::Ci::Pipeline.prepend_if_ee('EE::Enums::Ci::Pipeline') diff --git a/app/models/concerns/enums/commit_status.rb b/app/models/concerns/enums/commit_status.rb new file mode 100644 index 00000000000..faeed7276ab --- /dev/null +++ b/app/models/concerns/enums/commit_status.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Enums + module CommitStatus + # Returns the Hash to use for creating the `failure_reason` enum for + # `CommitStatus`. + def self.failure_reasons + { + unknown_failure: nil, + script_failure: 1, + api_failure: 2, + stuck_or_timeout_failure: 3, + runner_system_failure: 4, + missing_dependency_failure: 5, + runner_unsupported: 6, + stale_schedule: 7, + job_execution_timeout: 8, + archived_failure: 9, + unmet_prerequisites: 10, + scheduler_failure: 11, + data_integrity_failure: 12, + forward_deployment_failure: 13, + insufficient_bridge_permissions: 1_001, + downstream_bridge_project_not_found: 1_002, + invalid_bridge_trigger: 1_003, + bridge_pipeline_is_child_pipeline: 1_006, # not used anymore, but cannot be deleted because of old data + downstream_pipeline_creation_failed: 1_007, + secrets_provider_not_found: 1_008, + reached_max_descendant_pipelines_depth: 1_009 + } + end + end +end + +Enums::CommitStatus.prepend_if_ee('EE::Enums::CommitStatus') diff --git a/app/models/concerns/enums/internal_id.rb b/app/models/concerns/enums/internal_id.rb new file mode 100644 index 00000000000..2d51d232e93 --- /dev/null +++ b/app/models/concerns/enums/internal_id.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Enums + module InternalId + def self.usage_resources + # when adding new resource, make sure it doesn't conflict with EE usage_resources + { + issues: 0, + merge_requests: 1, + deployments: 2, + milestones: 3, + epics: 4, + ci_pipelines: 5, + operations_feature_flags: 6, + operations_user_lists: 7, + alert_management_alerts: 8, + sprints: 9 # iterations + } + end + end +end + +Enums::InternalId.prepend_if_ee('EE::Enums::InternalId') diff --git a/app/models/concerns/enums/prometheus_metric.rb b/app/models/concerns/enums/prometheus_metric.rb new file mode 100644 index 00000000000..e65a01990a3 --- /dev/null +++ b/app/models/concerns/enums/prometheus_metric.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Enums + module PrometheusMetric + def self.groups + { + # built-in groups + nginx_ingress_vts: -1, + ha_proxy: -2, + aws_elb: -3, + nginx: -4, + kubernetes: -5, + nginx_ingress: -6, + cluster_health: -100 + }.merge(custom_groups).freeze + end + + # custom/user groups + def self.custom_groups + { + business: 0, + response: 1, + system: 2, + custom: 3 + }.freeze + end + + def self.group_details + { + # built-in groups + nginx_ingress_vts: { + group_title: _('Response metrics (NGINX Ingress VTS)'), + required_metrics: %w(nginx_upstream_responses_total nginx_upstream_response_msecs_avg), + priority: 10 + }.freeze, + nginx_ingress: { + group_title: _('Response metrics (NGINX Ingress)'), + required_metrics: %w(nginx_ingress_controller_requests nginx_ingress_controller_ingress_upstream_latency_seconds_sum), + priority: 10 + }.freeze, + ha_proxy: { + group_title: _('Response metrics (HA Proxy)'), + required_metrics: %w(haproxy_frontend_http_requests_total haproxy_frontend_http_responses_total), + priority: 10 + }.freeze, + aws_elb: { + group_title: _('Response metrics (AWS ELB)'), + required_metrics: %w(aws_elb_request_count_sum aws_elb_latency_average aws_elb_httpcode_backend_5_xx_sum), + priority: 10 + }.freeze, + nginx: { + group_title: _('Response metrics (NGINX)'), + required_metrics: %w(nginx_server_requests nginx_server_requestMsec), + priority: 10 + }.freeze, + kubernetes: { + group_title: _('System metrics (Kubernetes)'), + required_metrics: %w(container_memory_usage_bytes container_cpu_usage_seconds_total), + priority: 5 + }.freeze, + cluster_health: { + group_title: _('Cluster Health'), + required_metrics: %w(container_memory_usage_bytes container_cpu_usage_seconds_total), + priority: 10 + }.freeze + }.merge(custom_group_details).freeze + end + + # custom/user groups + def self.custom_group_details + { + business: { + group_title: _('Business metrics (Custom)'), + priority: 0 + }.freeze, + response: { + group_title: _('Response metrics (Custom)'), + priority: -5 + }.freeze, + system: { + group_title: _('System metrics (Custom)'), + priority: -10 + }.freeze, + custom: { + group_title: _('Custom metrics'), + priority: 0 + } + }.freeze + end + end +end diff --git a/app/models/concerns/from_except.rb b/app/models/concerns/from_except.rb new file mode 100644 index 00000000000..b9ca9dda4b0 --- /dev/null +++ b/app/models/concerns/from_except.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module FromExcept + extend ActiveSupport::Concern + + class_methods do + # Produces a query that uses a FROM to select data using an EXCEPT. + # + # Example: + # groups = Group.from_except([group1.self_and_hierarchy, group2.self_and_hierarchy]) + # + # This would produce the following SQL query: + # + # SELECT * + # FROM ( + # SELECT "namespaces". * + # ... + # + # EXCEPT + # + # SELECT "namespaces". * + # ... + # ) groups; + # + # members - An Array of ActiveRecord::Relation objects to use in the except. + # + # remove_duplicates - A boolean indicating if duplicate entries should be + # removed. Defaults to true. + # + # alias_as - The alias to use for the sub query. Defaults to the name of the + # table of the current model. + # rubocop: disable Gitlab/Except + extend FromSetOperator + define_set_operator Gitlab::SQL::Except + # rubocop: enable Gitlab/Except + end +end diff --git a/app/models/concerns/from_intersect.rb b/app/models/concerns/from_intersect.rb new file mode 100644 index 00000000000..428e63eb45e --- /dev/null +++ b/app/models/concerns/from_intersect.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module FromIntersect + extend ActiveSupport::Concern + + class_methods do + # Produces a query that uses a FROM to select data using an INTERSECT. + # + # Example: + # groups = Group.from_intersect([group1.self_and_hierarchy, group2.self_and_hierarchy]) + # + # This would produce the following SQL query: + # + # SELECT * + # FROM ( + # SELECT "namespaces". * + # ... + # + # INTERSECT + # + # SELECT "namespaces". * + # ... + # ) groups; + # + # members - An Array of ActiveRecord::Relation objects to use in the intersect. + # + # remove_duplicates - A boolean indicating if duplicate entries should be + # removed. Defaults to true. + # + # alias_as - The alias to use for the sub query. Defaults to the name of the + # table of the current model. + # rubocop: disable Gitlab/Intersect + extend FromSetOperator + define_set_operator Gitlab::SQL::Intersect + # rubocop: enable Gitlab/Intersect + end +end diff --git a/app/models/concerns/from_set_operator.rb b/app/models/concerns/from_set_operator.rb new file mode 100644 index 00000000000..593fd251c5c --- /dev/null +++ b/app/models/concerns/from_set_operator.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module FromSetOperator + # Define a high level method to more easily work with the SQL set operations + # of UNION, INTERSECT, and EXCEPT as defined by Gitlab::SQL::Union, + # Gitlab::SQL::Intersect, and Gitlab::SQL::Except respectively. + def define_set_operator(operator) + method_name = 'from_' + operator.name.demodulize.downcase + method_name = method_name.to_sym + + raise "Trying to redefine method '#{method(method_name)}'" if methods.include?(method_name) + + define_method(method_name) do |members, remove_duplicates: true, alias_as: table_name| + operator_sql = operator.new(members, remove_duplicates: remove_duplicates).to_sql + + from(Arel.sql("(#{operator_sql}) #{alias_as}")) + end + end +end diff --git a/app/models/concerns/from_union.rb b/app/models/concerns/from_union.rb index e28dee34815..e25d603b802 100644 --- a/app/models/concerns/from_union.rb +++ b/app/models/concerns/from_union.rb @@ -35,13 +35,29 @@ module FromUnion # alias_as - The alias to use for the sub query. Defaults to the name of the # table of the current model. # rubocop: disable Gitlab/Union + extend FromSetOperator + define_set_operator Gitlab::SQL::Union + + alias_method :from_union_set_operator, :from_union def from_union(members, remove_duplicates: true, alias_as: table_name) + if Feature.enabled?(:sql_set_operators) + from_union_set_operator(members, remove_duplicates: remove_duplicates, alias_as: alias_as) + else + # The original from_union method. + standard_from_union(members, remove_duplicates: remove_duplicates, alias_as: alias_as) + end + end + + private + + def standard_from_union(members, remove_duplicates: true, alias_as: table_name) union = Gitlab::SQL::Union .new(members, remove_duplicates: remove_duplicates) .to_sql from(Arel.sql("(#{union}) #{alias_as}")) end + # rubocop: enable Gitlab/Union end end diff --git a/app/models/concerns/has_wiki.rb b/app/models/concerns/has_wiki.rb index 3e7cb940a62..df7bbe4dc08 100644 --- a/app/models/concerns/has_wiki.rb +++ b/app/models/concerns/has_wiki.rb @@ -25,10 +25,6 @@ module HasWiki wiki.repository_exists? end - def after_wiki_activity - true - end - private def check_wiki_path_conflict diff --git a/app/models/concerns/id_in_ordered.rb b/app/models/concerns/id_in_ordered.rb new file mode 100644 index 00000000000..b89409e6841 --- /dev/null +++ b/app/models/concerns/id_in_ordered.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module IdInOrdered + extend ActiveSupport::Concern + + included do + scope :id_in_ordered, -> (ids) do + raise ArgumentError, "ids must be an array of integers" unless ids.is_a?(Enumerable) && ids.all? { |id| id.is_a?(Integer) } + + # No need to sort if no more than 1 and the sorting code doesn't work + # with an empty array + return id_in(ids) unless ids.count > 1 + + id_attribute = arel_table[:id] + id_in(ids) + .order( + Arel.sql("array_position(ARRAY[#{ids.join(',')}], #{id_attribute.relation.name}.#{id_attribute.name})")) + end + end +end diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index dd5aedbb760..888e1b384a2 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -177,10 +177,41 @@ module Issuable assignees.count > 1 end - def supports_weight? + def allows_reviewers? false end + def supports_time_tracking? + is_a?(TimeTrackable) && !incident? + end + + def supports_severity? + incident? + end + + def incident? + is_a?(Issue) && super + end + + def supports_issue_type? + is_a?(Issue) + end + + def severity + return IssuableSeverity::DEFAULT unless incident? + + issuable_severity&.severity || IssuableSeverity::DEFAULT + end + + def update_severity(severity) + return unless incident? + + severity = severity.to_s.downcase + severity = IssuableSeverity::DEFAULT unless IssuableSeverity.severities.key?(severity) + + (issuable_severity || build_issuable_severity(issue_id: id)).update(severity: severity) + end + private def description_max_length_for_new_records_is_valid @@ -377,8 +408,12 @@ module Issuable Date.today == created_at.to_date end + def created_hours_ago + (Time.now.utc.to_i - created_at.utc.to_i) / 3600 + end + def new? - today? && created_at == updated_at + created_hours_ago < 24 end def open? diff --git a/app/models/concerns/loaded_in_group_list.rb b/app/models/concerns/loaded_in_group_list.rb index 79ff82d9f99..e624b9aa356 100644 --- a/app/models/concerns/loaded_in_group_list.rb +++ b/app/models/concerns/loaded_in_group_list.rb @@ -54,6 +54,7 @@ module LoadedInGroupList .where(members[:source_type].eq(Namespace.name)) .where(members[:source_id].eq(namespaces[:id])) .where(members[:requested_at].eq(nil)) + .where(members[:access_level].gt(Gitlab::Access::MINIMAL_ACCESS)) end end @@ -70,7 +71,7 @@ module LoadedInGroupList end def member_count - @member_count ||= try(:preloaded_member_count) || users.count + @member_count ||= try(:preloaded_member_count) || members.count end end diff --git a/app/models/concerns/mentionable/reference_regexes.rb b/app/models/concerns/mentionable/reference_regexes.rb index f44a674b3c9..307d58a3a3c 100644 --- a/app/models/concerns/mentionable/reference_regexes.rb +++ b/app/models/concerns/mentionable/reference_regexes.rb @@ -30,7 +30,7 @@ module Mentionable def self.external_pattern strong_memoize(:external_pattern) do issue_pattern = IssueTrackerService.reference_pattern - link_patterns = URI.regexp(%w(http https)) + link_patterns = URI::DEFAULT_PARSER.make_regexp(%w(http https)) reference_pattern(link_patterns, issue_pattern) end end diff --git a/app/models/concerns/milestoneable.rb b/app/models/concerns/milestoneable.rb index ccb334343ff..b1698bc2ee3 100644 --- a/app/models/concerns/milestoneable.rb +++ b/app/models/concerns/milestoneable.rb @@ -51,7 +51,7 @@ module Milestoneable # Overridden on EE module # def supports_milestone? - respond_to?(:milestone_id) + respond_to?(:milestone_id) && !incident? end end diff --git a/app/models/concerns/optimized_issuable_label_filter.rb b/app/models/concerns/optimized_issuable_label_filter.rb new file mode 100644 index 00000000000..7be4a26d4fa --- /dev/null +++ b/app/models/concerns/optimized_issuable_label_filter.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +module OptimizedIssuableLabelFilter + def by_label(items) + return items unless params.labels? + + return super if Feature.disabled?(:optimized_issuable_label_filter) + + target_model = items.model + + if params.filter_by_no_label? + items.where('NOT EXISTS (?)', optimized_any_label_query(target_model)) + elsif params.filter_by_any_label? + items.where('EXISTS (?)', optimized_any_label_query(target_model)) + else + issuables_with_selected_labels(items, target_model) + end + end + + # Taken from IssuableFinder + def count_by_state + return super if root_namespace.nil? + return super if Feature.disabled?(:optimized_issuable_label_filter) + + count_params = params.merge(state: nil, sort: nil, force_cte: true) + finder = self.class.new(current_user, count_params) + + state_counts = finder + .execute + .reorder(nil) + .group(:state_id) + .count + + counts = state_counts.transform_keys { |key| count_key(key) } + + counts[:all] = counts.values.sum + counts.with_indifferent_access + end + + private + + def issuables_with_selected_labels(items, target_model) + if root_namespace + all_label_ids = find_label_ids(root_namespace) + # Found less labels in the DB than we were searching for. Return nothing. + return items.none if all_label_ids.size != params.label_names.size + + all_label_ids.each do |label_ids| + items = items.where('EXISTS (?)', optimized_label_query_by_label_ids(target_model, label_ids)) + end + else + params.label_names.each do |label_name| + items = items.where('EXISTS (?)', optimized_label_query_by_label_name(target_model, label_name)) + end + end + + items + end + + def find_label_ids(root_namespace) + finder_params = { + include_subgroups: true, + include_ancestor_groups: true, + include_descendant_groups: true, + group: root_namespace, + title: params.label_names + } + + LabelsFinder + .new(nil, finder_params) + .execute(skip_authorization: true) + .pluck(:title, :id) + .group_by(&:first) + .values + .map { |labels| labels.map(&:last) } + end + + def root_namespace + strong_memoize(:root_namespace) do + (params.project || params.group)&.root_ancestor + end + end + + def optimized_any_label_query(target_model) + LabelLink + .where(target_type: target_model.name) + .where(LabelLink.arel_table['target_id'].eq(target_model.arel_table['id'])) + .limit(1) + end + + def optimized_label_query_by_label_ids(target_model, label_ids) + LabelLink + .where(target_type: target_model.name) + .where(LabelLink.arel_table['target_id'].eq(target_model.arel_table['id'])) + .where(label_id: label_ids) + .limit(1) + end + + def optimized_label_query_by_label_name(target_model, label_name) + LabelLink + .joins(:label) + .where(target_type: target_model.name) + .where(LabelLink.arel_table['target_id'].eq(target_model.arel_table['id'])) + .where(labels: { name: label_name }) + .limit(1) + end +end diff --git a/app/models/concerns/prometheus_adapter.rb b/app/models/concerns/prometheus_adapter.rb index adb6a59e11c..55c2bf96a94 100644 --- a/app/models/concerns/prometheus_adapter.rb +++ b/app/models/concerns/prometheus_adapter.rb @@ -3,6 +3,11 @@ module PrometheusAdapter extend ActiveSupport::Concern + # We should choose more conservative timeouts, but some queries we run are now busting our + # default timeouts, which are stricter. We should make those queries faster instead. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/232786 + DEFAULT_PROMETHEUS_REQUEST_TIMEOUT_SEC = 60.seconds + included do include ReactiveCaching @@ -15,6 +20,12 @@ module PrometheusAdapter raise NotImplementedError end + def prometheus_client_default_options + { + timeout: DEFAULT_PROMETHEUS_REQUEST_TIMEOUT_SEC + } + end + # This is a light-weight check if a prometheus client is properly configured. def configured? raise NotImplemented diff --git a/app/models/concerns/relative_positioning.rb b/app/models/concerns/relative_positioning.rb index d1f04609693..3cbc174536c 100644 --- a/app/models/concerns/relative_positioning.rb +++ b/app/models/concerns/relative_positioning.rb @@ -27,18 +27,7 @@ # module RelativePositioning extend ActiveSupport::Concern - - STEPS = 10 - IDEAL_DISTANCE = 2**(STEPS - 1) + 1 - - MIN_POSITION = Gitlab::Database::MIN_INT_VALUE - START_POSITION = 0 - MAX_POSITION = Gitlab::Database::MAX_INT_VALUE - - MAX_GAP = IDEAL_DISTANCE * 2 - MIN_GAP = 2 - - NoSpaceLeft = Class.new(StandardError) + include ::Gitlab::RelativePositioning class_methods do def move_nulls_to_end(objects) @@ -49,56 +38,10 @@ module RelativePositioning move_nulls(objects, at_end: false) end - # This method takes two integer values (positions) and - # calculates the position between them. The range is huge as - # the maximum integer value is 2147483647. - # - # We avoid open ranges by clamping the range to [MIN_POSITION, MAX_POSITION]. - # - # Then we handle one of three cases: - # - If the gap is too small, we raise NoSpaceLeft - # - If the gap is larger than MAX_GAP, we place the new position at most - # IDEAL_DISTANCE from the edge of the gap. - # - otherwise we place the new position at the midpoint. - # - # The new position will always satisfy: pos_before <= midpoint <= pos_after - # - # As a precondition, the gap between pos_before and pos_after MUST be >= 2. - # If the gap is too small, NoSpaceLeft is raised. - # - # This class method should only be called by instance methods of this module, which - # include handling for minimum gap size. - # - # @raises NoSpaceLeft - # @api private - def position_between(pos_before, pos_after) - pos_before ||= MIN_POSITION - pos_after ||= MAX_POSITION - - pos_before, pos_after = [pos_before, pos_after].sort - - gap_width = pos_after - pos_before - midpoint = [pos_after - 1, pos_before + (gap_width / 2)].min - - if gap_width < MIN_GAP - raise NoSpaceLeft - elsif gap_width > MAX_GAP - if pos_before == MIN_POSITION - pos_after - IDEAL_DISTANCE - elsif pos_after == MAX_POSITION - pos_before + IDEAL_DISTANCE - else - midpoint - end - else - midpoint - end - end - private # @api private - def gap_size(object, gaps:, at_end:, starting_from:) + def gap_size(context, gaps:, at_end:, starting_from:) total_width = IDEAL_DISTANCE * gaps size = if at_end && starting_from + total_width >= MAX_POSITION (MAX_POSITION - starting_from) / gaps @@ -108,23 +51,17 @@ module RelativePositioning IDEAL_DISTANCE end - # Shift max elements leftwards if there isn't enough space return [size, starting_from] if size >= MIN_GAP - order = at_end ? :desc : :asc - terminus = object - .send(:relative_siblings) # rubocop:disable GitlabSecurity/PublicSend - .where('relative_position IS NOT NULL') - .order(relative_position: order) - .first - if at_end - terminus.move_sequence_before(true) - max_relative_position = terminus.reset.relative_position + terminus = context.max_sibling + terminus.shift_left + max_relative_position = terminus.relative_position [[(MAX_POSITION - max_relative_position) / gaps, IDEAL_DISTANCE].min, max_relative_position] else - terminus.move_sequence_after(true) - min_relative_position = terminus.reset.relative_position + terminus = context.min_sibling + terminus.shift_right + min_relative_position = terminus.relative_position [[(min_relative_position - MIN_POSITION) / gaps, IDEAL_DISTANCE].min, min_relative_position] end end @@ -142,8 +79,9 @@ module RelativePositioning objects = objects.reject(&:relative_position) return 0 if objects.empty? - representative = objects.first - number_of_gaps = objects.size + 1 # 1 at left, one between each, and one at right + number_of_gaps = objects.size # 1 to the nearest neighbour, and one between each + representative = RelativePositioning.mover.context(objects.first) + position = if at_end representative.max_relative_position else @@ -152,16 +90,21 @@ module RelativePositioning position ||= START_POSITION # If there are no positioned siblings, start from START_POSITION - gap, position = gap_size(representative, gaps: number_of_gaps, at_end: at_end, starting_from: position) - - # Raise if we could not make enough space - raise NoSpaceLeft if gap < MIN_GAP + gap = 0 + attempts = 10 # consolidate up to 10 gaps to find enough space + while gap < 1 && attempts > 0 + gap, position = gap_size(representative, gaps: number_of_gaps, at_end: at_end, starting_from: position) + attempts -= 1 + end - indexed = objects.each_with_index.to_a - starting_from = at_end ? position : position - (gap * number_of_gaps) + # Allow placing items next to each other, if we have to. + gap = 1 if gap < MIN_GAP + delta = at_end ? gap : -gap + indexed = (at_end ? objects : objects.reverse).each_with_index # Some classes are polymorphic, and not all siblings are in the same table. by_model = indexed.group_by { |pair| pair.first.class } + lower_bound, upper_bound = at_end ? [position, MAX_POSITION] : [MIN_POSITION, position] by_model.each do |model, pairs| model.transaction do @@ -169,7 +112,8 @@ module RelativePositioning # These are known to be integers, one from the DB, and the other # calculated by us, and thus safe to interpolate values = batch.map do |obj, i| - pos = starting_from + gap * (i + 1) + desired_pos = position + delta * (i + 1) + pos = desired_pos.clamp(lower_bound, upper_bound) obj.relative_position = pos "(#{obj.id}, #{pos})" end.join(', ') @@ -192,306 +136,68 @@ module RelativePositioning end end - def min_relative_position(&block) - calculate_relative_position('MIN', &block) - end - - def max_relative_position(&block) - calculate_relative_position('MAX', &block) - end - - def prev_relative_position(ignoring: nil) - prev_pos = nil - - if self.relative_position - prev_pos = max_relative_position do |relation| - relation = relation.id_not_in(ignoring.id) if ignoring.present? - relation.where('relative_position < ?', self.relative_position) - end - end - - prev_pos - end - - def next_relative_position(ignoring: nil) - next_pos = nil - - if self.relative_position - next_pos = min_relative_position do |relation| - relation = relation.id_not_in(ignoring.id) if ignoring.present? - relation.where('relative_position > ?', self.relative_position) - end - end - - next_pos + def self.mover + ::Gitlab::RelativePositioning::Mover.new(START_POSITION, (MIN_POSITION..MAX_POSITION)) end def move_between(before, after) - return move_after(before) unless after - return move_before(after) unless before - - before, after = after, before if after.relative_position < before.relative_position - - pos_left = before.relative_position - pos_right = after.relative_position + before, after = [before, after].sort_by(&:relative_position) if before && after - if pos_right - pos_left < MIN_GAP - # Not enough room! Make space by shifting all previous elements to the left - # if there is enough space, else to the right - gap = after.send(:find_next_gap_before) # rubocop:disable GitlabSecurity/PublicSend - - if gap.present? - after.move_sequence_before(next_gap: gap) - pos_left -= optimum_delta_for_gap(gap) - else - before.move_sequence_after - pos_right = after.reset.relative_position - end - end - - new_position = self.class.position_between(pos_left, pos_right) - - self.relative_position = new_position + RelativePositioning.mover.move(self, before, after) + rescue ActiveRecord::QueryCanceled, NoSpaceLeft => e + could_not_move(e) + raise e end def move_after(before = self) - pos_before = before.relative_position - pos_after = before.next_relative_position(ignoring: self) - - if pos_before == MAX_POSITION || gap_too_small?(pos_after, pos_before) - gap = before.send(:find_next_gap_after) # rubocop:disable GitlabSecurity/PublicSend - - if gap.nil? - before.move_sequence_before(true) - pos_before = before.reset.relative_position - else - before.move_sequence_after(next_gap: gap) - pos_after += optimum_delta_for_gap(gap) - end - end - - self.relative_position = self.class.position_between(pos_before, pos_after) + RelativePositioning.mover.move(self, before, nil) + rescue ActiveRecord::QueryCanceled, NoSpaceLeft => e + could_not_move(e) + raise e end def move_before(after = self) - pos_after = after.relative_position - pos_before = after.prev_relative_position(ignoring: self) - - if pos_after == MIN_POSITION || gap_too_small?(pos_before, pos_after) - gap = after.send(:find_next_gap_before) # rubocop:disable GitlabSecurity/PublicSend - - if gap.nil? - after.move_sequence_after(true) - pos_after = after.reset.relative_position - else - after.move_sequence_before(next_gap: gap) - pos_before -= optimum_delta_for_gap(gap) - end - end - - self.relative_position = self.class.position_between(pos_before, pos_after) + RelativePositioning.mover.move(self, nil, after) + rescue ActiveRecord::QueryCanceled, NoSpaceLeft => e + could_not_move(e) + raise e end def move_to_end - max_pos = max_relative_position - - if max_pos.nil? - self.relative_position = START_POSITION - elsif gap_too_small?(max_pos, MAX_POSITION) - max = relative_siblings.order(Gitlab::Database.nulls_last_order('relative_position', 'DESC')).first - max.move_sequence_before(true) - max.reset - self.relative_position = self.class.position_between(max.relative_position, MAX_POSITION) - else - self.relative_position = self.class.position_between(max_pos, MAX_POSITION) - end + RelativePositioning.mover.move_to_end(self) + rescue NoSpaceLeft => e + could_not_move(e) + self.relative_position = MAX_POSITION + rescue ActiveRecord::QueryCanceled => e + could_not_move(e) + raise e end def move_to_start - min_pos = min_relative_position - - if min_pos.nil? - self.relative_position = START_POSITION - elsif gap_too_small?(min_pos, MIN_POSITION) - min = relative_siblings.order(Gitlab::Database.nulls_last_order('relative_position', 'ASC')).first - min.move_sequence_after(true) - min.reset - self.relative_position = self.class.position_between(MIN_POSITION, min.relative_position) - else - self.relative_position = self.class.position_between(MIN_POSITION, min_pos) - end - end - - # Moves the sequence before the current item to the middle of the next gap - # For example, we have - # - # 5 . . . . . 11 12 13 14 [15] 16 . 17 - # ----------- - # - # This moves the sequence [11 12 13 14] to [8 9 10 11], so we have: - # - # 5 . . 8 9 10 11 . . . [15] 16 . 17 - # --------- - # - # Creating a gap to the left of the current item. We can understand this as - # dividing the 5 spaces between 5 and 11 into two smaller gaps of 2 and 3. - # - # If `include_self` is true, the current item will also be moved, creating a - # gap to the right of the current item: - # - # 5 . . 8 9 10 11 [14] . . . 16 . 17 - # -------------- - # - # As an optimization, the gap can be precalculated and passed to this method. - # - # @api private - # @raises NoSpaceLeft if the sequence cannot be moved - def move_sequence_before(include_self = false, next_gap: find_next_gap_before) - raise NoSpaceLeft unless next_gap.present? - - delta = optimum_delta_for_gap(next_gap) - - move_sequence(next_gap[:start], relative_position, -delta, include_self) - end - - # Moves the sequence after the current item to the middle of the next gap - # For example, we have: - # - # 8 . 10 [11] 12 13 14 15 . . . . . 21 - # ----------- - # - # This moves the sequence [12 13 14 15] to [15 16 17 18], so we have: - # - # 8 . 10 [11] . . . 15 16 17 18 . . 21 - # ----------- - # - # Creating a gap to the right of the current item. We can understand this as - # dividing the 5 spaces between 15 and 21 into two smaller gaps of 3 and 2. - # - # If `include_self` is true, the current item will also be moved, creating a - # gap to the left of the current item: - # - # 8 . 10 . . . [14] 15 16 17 18 . . 21 - # ---------------- - # - # As an optimization, the gap can be precalculated and passed to this method. - # - # @api private - # @raises NoSpaceLeft if the sequence cannot be moved - def move_sequence_after(include_self = false, next_gap: find_next_gap_after) - raise NoSpaceLeft unless next_gap.present? - - delta = optimum_delta_for_gap(next_gap) - - move_sequence(relative_position, next_gap[:start], delta, include_self) - end - - private - - def gap_too_small?(pos_a, pos_b) - return false unless pos_a && pos_b - - (pos_a - pos_b).abs < MIN_GAP - end - - # Find the first suitable gap to the left of the current position. - # - # Satisfies the relations: - # - gap[:start] <= relative_position - # - abs(gap[:start] - gap[:end]) >= MIN_GAP - # - MIN_POSITION <= gap[:start] <= MAX_POSITION - # - MIN_POSITION <= gap[:end] <= MAX_POSITION - # - # Supposing that the current item is 13, and we have a sequence of items: - # - # 1 . . . 5 . . . . 11 12 [13] 14 . . 17 - # ^---------^ - # - # Then we return: `{ start: 11, end: 5 }` - # - # Here start refers to the end of the gap closest to the current item. - def find_next_gap_before - items_with_next_pos = scoped_items - .select('relative_position AS pos, LEAD(relative_position) OVER (ORDER BY relative_position DESC) AS next_pos') - .where('relative_position <= ?', relative_position) - .order(relative_position: :desc) - - find_next_gap(items_with_next_pos, MIN_POSITION) - end - - # Find the first suitable gap to the right of the current position. - # - # Satisfies the relations: - # - gap[:start] >= relative_position - # - abs(gap[:start] - gap[:end]) >= MIN_GAP - # - MIN_POSITION <= gap[:start] <= MAX_POSITION - # - MIN_POSITION <= gap[:end] <= MAX_POSITION - # - # Supposing the current item is 13, and that we have a sequence of items: - # - # 9 . . . [13] 14 15 . . . . 20 . . . 24 - # ^---------^ - # - # Then we return: `{ start: 15, end: 20 }` - # - # Here start refers to the end of the gap closest to the current item. - def find_next_gap_after - items_with_next_pos = scoped_items - .select('relative_position AS pos, LEAD(relative_position) OVER (ORDER BY relative_position ASC) AS next_pos') - .where('relative_position >= ?', relative_position) - .order(:relative_position) - - find_next_gap(items_with_next_pos, MAX_POSITION) - end - - def find_next_gap(items_with_next_pos, end_is_nil) - gap = self.class - .from(items_with_next_pos, :items) - .where('next_pos IS NULL OR ABS(pos::bigint - next_pos::bigint) >= ?', MIN_GAP) - .limit(1) - .pluck(:pos, :next_pos) - .first - - return if gap.nil? || gap.first == end_is_nil - - { start: gap.first, end: gap.second || end_is_nil } - end - - def optimum_delta_for_gap(gap) - delta = ((gap[:start] - gap[:end]) / 2.0).abs.ceil - - [delta, IDEAL_DISTANCE].min - end - - def move_sequence(start_pos, end_pos, delta, include_self = false) - relation = include_self ? scoped_items : relative_siblings - + RelativePositioning.mover.move_to_start(self) + rescue NoSpaceLeft => e + could_not_move(e) + self.relative_position = MIN_POSITION + rescue ActiveRecord::QueryCanceled => e + could_not_move(e) + raise e + end + + # This method is used during rebalancing - override it to customise the update + # logic: + def update_relative_siblings(relation, range, delta) relation - .where('relative_position BETWEEN ? AND ?', start_pos, end_pos) + .where(relative_position: range) .update_all("relative_position = relative_position + #{delta}") end - def calculate_relative_position(calculation) - # When calculating across projects, this is much more efficient than - # MAX(relative_position) without the GROUP BY, due to index usage: - # https://gitlab.com/gitlab-org/gitlab-foss/issues/54276#note_119340977 - relation = scoped_items - .order(Gitlab::Database.nulls_last_order('position', 'DESC')) - .group(self.class.relative_positioning_parent_column) - .limit(1) - - relation = yield relation if block_given? - - relation - .pluck(self.class.relative_positioning_parent_column, Arel.sql("#{calculation}(relative_position) AS position")) - .first&.last - end - - def relative_siblings(relation = scoped_items) - relation.id_not_in(id) + # This method is used to exclude the current self (or another object) + # from a relation. Customize this if `id <> :id` is not sufficient + def exclude_self(relation, excluded: self) + relation.id_not_in(excluded.id) end - def scoped_items - self.class.relative_positioning_query_base(self) + # Override if you want to be notified of failures to move + def could_not_move(exception) end end diff --git a/app/models/concerns/resolvable_discussion.rb b/app/models/concerns/resolvable_discussion.rb index 5174ae05d15..3e1e5faee54 100644 --- a/app/models/concerns/resolvable_discussion.rb +++ b/app/models/concerns/resolvable_discussion.rb @@ -31,7 +31,6 @@ module ResolvableDiscussion delegate :resolved_at, :resolved_by, :resolved_by_push?, - to: :last_resolved_note, allow_nil: true end diff --git a/app/models/concerns/storage/legacy_namespace.rb b/app/models/concerns/storage/legacy_namespace.rb index 250889fdf8b..71b976c6f11 100644 --- a/app/models/concerns/storage/legacy_namespace.rb +++ b/app/models/concerns/storage/legacy_namespace.rb @@ -23,10 +23,22 @@ module Storage former_parent_full_path = parent_was&.full_path parent_full_path = parent&.full_path Gitlab::UploadsTransfer.new.move_namespace(path, former_parent_full_path, parent_full_path) - Gitlab::PagesTransfer.new.move_namespace(path, former_parent_full_path, parent_full_path) + + if any_project_with_pages_deployed? + run_after_commit do + Gitlab::PagesTransfer.new.async.move_namespace(path, former_parent_full_path, parent_full_path) + end + end else Gitlab::UploadsTransfer.new.rename_namespace(full_path_before_last_save, full_path) - Gitlab::PagesTransfer.new.rename_namespace(full_path_before_last_save, full_path) + + if any_project_with_pages_deployed? + full_path_was = full_path_before_last_save + + run_after_commit do + Gitlab::PagesTransfer.new.async.rename_namespace(full_path_was, full_path) + end + end end # If repositories moved successfully we need to diff --git a/app/models/concerns/timebox.rb b/app/models/concerns/timebox.rb index 8927e42dd97..3e2cf9031d0 100644 --- a/app/models/concerns/timebox.rb +++ b/app/models/concerns/timebox.rb @@ -75,8 +75,8 @@ module Timebox scope :within_timeframe, -> (start_date, end_date) do where('start_date is not NULL or due_date is not NULL') - .where('start_date is NULL or start_date <= ?', end_date) - .where('due_date is NULL or due_date >= ?', start_date) + .where('start_date is NULL or start_date <= ?', end_date) + .where('due_date is NULL or due_date >= ?', start_date) end strip_attributes :title @@ -195,6 +195,10 @@ module Timebox end end + def weight_available? + resource_parent&.feature_available?(:issue_weights) + end + private def timebox_format_reference(format = :iid) |