diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-03-16 21:18:33 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-03-16 21:18:33 +0300 |
commit | f64a639bcfa1fc2bc89ca7db268f594306edfd7c (patch) | |
tree | a2c3c2ebcc3b45e596949db485d6ed18ffaacfa1 /app/models | |
parent | bfbc3e0d6583ea1a91f627528bedc3d65ba4b10f (diff) |
Add latest changes from gitlab-org/gitlab@13-10-stable-eev13.10.0-rc40
Diffstat (limited to 'app/models')
91 files changed, 884 insertions, 539 deletions
diff --git a/app/models/alert_management/http_integration.rb b/app/models/alert_management/http_integration.rb index 2d9a2d7031c..e98c770c364 100644 --- a/app/models/alert_management/http_integration.rb +++ b/app/models/alert_management/http_integration.rb @@ -27,6 +27,7 @@ module AlertManagement before_validation :prevent_token_assignment before_validation :prevent_endpoint_identifier_assignment before_validation :ensure_token + before_validation :ensure_payload_example_not_nil scope :for_endpoint_identifier, -> (endpoint_identifier) { where(endpoint_identifier: endpoint_identifier) } scope :active, -> { where(active: true) } @@ -74,5 +75,9 @@ module AlertManagement self.endpoint_identifier = endpoint_identifier_was end end + + def ensure_payload_example_not_nil + self.payload_example ||= {} + end end end diff --git a/app/models/analytics/instance_statistics.rb b/app/models/analytics/instance_statistics.rb deleted file mode 100644 index df7b26e4fa6..00000000000 --- a/app/models/analytics/instance_statistics.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -module Analytics - module InstanceStatistics - def self.table_name_prefix - 'analytics_instance_statistics_' - end - end -end diff --git a/app/models/analytics/instance_statistics/measurement.rb b/app/models/analytics/usage_trends/measurement.rb index c8b76e005ef..ad0272699c2 100644 --- a/app/models/analytics/instance_statistics/measurement.rb +++ b/app/models/analytics/usage_trends/measurement.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true module Analytics - module InstanceStatistics + module UsageTrends class Measurement < ApplicationRecord - EXPERIMENTAL_IDENTIFIERS = %i[pipelines_succeeded pipelines_failed pipelines_canceled pipelines_skipped].freeze + self.table_name = 'analytics_instance_statistics_measurements' enum identifier: { projects: 1, @@ -58,4 +58,4 @@ module Analytics end end -Analytics::InstanceStatistics::Measurement.prepend_if_ee('EE::Analytics::InstanceStatistics::Measurement') +Analytics::UsageTrends::Measurement.prepend_if_ee('EE::Analytics::UsageTrends::Measurement') diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 4959401eb27..44eb2fefb3f 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -25,10 +25,6 @@ class ApplicationSetting < ApplicationRecord alias_attribute :instance_group_id, :instance_administrators_group_id alias_attribute :instance_administrators_group, :instance_group - def self.repository_storages_weighted_attributes - @repository_storages_weighted_atributes ||= Gitlab.config.repositories.storages.keys.map { |k| "repository_storages_weighted_#{k}".to_sym }.freeze - end - def self.kroki_formats_attributes { blockdiag: { @@ -44,7 +40,6 @@ class ApplicationSetting < ApplicationRecord end store_accessor :kroki_formats, *ApplicationSetting.kroki_formats_attributes.keys, prefix: true - store_accessor :repository_storages_weighted, *Gitlab.config.repositories.storages.keys, prefix: true # Include here so it can override methods from # `add_authentication_token_field` @@ -503,6 +498,7 @@ class ApplicationSetting < ApplicationRecord inclusion: { in: [true, false], message: _('must be a boolean value') } before_validation :ensure_uuid! + before_validation :coerce_repository_storages_weighted, if: :repository_storages_weighted_changed? before_save :ensure_runners_registration_token before_save :ensure_health_check_access_token @@ -583,12 +579,6 @@ class ApplicationSetting < ApplicationRecord recaptcha_enabled || login_recaptcha_protection_enabled end - repository_storages_weighted_attributes.each do |attribute| - define_method :"#{attribute}=" do |value| - super(value.to_i) - end - end - kroki_formats_attributes.keys.each do |key| define_method :"kroki_formats_#{key}=" do |value| super(::Gitlab::Utils.to_boolean(value)) diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 08c16930b13..c067199b52c 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -123,7 +123,7 @@ module ApplicationSettingImplementation raw_blob_request_limit: 300, recaptcha_enabled: false, repository_checks_enabled: true, - repository_storages_weighted: { default: 100 }, + repository_storages_weighted: { 'default' => 100 }, repository_storages: ['default'], require_admin_approval_after_user_signup: true, require_two_factor_authentication: false, @@ -298,10 +298,6 @@ module ApplicationSettingImplementation Array(read_attribute(:repository_storages)) end - def repository_storages_weighted - read_attribute(:repository_storages_weighted) - end - def commit_email_hostname super.presence || self.class.default_commit_email_hostname end @@ -333,9 +329,10 @@ module ApplicationSettingImplementation def normalized_repository_storage_weights strong_memoize(:normalized_repository_storage_weights) do - weights_total = repository_storages_weighted.values.reduce(:+) + repository_storages_weights = repository_storages_weighted.slice(*Gitlab.config.repositories.storages.keys) + weights_total = repository_storages_weights.values.reduce(:+) - repository_storages_weighted.transform_values do |w| + repository_storages_weights.transform_values do |w| next w if weights_total == 0 w.to_f / weights_total @@ -473,16 +470,20 @@ module ApplicationSettingImplementation invalid.empty? end + def coerce_repository_storages_weighted + repository_storages_weighted.transform_values!(&:to_i) + end + def check_repository_storages_weighted invalid = repository_storages_weighted.keys - Gitlab.config.repositories.storages.keys - errors.add(:repository_storages_weighted, "can't include: %{invalid_storages}" % { invalid_storages: invalid.join(", ") }) unless + errors.add(:repository_storages_weighted, _("can't include: %{invalid_storages}") % { invalid_storages: invalid.join(", ") }) unless invalid.empty? repository_storages_weighted.each do |key, val| next unless val.present? - errors.add(:"repository_storages_weighted_#{key}", "value must be an integer") unless val.is_a?(Integer) - errors.add(:"repository_storages_weighted_#{key}", "value must be between 0 and 100") unless val.between?(0, 100) + errors.add(:repository_storages_weighted, _("value for '%{storage}' must be an integer") % { storage: key }) unless val.is_a?(Integer) + errors.add(:repository_storages_weighted, _("value for '%{storage}' must be between 0 and 100") % { storage: key }) unless val.between?(0, 100) end end diff --git a/app/models/board.rb b/app/models/board.rb index 85fad762ebe..418ea67fc6a 100644 --- a/app/models/board.rb +++ b/app/models/board.rb @@ -7,6 +7,7 @@ class Board < ApplicationRecord has_many :lists, -> { ordered }, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_many :destroyable_lists, -> { destroyable.ordered }, class_name: "List" + validates :name, presence: true validates :project, presence: true, if: :project_needed? validates :group, presence: true, unless: :project diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index 16224fde502..9127dab56a6 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -37,8 +37,9 @@ class BulkImports::Entity < ApplicationRecord validates :project, absence: true, if: :group validates :group, absence: true, if: :project - validates :source_type, :source_full_path, :destination_name, - :destination_namespace, presence: true + validates :source_type, :source_full_path, :destination_name, presence: true + validates :destination_namespace, exclusion: [nil], if: :group + validates :destination_namespace, presence: true, if: :project validate :validate_parent_is_a_group, if: :parent validate :validate_imported_entity_type @@ -117,8 +118,8 @@ class BulkImports::Entity < ApplicationRecord if source.self_and_descendants.any? { |namespace| namespace.full_path == destination_namespace } errors.add( - :destination_namespace, - s_('BulkImport|destination group cannot be part of the source group tree') + :base, + s_('BulkImport|Import failed: Destination cannot be a subgroup of the source group. Change the destination and try again.') ) end end diff --git a/app/models/bulk_imports/tracker.rb b/app/models/bulk_imports/tracker.rb index 02e0904e1af..182c0bbaa8a 100644 --- a/app/models/bulk_imports/tracker.rb +++ b/app/models/bulk_imports/tracker.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -# This model is responsible for keeping track of the requests/pagination -# happening during a Group Migration (BulkImport). class BulkImports::Tracker < ApplicationRecord self.table_name = 'bulk_import_trackers' @@ -15,4 +13,32 @@ class BulkImports::Tracker < ApplicationRecord uniqueness: { scope: :bulk_import_entity_id } validates :next_page, presence: { if: :has_next_page? } + + validates :stage, presence: true + + state_machine :status, initial: :created do + state :created, value: 0 + state :started, value: 1 + state :finished, value: 2 + state :failed, value: -1 + state :skipped, value: -2 + + event :start do + transition created: :started + end + + event :finish do + transition started: :finished + transition failed: :failed + transition skipped: :skipped + end + + event :skip do + transition any => :skipped + end + + event :fail_op do + transition any => :failed + end + end end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index db151126caf..824e35a6480 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -486,6 +486,10 @@ module Ci self.options.fetch(:environment, {}).fetch(:action, 'start') if self.options end + def environment_deployment_tier + self.options.dig(:environment, :deployment_tier) if self.options + end + def outdated_deployment? success? && !deployment.try(:last?) end @@ -510,7 +514,6 @@ module Ci .concat(scoped_variables) .concat(job_variables) .concat(persisted_environment_variables) - .to_runner_variables end end @@ -523,6 +526,7 @@ module Ci .append(key: 'CI_JOB_ID', value: id.to_s) .append(key: 'CI_JOB_URL', value: Gitlab::Routing.url_helpers.project_job_url(project, self)) .append(key: 'CI_JOB_TOKEN', value: token.to_s, public: false, masked: true) + .append(key: 'CI_JOB_STARTED_AT', value: started_at&.iso8601) .append(key: 'CI_BUILD_ID', value: id.to_s) .append(key: 'CI_BUILD_TOKEN', value: token.to_s, public: false, masked: true) .append(key: 'CI_REGISTRY_USER', value: ::Gitlab::Auth::CI_JOB_USER) @@ -564,7 +568,10 @@ module Ci end def features - { trace_sections: true } + { + trace_sections: true, + failure_reasons: self.class.failure_reasons.keys + } end def merge_request @@ -691,7 +698,7 @@ module Ci end def any_runners_online? - project.any_runners? { |runner| runner.active? && runner.online? && runner.can_pick?(self) } + project.any_active_runners? { |runner| runner.match_build_if_online?(self) } end def stuck? @@ -810,14 +817,15 @@ module Ci end def cache - cache = options[:cache] + cache = Array.wrap(options[:cache]) - if cache && project.jobs_cache_index - cache = cache.merge( - key: "#{cache[:key]}-#{project.jobs_cache_index}") + if project.jobs_cache_index + cache = cache.map do |single_cache| + single_cache.merge(key: "#{single_cache[:key]}-#{project.jobs_cache_index}") + end end - [cache] + cache end def credentials @@ -983,7 +991,7 @@ module Ci # TODO: Have `debug_mode?` check against data on sent back from runner # to capture all the ways that variables can be set. # See (https://gitlab.com/gitlab-org/gitlab/-/issues/290955) - variables.any? { |variable| variable[:key] == 'CI_DEBUG_TRACE' && variable[:value].casecmp('true') == 0 } + variables['CI_DEBUG_TRACE']&.value&.casecmp('true') == 0 end def drop_with_exit_code!(failure_reason, exit_code) diff --git a/app/models/ci/build_need.rb b/app/models/ci/build_need.rb index fac615f97b9..7bc70f9f1e1 100644 --- a/app/models/ci/build_need.rb +++ b/app/models/ci/build_need.rb @@ -10,6 +10,7 @@ module Ci validates :build, presence: true validates :name, presence: true, length: { maximum: 128 } + validates :optional, inclusion: { in: [true, false] } scope :scoped_build, -> { where('ci_builds.id=ci_build_needs.build_id') } scope :artifacts, -> { where(artifacts: true) } diff --git a/app/models/ci/build_report_result.rb b/app/models/ci/build_report_result.rb index 530233ad5c0..eb6a0700006 100644 --- a/app/models/ci/build_report_result.rb +++ b/app/models/ci/build_report_result.rb @@ -38,6 +38,10 @@ module Ci tests.dig("skipped").to_i end + def suite_error + tests.dig("suite_error") + end + def tests_total [tests_success, tests_failed, tests_errored, tests_skipped].sum end diff --git a/app/models/ci/daily_build_group_report_result.rb b/app/models/ci/daily_build_group_report_result.rb index 23c96e63724..5dcf575abd7 100644 --- a/app/models/ci/daily_build_group_report_result.rb +++ b/app/models/ci/daily_build_group_report_result.rb @@ -4,7 +4,6 @@ module Ci class DailyBuildGroupReportResult < ApplicationRecord extend Gitlab::Ci::Model - REPORT_WINDOW = 90.days PARAM_TYPES = %w[coverage].freeze belongs_to :last_pipeline, class_name: 'Ci::Pipeline', foreign_key: :last_pipeline_id @@ -13,13 +12,11 @@ module Ci validates :data, json_schema: { filename: "daily_build_group_report_result_data" } - scope :with_included_projects, -> { includes(:project) } scope :by_ref_path, -> (ref_path) { where(ref_path: ref_path) } scope :by_projects, -> (ids) { where(project_id: ids) } scope :by_group, -> (group_id) { where(group_id: group_id) } scope :with_coverage, -> { where("(data->'coverage') IS NOT NULL") } scope :with_default_branch, -> { where(default_branch: true) } - scope :by_date, -> (start_date) { where(date: report_window(start_date)..Date.current) } scope :by_dates, -> (start_date, end_date) { where(date: start_date..end_date) } scope :ordered_by_date_and_group_name, -> { order(date: :desc, group_name: :asc) } @@ -29,17 +26,6 @@ module Ci def upsert_reports(data) upsert_all(data, unique_by: :index_daily_build_group_report_results_unique_columns) if data.any? end - - def recent_results(attrs, limit: nil) - where(attrs).order(date: :desc, group_name: :asc).limit(limit) - end - - def report_window(start_date) - default_date = REPORT_WINDOW.ago.to_date - date = Date.parse(start_date) rescue default_date - - [date, default_date].max - end end end end diff --git a/app/models/ci/deleted_object.rb b/app/models/ci/deleted_object.rb index e74946eda16..2942a153e05 100644 --- a/app/models/ci/deleted_object.rb +++ b/app/models/ci/deleted_object.rb @@ -17,9 +17,9 @@ module Ci .lock('FOR UPDATE SKIP LOCKED') end - def self.bulk_import(artifacts) + def self.bulk_import(artifacts, pick_up_at = nil) attributes = artifacts.each.with_object([]) do |artifact, accumulator| - record = artifact.to_deleted_object_attrs + record = artifact.to_deleted_object_attrs(pick_up_at) accumulator << record if record[:store_dir] && record[:file] end diff --git a/app/models/ci/group.rb b/app/models/ci/group.rb index c7c0ec61e62..4ba09fd8152 100644 --- a/app/models/ci/group.rb +++ b/app/models/ci/group.rb @@ -39,7 +39,7 @@ module Ci def status_struct strong_memoize(:status_struct) do Gitlab::Ci::Status::Composite - .new(@jobs) + .new(@jobs, project: project) end end diff --git a/app/models/ci/group_variable.rb b/app/models/ci/group_variable.rb index 1e1dd68ee6c..2928ce801ad 100644 --- a/app/models/ci/group_variable.rb +++ b/app/models/ci/group_variable.rb @@ -6,16 +6,18 @@ module Ci include Ci::HasVariable include Presentable include Ci::Maskable + prepend HasEnvironmentScope belongs_to :group, class_name: "::Group" alias_attribute :secret_value, :value validates :key, uniqueness: { - scope: :group_id, + scope: [:group_id, :environment_scope], message: "(%{value}) has already been taken" } scope :unprotected, -> { where(protected: false) } + scope :by_environment_scope, -> (environment_scope) { where(environment_scope: environment_scope) } end end diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index f927111758a..d5e88f2be5b 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -136,11 +136,7 @@ module Ci scope :for_sha, ->(sha, project_id) { joins(job: :pipeline).where(ci_pipelines: { sha: sha, project_id: project_id }) } scope :for_job_name, ->(name) { joins(:job).where(ci_builds: { name: name }) } - scope :with_job, -> do - if Feature.enabled?(:non_public_artifacts, type: :development) - joins(:job).includes(:job) - end - end + scope :with_job, -> { joins(:job).includes(:job) } scope :with_file_types, -> (file_types) do types = self.file_types.select { |file_type| file_types.include?(file_type) }.values @@ -311,12 +307,12 @@ module Ci max_size&.megabytes.to_i end - def to_deleted_object_attrs + def to_deleted_object_attrs(pick_up_at = nil) { file_store: file_store, store_dir: file.store_dir.to_s, file: file_identifier, - pick_up_at: expire_at || Time.current + pick_up_at: pick_up_at || expire_at || Time.current } end diff --git a/app/models/ci/legacy_stage.rb b/app/models/ci/legacy_stage.rb index df4368eccd5..ffd3d3fcd88 100644 --- a/app/models/ci/legacy_stage.rb +++ b/app/models/ci/legacy_stage.rb @@ -32,7 +32,7 @@ module Ci end def status - @status ||= statuses.latest.composite_status + @status ||= statuses.latest.composite_status(project: project) end def detailed_status(current_user) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 3be107ea2e1..b63ec0c8a97 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -228,7 +228,7 @@ module Ci pipeline.run_after_commit do PipelineHooksWorker.perform_async(pipeline.id) - ExpirePipelineCacheWorker.perform_async(pipeline.id) if pipeline.cacheable? + ExpirePipelineCacheWorker.perform_async(pipeline.id) end end @@ -573,7 +573,7 @@ module Ci end def cancel_running(retries: nil) - retry_optimistic_lock(cancelable_statuses, retries) do |cancelable| + retry_optimistic_lock(cancelable_statuses, retries, name: 'ci_pipeline_cancel_running') do |cancelable| cancelable.find_each do |job| yield(job) if block_given? job.cancel @@ -693,14 +693,6 @@ module Ci .exists? end - # TODO: this logic is duplicate with Pipeline::Chain::Config::Content - # we should persist this is `ci_pipelines.config_path` - def config_path - return unless repository_source? || unknown_source? - - project.ci_config_path_or_default - end - def has_yaml_errors? yaml_errors.present? end @@ -744,7 +736,7 @@ module Ci end def set_status(new_status) - retry_optimistic_lock(self) do + retry_optimistic_lock(self, name: 'ci_pipeline_set_status') do case new_status when 'created' then nil when 'waiting_for_resource' then request_resource @@ -785,8 +777,7 @@ module Ci Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'CI_PIPELINE_IID', value: iid.to_s) variables.append(key: 'CI_PIPELINE_SOURCE', value: source.to_s) - - variables.append(key: 'CI_CONFIG_PATH', value: config_path) + variables.append(key: 'CI_PIPELINE_CREATED_AT', value: created_at&.iso8601) variables.concat(predefined_commit_variables) @@ -938,6 +929,12 @@ module Ci .first end + def self_with_ancestors_and_descendants(same_project: false) + ::Gitlab::Ci::PipelineObjectHierarchy + .new(self.class.unscoped.where(id: id), options: { same_project: same_project }) + .all_objects + end + def bridge_triggered? source_bridge.present? end @@ -1117,7 +1114,7 @@ module Ci detached_merge_request_pipeline? && !merge_request_ref? end - def merge_request_pipeline? + def merged_result_pipeline? merge_request? && target_sha.present? end @@ -1157,7 +1154,7 @@ module Ci return unless merge_request? strong_memoize(:merge_request_event_type) do - if merge_request_pipeline? + if merged_result_pipeline? :merged_result elsif detached_merge_request_pipeline? :detached @@ -1169,10 +1166,6 @@ module Ci @persistent_ref ||= PersistentRef.new(pipeline: self) end - def cacheable? - !dangling? - end - def dangling? Enums::Ci::Pipeline.dangling_sources.key?(source.to_sym) end @@ -1247,7 +1240,7 @@ module Ci def merge_request_diff_sha return unless merge_request? - if merge_request_pipeline? + if merged_result_pipeline? source_sha else sha diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb index fae65ed0632..0ad1ed2fce8 100644 --- a/app/models/ci/processable.rb +++ b/app/models/ci/processable.rb @@ -120,10 +120,6 @@ module Ci raise NotImplementedError end - def scoped_variables_hash - raise NotImplementedError - end - override :all_met_to_become_pending? def all_met_to_become_pending? super && !with_resource_group? diff --git a/app/models/ci/ref.rb b/app/models/ci/ref.rb index 713a0bf9c45..3d71a5f2c96 100644 --- a/app/models/ci/ref.rb +++ b/app/models/ci/ref.rb @@ -62,7 +62,7 @@ module Ci end def update_status_by!(pipeline) - retry_lock(self) do + retry_lock(self, name: 'ci_ref_update_status_by') do next unless last_finished_pipeline_id == pipeline.id case pipeline.status diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 44a00e36bcc..d1a20bc93c3 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -9,6 +9,7 @@ module Ci include FromUnion include TokenAuthenticatable include IgnorableColumns + include FeatureGate add_authentication_token_field :token, encrypted: -> { Feature.enabled?(:ci_runners_tokens_optional_encryption, default_enabled: true) ? :optional : :required } @@ -251,10 +252,21 @@ module Ci runner_projects.any? end + # TODO: remove this method in favor of `matches_build?` once feature flag is removed + # https://gitlab.com/gitlab-org/gitlab/-/issues/323317 def can_pick?(build) - return false if self.ref_protected? && !build.protected? + if Feature.enabled?(:ci_runners_short_circuit_assignable_for, self, default_enabled: :yaml) + matches_build?(build) + else + # Run `matches_build?` checks before, since they are cheaper than + # `assignable_for?`. + # + matches_build?(build) && assignable_for?(build.project_id) + end + end - assignable_for?(build.project_id) && accepting_tags?(build) + def match_build_if_online?(build) + active? && online? && can_pick?(build) end def only_for?(project) @@ -265,6 +277,16 @@ module Ci token[0...8] if token end + def tag_list + return super unless Feature.enabled?(:ci_preload_runner_tags, default_enabled: :yaml) + + if tags.loaded? + tags.map(&:name) + else + super + end + end + def has_tags? tag_list.any? end @@ -304,8 +326,10 @@ module Ci end def pick_build!(build) - if can_pick?(build) - tick_runner_queue + if Feature.enabled?(:ci_reduce_queries_when_ticking_runner_queue, self, default_enabled: :yaml) + tick_runner_queue if matches_build?(build) + else + tick_runner_queue if can_pick?(build) end end @@ -341,6 +365,8 @@ module Ci end end + # TODO: remove this method once feature flag ci_runners_short_circuit_assignable_for + # is removed. https://gitlab.com/gitlab-org/gitlab/-/issues/323317 def assignable_for?(project_id) self.class.owned_or_instance_wide(project_id).where(id: self.id).any? end @@ -369,6 +395,12 @@ module Ci end end + def matches_build?(build) + return false if self.ref_protected? && !build.protected? + + accepting_tags?(build) + end + def accepting_tags?(build) (run_untagged? || build.has_tags?) && (build.tag_list - tag_list).empty? end diff --git a/app/models/ci/runner_namespace.rb b/app/models/ci/runner_namespace.rb index 6903e8a21a1..e6c1899c89d 100644 --- a/app/models/ci/runner_namespace.rb +++ b/app/models/ci/runner_namespace.rb @@ -4,10 +4,17 @@ module Ci class RunnerNamespace < ApplicationRecord extend Gitlab::Ci::Model - belongs_to :runner, inverse_of: :runner_namespaces, validate: true + belongs_to :runner, inverse_of: :runner_namespaces belongs_to :namespace, inverse_of: :runner_namespaces, class_name: '::Namespace' belongs_to :group, class_name: '::Group', foreign_key: :namespace_id validates :runner_id, uniqueness: { scope: :namespace_id } + validate :group_runner_type + + private + + def group_runner_type + errors.add(:runner, 'is not a group runner') unless runner&.group_type? + end end end diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index ae80692d598..03a97355574 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -84,7 +84,7 @@ module Ci end def set_status(new_status) - retry_optimistic_lock(self) do + retry_optimistic_lock(self, name: 'ci_stage_set_status') do case new_status when 'created' then nil when 'waiting_for_resource' then request_resource @@ -138,7 +138,7 @@ module Ci end def latest_stage_status - statuses.latest.composite_status || 'skipped' + statuses.latest.composite_status(project: project) || 'skipped' end end end diff --git a/app/models/ci/variable.rb b/app/models/ci/variable.rb index 13358b95a47..84505befc5c 100644 --- a/app/models/ci/variable.rb +++ b/app/models/ci/variable.rb @@ -18,7 +18,6 @@ module Ci } scope :unprotected, -> { where(protected: false) } - scope :by_key, -> (key) { where(key: key) } scope :by_environment_scope, -> (environment_scope) { where(environment_scope: environment_scope) } end end diff --git a/app/models/clusters/agent_token.rb b/app/models/clusters/agent_token.rb index b260822f784..9d79887b574 100644 --- a/app/models/clusters/agent_token.rb +++ b/app/models/clusters/agent_token.rb @@ -7,9 +7,12 @@ module Clusters self.table_name = 'cluster_agent_tokens' - belongs_to :agent, class_name: 'Clusters::Agent' + belongs_to :agent, class_name: 'Clusters::Agent', optional: false belongs_to :created_by_user, class_name: 'User', optional: true before_save :ensure_token + + validates :description, length: { maximum: 1024 } + validates :name, presence: true, length: { maximum: 255 }, on: :create end end diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index f87eccecf9f..8a49d476ba7 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Runner < ApplicationRecord - VERSION = '0.25.0' + VERSION = '0.26.0' self.table_name = 'clusters_applications_runners' diff --git a/app/models/clusters/instance.rb b/app/models/clusters/instance.rb index 94fadace01c..2a09ba11564 100644 --- a/app/models/clusters/instance.rb +++ b/app/models/clusters/instance.rb @@ -6,10 +6,6 @@ module Clusters Clusters::Cluster.instance_type end - def feature_available?(feature) - ::Feature.enabled?(feature, type: :licensed, default_enabled: true) - end - def flipper_id self.class.to_s end diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index ea2f425c5f6..524429bf12a 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -52,7 +52,6 @@ class CommitStatus < ApplicationRecord scope :before_stage, -> (index) { where('stage_idx < ?', index) } scope :for_stage, -> (index) { where(stage_idx: index) } scope :after_stage, -> (index) { where('stage_idx > ?', index) } - scope :for_ids, -> (ids) { where(id: ids) } scope :for_ref, -> (ref) { where(ref: ref) } scope :by_name, -> (name) { where(name: name) } scope :in_pipelines, ->(pipelines) { where(pipeline: pipelines) } @@ -85,6 +84,8 @@ class CommitStatus < ApplicationRecord # extend this `Hash` with new values. enum_with_nil failure_reason: Enums::Ci::CommitStatus.failure_reasons + default_value_for :retried, false + ## # We still create some CommitStatuses outside of CreatePipelineService. # @@ -291,6 +292,14 @@ class CommitStatus < ApplicationRecord failed? && !unrecoverable_failure? end + def update_older_statuses_retried! + self.class + .latest + .where(name: name) + .where.not(id: id) + .update_all(retried: true, processed: true) + end + private def unrecoverable_failure? diff --git a/app/models/commit_with_pipeline.rb b/app/models/commit_with_pipeline.rb deleted file mode 100644 index 7f952fb77a0..00000000000 --- a/app/models/commit_with_pipeline.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -class Ci::CommitWithPipeline < SimpleDelegator - include Presentable - - def initialize(commit) - @latest_pipelines = {} - super(commit) - end - - def pipelines - project.ci_pipelines.where(sha: sha) - end - - def last_pipeline - strong_memoize(:last_pipeline) do - pipelines.last - end - end - - def latest_pipeline(ref = nil) - @latest_pipelines.fetch(ref) do |ref| - @latest_pipelines[ref] = latest_pipeline_for_project(ref, project) - end - end - - def latest_pipeline_for_project(ref, pipeline_project) - pipeline_project.ci_pipelines.latest_pipeline_per_commit(id, ref)[id] - end - - def set_latest_pipeline_for_ref(ref, pipeline) - @latest_pipelines[ref] = pipeline - end - - def status(ref = nil) - latest_pipeline(ref)&.status - end -end diff --git a/app/models/concerns/analytics/cycle_analytics/stage.rb b/app/models/concerns/analytics/cycle_analytics/stage.rb index 080ff07ec0c..f1c39dda49d 100644 --- a/app/models/concerns/analytics/cycle_analytics/stage.rb +++ b/app/models/concerns/analytics/cycle_analytics/stage.rb @@ -49,14 +49,6 @@ module Analytics end end - def start_event_identifier - backward_compatible_identifier(:start_event_identifier) || super - end - - def end_event_identifier - backward_compatible_identifier(:end_event_identifier) || super - end - def start_event_label_based? start_event_identifier && start_event.label_based? end @@ -136,17 +128,6 @@ module Analytics .id_in(label_id) .exists? end - - # Temporary, will be removed in 13.10 - def backward_compatible_identifier(attribute_name) - removed_identifier = 6 # References IssueFirstMentionedInCommit removed on https://gitlab.com/gitlab-org/gitlab/-/merge_requests/51975 - replacement_identifier = :issue_first_mentioned_in_commit - - # ActiveRecord returns nil if the column value is not part of the Enum definition - if self[attribute_name].nil? && read_attribute_before_type_cast(attribute_name) == removed_identifier - replacement_identifier - end - end end end end diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb index d342b526677..c106c08c04a 100644 --- a/app/models/concerns/avatarable.rb +++ b/app/models/concerns/avatarable.rb @@ -20,6 +20,7 @@ module Avatarable mount_uploader :avatar, AvatarUploader after_initialize :add_avatar_to_batch + after_commit :clear_avatar_caches end module ShadowMethods @@ -127,4 +128,11 @@ module Avatarable def avatar_mounter strong_memoize(:avatar_mounter) { _mounter(:avatar) } end + + def clear_avatar_caches + return unless respond_to?(:verified_emails) && verified_emails.any? && avatar_changed? + return unless Feature.enabled?(:avatar_cache_for_email, self, type: :development) + + Gitlab::AvatarCache.delete_by_email(*verified_emails) + end end diff --git a/app/models/concerns/boards/listable.rb b/app/models/concerns/boards/listable.rb index b7c0a8b3489..d6863e87261 100644 --- a/app/models/concerns/boards/listable.rb +++ b/app/models/concerns/boards/listable.rb @@ -13,6 +13,14 @@ module Boards scope :ordered, -> { order(:list_type, :position) } scope :destroyable, -> { where(list_type: list_types.slice(*destroyable_types).values) } scope :movable, -> { where(list_type: list_types.slice(*movable_types).values) } + + class << self + def preload_preferences_for_user(lists, user) + return unless user + + lists.each { |list| list.preferences_for(user) } + end + end end class_methods do @@ -33,6 +41,18 @@ module Boards self.class.movable_types.include?(list_type&.to_sym) end + def collapsed?(user) + preferences = preferences_for(user) + + preferences.collapsed? + end + + def update_preferences_for(user, preferences = {}) + return unless user + + preferences_for(user).update(preferences) + end + def title if label? label.name diff --git a/app/models/concerns/ci/contextable.rb b/app/models/concerns/ci/contextable.rb index c8b55e7b39f..bdba2d3e251 100644 --- a/app/models/concerns/ci/contextable.rb +++ b/app/models/concerns/ci/contextable.rb @@ -20,7 +20,7 @@ module Ci variables.concat(user_variables) variables.concat(dependency_variables) if dependencies variables.concat(secret_instance_variables) - variables.concat(secret_group_variables) + variables.concat(secret_group_variables(environment: environment)) variables.concat(secret_project_variables(environment: environment)) variables.concat(trigger_request.user_variables) if trigger_request variables.concat(pipeline.variables) @@ -29,14 +29,6 @@ module Ci end ## - # Regular Ruby hash of scoped variables, without duplicates that are - # possible to be present in an array of hashes returned from `variables`. - # - def scoped_variables_hash - scoped_variables.to_hash - end - - ## # Variables that do not depend on the environment name. # def simple_variables @@ -93,13 +85,13 @@ module Ci project.ci_instance_variables_for(ref: git_ref) end - def secret_group_variables + def secret_group_variables(environment: expanded_environment_name) return [] unless project.group - project.group.ci_variables_for(git_ref, project) + project.group.ci_variables_for(git_ref, project, environment: environment) end - def secret_project_variables(environment: persisted_environment) + def secret_project_variables(environment: expanded_environment_name) project.ci_variables_for(ref: git_ref, environment: environment) end diff --git a/app/models/concerns/ci/has_status.rb b/app/models/concerns/ci/has_status.rb index 1cc2e8a51e3..0412f7a072b 100644 --- a/app/models/concerns/ci/has_status.rb +++ b/app/models/concerns/ci/has_status.rb @@ -20,9 +20,11 @@ module Ci UnknownStatusError = Class.new(StandardError) class_methods do - def composite_status + # The parameter `project` is only used for the feature flag check, and will be removed with + # https://gitlab.com/gitlab-org/gitlab/-/issues/321972 + def composite_status(project: nil) Gitlab::Ci::Status::Composite - .new(all, with_allow_failure: columns_hash.key?('allow_failure')) + .new(all, with_allow_failure: columns_hash.key?('allow_failure'), project: project) .status end diff --git a/app/models/concerns/ci/has_variable.rb b/app/models/concerns/ci/has_variable.rb index 9bf2b409080..7309469c77e 100644 --- a/app/models/concerns/ci/has_variable.rb +++ b/app/models/concerns/ci/has_variable.rb @@ -16,6 +16,7 @@ module Ci format: { with: /\A[a-zA-Z0-9_]+\z/, message: "can contain only letters, digits and '_'." } + scope :by_key, -> (key) { where(key: key) } scope :order_key_asc, -> { reorder(key: :asc) } attr_encrypted :value, diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 83ff5b16efe..e1be0665452 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -86,6 +86,7 @@ module Issuable before_validation :truncate_description_on_import! scope :authored, ->(user) { where(author_id: user) } + scope :not_authored, ->(user) { where.not(author_id: user) } scope :recent, -> { reorder(id: :desc) } scope :of_projects, ->(ids) { where(project_id: ids) } scope :opened, -> { with_state(:opened) } diff --git a/app/models/concerns/project_features_compatibility.rb b/app/models/concerns/project_features_compatibility.rb index 07bec07e556..7c774d8bad7 100644 --- a/app/models/concerns/project_features_compatibility.rb +++ b/app/models/concerns/project_features_compatibility.rb @@ -34,6 +34,10 @@ module ProjectFeaturesCompatibility write_feature_attribute_boolean(:snippets_access_level, value) end + def security_and_compliance_enabled=(value) + write_feature_attribute_boolean(:security_and_compliance_access_level, value) + end + def repository_access_level=(value) write_feature_attribute_string(:repository_access_level, value) end @@ -78,6 +82,14 @@ module ProjectFeaturesCompatibility write_feature_attribute_string(:operations_access_level, value) end + def security_and_compliance_access_level=(value) + write_feature_attribute_string(:security_and_compliance_access_level, value) + end + + def container_registry_access_level=(value) + write_feature_attribute_string(:container_registry_access_level, value) + end + private def write_feature_attribute_boolean(field, value) diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb index f4c914c6a3a..aea48a5ec20 100644 --- a/app/models/custom_emoji.rb +++ b/app/models/custom_emoji.rb @@ -6,6 +6,7 @@ class CustomEmoji < ApplicationRecord belongs_to :namespace, inverse_of: :custom_emoji belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id' + belongs_to :creator, class_name: "User", inverse_of: :created_custom_emoji # For now only external emoji are supported. See https://gitlab.com/gitlab-org/gitlab/-/issues/230467 validates :external, inclusion: { in: [true] } @@ -15,6 +16,7 @@ class CustomEmoji < ApplicationRecord validate :valid_emoji_name validates :group, presence: true + validates :creator, presence: true validates :name, uniqueness: { scope: [:namespace_id, :name] }, presence: true, diff --git a/app/models/dependency_proxy.rb b/app/models/dependency_proxy.rb index 9cbaf7e9884..0ed17921aaa 100644 --- a/app/models/dependency_proxy.rb +++ b/app/models/dependency_proxy.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true module DependencyProxy URL_SUFFIX = '/dependency_proxy/containers' + DISTRIBUTION_API_VERSION = 'registry/2.0' def self.table_name_prefix 'dependency_proxy_' diff --git a/app/models/dependency_proxy/manifest.rb b/app/models/dependency_proxy/manifest.rb index f3c7f34e0d7..d613d5708f0 100644 --- a/app/models/dependency_proxy/manifest.rb +++ b/app/models/dependency_proxy/manifest.rb @@ -12,5 +12,10 @@ class DependencyProxy::Manifest < ApplicationRecord mount_file_store_uploader DependencyProxy::FileUploader - scope :find_or_initialize_by_file_name, ->(file_name) { find_or_initialize_by(file_name: file_name) } + def self.find_or_initialize_by_file_name_or_digest(file_name:, digest:) + result = find_by(file_name: file_name) || find_by(digest: digest) + return result if result + + new(file_name: file_name, digest: digest) + end end diff --git a/app/models/environment.rb b/app/models/environment.rb index 4f7f688a040..d89909a71a2 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -39,6 +39,7 @@ class Environment < ApplicationRecord before_validation :generate_slug, if: ->(env) { env.slug.blank? } before_save :set_environment_type + before_save :ensure_environment_tier after_save :clear_reactive_cache! validates :name, @@ -87,6 +88,7 @@ class Environment < ApplicationRecord end scope :for_project, -> (project) { where(project_id: project) } + scope :for_tier, -> (tier) { where(tier: tier).where('tier IS NOT NULL') } scope :with_deployment, -> (sha) { where('EXISTS (?)', Deployment.select(1).where('deployments.environment_id = environments.id').where(sha: sha)) } scope :unfoldered, -> { where(environment_type: nil) } scope :with_rank, -> do @@ -94,6 +96,30 @@ class Environment < ApplicationRecord end scope :for_id, -> (id) { where(id: id) } + scope :stopped_review_apps, -> (before, limit) do + stopped + .in_review_folder + .where("created_at < ?", before) + .order("created_at ASC") + .limit(limit) + end + + scope :scheduled_for_deletion, -> do + where.not(auto_delete_at: nil) + end + + scope :not_scheduled_for_deletion, -> do + where(auto_delete_at: nil) + end + + enum tier: { + production: 0, + staging: 1, + testing: 2, + development: 3, + other: 4 + } + state_machine :state, initial: :available do event :start do transition stopped: :available @@ -137,6 +163,10 @@ class Environment < ApplicationRecord self.state_machine.states.map(&:name) end + def self.schedule_to_delete(at_time = 1.week.from_now) + update_all(auto_delete_at: at_time) + end + class << self ## # This method returns stop actions (jobs) for multiple environments within one @@ -242,7 +272,7 @@ class Environment < ApplicationRecord def cancel_deployment_jobs! jobs = active_deployments.with_deployable jobs.each do |deployment| - Gitlab::OptimisticLocking.retry_lock(deployment.deployable) do |deployable| + Gitlab::OptimisticLocking.retry_lock(deployment.deployable, name: 'environment_cancel_deployment_jobs') do |deployable| deployable.cancel! if deployable&.cancelable? end rescue => e @@ -429,6 +459,22 @@ class Environment < ApplicationRecord def generate_slug self.slug = Gitlab::Slug::Environment.new(name).generate end + + def ensure_environment_tier + self.tier ||= guess_tier + end + + # Guessing the tier of the environment if it's not explicitly specified by users. + # See https://en.wikipedia.org/wiki/Deployment_environment for industry standard deployment environments + def guess_tier + case name + when %r{dev|review|trunk}i then self.class.tiers[:development] + when %r{test|qc}i then self.class.tiers[:testing] + when %r{st(a|)g|mod(e|)l|pre|demo}i then self.class.tiers[:staging] + when %r{pr(o|)d|live}i then self.class.tiers[:production] + else self.class.tiers[:other] + end + end end Environment.prepend_if_ee('EE::Environment') diff --git a/app/models/error_tracking/project_error_tracking_setting.rb b/app/models/error_tracking/project_error_tracking_setting.rb index fa32c8a5450..9a9fbc6a801 100644 --- a/app/models/error_tracking/project_error_tracking_setting.rb +++ b/app/models/error_tracking/project_error_tracking_setting.rb @@ -77,7 +77,7 @@ module ErrorTracking def sentry_client strong_memoize(:sentry_client) do - Sentry::Client.new(api_url, token) + ErrorTracking::SentryClient.new(api_url, token) end end @@ -168,13 +168,13 @@ module ErrorTracking def handle_exceptions yield - rescue Sentry::Client::Error => e + rescue ErrorTracking::SentryClient::Error => e { error: e.message, error_type: SENTRY_API_ERROR_TYPE_NON_20X_RESPONSE } - rescue Sentry::Client::MissingKeysError => e + rescue ErrorTracking::SentryClient::MissingKeysError => e { error: e.message, error_type: SENTRY_API_ERROR_TYPE_MISSING_KEYS } - rescue Sentry::Client::ResponseInvalidSizeError => e + rescue ErrorTracking::SentryClient::ResponseInvalidSizeError => e { error: e.message, error_type: SENTRY_API_ERROR_INVALID_SIZE } - rescue Sentry::Client::BadRequestError => e + rescue ErrorTracking::SentryClient::BadRequestError => e { error: e.message, error_type: SENTRY_API_ERROR_TYPE_BAD_REQUEST } rescue StandardError => e Gitlab::ErrorTracking.track_exception(e) diff --git a/app/models/experiment.rb b/app/models/experiment.rb index 354b1e0b6b9..ac8b6516d02 100644 --- a/app/models/experiment.rb +++ b/app/models/experiment.rb @@ -14,22 +14,30 @@ class Experiment < ApplicationRecord find_or_create_by!(name: name).record_group_and_variant!(group, variant) end - def self.record_conversion_event(name, user) - find_or_create_by!(name: name).record_conversion_event_for_user(user) + def self.record_conversion_event(name, user, context = {}) + find_or_create_by!(name: name).record_conversion_event_for_user(user, context) end # Create or update the recorded experiment_user row for the user in this experiment. def record_user_and_group(user, group_type, context = {}) experiment_user = experiment_users.find_or_initialize_by(user: user) - merged_context = experiment_user.context.deep_merge(context.deep_stringify_keys) - experiment_user.update!(group_type: group_type, context: merged_context) + experiment_user.update!(group_type: group_type, context: merged_context(experiment_user, context)) end - def record_conversion_event_for_user(user) - experiment_users.find_by(user: user, converted_at: nil)&.touch(:converted_at) + def record_conversion_event_for_user(user, context = {}) + experiment_user = experiment_users.find_by(user: user) + return unless experiment_user + + experiment_user.update!(converted_at: Time.current, context: merged_context(experiment_user, context)) end def record_group_and_variant!(group, variant) experiment_subjects.find_or_initialize_by(group: group).update!(variant: variant) end + + private + + def merged_context(experiment_user, new_context) + experiment_user.context.deep_merge(new_context.deep_stringify_keys) + end end diff --git a/app/models/group.rb b/app/models/group.rb index 1eaa4499eb5..9f8a9996f31 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -33,7 +33,6 @@ class Group < Namespace has_many :members_and_requesters, as: :source, class_name: 'GroupMember' has_many :milestones - has_many :iterations has_many :services has_many :shared_group_links, foreign_key: :shared_with_group_id, class_name: 'GroupGroupLink' has_many :shared_with_group_links, foreign_key: :shared_group_id, class_name: 'GroupGroupLink' @@ -341,6 +340,13 @@ class Group < Namespace has_owner?(user) && members_with_parents.owners.size == 1 end + def last_blocked_owner?(user) + return false if members_with_parents.owners.any? + + blocked_owners = members.blocked.where(access_level: Gitlab::Access::OWNER) + blocked_owners.size == 1 && blocked_owners.exists?(user_id: user) + end + def ldap_synced? false end @@ -364,15 +370,30 @@ class Group < Namespace # rubocop: enable CodeReuse/ServiceClass # rubocop: disable CodeReuse/ServiceClass - def refresh_members_authorized_projects(blocking: true, priority: UserProjectAccessChangedService::HIGH_PRIORITY) + def refresh_members_authorized_projects( + blocking: true, + priority: UserProjectAccessChangedService::HIGH_PRIORITY, + direct_members_only: false + ) + + user_ids = if direct_members_only + users_ids_of_direct_members + else + user_ids_for_project_authorizations + end + UserProjectAccessChangedService - .new(user_ids_for_project_authorizations) + .new(user_ids) .execute(blocking: blocking, priority: priority) end # rubocop: enable CodeReuse/ServiceClass + def users_ids_of_direct_members + direct_members.pluck(:user_id) + end + def user_ids_for_project_authorizations - members_with_parents.pluck(:user_id) + members_with_parents.pluck(Arel.sql('DISTINCT members.user_id')) end def self_and_ancestors_ids @@ -381,6 +402,12 @@ class Group < Namespace end end + def direct_members + GroupMember.active_without_invites_and_requests + .non_minimal_access + .where(source_id: id) + end + def members_with_parents # Avoids an unnecessary SELECT when the group has no parents source_ids = @@ -485,7 +512,7 @@ class Group < Namespace # @param only_concrete_membership [Bool] whether require admin concrete membership status def max_member_access_for_user(user, only_concrete_membership: false) return GroupMember::NO_ACCESS unless user - return GroupMember::OWNER if user.admin? && !only_concrete_membership + return GroupMember::OWNER if user.can_admin_all_resources? && !only_concrete_membership max_member_access = members_with_parents.where(user_id: user) .reorder(access_level: :desc) @@ -505,15 +532,11 @@ class Group < Namespace } end - def ci_variables_for(ref, project) - cache_key = "ci_variables_for:group:#{self&.id}:project:#{project&.id}:ref:#{ref}" + def ci_variables_for(ref, project, environment: nil) + cache_key = "ci_variables_for:group:#{self&.id}:project:#{project&.id}:ref:#{ref}:environment:#{environment}" ::Gitlab::SafeRequestStore.fetch(cache_key) do - list_of_ids = [self] + ancestors - variables = Ci::GroupVariable.where(group: list_of_ids) - variables = variables.unprotected unless project.protected_for?(ref) - variables = variables.group_by(&:group_id) - list_of_ids.reverse.flat_map { |group| variables[group.id] }.compact + uncached_ci_variables_for(ref, project, environment: environment) end end @@ -755,6 +778,23 @@ class Group < Namespace def enable_shared_runners! update!(shared_runners_enabled: true) end + + def uncached_ci_variables_for(ref, project, environment: nil) + list_of_ids = [self] + ancestors + variables = Ci::GroupVariable.where(group: list_of_ids) + variables = variables.unprotected unless project.protected_for?(ref) + + if Feature.enabled?(:scoped_group_variables, self, default_enabled: :yaml) + variables = if environment + variables.on_environment(environment) + else + variables.where(environment_scope: '*') + end + end + + variables = variables.group_by(&:group_id) + list_of_ids.reverse.flat_map { |group| variables[group.id] }.compact + end end Group.prepend_if_ee('EE::Group') diff --git a/app/models/hooks/web_hook_log.rb b/app/models/hooks/web_hook_log.rb index 03f1797f4f4..e2230a2d644 100644 --- a/app/models/hooks/web_hook_log.rb +++ b/app/models/hooks/web_hook_log.rb @@ -6,6 +6,8 @@ class WebHookLog < ApplicationRecord include DeleteWithLimit include CreatedAtFilterable + self.primary_key = :id + belongs_to :web_hook serialize :request_headers, Hash # rubocop:disable Cop/ActiveRecordSerialize diff --git a/app/models/hooks/web_hook_log_partitioned.rb b/app/models/hooks/web_hook_log_partitioned.rb new file mode 100644 index 00000000000..b4b150afb6a --- /dev/null +++ b/app/models/hooks/web_hook_log_partitioned.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +# This model is not yet intended to be used. +# It is in a transitioning phase while we are partitioning +# the web_hook_logs table on the database-side. +# Please refer to https://gitlab.com/groups/gitlab-org/-/epics/5558 +# for details. +# rubocop:disable Gitlab/NamespacedClass: This is a temporary class with no relevant namespace +# WebHook, WebHookLog and all hooks are defined outside of a namespace +class WebHookLogPartitioned < ApplicationRecord + include PartitionedTable + + self.table_name = 'web_hook_logs_part_0c5294f417' + self.primary_key = :id + + partitioned_by :created_at, strategy: :monthly +end diff --git a/app/models/issue.rb b/app/models/issue.rb index 79d0229a281..2f2d24cbe93 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -438,6 +438,14 @@ class Issue < ApplicationRecord issue_type_supports?(:assignee) end + def email_participants_emails + issue_email_participants.pluck(:email) + end + + def email_participants_emails_downcase + issue_email_participants.pluck(IssueEmailParticipant.arel_table[:email].lower) + end + private def ensure_metrics diff --git a/app/models/issue_email_participant.rb b/app/models/issue_email_participant.rb index 8eb9b6a8152..76a96151350 100644 --- a/app/models/issue_email_participant.rb +++ b/app/models/issue_email_participant.rb @@ -3,7 +3,7 @@ class IssueEmailParticipant < ApplicationRecord belongs_to :issue - validates :email, presence: true, uniqueness: { scope: [:issue_id] } + validates :email, uniqueness: { scope: [:issue_id], case_sensitive: false } validates :issue, presence: true validate :validate_email_format diff --git a/app/models/iteration.rb b/app/models/iteration.rb index 7a35bb1cd1f..7483d04aab8 100644 --- a/app/models/iteration.rb +++ b/app/models/iteration.rb @@ -1,140 +1,16 @@ # frozen_string_literal: true +# Placeholder class for model that is implemented in EE class Iteration < ApplicationRecord self.table_name = 'sprints' - attr_accessor :skip_future_date_validation - attr_accessor :skip_project_validation - - STATE_ENUM_MAP = { - upcoming: 1, - started: 2, - closed: 3 - }.with_indifferent_access.freeze - - include AtomicInternalId - - belongs_to :project - belongs_to :group - - has_internal_id :iid, scope: :project - has_internal_id :iid, scope: :group - - validates :start_date, presence: true - validates :due_date, presence: true - - validate :dates_do_not_overlap, if: :start_or_due_dates_changed? - validate :future_date, if: :start_or_due_dates_changed?, unless: :skip_future_date_validation - validate :no_project, unless: :skip_project_validation - - scope :upcoming, -> { with_state(:upcoming) } - scope :started, -> { with_state(:started) } - scope :closed, -> { with_state(:closed) } - - scope :within_timeframe, -> (start_date, end_date) do - where('start_date IS NOT NULL OR due_date IS NOT NULL') - .where('start_date IS NULL OR start_date <= ?', end_date) - .where('due_date IS NULL OR due_date >= ?', start_date) + def self.reference_prefix + '*iteration:' end - scope :start_date_passed, -> { where('start_date <= ?', Date.current).where('due_date >= ?', Date.current) } - scope :due_date_passed, -> { where('due_date < ?', Date.current) } - - state_machine :state_enum, initial: :upcoming do - event :start do - transition upcoming: :started - end - - event :close do - transition [:upcoming, :started] => :closed - end - - state :upcoming, value: Iteration::STATE_ENUM_MAP[:upcoming] - state :started, value: Iteration::STATE_ENUM_MAP[:started] - state :closed, value: Iteration::STATE_ENUM_MAP[:closed] - end - - # Alias to state machine .with_state_enum method - # This needs to be defined after the state machine block to avoid errors - class << self - alias_method :with_state, :with_state_enum - alias_method :with_states, :with_state_enums - - def filter_by_state(iterations, state) - case state - when 'closed' then iterations.closed - when 'started' then iterations.started - when 'upcoming' then iterations.upcoming - when 'opened' then iterations.started.or(iterations.upcoming) - when 'all' then iterations - else raise ArgumentError, "Unknown state filter: #{state}" - end - end - - def reference_prefix - '*iteration:' - end - - def reference_pattern - nil - end - end - - def state - STATE_ENUM_MAP.key(state_enum) - end - - def state=(value) - self.state_enum = STATE_ENUM_MAP[value] - end - - def resource_parent - group || project - end - - private - - def parent_group - group || project.group - end - - def start_or_due_dates_changed? - start_date_changed? || due_date_changed? - end - - # ensure dates do not overlap with other Iterations in the same group/project tree - def dates_do_not_overlap - iterations = if parent_group.present? && resource_parent.is_a?(Project) - Iteration.where(group: parent_group.self_and_ancestors).or(project.iterations) - elsif parent_group.present? - Iteration.where(group: parent_group.self_and_ancestors) - else - project.iterations - end - - return unless iterations.where.not(id: self.id).within_timeframe(start_date, due_date).exists? - - errors.add(:base, s_("Iteration|Dates cannot overlap with other existing Iterations")) - end - - # ensure dates are in the future - def future_date - if start_date_changed? - errors.add(:start_date, s_("Iteration|cannot be in the past")) if start_date < Date.current - errors.add(:start_date, s_("Iteration|cannot be more than 500 years in the future")) if start_date > 500.years.from_now - end - - if due_date_changed? - errors.add(:due_date, s_("Iteration|cannot be in the past")) if due_date < Date.current - errors.add(:due_date, s_("Iteration|cannot be more than 500 years in the future")) if due_date > 500.years.from_now - end - end - - def no_project - return unless project_id.present? - - errors.add(:project_id, s_("is not allowed. We do not currently support project-level iterations")) + def self.reference_pattern + nil end end -Iteration.prepend_if_ee('EE::Iteration') +Iteration.prepend_if_ee('::EE::Iteration') diff --git a/app/models/label.rb b/app/models/label.rb index 7a31b095cfc..26faaa90df3 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -131,6 +131,10 @@ class Label < ApplicationRecord nil end + def self.ids_on_board(board_id) + on_board(board_id).pluck(:label_id) + end + # Searches for labels with a matching title or description. # # This method uses ILIKE on PostgreSQL. diff --git a/app/models/list.rb b/app/models/list.rb index 49834af3dfb..e1954ed72c4 100644 --- a/app/models/list.rb +++ b/app/models/list.rb @@ -14,17 +14,10 @@ class List < ApplicationRecord validates :label_id, uniqueness: { scope: :board_id }, if: :label? scope :preload_associated_models, -> { preload(:board, label: :priorities) } + scope :without_types, ->(list_types) { where.not(list_type: list_types) } alias_method :preferences, :list_user_preferences - class << self - def preload_preferences_for_user(lists, user) - return unless user - - lists.each { |list| list.preferences_for(user) } - end - end - def preferences_for(user) return preferences.build unless user @@ -38,18 +31,6 @@ class List < ApplicationRecord end end - def update_preferences_for(user, preferences = {}) - return unless user - - preferences_for(user).update(preferences) - end - - def collapsed?(user) - preferences = preferences_for(user) - - preferences.collapsed? - end - def as_json(options = {}) super(options).tap do |json| json[:collapsed] = false diff --git a/app/models/member.rb b/app/models/member.rb index 62fe757683f..38574d67cb6 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -75,7 +75,20 @@ class Member < ApplicationRecord left_join_users .where(user_ok) - .where(requested_at: nil) + .non_request + .non_minimal_access + .reorder(nil) + end + + scope :blocked, -> do + is_external_invite = arel_table[:user_id].eq(nil).and(arel_table[:invite_token].not_eq(nil)) + user_is_blocked = User.arel_table[:state].eq(:blocked) + + user_ok = Arel::Nodes::Grouping.new(is_external_invite).or(user_is_blocked) + + left_join_users + .where(user_ok) + .non_request .non_minimal_access .reorder(nil) end diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 1374e8a814a..ac50e5c5107 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -36,6 +36,10 @@ class MergeRequest < ApplicationRecord SORTING_PREFERENCE_FIELD = :merge_requests_sort + ALLOWED_TO_USE_MERGE_BASE_PIPELINE_FOR_COMPARISON = { + 'Ci::CompareCodequalityReportsService' => ->(project) { ::Gitlab::Ci::Features.display_codequality_backend_comparison?(project) } + }.freeze + belongs_to :target_project, class_name: "Project" belongs_to :source_project, class_name: "Project" belongs_to :merge_user, class_name: "User" @@ -187,9 +191,13 @@ class MergeRequest < ApplicationRecord end state_machine :merge_status, initial: :unchecked do + event :mark_as_preparing do + transition unchecked: :preparing + end + event :mark_as_unchecked do - transition [:can_be_merged, :checking, :unchecked] => :unchecked - transition [:cannot_be_merged, :cannot_be_merged_rechecking, :cannot_be_merged_recheck] => :cannot_be_merged_recheck + transition [:preparing, :can_be_merged, :checking] => :unchecked + transition [:cannot_be_merged, :cannot_be_merged_rechecking] => :cannot_be_merged_recheck end event :mark_as_checking do @@ -205,6 +213,7 @@ class MergeRequest < ApplicationRecord transition [:unchecked, :cannot_be_merged_recheck, :checking, :cannot_be_merged_rechecking] => :cannot_be_merged end + state :preparing state :unchecked state :cannot_be_merged_recheck state :checking @@ -233,7 +242,7 @@ class MergeRequest < ApplicationRecord # Returns current merge_status except it returns `cannot_be_merged_rechecking` as `checking` # to avoid exposing unnecessary internal state def public_merge_status - cannot_be_merged_rechecking? ? 'checking' : merge_status + cannot_be_merged_rechecking? || preparing? ? 'checking' : merge_status end validates :source_project, presence: true, unless: [:allow_broken, :importing?, :closed_or_merged_without_fork?] @@ -301,10 +310,28 @@ class MergeRequest < ApplicationRecord end scope :by_target_branch, ->(branch_name) { where(target_branch: branch_name) } scope :order_merged_at, ->(direction) do - query = join_metrics.order(Gitlab::Database.nulls_last_order('merge_request_metrics.merged_at', direction)) - - # Add `merge_request_metrics.merged_at` to the `SELECT` in order to make the keyset pagination work. - query.select(*query.arel.projections, MergeRequest::Metrics.arel_table[:merged_at].as('"merge_request_metrics.merged_at"')) + reverse_direction = { 'ASC' => 'DESC', 'DESC' => 'ASC' } + reversed_direction = reverse_direction[direction] || raise("Unknown sort direction was given: #{direction}") + + order = Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'merge_request_metrics_merged_at', + column_expression: MergeRequest::Metrics.arel_table[:merged_at], + order_expression: Gitlab::Database.nulls_last_order('merge_request_metrics.merged_at', direction), + reversed_order_expression: Gitlab::Database.nulls_first_order('merge_request_metrics.merged_at', reversed_direction), + order_direction: direction, + nullable: :nulls_last, + distinct: false, + add_to_projections: true + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'merge_request_metrics_id', + order_expression: MergeRequest::Metrics.arel_table[:id].desc, + add_to_projections: true + ) + ]) + + order.apply_cursor_conditions(join_metrics).order(order) end scope :order_merged_at_asc, -> { order_merged_at('ASC') } scope :order_merged_at_desc, -> { order_merged_at('DESC') } @@ -317,6 +344,8 @@ class MergeRequest < ApplicationRecord scope :preload_author, -> { preload(:author) } scope :preload_approved_by_users, -> { preload(:approved_by_users) } scope :preload_metrics, -> (relation) { preload(metrics: relation) } + scope :preload_project_and_latest_diff, -> { preload(:source_project, :latest_merge_request_diff) } + scope :preload_latest_diff_commit, -> { preload(latest_merge_request_diff: :merge_request_diff_commits) } scope :with_web_entity_associations, -> { preload(:author, :target_project) } scope :with_auto_merge_enabled, -> do @@ -374,8 +403,7 @@ class MergeRequest < ApplicationRecord alias_attribute :auto_merge_enabled, :merge_when_pipeline_succeeds alias_method :issuing_parent, :target_project - delegate :active?, :builds_with_coverage, to: :head_pipeline, prefix: true, allow_nil: true - delegate :success?, :active?, to: :actual_head_pipeline, prefix: true, allow_nil: true + delegate :builds_with_coverage, to: :head_pipeline, prefix: true, allow_nil: true RebaseLockTimeout = Class.new(StandardError) @@ -401,8 +429,8 @@ class MergeRequest < ApplicationRecord def self.sort_by_attribute(method, excluded_labels: []) case method.to_s - when 'merged_at', 'merged_at_asc' then order_merged_at_asc.with_order_id_desc - when 'merged_at_desc' then order_merged_at_desc.with_order_id_desc + when 'merged_at', 'merged_at_asc' then order_merged_at_asc + when 'merged_at_desc' then order_merged_at_desc else super end @@ -435,6 +463,18 @@ class MergeRequest < ApplicationRecord target_project.latest_pipeline(target_branch, sha) end + def head_pipeline_active? + !!head_pipeline&.active? + end + + def actual_head_pipeline_active? + !!actual_head_pipeline&.active? + end + + def actual_head_pipeline_success? + !!actual_head_pipeline&.success? + end + # Pattern used to extract `!123` merge request references from text # # This pattern supports cross-project references. @@ -1026,6 +1066,7 @@ class MergeRequest < ApplicationRecord def work_in_progress? self.class.work_in_progress?(title) end + alias_method :draft?, :work_in_progress? def wipless_title self.class.wipless_title(self.title) @@ -1264,7 +1305,14 @@ class MergeRequest < ApplicationRecord # Returns the oldest multi-line commit message, or the MR title if none found def default_squash_commit_message strong_memoize(:default_squash_commit_message) do - recent_commits.without_merge_commits.reverse_each.find(&:description?)&.safe_message || title + first_multiline_commit&.safe_message || title + end + end + + # Returns the oldest multi-line commit + def first_multiline_commit + strong_memoize(:first_multiline_commit) do + recent_commits.without_merge_commits.reverse_each.find(&:description?) end end @@ -1550,7 +1598,7 @@ class MergeRequest < ApplicationRecord def compare_reports(service_class, current_user = nil, report_type = nil ) with_reactive_cache(service_class.name, current_user&.id, report_type) do |data| unless service_class.new(project, current_user, id: id, report_type: report_type) - .latest?(base_pipeline, actual_head_pipeline, data) + .latest?(comparison_base_pipeline(service_class.name), actual_head_pipeline, data) raise InvalidateReactiveCache end @@ -1586,7 +1634,7 @@ class MergeRequest < ApplicationRecord raise NameError, service_class unless service_class < Ci::CompareReportsBaseService current_user = User.find_by(id: current_user_id) - service_class.new(project, current_user, id: id, report_type: report_type).execute(base_pipeline, actual_head_pipeline) + service_class.new(project, current_user, id: id, report_type: report_type).execute(comparison_base_pipeline(identifier), actual_head_pipeline) end def all_commits @@ -1710,6 +1758,14 @@ class MergeRequest < ApplicationRecord end end + def use_merge_base_pipeline_for_comparison?(service_class) + ALLOWED_TO_USE_MERGE_BASE_PIPELINE_FOR_COMPARISON[service_class]&.call(project) + end + + def comparison_base_pipeline(service_class) + (use_merge_base_pipeline_for_comparison?(service_class) && merge_base_pipeline) || base_pipeline + end + def base_pipeline @base_pipeline ||= project.ci_pipelines .order(id: :desc) @@ -1830,6 +1886,12 @@ class MergeRequest < ApplicationRecord } end + def includes_ci_config? + return false unless diff_stats + + diff_stats.map(&:path).include?(project.ci_config_path_or_default) + end + private def missing_report_error(report_type) @@ -1837,11 +1899,7 @@ class MergeRequest < ApplicationRecord end def with_rebase_lock - if Feature.enabled?(:merge_request_rebase_nowait_lock, default_enabled: true) - with_retried_nowait_lock { yield } - else - with_lock(true) { yield } - end + with_retried_nowait_lock { yield } end # If the merge request is idle in transaction or has a SELECT FOR diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 3342fb1fce9..3f7ccdb977e 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -63,12 +63,11 @@ class Namespace < ApplicationRecord validates :max_artifacts_size, numericality: { only_integer: true, greater_than: 0, allow_nil: true } + validate :validate_parent_type, if: -> { Feature.enabled?(:validate_namespace_parent_type) } validate :nesting_level_allowed validate :changing_shared_runners_enabled_is_allowed validate :changing_allow_descendants_override_disabled_shared_runners_is_allowed - validates_associated :runners - delegate :name, to: :owner, allow_nil: true, prefix: true delegate :avatar_url, to: :owner, allow_nil: true @@ -84,6 +83,8 @@ class Namespace < ApplicationRecord before_destroy(prepend: true) { prepare_for_destroy } after_destroy :rm_dir + before_save :ensure_delayed_project_removal_assigned_to_namespace_settings, if: :delayed_project_removal_changed? + scope :for_user, -> { where('type IS NULL') } scope :sort_by_type, -> { order(Gitlab::Database.nulls_first_order(:type)) } scope :include_route, -> { includes(:route) } @@ -164,6 +165,10 @@ class Namespace < ApplicationRecord name = host.delete_suffix(gitlab_host) Namespace.where(parent_id: nil).by_path(name) end + + def top_most + where(parent_id: nil) + end end def package_settings @@ -255,11 +260,12 @@ class Namespace < ApplicationRecord # Includes projects from this namespace and projects from all subgroups # that belongs to this namespace def all_projects - if Feature.enabled?(:recursive_approach_for_all_projects) - namespace = user? ? self : self_and_descendants - Project.where(namespace: namespace) + return Project.where(namespace: self) if user? + + if Feature.enabled?(:recursive_namespace_lookup_as_inner_join, self) + Project.joins("INNER JOIN (#{self_and_descendants.select(:id).to_sql}) namespaces ON namespaces.id=projects.namespace_id") else - Project.inside_path(full_path) + Project.where(namespace: self_and_descendants) end end @@ -400,8 +406,19 @@ class Namespace < ApplicationRecord !has_parent? end + def recent? + created_at >= 90.days.ago + end + private + def ensure_delayed_project_removal_assigned_to_namespace_settings + return if Feature.disabled?(:migrate_delayed_project_removal, default_enabled: true) + + self.namespace_settings || build_namespace_settings + namespace_settings.delayed_project_removal = delayed_project_removal + end + def all_projects_with_pages if all_projects.pages_metadata_not_migrated.exists? Gitlab::BackgroundMigration::MigratePagesMetadata.new.perform_on_relation( @@ -437,6 +454,16 @@ class Namespace < ApplicationRecord end end + def validate_parent_type + return unless has_parent? + + if user? + errors.add(:parent_id, 'a user namespace cannot have a parent') + elsif group? + errors.add(:parent_id, 'a group cannot have a user namespace as its parent') if parent.user? + end + end + def sync_share_with_group_lock_with_parent if parent&.share_with_group_lock? self.share_with_group_lock = true diff --git a/app/models/namespace/root_storage_statistics.rb b/app/models/namespace/root_storage_statistics.rb index 90aeee7a4f1..0c91ae760b2 100644 --- a/app/models/namespace/root_storage_statistics.rb +++ b/app/models/namespace/root_storage_statistics.rb @@ -57,8 +57,7 @@ class Namespace::RootStorageStatistics < ApplicationRecord end def attributes_from_personal_snippets - # Return if the type of namespace does not belong to a user - return {} unless namespace.type.nil? + return {} unless namespace.user? from_personal_snippets.take.slice(SNIPPETS_SIZE_STAT_NAME) end @@ -70,3 +69,5 @@ class Namespace::RootStorageStatistics < ApplicationRecord .select("COALESCE(SUM(s.repository_size), 0) AS #{SNIPPETS_SIZE_STAT_NAME}") end end + +Namespace::RootStorageStatistics.prepend_if_ee('EE::Namespace::RootStorageStatistics') diff --git a/app/models/namespaces/traversal/recursive.rb b/app/models/namespaces/traversal/recursive.rb index c46cc521735..d74b7883830 100644 --- a/app/models/namespaces/traversal/recursive.rb +++ b/app/models/namespaces/traversal/recursive.rb @@ -15,8 +15,7 @@ module Namespaces # Returns all ancestors, self, and descendants of the current namespace. def self_and_hierarchy - Gitlab::ObjectHierarchy - .new(self.class.where(id: id)) + object_hierarchy(self.class.where(id: id)) .all_objects end @@ -24,38 +23,38 @@ module Namespaces def ancestors return self.class.none unless parent_id - Gitlab::ObjectHierarchy - .new(self.class.where(id: parent_id)) + object_hierarchy(self.class.where(id: parent_id)) .base_and_ancestors end # returns all ancestors upto but excluding the given namespace # when no namespace is given, all ancestors upto the top are returned def ancestors_upto(top = nil, hierarchy_order: nil) - Gitlab::ObjectHierarchy.new(self.class.where(id: id)) + object_hierarchy(self.class.where(id: id)) .ancestors(upto: top, hierarchy_order: hierarchy_order) end def self_and_ancestors(hierarchy_order: nil) return self.class.where(id: id) unless parent_id - Gitlab::ObjectHierarchy - .new(self.class.where(id: id)) + object_hierarchy(self.class.where(id: id)) .base_and_ancestors(hierarchy_order: hierarchy_order) end # Returns all the descendants of the current namespace. def descendants - Gitlab::ObjectHierarchy - .new(self.class.where(parent_id: id)) + object_hierarchy(self.class.where(parent_id: id)) .base_and_descendants end def self_and_descendants - Gitlab::ObjectHierarchy - .new(self.class.where(id: id)) + object_hierarchy(self.class.where(id: id)) .base_and_descendants end + + def object_hierarchy(ancestors_base) + Gitlab::ObjectHierarchy.new(ancestors_base, options: { use_distinct: Feature.enabled?(:use_distinct_in_object_hierarchy, self) }) + end end end end diff --git a/app/models/note.rb b/app/models/note.rb index fdc972d9726..fb540d692d1 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -30,7 +30,6 @@ class Note < ApplicationRecord # Aliases to make application_helper#edited_time_ago_with_tooltip helper work properly with notes. # See https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/10392/diffs#note_28719102 - alias_attribute :last_edited_at, :updated_at alias_attribute :last_edited_by, :updated_by # Attribute containing rendered and redacted Markdown as generated by @@ -319,6 +318,7 @@ class Note < ApplicationRecord def noteable_assignee_or_author?(user) return false unless user + return false unless noteable.respond_to?(:author_id) return noteable.assignee_or_author?(user) if [MergeRequest, Issue].include?(noteable.class) noteable.author_id == user.id @@ -348,7 +348,13 @@ class Note < ApplicationRecord !system? end - # Since we're using `updated_at` as `last_edited_at`, it could be touched by transforming / resolving a note. + # We used `last_edited_at` as an alias of `updated_at` before. + # This makes it compatible with the previous way without data migration. + def last_edited_at + super || updated_at + end + + # Since we used `updated_at` as `last_edited_at`, it could be touched by transforming / resolving a note. # This makes sure it is only marked as edited when the note body is updated. def edited? return false if updated_by.blank? @@ -546,7 +552,7 @@ class Note < ApplicationRecord end def skip_notification? - review.present? || author.blocked? || author.ghost? + review.present? || !author.can_trigger_notifications? end private diff --git a/app/models/notification_setting.rb b/app/models/notification_setting.rb index 82e39e4f207..72813b17501 100644 --- a/app/models/notification_setting.rb +++ b/app/models/notification_setting.rb @@ -49,7 +49,8 @@ class NotificationSetting < ApplicationRecord :failed_pipeline, :fixed_pipeline, :success_pipeline, - :moved_project + :moved_project, + :merge_when_pipeline_succeeds ].freeze def self.email_events(source = nil) diff --git a/app/models/onboarding_progress.rb b/app/models/onboarding_progress.rb index 8a444f8934e..be76c3dbf9d 100644 --- a/app/models/onboarding_progress.rb +++ b/app/models/onboarding_progress.rb @@ -17,6 +17,7 @@ class OnboardingProgress < ApplicationRecord :code_owners_enabled, :scoped_label_created, :security_scan_enabled, + :issue_created, :issue_auto_closed, :repository_imported, :repository_mirrored @@ -66,6 +67,13 @@ class OnboardingProgress < ApplicationRecord where(namespace: namespace).where.not(action_column => nil).exists? end + def not_completed?(namespace_id, action) + return unless ACTIONS.include?(action) + + action_column = column_name(action) + where(namespace_id: namespace_id).where(action_column => nil).exists? + end + def column_name(action) :"#{action}_at" end diff --git a/app/models/packages/maven/metadatum.rb b/app/models/packages/maven/metadatum.rb index b7f27fb9e06..7aed274216b 100644 --- a/app/models/packages/maven/metadatum.rb +++ b/app/models/packages/maven/metadatum.rb @@ -18,6 +18,13 @@ class Packages::Maven::Metadatum < ApplicationRecord validate :maven_package_type + scope :for_package_ids, -> (package_ids) { where(package_id: package_ids) } + scope :order_created, -> { reorder('created_at ASC') } + + def self.pluck_app_name + pluck(:app_name) + end + private def maven_package_type diff --git a/app/models/packages/nuget.rb b/app/models/packages/nuget.rb index 42c167e9b7f..f152eedb8fc 100644 --- a/app/models/packages/nuget.rb +++ b/app/models/packages/nuget.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true module Packages module Nuget + TEMPORARY_PACKAGE_NAME = 'NuGet.Temporary.Package' + def self.table_name_prefix 'packages_nuget_' end diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb index 391540634be..993d1123c86 100644 --- a/app/models/packages/package.rb +++ b/app/models/packages/package.rb @@ -40,11 +40,11 @@ class Packages::Package < ApplicationRecord validate :unique_debian_package_name, if: :debian_package? validate :valid_conan_package_recipe, if: :conan? - validate :valid_npm_package_name, if: :npm? validate :valid_composer_global_name, if: :composer? validate :package_already_taken, if: :npm? validates :name, format: { with: Gitlab::Regex.conan_recipe_component_regex }, if: :conan? validates :name, format: { with: Gitlab::Regex.generic_package_name_regex }, if: :generic? + validates :name, format: { with: Gitlab::Regex.npm_package_name_regex }, if: :npm? validates :name, format: { with: Gitlab::Regex.nuget_package_name_regex }, if: :nuget? validates :name, format: { with: Gitlab::Regex.debian_package_name_regex }, if: :debian_package? validates :name, inclusion: { in: %w[incoming] }, if: :debian_incoming? @@ -91,6 +91,12 @@ class Packages::Package < ApplicationRecord joins(:conan_metadatum).where(packages_conan_metadata: { package_username: package_username }) end + scope :with_debian_codename, -> (codename) do + debian + .joins(:debian_distribution) + .where(Packages::Debian::ProjectDistribution.table_name => { codename: codename }) + end + scope :preload_debian_file_metadata, -> { preload(package_files: :debian_file_metadatum) } scope :with_composer_target, -> (target) do includes(:composer_metadatum) .joins(:composer_metadatum) @@ -98,12 +104,12 @@ class Packages::Package < ApplicationRecord end scope :preload_composer, -> { preload(:composer_metadatum) } - scope :without_nuget_temporary_name, -> { where.not(name: Packages::Nuget::CreatePackageService::TEMPORARY_PACKAGE_NAME) } + scope :without_nuget_temporary_name, -> { where.not(name: Packages::Nuget::TEMPORARY_PACKAGE_NAME) } scope :has_version, -> { where.not(version: nil) } scope :processed, -> do where.not(package_type: :nuget).or( - where.not(name: Packages::Nuget::CreatePackageService::TEMPORARY_PACKAGE_NAME) + where.not(name: Packages::Nuget::TEMPORARY_PACKAGE_NAME) ) end scope :preload_files, -> { preload(:package_files) } @@ -126,6 +132,8 @@ class Packages::Package < ApplicationRecord scope :order_project_path_desc, -> { joins(:project).reorder('projects.path DESC, id DESC') } scope :order_by_package_file, -> { joins(:package_files).order('packages_package_files.created_at ASC') } + after_commit :update_composer_cache, on: :destroy, if: -> { composer? } + def self.for_projects(projects) return none unless projects.any? @@ -218,8 +226,20 @@ class Packages::Package < ApplicationRecord end end + def sync_maven_metadata(user) + return unless maven? && version? && user + + ::Packages::Maven::Metadata::SyncWorker.perform_async(user.id, project.id, name) + end + private + def update_composer_cache + return unless composer? + + ::Packages::Composer::CacheUpdateWorker.perform_async(project_id, name, composer_metadatum.version_cache_sha) # rubocop:disable CodeReuse/Worker + end + def composer_tag_version? composer? && !Gitlab::Regex.composer_dev_version_regex.match(version.to_s) end @@ -247,14 +267,6 @@ class Packages::Package < ApplicationRecord end end - def valid_npm_package_name - return unless project&.root_namespace - - unless name =~ %r{\A@#{project.root_namespace.path}/[^/]+\z} - errors.add(:name, 'is not valid') - end - end - def package_already_taken return unless project diff --git a/app/models/packages/package_file.rb b/app/models/packages/package_file.rb index 9059f61b5de..23a7144e2bb 100644 --- a/app/models/packages/package_file.rb +++ b/app/models/packages/package_file.rb @@ -30,6 +30,10 @@ class Packages::PackageFile < ApplicationRecord scope :preload_conan_file_metadata, -> { preload(:conan_file_metadatum) } scope :preload_debian_file_metadata, -> { preload(:debian_file_metadatum) } + scope :for_rubygem_with_file_name, ->(project, file_name) do + joins(:package).merge(project.packages.rubygems).with_file_name(file_name) + end + scope :with_conan_file_type, ->(file_type) do joins(:conan_file_metadatum) .where(packages_conan_file_metadata: { conan_file_type: ::Packages::Conan::FileMetadatum.conan_file_types[file_type] }) diff --git a/app/models/packages/rubygems.rb b/app/models/packages/rubygems.rb new file mode 100644 index 00000000000..1aa6b16f47e --- /dev/null +++ b/app/models/packages/rubygems.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true +module Packages + module Rubygems + TEMPORARY_PACKAGE_NAME = 'Gem.Temporary.Package' + + def self.table_name_prefix + 'packages_rubygems_' + end + end +end diff --git a/app/models/packages/rubygems/metadatum.rb b/app/models/packages/rubygems/metadatum.rb index 42db1f3defc..d4e5feb7c98 100644 --- a/app/models/packages/rubygems/metadatum.rb +++ b/app/models/packages/rubygems/metadatum.rb @@ -3,7 +3,6 @@ module Packages module Rubygems class Metadatum < ApplicationRecord - self.table_name = 'packages_rubygems_metadata' self.primary_key = :package_id belongs_to :package, -> { where(package_type: :rubygems) }, inverse_of: :rubygems_metadatum diff --git a/app/models/pages/lookup_path.rb b/app/models/pages/lookup_path.rb index c6781f8f6e3..33771580be2 100644 --- a/app/models/pages/lookup_path.rb +++ b/app/models/pages/lookup_path.rb @@ -43,8 +43,6 @@ module Pages def deployment strong_memoize(:deployment) do - next unless Feature.enabled?(:pages_serve_from_deployments, project, default_enabled: true) - project.pages_metadatum.pages_deployment end end @@ -52,9 +50,9 @@ module Pages def zip_source return unless deployment&.file - return if deployment.file.file_storage? && !Feature.enabled?(:pages_serve_with_zip_file_protocol, project) + return if deployment.file.file_storage? && !Feature.enabled?(:pages_serve_with_zip_file_protocol, project, default_enabled: true) - return if deployment.migrated? && !Feature.enabled?(:pages_serve_from_migrated_zip, project) + return if deployment.migrated? && !Feature.enabled?(:pages_serve_from_migrated_zip, project, default_enabled: true) global_id = ::Gitlab::GlobalId.build(deployment, id: deployment.id).to_s diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index 3b07551fe05..ad2f4525171 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -4,6 +4,7 @@ class PersonalAccessToken < ApplicationRecord include Expirable include TokenAuthenticatable include Sortable + include EachBatch extend ::Gitlab::Utils::Override add_authentication_token_field :token, digest: true @@ -97,6 +98,10 @@ class PersonalAccessToken < ApplicationRecord end def set_default_scopes + # When only loading a select set of attributes, for example using `EachBatch`, + # the `scopes` attribute is not present, so we can't initialize it. + return unless has_attribute?(:scopes) + self.scopes = Gitlab::Auth::DEFAULT_SCOPES if self.scopes.empty? end diff --git a/app/models/project.rb b/app/models/project.rb index 2b9b7dcf733..274dae8fd65 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -95,6 +95,9 @@ class Project < ApplicationRecord before_save :ensure_runners_token + # https://api.rubyonrails.org/v6.0.3.4/classes/ActiveRecord/AttributeMethods/Dirty.html#method-i-will_save_change_to_attribute-3F + before_update :set_container_registry_access_level, if: :will_save_change_to_container_registry_enabled? + after_save :update_project_statistics, if: :saved_change_to_namespace_id? after_save :create_import_state, if: ->(project) { project.import? && project.import_state.nil? } @@ -345,7 +348,7 @@ class Project < ApplicationRecord has_many :daily_build_group_report_results, class_name: 'Ci::DailyBuildGroupReportResult' - has_many :repository_storage_moves, class_name: 'ProjectRepositoryStorageMove', inverse_of: :container + has_many :repository_storage_moves, class_name: 'Projects::RepositoryStorageMove', inverse_of: :container has_many :webide_pipelines, -> { webide_source }, class_name: 'Ci::Pipeline', inverse_of: :project has_many :reviews, inverse_of: :project @@ -392,7 +395,9 @@ class Project < ApplicationRecord :merge_requests_access_level, :forking_access_level, :issues_access_level, :wiki_access_level, :snippets_access_level, :builds_access_level, :repository_access_level, :pages_access_level, :metrics_dashboard_access_level, :analytics_access_level, - :operations_enabled?, :operations_access_level, to: :project_feature, allow_nil: true + :operations_enabled?, :operations_access_level, :security_and_compliance_access_level, + :container_registry_access_level, + to: :project_feature, allow_nil: true delegate :show_default_award_emojis, :show_default_award_emojis=, :show_default_award_emojis?, to: :project_setting, allow_nil: true @@ -491,10 +496,22 @@ class Project < ApplicationRecord { column: arel_table["description"], multiplier: 0.2 } ]) - query = reorder(order_expression.desc, arel_table['id'].desc) + order = Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'similarity', + column_expression: order_expression, + order_expression: order_expression.desc, + order_direction: :desc, + distinct: false, + add_to_projections: true + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id', + order_expression: Project.arel_table[:id].desc + ) + ]) - query = query.select(*query.arel.projections, order_expression.as('similarity')) if include_in_select - query + order.apply_cursor_conditions(reorder(order)) end scope :with_packages, -> { joins(:packages) } @@ -1696,8 +1713,8 @@ class Project < ApplicationRecord end end - def any_runners?(&block) - active_runners.any?(&block) + def any_active_runners?(&block) + active_runners_with_tags.any?(&block) end def valid_runners_token?(token) @@ -1986,6 +2003,7 @@ class Project < ApplicationRecord .append(key: 'CI_PROJECT_REPOSITORY_LANGUAGES', value: repository_languages.map(&:name).join(',').downcase) .append(key: 'CI_DEFAULT_BRANCH', value: default_branch) .append(key: 'CI_PROJECT_CONFIG_PATH', value: ci_config_path_or_default) + .append(key: 'CI_CONFIG_PATH', value: ci_config_path_or_default) end def predefined_ci_server_variables @@ -2023,10 +2041,12 @@ class Project < ApplicationRecord Gitlab::Ci::Variables::Collection.new.tap do |variables| break variables unless Gitlab.config.dependency_proxy.enabled - variables.append(key: 'CI_DEPENDENCY_PROXY_SERVER', value: "#{Gitlab.config.gitlab.host}:#{Gitlab.config.gitlab.port}") + variables.append(key: 'CI_DEPENDENCY_PROXY_SERVER', value: Gitlab.host_with_port) variables.append( key: 'CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX', - value: "#{Gitlab.config.gitlab.host}:#{Gitlab.config.gitlab.port}/#{namespace.root_ancestor.path}#{DependencyProxy::URL_SUFFIX}" + # The namespace path can include uppercase letters, which + # Docker doesn't allow. The proxy expects it to be downcased. + value: "#{Gitlab.host_with_port}/#{namespace.root_ancestor.path.downcase}#{DependencyProxy::URL_SUFFIX}" ) end end @@ -2532,8 +2552,26 @@ class Project < ApplicationRecord Projects::GitGarbageCollectWorker end + def inherited_issuable_templates_enabled? + Feature.enabled?(:inherited_issuable_templates, self, default_enabled: :yaml) + end + private + def set_container_registry_access_level + # changes_to_save = { 'container_registry_enabled' => [value_before_update, value_after_update] } + value = changes_to_save['container_registry_enabled'][1] + + access_level = + if value + ProjectFeature::ENABLED + else + ProjectFeature::DISABLED + end + + project_feature.update!(container_registry_access_level: access_level) + end + def find_service(services, name) services.find { |service| service.to_param == name } end @@ -2694,6 +2732,12 @@ class Project < ApplicationRecord def cache_has_external_issue_tracker update_column(:has_external_issue_tracker, services.external_issue_trackers.any?) if Gitlab::Database.read_write? end + + def active_runners_with_tags + strong_memoize(:active_runners_with_tags) do + active_runners.with_tags + end + end end Project.prepend_if_ee('EE::Project') diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb index 7b204cfb1c0..a598bf3f60c 100644 --- a/app/models/project_feature.rb +++ b/app/models/project_feature.rb @@ -3,7 +3,23 @@ class ProjectFeature < ApplicationRecord include Featurable - FEATURES = %i(issues forking merge_requests wiki snippets builds repository pages metrics_dashboard analytics operations).freeze + FEATURES = %i[ + issues + forking + merge_requests + wiki + snippets + builds + repository + pages + metrics_dashboard + analytics + operations + security_and_compliance + container_registry + ].freeze + + EXPORTABLE_FEATURES = (FEATURES - [:security_and_compliance]).freeze set_available_features(FEATURES) @@ -27,6 +43,8 @@ class ProjectFeature < ApplicationRecord end end + before_create :set_container_registry_access_level + # Default scopes force us to unscope here since a service may need to check # permissions for a project in pending_delete # http://stackoverflow.com/questions/1540645/how-to-disable-default-scope-for-a-belongs-to @@ -37,16 +55,17 @@ class ProjectFeature < ApplicationRecord validate :repository_children_level validate :allowed_access_levels - default_value_for :builds_access_level, value: ENABLED, allows_nil: false - default_value_for :issues_access_level, value: ENABLED, allows_nil: false - default_value_for :forking_access_level, value: ENABLED, allows_nil: false - default_value_for :merge_requests_access_level, value: ENABLED, allows_nil: false - default_value_for :snippets_access_level, value: ENABLED, allows_nil: false - default_value_for :wiki_access_level, value: ENABLED, allows_nil: false - default_value_for :repository_access_level, value: ENABLED, allows_nil: false - default_value_for :analytics_access_level, value: ENABLED, allows_nil: false + default_value_for :builds_access_level, value: ENABLED, allows_nil: false + default_value_for :issues_access_level, value: ENABLED, allows_nil: false + default_value_for :forking_access_level, value: ENABLED, allows_nil: false + default_value_for :merge_requests_access_level, value: ENABLED, allows_nil: false + default_value_for :snippets_access_level, value: ENABLED, allows_nil: false + default_value_for :wiki_access_level, value: ENABLED, allows_nil: false + default_value_for :repository_access_level, value: ENABLED, allows_nil: false + default_value_for :analytics_access_level, value: ENABLED, allows_nil: false default_value_for :metrics_dashboard_access_level, value: PRIVATE, allows_nil: false - default_value_for :operations_access_level, value: ENABLED, allows_nil: false + default_value_for :operations_access_level, value: ENABLED, allows_nil: false + default_value_for :security_and_compliance_access_level, value: PRIVATE, allows_nil: false default_value_for(:pages_access_level, allows_nil: false) do |feature| if ::Gitlab::Pages.access_control_is_forced? @@ -70,6 +89,15 @@ class ProjectFeature < ApplicationRecord private + def set_container_registry_access_level + self.container_registry_access_level = + if project&.container_registry_enabled + ENABLED + else + DISABLED + end + end + # Validates builds and merge requests access level # which cannot be higher than repository access level def repository_children_level diff --git a/app/models/project_repository_storage_move.rb b/app/models/project_repository_storage_move.rb index 1e3782a1fb5..e54489ddb88 100644 --- a/app/models/project_repository_storage_move.rb +++ b/app/models/project_repository_storage_move.rb @@ -1,34 +1,13 @@ # frozen_string_literal: true -# ProjectRepositoryStorageMove are details of repository storage moves for a -# project. For example, moving a project to another gitaly node to help -# balance storage capacity. -class ProjectRepositoryStorageMove < ApplicationRecord - extend ::Gitlab::Utils::Override - include RepositoryStorageMovable - - belongs_to :container, class_name: 'Project', inverse_of: :repository_storage_moves, foreign_key: :project_id - alias_attribute :project, :container - scope :with_projects, -> { includes(container: :route) } - - override :update_repository_storage - def update_repository_storage(new_storage) - container.update_column(:repository_storage, new_storage) - end - - override :schedule_repository_storage_update_worker - def schedule_repository_storage_update_worker - ProjectUpdateRepositoryStorageWorker.perform_async( - project_id, - destination_storage_name, - id - ) - end - - private - - override :error_key - def error_key - :project - end +# This is a compatibility class to avoid calling a non-existent +# class from sidekiq during deployment. +# +# This class was moved to a namespace in https://gitlab.com/gitlab-org/gitlab/-/issues/299853. +# we cannot remove this class entirely because there can be jobs +# referencing it. +# +# We can get rid of this class in 14.0 +# https://gitlab.com/gitlab-org/gitlab/-/issues/322393 +class ProjectRepositoryStorageMove < Projects::RepositoryStorageMove end diff --git a/app/models/project_services/chat_notification_service.rb b/app/models/project_services/chat_notification_service.rb index 1d50d5cf19e..cf7cad09676 100644 --- a/app/models/project_services/chat_notification_service.rb +++ b/app/models/project_services/chat_notification_service.rb @@ -97,9 +97,12 @@ class ChatNotificationService < Service opts[:channel] = channels if channels.present? opts[:username] = username if username - return false unless notify(message, opts) + if notify(message, opts) + log_usage(event_type, user_id_from_hook_data(data)) + return true + end - true + false end def event_channel_names @@ -120,6 +123,10 @@ class ChatNotificationService < Service private + def log_usage(_, _) + # Implement in child class + end + def labels_to_be_notified_list return [] if labels_to_be_notified.nil? @@ -136,6 +143,10 @@ class ChatNotificationService < Service (labels_to_be_notified_list & label_titles).any? end + def user_id_from_hook_data(data) + data.dig(:user, :id) || data[:user_id] + end + # every notifier must implement this independently def notify(message, opts) raise NotImplementedError diff --git a/app/models/project_services/discord_service.rb b/app/models/project_services/discord_service.rb index 941b7f64263..37bbb9b8752 100644 --- a/app/models/project_services/discord_service.rb +++ b/app/models/project_services/discord_service.rb @@ -59,6 +59,9 @@ class DiscordService < ChatNotificationService embed.description = (message.pretext + "\n" + Array.wrap(message.attachments).join("\n")).gsub(ATTACHMENT_REGEX, " \\k<entry> - \\k<name>\n") end end + rescue RestClient::Exception => error + log_error(error.message) + false end def custom_data(data) diff --git a/app/models/project_services/mattermost_service.rb b/app/models/project_services/mattermost_service.rb index c1055db78e5..9cff979fcf2 100644 --- a/app/models/project_services/mattermost_service.rb +++ b/app/models/project_services/mattermost_service.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class MattermostService < ChatNotificationService - include ::SlackService::Notifier + include SlackMattermost::Notifier def title 'Mattermost notifications' diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb index ab043227832..b8869547a37 100644 --- a/app/models/project_services/prometheus_service.rb +++ b/app/models/project_services/prometheus_service.rb @@ -46,7 +46,7 @@ class PrometheusService < MonitoringService end def description - s_('PrometheusService|Time-series monitoring service') + s_('PrometheusService|Monitor application health with Prometheus metrics and dashboards') end def self.to_param @@ -59,20 +59,23 @@ class PrometheusService < MonitoringService type: 'checkbox', name: 'manual_configuration', title: s_('PrometheusService|Active'), + help: s_('PrometheusService|Select this checkbox to override the auto configuration settings with your own settings.'), required: true }, { type: 'text', name: 'api_url', title: 'API URL', - placeholder: s_('PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/'), + placeholder: s_('PrometheusService|https://prometheus.example.com/'), + help: s_('PrometheusService|The Prometheus API base URL.'), required: true }, { type: 'text', name: 'google_iap_audience_client_id', title: 'Google IAP Audience Client ID', - placeholder: s_('PrometheusService|Client ID of the IAP secured resource (looks like IAP_CLIENT_ID.apps.googleusercontent.com)'), + placeholder: s_('PrometheusService|IAP_CLIENT_ID.apps.googleusercontent.com'), + help: s_('PrometheusService|PrometheusService|The ID of the IAP-secured resource.'), autocomplete: 'off', required: false }, @@ -80,7 +83,8 @@ class PrometheusService < MonitoringService type: 'textarea', name: 'google_iap_service_account_json', title: 'Google IAP Service Account JSON', - placeholder: s_('PrometheusService|Contents of the credentials.json file of your service account, like: { "type": "service_account", "project_id": ... }'), + placeholder: s_('PrometheusService|{ "type": "service_account", "project_id": ... }'), + help: s_('PrometheusService|The contents of the credentials.json file of your service account.'), required: false } ] diff --git a/app/models/project_services/slack_mattermost/notifier.rb b/app/models/project_services/slack_mattermost/notifier.rb new file mode 100644 index 00000000000..1a78cea5933 --- /dev/null +++ b/app/models/project_services/slack_mattermost/notifier.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module SlackMattermost + module Notifier + private + + def notify(message, opts) + # See https://gitlab.com/gitlab-org/slack-notifier/#custom-http-client + notifier = Slack::Messenger.new(webhook, opts.merge(http_client: HTTPClient)) + notifier.ping( + message.pretext, + attachments: message.attachments, + fallback: message.fallback + ) + end + + class HTTPClient + def self.post(uri, params = {}) + params.delete(:http_options) # these are internal to the client and we do not want them + Gitlab::HTTP.post(uri, body: params) + end + end + end +end diff --git a/app/models/project_services/slack_service.rb b/app/models/project_services/slack_service.rb index 79245e84238..f42b3de39d5 100644 --- a/app/models/project_services/slack_service.rb +++ b/app/models/project_services/slack_service.rb @@ -1,6 +1,14 @@ # frozen_string_literal: true class SlackService < ChatNotificationService + include SlackMattermost::Notifier + extend ::Gitlab::Utils::Override + + SUPPORTED_EVENTS_FOR_USAGE_LOG = %w[ + push issue confidential_issue merge_request note confidential_note + tag_push wiki_page deployment + ].freeze + prop_accessor EVENT_CHANNEL['alert'] def title @@ -36,26 +44,14 @@ class SlackService < ChatNotificationService super end - module Notifier - private + override :log_usage + def log_usage(event, user_id) + return unless user_id - def notify(message, opts) - # See https://gitlab.com/gitlab-org/slack-notifier/#custom-http-client - notifier = Slack::Messenger.new(webhook, opts.merge(http_client: HTTPClient)) - notifier.ping( - message.pretext, - attachments: message.attachments, - fallback: message.fallback - ) - end + return unless SUPPORTED_EVENTS_FOR_USAGE_LOG.include?(event) - class HTTPClient - def self.post(uri, params = {}) - params.delete(:http_options) # these are internal to the client and we do not want them - Gitlab::HTTP.post(uri, body: params) - end - end - end + key = "i_ecosystem_slack_service_#{event}_notification" - include Notifier + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(key, values: user_id) + end end diff --git a/app/models/project_services/unify_circuit_service.rb b/app/models/project_services/unify_circuit_service.rb index 1e12179e62a..1a0eebe7d64 100644 --- a/app/models/project_services/unify_circuit_service.rb +++ b/app/models/project_services/unify_circuit_service.rb @@ -47,7 +47,7 @@ class UnifyCircuitService < ChatNotificationService def notify(message, opts) response = Gitlab::HTTP.post(webhook, body: { subject: message.project_name, - text: message.pretext, + text: message.summary, markdown: true }.to_json) diff --git a/app/models/project_services/webex_teams_service.rb b/app/models/project_services/webex_teams_service.rb index 1d791b19486..4e8281f4e81 100644 --- a/app/models/project_services/webex_teams_service.rb +++ b/app/models/project_services/webex_teams_service.rb @@ -46,7 +46,7 @@ class WebexTeamsService < ChatNotificationService def notify(message, opts) header = { 'Content-Type' => 'application/json' } - response = Gitlab::HTTP.post(webhook, headers: header, body: { markdown: message.pretext }.to_json) + response = Gitlab::HTTP.post(webhook, headers: header, body: { markdown: message.summary }.to_json) response if response.success? end diff --git a/app/models/projects/repository_storage_move.rb b/app/models/projects/repository_storage_move.rb new file mode 100644 index 00000000000..f4411e0b4fd --- /dev/null +++ b/app/models/projects/repository_storage_move.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +# Projects::RepositoryStorageMove are details of repository storage moves for a +# project. For example, moving a project to another gitaly node to help +# balance storage capacity. +module Projects + class RepositoryStorageMove < ApplicationRecord + extend ::Gitlab::Utils::Override + include RepositoryStorageMovable + + self.table_name = 'project_repository_storage_moves' + + belongs_to :container, class_name: 'Project', inverse_of: :repository_storage_moves, foreign_key: :project_id + alias_attribute :project, :container + scope :with_projects, -> { includes(container: :route) } + + override :update_repository_storage + def update_repository_storage(new_storage) + container.update_column(:repository_storage, new_storage) + end + + override :schedule_repository_storage_update_worker + def schedule_repository_storage_update_worker + Projects::UpdateRepositoryStorageWorker.perform_async( + project_id, + destination_storage_name, + id + ) + end + + private + + override :error_key + def error_key + :project + end + end +end diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index ad418a47476..cbbdd091feb 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -7,6 +7,9 @@ class ProtectedBranch < ApplicationRecord scope :requiring_code_owner_approval, -> { where(code_owner_approval_required: true) } + scope :allowing_force_push, + -> { where(allow_force_push: true) } + protected_ref_access_levels :merge, :push def self.protected_ref_accessible_to?(ref, user, project:, action:, protected_refs: nil) @@ -26,6 +29,12 @@ class ProtectedBranch < ApplicationRecord self.matching(ref_name, protected_refs: protected_refs(project)).present? end + def self.allow_force_push?(project, ref_name) + return false unless ::Feature.enabled?(:allow_force_push_to_protected_branches, project) + + project.protected_branches.allowing_force_push.matching(ref_name).any? + end + def self.any_protected?(project, ref_names) protected_refs(project).any? do |protected_ref| ref_names.any? do |ref_name| diff --git a/app/models/snippet.rb b/app/models/snippet.rb index ab8782ed87f..5fdd4551982 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -45,7 +45,7 @@ class Snippet < ApplicationRecord has_many :notes, as: :noteable, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :user_mentions, class_name: "SnippetUserMention", dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_one :snippet_repository, inverse_of: :snippet - has_many :repository_storage_moves, class_name: 'SnippetRepositoryStorageMove', inverse_of: :container + has_many :repository_storage_moves, class_name: 'Snippets::RepositoryStorageMove', inverse_of: :container # We need to add the `dependent` in order to call the after_destroy callback has_one :statistics, class_name: 'SnippetStatistics', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -82,6 +82,7 @@ class Snippet < ApplicationRecord scope :fresh, -> { order("created_at DESC") } scope :inc_author, -> { includes(:author) } scope :inc_relations_for_view, -> { includes(author: :status) } + scope :inc_statistics, -> { includes(:statistics) } scope :with_statistics, -> { joins(:statistics) } scope :inc_projects_namespace_route, -> { includes(project: [:route, :namespace]) } @@ -216,8 +217,10 @@ class Snippet < ApplicationRecord def blobs return [] unless repository_exists? - branch = default_branch - list_files(branch).map { |file| Blob.lazy(repository, branch, file) } + files = list_files(default_branch) + items = files.map { |file| [default_branch, file] } + + repository.blobs_at(items).compact end def hook_attrs diff --git a/app/models/snippet_repository_storage_move.rb b/app/models/snippet_repository_storage_move.rb index bb157c08995..8234905a7e1 100644 --- a/app/models/snippet_repository_storage_move.rb +++ b/app/models/snippet_repository_storage_move.rb @@ -1,28 +1,13 @@ # frozen_string_literal: true -# SnippetRepositoryStorageMove are details of repository storage moves for a -# snippet. For example, moving a snippet to another gitaly node to help -# balance storage capacity. -class SnippetRepositoryStorageMove < ApplicationRecord - extend ::Gitlab::Utils::Override - include RepositoryStorageMovable - - belongs_to :container, class_name: 'Snippet', inverse_of: :repository_storage_moves, foreign_key: :snippet_id - alias_attribute :snippet, :container - - override :schedule_repository_storage_update_worker - def schedule_repository_storage_update_worker - SnippetUpdateRepositoryStorageWorker.perform_async( - snippet_id, - destination_storage_name, - id - ) - end - - private - - override :error_key - def error_key - :snippet - end +# This is a compatibility class to avoid calling a non-existent +# class from sidekiq during deployment. +# +# This class was moved to a namespace in https://gitlab.com/gitlab-org/gitlab/-/issues/299853. +# we cannot remove this class entirely because there can be jobs +# referencing it. +# +# We can get rid of this class in 14.0 +# https://gitlab.com/gitlab-org/gitlab/-/issues/322393 +class SnippetRepositoryStorageMove < Snippets::RepositoryStorageMove end diff --git a/app/models/snippets/repository_storage_move.rb b/app/models/snippets/repository_storage_move.rb new file mode 100644 index 00000000000..3d6e1b0ccea --- /dev/null +++ b/app/models/snippets/repository_storage_move.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +# Snippets::RepositoryStorageMove are details of repository storage moves for a +# snippet. For example, moving a snippet to another gitaly node to help +# balance storage capacity. +module Snippets + class RepositoryStorageMove < ApplicationRecord + extend ::Gitlab::Utils::Override + include RepositoryStorageMovable + + self.table_name = 'snippet_repository_storage_moves' + + belongs_to :container, class_name: 'Snippet', inverse_of: :repository_storage_moves, foreign_key: :snippet_id + alias_attribute :snippet, :container + + override :schedule_repository_storage_update_worker + def schedule_repository_storage_update_worker + Snippets::UpdateRepositoryStorageWorker.perform_async( + snippet_id, + destination_storage_name, + id + ) + end + + private + + override :error_key + def error_key + :snippet + end + end +end diff --git a/app/models/todo.rb b/app/models/todo.rb index 12dc9ce0fe6..176d5e56fc0 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -55,7 +55,6 @@ class Todo < ApplicationRecord validates :project, presence: true, unless: :group_id validates :group, presence: true, unless: :project_id - scope :for_ids, -> (ids) { where(id: ids) } scope :pending, -> { with_state(:pending) } scope :done, -> { with_state(:done) } scope :for_action, -> (action) { where(action: action) } diff --git a/app/models/user.rb b/app/models/user.rb index 1f8b680c7e5..11046bdabe4 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -179,6 +179,7 @@ class User < ApplicationRecord has_many :merge_request_reviewers, inverse_of: :reviewer has_many :assigned_issues, class_name: "Issue", through: :issue_assignees, source: :issue has_many :assigned_merge_requests, class_name: "MergeRequest", through: :merge_request_assignees, source: :merge_request + has_many :created_custom_emoji, class_name: 'CustomEmoji', inverse_of: :creator has_many :bulk_imports @@ -271,7 +272,7 @@ class User < ApplicationRecord enum layout: { fixed: 0, fluid: 1 } # User's Dashboard preference - enum dashboard: { projects: 0, stars: 1, project_activity: 2, starred_project_activity: 3, groups: 4, todos: 5, issues: 6, merge_requests: 7, operations: 8 } + enum dashboard: { projects: 0, stars: 1, project_activity: 2, starred_project_activity: 3, groups: 4, todos: 5, issues: 6, merge_requests: 7, operations: 8, followed_user_activity: 9 } # User's Project preference enum project_view: { readme: 0, activity: 1, files: 2 } @@ -293,6 +294,7 @@ class User < ApplicationRecord :setup_for_company, :setup_for_company=, :render_whitespace_in_code, :render_whitespace_in_code=, :experience_level, :experience_level=, + :markdown_surround_selection, :markdown_surround_selection=, to: :user_preference delegate :path, to: :namespace, allow_nil: true, prefix: true @@ -359,6 +361,7 @@ class User < ApplicationRecord scope :blocked, -> { with_states(:blocked, :ldap_blocked) } scope :blocked_pending_approval, -> { with_states(:blocked_pending_approval) } scope :external, -> { where(external: true) } + scope :non_external, -> { where(external: false) } scope :confirmed, -> { where.not(confirmed_at: nil) } scope :active, -> { with_state(:active).non_internal } scope :active_without_ghosts, -> { with_state(:active).without_ghosts } @@ -937,11 +940,7 @@ class User < ApplicationRecord # Returns the groups a user has access to, either through a membership or a project authorization def authorized_groups Group.unscoped do - if Feature.enabled?(:shared_group_membership_auth, self) - authorized_groups_with_shared_membership - else - authorized_groups_without_shared_membership - end + authorized_groups_with_shared_membership end end @@ -1705,6 +1704,10 @@ class User < ApplicationRecord can?(:read_all_resources) end + def can_admin_all_resources? + can?(:admin_all_resources) + end + def update_two_factor_requirement periods = expanded_groups_requiring_two_factor_authentication.pluck(:two_factor_grace_period) @@ -1855,6 +1858,14 @@ class User < ApplicationRecord created_at > Devise.confirm_within.ago end + def find_or_initialize_callout(feature_name) + callouts.find_or_initialize_by(feature_name: ::UserCallout.feature_names[feature_name]) + end + + def can_trigger_notifications? + confirmed? && !blocked? && !ghost? + end + protected # override, from Devise::Validatable diff --git a/app/models/user_callout.rb b/app/models/user_callout.rb index d93fe611538..bb5a9dceaeb 100644 --- a/app/models/user_callout.rb +++ b/app/models/user_callout.rb @@ -7,7 +7,7 @@ class UserCallout < ApplicationRecord gke_cluster_integration: 1, gcp_signup_offer: 2, cluster_security_warning: 3, - gold_trial: 4, # EE-only + ultimate_trial: 4, # EE-only geo_enable_hashed_storage: 5, # EE-only geo_migrate_hashed_storage: 6, # EE-only canary_deployment: 7, # EE-only diff --git a/app/models/user_preference.rb b/app/models/user_preference.rb index 49b93ffaf66..0bf8c8f901d 100644 --- a/app/models/user_preference.rb +++ b/app/models/user_preference.rb @@ -27,6 +27,7 @@ class UserPreference < ApplicationRecord default_value_for :time_display_relative, value: true, allows_nil: false default_value_for :time_format_in_24h, value: false, allows_nil: false default_value_for :render_whitespace_in_code, value: false, allows_nil: false + default_value_for :markdown_surround_selection, value: true, allows_nil: false class << self def notes_filters diff --git a/app/models/users_statistics.rb b/app/models/users_statistics.rb index 37a430015e5..d724b06a996 100644 --- a/app/models/users_statistics.rb +++ b/app/models/users_statistics.rb @@ -70,3 +70,5 @@ class UsersStatistics < ApplicationRecord end end end + +UsersStatistics.prepend_if_ee('EE::UsersStatistics') diff --git a/app/models/wiki.rb b/app/models/wiki.rb index 45747c0b03c..df31c54bd0f 100644 --- a/app/models/wiki.rb +++ b/app/models/wiki.rb @@ -159,8 +159,17 @@ class Wiki find_page(SIDEBAR, version) end - def find_file(name, version = nil) - wiki.file(name, version) + def find_file(name, version = 'HEAD', load_content: true) + if Feature.enabled?(:gitaly_find_file, user, default_enabled: :yaml) + data_limit = load_content ? -1 : 0 + blobs = repository.blobs_at([[version, name]], blob_size_limit: data_limit) + + return if blobs.empty? + + Gitlab::Git::WikiFile.from_blob(blobs.first) + else + wiki.file(name, version) + end end def create_page(title, content, format = :markdown, message = nil) diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index 989128987d5..3b9a7ded83e 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -205,14 +205,15 @@ class WikiPage last_commit_sha = attrs.delete(:last_commit_sha) if last_commit_sha && last_commit_sha != self.last_commit_sha - raise PageChangedError + raise PageChangedError, s_( + 'WikiPageConflictMessage|Someone edited the page the same time you did. Please check out %{wikiLinkStart}the page%{wikiLinkEnd} and make sure your changes will not unintentionally remove theirs.') end update_attributes(attrs) if title.present? && title_changed? && wiki.find_page(title).present? attributes[:title] = page.title - raise PageRenameError + raise PageRenameError, s_('WikiEdit|There is already a page with the same title in that path.') end save do diff --git a/app/models/zoom_meeting.rb b/app/models/zoom_meeting.rb index c8b510c4779..f684f9e6fe0 100644 --- a/app/models/zoom_meeting.rb +++ b/app/models/zoom_meeting.rb @@ -10,7 +10,7 @@ class ZoomMeeting < ApplicationRecord validates :project, presence: true, unless: :importing? validates :issue, presence: true, unless: :importing? - validates :url, presence: true, length: { maximum: 255 }, zoom_url: true + validates :url, presence: true, length: { maximum: 255 }, 'gitlab/utils/zoom_url': true validates :issue, same_project_association: true, unless: :importing? enum issue_status: { |