diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-04-21 02:50:22 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-04-21 02:50:22 +0300 |
| commit | 9dc93a4519d9d5d7be48ff274127136236a3adb3 (patch) | |
| tree | 70467ae3692a0e35e5ea56bcb803eb512a10bedb /app/models | |
| parent | 4b0f34b6d759d6299322b3a54453e930c6121ff0 (diff) | |
Add latest changes from gitlab-org/gitlab@13-11-stable-eev13.11.0-rc43
Diffstat (limited to 'app/models')
182 files changed, 3019 insertions, 676 deletions
diff --git a/app/models/ability.rb b/app/models/ability.rb index 514e923c380..ba46a98b951 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -58,7 +58,8 @@ class Ability def allowed?(user, action, subject = :global, opts = {}) if subject.is_a?(Hash) - opts, subject = subject, :global + opts = subject + subject = :global end policy = policy_for(user, subject) diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 44d1b6cf907..1bbace791ed 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -42,10 +42,6 @@ class ApplicationRecord < ActiveRecord::Base false end - def self.at_most(count) - limit(count) - end - def self.safe_find_or_create_by!(*args, &block) safe_find_or_create_by(*args, &block).tap do |record| raise ActiveRecord::RecordNotFound unless record.present? @@ -56,9 +52,9 @@ class ApplicationRecord < ActiveRecord::Base # Start a new transaction with a shorter-than-usual statement timeout. This is # currently one third of the default 15-second timeout - def self.with_fast_statement_timeout + def self.with_fast_read_statement_timeout(timeout_ms = 5000) transaction(requires_new: true) do - connection.exec_query("SET LOCAL statement_timeout = 5000") + connection.exec_query("SET LOCAL statement_timeout = #{timeout_ms}") yield end @@ -83,3 +79,5 @@ class ApplicationRecord < ActiveRecord::Base enum(enum_mod.key => values) end end + +ApplicationRecord.prepend_if_ee('EE::ApplicationRecordHelpers') diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 44eb2fefb3f..f405f5ca5d3 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -465,6 +465,16 @@ class ApplicationSetting < ApplicationRecord length: { maximum: 100, message: N_('is too long (maximum is 100 entries)') }, allow_nil: false + validates :admin_mode, + inclusion: { in: [true, false], message: _('must be a boolean value') } + + validates :external_pipeline_validation_service_url, + addressable_url: true, allow_blank: true + + validates :external_pipeline_validation_service_timeout, + allow_nil: true, + numericality: { only_integer: true, greater_than: 0 } + attr_encrypted :asset_proxy_secret_key, mode: :per_attribute_iv, key: Settings.attr_encrypted_db_key_base_truncated, @@ -493,6 +503,7 @@ class ApplicationSetting < ApplicationRecord attr_encrypted :ci_jwt_signing_key, encryption_options_base_truncated_aes_256_gcm attr_encrypted :secret_detection_token_revocation_token, encryption_options_base_truncated_aes_256_gcm attr_encrypted :cloud_license_auth_token, encryption_options_base_truncated_aes_256_gcm + attr_encrypted :external_pipeline_validation_service_token, encryption_options_base_truncated_aes_256_gcm validates :disable_feed_token, inclusion: { in: [true, false], message: _('must be a boolean value') } diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index c067199b52c..66a8d1f8105 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -35,6 +35,7 @@ module ApplicationSettingImplementation class_methods do def defaults { + admin_mode: false, after_sign_up_text: nil, akismet_enabled: false, allow_local_requests_from_system_hooks: true, @@ -71,6 +72,9 @@ module ApplicationSettingImplementation eks_secret_access_key: nil, email_restrictions_enabled: false, email_restrictions: nil, + external_pipeline_validation_service_timeout: nil, + external_pipeline_validation_service_token: nil, + external_pipeline_validation_service_url: nil, first_day_of_week: 0, gitaly_timeout_default: 55, gitaly_timeout_fast: 10, @@ -434,11 +438,14 @@ module ApplicationSettingImplementation def parse_addr_and_port(str) case str when /\A\[(?<address> .* )\]:(?<port> \d+ )\z/x # string like "[::1]:80" - address, port = $~[:address], $~[:port] + address = $~[:address] + port = $~[:port] when /\A(?<address> [^:]+ ):(?<port> \d+ )\z/x # string like "127.0.0.1:80" - address, port = $~[:address], $~[:port] + address = $~[:address] + port = $~[:port] else # string with no port number - address, port = str, nil + address = str + port = nil end [address, port&.to_i] diff --git a/app/models/audit_event_archived.rb b/app/models/audit_event_archived.rb deleted file mode 100644 index 3119f56fbcc..00000000000 --- a/app/models/audit_event_archived.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -# This model is not intended to be used. -# It is a temporary reference to the pre-partitioned -# audit_events table. -# Please refer to https://gitlab.com/groups/gitlab-org/-/epics/3206 -# for details. -class AuditEventArchived < ApplicationRecord - self.table_name = 'audit_events_archived' -end diff --git a/app/models/blob.rb b/app/models/blob.rb index 8a9db8b45ea..2185233a1ac 100644 --- a/app/models/blob.rb +++ b/app/models/blob.rb @@ -2,6 +2,7 @@ # Blob is a Rails-specific wrapper around Gitlab::Git::Blob, SnippetBlob and Ci::ArtifactBlob class Blob < SimpleDelegator + include GlobalID::Identification include Presentable include BlobLanguageFromGitAttributes include BlobActiveModel diff --git a/app/models/blob_viewer/dependency_manager.rb b/app/models/blob_viewer/dependency_manager.rb index 1be7120a955..a851f22bfcd 100644 --- a/app/models/blob_viewer/dependency_manager.rb +++ b/app/models/blob_viewer/dependency_manager.rb @@ -33,8 +33,8 @@ module BlobViewer @json_data ||= begin prepare! Gitlab::Json.parse(blob.data) - rescue - {} + rescue + {} end end diff --git a/app/models/board.rb b/app/models/board.rb index 418ea67fc6a..b26a9461ffc 100644 --- a/app/models/board.rb +++ b/app/models/board.rb @@ -34,14 +34,6 @@ class Board < ApplicationRecord project_id.present? end - def backlog_list - lists.merge(List.backlog).take - end - - def closed_list - lists.merge(List.closed).take - end - def scoped? false end diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index 9127dab56a6..04af1145769 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -68,25 +68,6 @@ class BulkImports::Entity < ApplicationRecord end end - def update_tracker_for(relation:, has_next_page:, next_page: nil) - attributes = { - relation: relation, - has_next_page: has_next_page, - next_page: next_page, - bulk_import_entity_id: id - } - - trackers.upsert(attributes, unique_by: %i[bulk_import_entity_id relation]) - end - - def has_next_page?(relation) - trackers.find_by(relation: relation)&.has_next_page - end - - def next_page_for(relation) - trackers.find_by(relation: relation)&.next_page - end - private def validate_parent_is_a_group diff --git a/app/models/bulk_imports/stage.rb b/app/models/bulk_imports/stage.rb new file mode 100644 index 00000000000..050c2c76ce8 --- /dev/null +++ b/app/models/bulk_imports/stage.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module BulkImports + class Stage + include Singleton + + CONFIG = { + group: { + pipeline: BulkImports::Groups::Pipelines::GroupPipeline, + stage: 0 + }, + subgroups: { + pipeline: BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, + stage: 1 + }, + members: { + pipeline: BulkImports::Groups::Pipelines::MembersPipeline, + stage: 1 + }, + labels: { + pipeline: BulkImports::Groups::Pipelines::LabelsPipeline, + stage: 1 + }, + milestones: { + pipeline: BulkImports::Groups::Pipelines::MilestonesPipeline, + stage: 1 + }, + badges: { + pipeline: BulkImports::Groups::Pipelines::BadgesPipeline, + stage: 1 + }, + finisher: { + pipeline: BulkImports::Groups::Pipelines::EntityFinisher, + stage: 2 + } + }.freeze + + def self.pipelines + instance.pipelines + end + + def self.pipeline_exists?(name) + pipelines.any? do |(_, pipeline)| + pipeline.to_s == name.to_s + end + end + + def pipelines + @pipelines ||= config + .values + .sort_by { |entry| entry[:stage] } + .map do |entry| + [entry[:stage], entry[:pipeline]] + end + end + + private + + def config + @config ||= CONFIG + end + end +end + +::BulkImports::Stage.prepend_if_ee('::EE::BulkImports::Stage') diff --git a/app/models/bulk_imports/tracker.rb b/app/models/bulk_imports/tracker.rb index 182c0bbaa8a..282ba9e19ac 100644 --- a/app/models/bulk_imports/tracker.rb +++ b/app/models/bulk_imports/tracker.rb @@ -3,6 +3,8 @@ class BulkImports::Tracker < ApplicationRecord self.table_name = 'bulk_import_trackers' + alias_attribute :pipeline_name, :relation + belongs_to :entity, class_name: 'BulkImports::Entity', foreign_key: :bulk_import_entity_id, @@ -16,6 +18,29 @@ class BulkImports::Tracker < ApplicationRecord validates :stage, presence: true + DEFAULT_PAGE_SIZE = 500 + + scope :next_pipeline_trackers_for, -> (entity_id) { + entity_scope = where(bulk_import_entity_id: entity_id) + next_stage_scope = entity_scope.with_status(:created).select('MIN(stage)') + + entity_scope.where(stage: next_stage_scope) + } + + def self.stage_running?(entity_id, stage) + where(stage: stage, bulk_import_entity_id: entity_id) + .with_status(:created, :started) + .exists? + end + + def pipeline_class + unless BulkImports::Stage.pipeline_exists?(pipeline_name) + raise NameError.new("'#{pipeline_name}' is not a valid BulkImport Pipeline") + end + + pipeline_name.constantize + end + state_machine :status, initial: :created do state :created, value: 0 state :started, value: 1 diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 824e35a6480..3d8e9f4c126 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -14,8 +14,6 @@ module Ci BuildArchivedError = Class.new(StandardError) - ignore_columns :artifacts_file, :artifacts_file_store, :artifacts_metadata, :artifacts_metadata_store, :artifacts_size, :commands, remove_after: '2019-12-15', remove_with: '12.7' - belongs_to :project, inverse_of: :builds belongs_to :runner belongs_to :trigger_request @@ -35,6 +33,7 @@ module Ci }.freeze DEGRADATION_THRESHOLD_VARIABLE_NAME = 'DEGRADATION_THRESHOLD' + RUNNERS_STATUS_CACHE_EXPIRATION = 1.minute has_one :deployment, as: :deployable, class_name: 'Deployment' has_one :pending_state, class_name: 'Ci::BuildPendingState', inverse_of: :build @@ -75,7 +74,14 @@ module Ci return unless has_environment? strong_memoize(:persisted_environment) do - Environment.find_by(name: expanded_environment_name, project: project) + # This code path has caused N+1s in the past, since environments are only indirectly + # associated to builds and pipelines; see https://gitlab.com/gitlab-org/gitlab/-/issues/326445 + # We therefore batch-load them to prevent dormant N+1s until we found a proper solution. + BatchLoader.for(expanded_environment_name).batch(key: project_id) do |names, loader, args| + Environment.where(name: names, project: args[:key]).find_each do |environment| + loader.call(environment.name, environment) + end + end end end @@ -88,8 +94,7 @@ module Ci validates :ref, presence: true scope :not_interruptible, -> do - joins(:metadata).where('ci_builds_metadata.id NOT IN (?)', - Ci::BuildMetadata.scoped_build.with_interruptible.select(:id)) + joins(:metadata).where.not('ci_builds_metadata.id' => Ci::BuildMetadata.scoped_build.with_interruptible.select(:id)) end scope :unstarted, -> { where(runner_id: nil) } @@ -319,7 +324,7 @@ module Ci end end - before_transition any => [:failed] do |build| + after_transition any => [:failed] do |build| next unless build.project next unless build.deployment @@ -372,11 +377,11 @@ module Ci end def other_manual_actions - pipeline.manual_actions.where.not(name: name) + pipeline.manual_actions.reject { |action| action.name == self.name } end def other_scheduled_actions - pipeline.scheduled_actions.where.not(name: name) + pipeline.scheduled_actions.reject { |action| action.name == self.name } end def pages_generator? @@ -698,7 +703,23 @@ module Ci end def any_runners_online? - project.any_active_runners? { |runner| runner.match_build_if_online?(self) } + if Feature.enabled?(:runners_cached_states, project, default_enabled: :yaml) + cache_for_online_runners do + project.any_online_runners? { |runner| runner.match_build_if_online?(self) } + end + else + project.any_active_runners? { |runner| runner.match_build_if_online?(self) } + end + end + + def any_runners_available? + if Feature.enabled?(:runners_cached_states, project, default_enabled: :yaml) + cache_for_available_runners do + project.active_runners.exists? + end + else + project.any_active_runners? + end end def stuck? @@ -1103,6 +1124,20 @@ module Ci .to_a .include?(exit_code) end + + def cache_for_online_runners(&block) + Rails.cache.fetch( + ['has-online-runners', id], + expires_in: RUNNERS_STATUS_CACHE_EXPIRATION + ) { yield } + end + + def cache_for_available_runners(&block) + Rails.cache.fetch( + ['has-available-runners', project.id], + expires_in: RUNNERS_STATUS_CACHE_EXPIRATION + ) { yield } + end end end diff --git a/app/models/ci/build_dependencies.rb b/app/models/ci/build_dependencies.rb index b50ecf99439..8ae921f1416 100644 --- a/app/models/ci/build_dependencies.rb +++ b/app/models/ci/build_dependencies.rb @@ -21,8 +21,7 @@ module Ci deps = model_class.where(pipeline_id: processable.pipeline_id).latest deps = from_previous_stages(deps) deps = from_needs(deps) - deps = from_dependencies(deps) - deps + from_dependencies(deps) end # Dependencies from the same parent-pipeline hierarchy excluding diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index d4f9f78a1ac..7e03d709f24 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -30,9 +30,9 @@ module Ci fog: 3 }.freeze - STORE_TYPES = DATA_STORES.keys.map do |store| + STORE_TYPES = DATA_STORES.keys.to_h do |store| [store, "Ci::BuildTraceChunks::#{store.capitalize}".constantize] - end.to_h.freeze + end.freeze enum data_store: DATA_STORES diff --git a/app/models/ci/build_trace_chunks/redis.rb b/app/models/ci/build_trace_chunks/redis.rb index 58d50b39c11..003ec107895 100644 --- a/app/models/ci/build_trace_chunks/redis.rb +++ b/app/models/ci/build_trace_chunks/redis.rb @@ -4,7 +4,7 @@ module Ci module BuildTraceChunks class Redis CHUNK_REDIS_TTL = 1.week - LUA_APPEND_CHUNK = <<~EOS.freeze + LUA_APPEND_CHUNK = <<~EOS local key, new_data, offset = KEYS[1], ARGV[1], ARGV[2] local length = new_data:len() local expire = #{CHUNK_REDIS_TTL.seconds} diff --git a/app/models/ci/group.rb b/app/models/ci/group.rb index 4ba09fd8152..47b91fcf2ce 100644 --- a/app/models/ci/group.rb +++ b/app/models/ci/group.rb @@ -22,6 +22,13 @@ module Ci @jobs = jobs end + def ==(other) + other.present? && other.is_a?(self.class) && + project == other.project && + stage == other.stage && + name == other.name + end + def status strong_memoize(:status) do status_struct.status diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index d5e88f2be5b..50e21a1c323 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -131,8 +131,6 @@ module Ci update_project_statistics project_statistics_name: :build_artifacts_size scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) } - scope :with_files_stored_locally, -> { where(file_store: ::JobArtifactUploader::Store::LOCAL) } - scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) } scope :for_sha, ->(sha, project_id) { joins(job: :pipeline).where(ci_pipelines: { sha: sha, project_id: project_id }) } scope :for_job_name, ->(name) { joins(:job).where(ci_builds: { name: name }) } @@ -292,8 +290,12 @@ module Ci end end + def archived_trace_exists? + file&.file&.exists? + end + def self.archived_trace_exists_for?(job_id) - where(job_id: job_id).trace.take&.file&.file&.exists? + where(job_id: job_id).trace.take&.archived_trace_exists? end def self.max_artifact_size(type:, project:) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index b63ec0c8a97..c9ab69317e1 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -286,9 +286,11 @@ module Ci end after_transition any => [:failed] do |pipeline| - next unless pipeline.auto_devops_source? + pipeline.run_after_commit do + ::Gitlab::Ci::Pipeline::Metrics.pipeline_failure_reason_counter.increment(reason: pipeline.failure_reason) - pipeline.run_after_commit { AutoDevops::DisableWorker.perform_async(pipeline.id) } + AutoDevops::DisableWorker.perform_async(pipeline.id) if pipeline.auto_devops_source? + end end end @@ -309,6 +311,7 @@ module Ci scope :created_after, -> (time) { where('ci_pipelines.created_at > ?', time) } scope :created_before_id, -> (id) { where('ci_pipelines.id < ?', id) } scope :before_pipeline, -> (pipeline) { created_before_id(pipeline.id).outside_pipeline_family(pipeline) } + scope :eager_load_project, -> { eager_load(project: [:route, { namespace: :route }]) } scope :outside_pipeline_family, ->(pipeline) do where.not(id: pipeline.same_family_pipeline_ids) @@ -393,26 +396,13 @@ module Ci # given we simply get the latest pipelines for the commits, regardless # of what refs the pipelines belong to. def self.latest_pipeline_per_commit(commits, ref = nil) - p1 = arel_table - p2 = arel_table.alias - - # This LEFT JOIN will filter out all but the newest row for every - # combination of (project_id, sha) or (project_id, sha, ref) if a ref is - # given. - cond = p1[:sha].eq(p2[:sha]) - .and(p1[:project_id].eq(p2[:project_id])) - .and(p1[:id].lt(p2[:id])) - - cond = cond.and(p1[:ref].eq(p2[:ref])) if ref - join = p1.join(p2, Arel::Nodes::OuterJoin).on(cond) + sql = select('DISTINCT ON (sha) *') + .where(sha: commits) + .order(:sha, id: :desc) - relation = where(sha: commits) - .where(p2[:id].eq(nil)) - .joins(join.join_sources) + sql = sql.where(ref: ref) if ref - relation = relation.where(ref: ref) if ref - - relation.each_with_object({}) do |pipeline, hash| + sql.each_with_object({}) do |pipeline, hash| hash[pipeline.sha] = pipeline end end @@ -445,6 +435,10 @@ module Ci @auto_devops_pipelines_completed_total ||= Gitlab::Metrics.counter(:auto_devops_pipelines_completed_total, 'Number of completed auto devops pipelines') end + def uses_needs? + builds.where(scheduling_type: :dag).any? + end + def stages_count statuses.select(:stage).distinct.count end @@ -510,6 +504,12 @@ module Ci end end + def git_author_full_text + strong_memoize(:git_author_full_text) do + commit.try(:author_full_text) + end + end + def git_commit_message strong_memoize(:git_commit_message) do commit.try(:message) @@ -573,10 +573,18 @@ module Ci end def cancel_running(retries: nil) - retry_optimistic_lock(cancelable_statuses, retries, name: 'ci_pipeline_cancel_running') do |cancelable| - cancelable.find_each do |job| - yield(job) if block_given? - job.cancel + commit_status_relations = [:project, :pipeline] + ci_build_relations = [:deployment, :taggings] + + retry_optimistic_lock(cancelable_statuses, retries, name: 'ci_pipeline_cancel_running') do |cancelables| + cancelables.find_in_batches do |batch| + ActiveRecord::Associations::Preloader.new.preload(batch, commit_status_relations) + ActiveRecord::Associations::Preloader.new.preload(batch.select { |job| job.is_a?(Ci::Build) }, ci_build_relations) + + batch.each do |job| + yield(job) if block_given? + job.cancel + end end end end @@ -664,7 +672,9 @@ module Ci end def has_kubernetes_active? - project.deployment_platform&.active? + strong_memoize(:has_kubernetes_active) do + project.deployment_platform&.active? + end end def freeze_period? @@ -822,6 +832,7 @@ module Ci variables.append(key: 'CI_COMMIT_DESCRIPTION', value: git_commit_description.to_s) variables.append(key: 'CI_COMMIT_REF_PROTECTED', value: (!!protected_ref?).to_s) variables.append(key: 'CI_COMMIT_TIMESTAMP', value: git_commit_timestamp.to_s) + variables.append(key: 'CI_COMMIT_AUTHOR', value: git_author_full_text.to_s) # legacy variables variables.append(key: 'CI_BUILD_REF', value: sha) diff --git a/app/models/ci/pipeline_artifact.rb b/app/models/ci/pipeline_artifact.rb index f538a4cd808..9dfe4252e95 100644 --- a/app/models/ci/pipeline_artifact.rb +++ b/app/models/ci/pipeline_artifact.rb @@ -57,3 +57,5 @@ module Ci end end end + +Ci::PipelineArtifact.prepend_ee_mod diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb index 2fae077dd87..3c17246bc34 100644 --- a/app/models/ci/pipeline_schedule.rb +++ b/app/models/ci/pipeline_schedule.rb @@ -7,6 +7,7 @@ module Ci include StripAttribute include Schedulable include Limitable + include EachBatch self.limit_name = 'ci_pipeline_schedules' self.limit_scope = :project @@ -28,6 +29,7 @@ module Ci scope :active, -> { where(active: true) } scope :inactive, -> { where(active: false) } scope :preloaded, -> { preload(:owner, project: [:route]) } + scope :owned_by, ->(user) { where(owner: user) } accepts_nested_attributes_for :variables, allow_destroy: true diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb index 0ad1ed2fce8..3b61840805a 100644 --- a/app/models/ci/processable.rb +++ b/app/models/ci/processable.rb @@ -165,7 +165,13 @@ module Ci end def all_dependencies - dependencies.all + if Feature.enabled?(:preload_associations_jobs_request_api_endpoint, project, default_enabled: :yaml) + strong_memoize(:all_dependencies) do + dependencies.all + end + else + dependencies.all + end end private diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index d1a20bc93c3..05126853e0f 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -45,8 +45,6 @@ module Ci FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level maximum_timeout_human_readable].freeze MINUTES_COST_FACTOR_FIELDS = %i[public_projects_minutes_cost_factor private_projects_minutes_cost_factor].freeze - ignore_column :is_shared, remove_after: '2019-12-15', remove_with: '12.6' - has_many :builds has_many :runner_projects, inverse_of: :runner, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :runner_projects diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 03a97355574..9dd75150ac7 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -14,11 +14,20 @@ module Ci has_many :statuses, class_name: 'CommitStatus', foreign_key: :stage_id has_many :latest_statuses, -> { ordered.latest }, class_name: 'CommitStatus', foreign_key: :stage_id + has_many :retried_statuses, -> { ordered.retried }, class_name: 'CommitStatus', foreign_key: :stage_id has_many :processables, class_name: 'Ci::Processable', foreign_key: :stage_id has_many :builds, foreign_key: :stage_id has_many :bridges, foreign_key: :stage_id scope :ordered, -> { order(position: :asc) } + scope :in_pipelines, ->(pipelines) { where(pipeline: pipelines) } + scope :by_name, ->(names) { where(name: names) } + scope :with_latest_and_retried_statuses, -> do + includes( + latest_statuses: [:pipeline, project: :namespace], + retried_statuses: [:pipeline, project: :namespace] + ) + end with_options unless: :importing? do validates :project, presence: true @@ -35,7 +44,7 @@ module Ci next if position.present? self.position = statuses.select(:stage_idx) - .where('stage_idx IS NOT NULL') + .where.not(stage_idx: nil) .group(:stage_idx) .order('COUNT(*) DESC') .first&.stage_idx.to_i diff --git a/app/models/ci/test_case.rb b/app/models/ci/test_case.rb deleted file mode 100644 index 19ecc177436..00000000000 --- a/app/models/ci/test_case.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Ci - class TestCase < ApplicationRecord - extend Gitlab::Ci::Model - - validates :project, :key_hash, presence: true - - has_many :test_case_failures, class_name: 'Ci::TestCaseFailure' - - belongs_to :project - - scope :by_project_and_keys, -> (project, keys) { where(project_id: project.id, key_hash: keys) } - - class << self - def find_or_create_by_batch(project, test_case_keys) - # Insert records first. Existing ones will be skipped. - insert_all(test_case_attrs(project, test_case_keys)) - - # Find all matching records now that we are sure they all are persisted. - by_project_and_keys(project, test_case_keys) - end - - private - - def test_case_attrs(project, test_case_keys) - # NOTE: Rails 6.1 will add support for insert_all on relation so that - # we will be able to do project.test_cases.insert_all. - test_case_keys.map do |hashed_key| - { project_id: project.id, key_hash: hashed_key } - end - end - end - end -end diff --git a/app/models/ci/test_case_failure.rb b/app/models/ci/test_case_failure.rb deleted file mode 100644 index 8867b954240..00000000000 --- a/app/models/ci/test_case_failure.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Ci - class TestCaseFailure < ApplicationRecord - extend Gitlab::Ci::Model - - REPORT_WINDOW = 14.days - - validates :test_case, :build, :failed_at, presence: true - - belongs_to :test_case, class_name: "Ci::TestCase", foreign_key: :test_case_id - belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id - - def self.recent_failures_count(project:, test_case_keys:, date_range: REPORT_WINDOW.ago..Time.current) - joins(:test_case) - .where( - ci_test_cases: { - project_id: project.id, - key_hash: test_case_keys - }, - ci_test_case_failures: { - failed_at: date_range - } - ) - .group(:key_hash) - .count('ci_test_case_failures.id') - end - end -end diff --git a/app/models/ci/unit_test.rb b/app/models/ci/unit_test.rb new file mode 100644 index 00000000000..81623b4f6ad --- /dev/null +++ b/app/models/ci/unit_test.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Ci + class UnitTest < ApplicationRecord + extend Gitlab::Ci::Model + + MAX_NAME_SIZE = 255 + MAX_SUITE_NAME_SIZE = 255 + + validates :project, :key_hash, :name, :suite_name, presence: true + + has_many :unit_test_failures, class_name: 'Ci::UnitTestFailure' + + belongs_to :project + + scope :by_project_and_keys, -> (project, keys) { where(project_id: project.id, key_hash: keys) } + + class << self + def find_or_create_by_batch(project, unit_test_attrs) + # Insert records first. Existing ones will be skipped. + insert_all(build_insert_attrs(project, unit_test_attrs)) + + # Find all matching records now that we are sure they all are persisted. + by_project_and_keys(project, gather_keys(unit_test_attrs)) + end + + private + + def build_insert_attrs(project, unit_test_attrs) + # NOTE: Rails 6.1 will add support for insert_all on relation so that + # we will be able to do project.test_cases.insert_all. + unit_test_attrs.map do |attrs| + attrs.merge( + project_id: project.id, + name: attrs[:name].truncate(MAX_NAME_SIZE), + suite_name: attrs[:suite_name].truncate(MAX_SUITE_NAME_SIZE) + ) + end + end + + def gather_keys(unit_test_attrs) + unit_test_attrs.map { |attrs| attrs[:key_hash] } + end + end + end +end diff --git a/app/models/ci/unit_test_failure.rb b/app/models/ci/unit_test_failure.rb new file mode 100644 index 00000000000..653a56bd2b3 --- /dev/null +++ b/app/models/ci/unit_test_failure.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Ci + class UnitTestFailure < ApplicationRecord + extend Gitlab::Ci::Model + + REPORT_WINDOW = 14.days + + validates :unit_test, :build, :failed_at, presence: true + + belongs_to :unit_test, class_name: "Ci::UnitTest", foreign_key: :unit_test_id + belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id + + def self.recent_failures_count(project:, unit_test_keys:, date_range: REPORT_WINDOW.ago..Time.current) + joins(:unit_test) + .where( + ci_unit_tests: { + project_id: project.id, + key_hash: unit_test_keys + }, + ci_unit_test_failures: { + failed_at: date_range + } + ) + .group(:key_hash) + .count('ci_unit_test_failures.id') + end + end +end diff --git a/app/models/clusters/agent_token.rb b/app/models/clusters/agent_token.rb index 9d79887b574..d42279502c5 100644 --- a/app/models/clusters/agent_token.rb +++ b/app/models/clusters/agent_token.rb @@ -2,17 +2,45 @@ module Clusters class AgentToken < ApplicationRecord + include RedisCacheable include TokenAuthenticatable + add_authentication_token_field :token, encrypted: :required, token_generator: -> { Devise.friendly_token(50) } + cached_attr_reader :last_contacted_at self.table_name = 'cluster_agent_tokens' + # The `UPDATE_USED_COLUMN_EVERY` defines how often the token DB entry can be updated + UPDATE_USED_COLUMN_EVERY = (40.minutes..55.minutes).freeze + belongs_to :agent, class_name: 'Clusters::Agent', optional: false belongs_to :created_by_user, class_name: 'User', optional: true before_save :ensure_token validates :description, length: { maximum: 1024 } - validates :name, presence: true, length: { maximum: 255 }, on: :create + validates :name, presence: true, length: { maximum: 255 } + + def track_usage + track_values = { last_used_at: Time.current.utc } + + cache_attributes(track_values) + + # Use update_column so updated_at is skipped + update_columns(track_values) if can_update_track_values? + end + + private + + def can_update_track_values? + # Use a random threshold to prevent beating DB updates. + last_used_at_max_age = Random.rand(UPDATE_USED_COLUMN_EVERY) + + real_last_used_at = read_attribute(:last_used_at) + + # Handle too many updates from high token traffic + real_last_used_at.nil? || + (Time.current - real_last_used_at) >= last_used_at_max_age + end end end diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb index 55a9a0ccb81..b9c136abab4 100644 --- a/app/models/clusters/applications/prometheus.rb +++ b/app/models/clusters/applications/prometheus.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Prometheus < ApplicationRecord - include PrometheusAdapter + include ::Clusters::Concerns::PrometheusClient VERSION = '10.4.1' @@ -32,7 +32,7 @@ module Clusters end state_machine :status do - after_transition any => [:installed] do |application| + after_transition any => [:installed, :externally_installed] do |application| application.run_after_commit do Clusters::Applications::ActivateServiceWorker .perform_async(application.cluster_id, ::PrometheusService.to_param) # rubocop:disable CodeReuse/ServiceClass @@ -58,14 +58,6 @@ module Clusters 'https://gitlab-org.gitlab.io/cluster-integration/helm-stable-archive' end - def service_name - 'prometheus-prometheus-server' - end - - def service_port - 80 - end - def install_command helm_command_module::InstallCommand.new( name: name, @@ -106,29 +98,6 @@ module Clusters files.merge('values.yaml': replaced_values) end - def prometheus_client - return unless kube_client - - proxy_url = kube_client.proxy_url('service', service_name, service_port, Gitlab::Kubernetes::Helm::NAMESPACE) - - # ensures headers containing auth data are appended to original k8s client options - options = kube_client.rest_client.options - .merge(prometheus_client_default_options) - .merge(headers: kube_client.headers) - Gitlab::PrometheusClient.new(proxy_url, options) - rescue Kubeclient::HttpError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::ENETUNREACH - # If users have mistakenly set parameters or removed the depended clusters, - # `proxy_url` could raise an exception because gitlab can not communicate with the cluster. - # Since `PrometheusAdapter#can_query?` is eargely loaded on environement pages in gitlab, - # we need to silence the exceptions - end - - def configured? - kube_client.present? && available? - rescue Gitlab::UrlBlocker::BlockedUrlError - false - end - def generate_alert_manager_token! unless alert_manager_token.present? update!(alert_manager_token: generate_token) @@ -146,10 +115,6 @@ module Clusters .perform_async(cluster_id, ::PrometheusService.to_param) # rubocop:disable CodeReuse/ServiceClass end - def kube_client - cluster&.kubeclient&.core_client - end - def install_knative_metrics return [] unless cluster.application_knative_available? diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index 8a49d476ba7..bc80bcd0b06 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Runner < ApplicationRecord - VERSION = '0.26.0' + VERSION = '0.27.0' self.table_name = 'clusters_applications_runners' diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index a34d8a6b98d..a1e2aa194a0 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -51,6 +51,8 @@ module Clusters has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', inverse_of: :cluster, autosave: true + has_one :integration_prometheus, class_name: 'Clusters::Integrations::Prometheus', inverse_of: :cluster + def self.has_one_cluster_application(name) # rubocop:disable Naming/PredicateName application = APPLICATIONS[name.to_s] has_one application.association_name, class_name: application.to_s, inverse_of: :cluster # rubocop:disable Rails/ReflectionClassName @@ -100,7 +102,6 @@ module Clusters delegate :rbac?, to: :platform_kubernetes, prefix: true, allow_nil: true delegate :available?, to: :application_helm, prefix: true, allow_nil: true delegate :available?, to: :application_ingress, prefix: true, allow_nil: true - delegate :available?, to: :application_prometheus, prefix: true, allow_nil: true delegate :available?, to: :application_knative, prefix: true, allow_nil: true delegate :available?, to: :application_elastic_stack, prefix: true, allow_nil: true delegate :external_ip, to: :application_ingress, prefix: true, allow_nil: true @@ -148,6 +149,9 @@ module Clusters scope :with_management_project, -> { where.not(management_project: nil) } scope :for_project_namespace, -> (namespace_id) { joins(:projects).where(projects: { namespace_id: namespace_id }) } + + # with_application_prometheus scope is deprecated, and scheduled for removal + # in %14.0. See https://gitlab.com/groups/gitlab-org/-/epics/4280 scope :with_application_prometheus, -> { includes(:application_prometheus).joins(:application_prometheus) } scope :with_project_http_integrations, -> (project_ids) do conditions = { projects: :alert_management_http_integrations } @@ -276,6 +280,10 @@ module Clusters public_send(association_name) || public_send("build_#{association_name}") # rubocop:disable GitlabSecurity/PublicSend end + def find_or_build_integration_prometheus + integration_prometheus || build_integration_prometheus + end + def provider if gcp? provider_gcp @@ -361,8 +369,12 @@ module Clusters end end + def application_prometheus_available? + integration_prometheus&.available? || application_prometheus&.available? + end + def prometheus_adapter - application_prometheus + integration_prometheus || application_prometheus end private diff --git a/app/models/clusters/clusters_hierarchy.rb b/app/models/clusters/clusters_hierarchy.rb index c9c18d8c96a..125783e6ee1 100644 --- a/app/models/clusters/clusters_hierarchy.rb +++ b/app/models/clusters/clusters_hierarchy.rb @@ -16,7 +16,7 @@ module Clusters model .unscoped - .where('clusters.id IS NOT NULL') + .where.not('clusters.id' => nil) .with .recursive(cte.to_arel) .from(cte_alias) diff --git a/app/models/clusters/concerns/application_status.rb b/app/models/clusters/concerns/application_status.rb index 95ac95448dd..7485ee079ce 100644 --- a/app/models/clusters/concerns/application_status.rb +++ b/app/models/clusters/concerns/application_status.rb @@ -9,6 +9,7 @@ module Clusters scope :available, -> do where( status: [ + self.state_machines[:status].states[:externally_installed].value, self.state_machines[:status].states[:installed].value, self.state_machines[:status].states[:updated].value ] @@ -28,6 +29,7 @@ module Clusters state :uninstalling, value: 7 state :uninstall_errored, value: 8 state :uninstalled, value: 10 + state :externally_installed, value: 11 # Used for applications that are pre-installed by the cluster, # e.g. Knative in GCP Cloud Run enabled clusters @@ -37,7 +39,7 @@ module Clusters state :pre_installed, value: 9 event :make_externally_installed do - transition any => :installed + transition any => :externally_installed end event :make_externally_uninstalled do @@ -79,7 +81,7 @@ module Clusters transition [:scheduled] => :uninstalling end - before_transition any => [:scheduled, :installed, :uninstalled] do |application, _| + before_transition any => [:scheduled, :installed, :uninstalled, :externally_installed] do |application, _| application.status_reason = nil end @@ -114,7 +116,7 @@ module Clusters end def available? - pre_installed? || installed? || updated? + pre_installed? || installed? || externally_installed? || updated? end def update_in_progress? diff --git a/app/models/clusters/concerns/application_version.rb b/app/models/clusters/concerns/application_version.rb index 6c0b014662c..dab0bd23e2e 100644 --- a/app/models/clusters/concerns/application_version.rb +++ b/app/models/clusters/concerns/application_version.rb @@ -5,11 +5,17 @@ module Clusters module ApplicationVersion extend ActiveSupport::Concern + EXTERNAL_VERSION = 'EXTERNALLY_INSTALLED' + included do state_machine :status do before_transition any => [:installed, :updated] do |application| application.version = application.class.const_get(:VERSION, false) end + + before_transition any => [:externally_installed] do |application| + application.version = EXTERNAL_VERSION + end end end diff --git a/app/models/clusters/concerns/prometheus_client.rb b/app/models/clusters/concerns/prometheus_client.rb new file mode 100644 index 00000000000..10cb307addd --- /dev/null +++ b/app/models/clusters/concerns/prometheus_client.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Clusters + module Concerns + module PrometheusClient + extend ActiveSupport::Concern + + included do + include PrometheusAdapter + + def service_name + 'prometheus-prometheus-server' + end + + def service_port + 80 + end + + def prometheus_client + return unless kube_client + + proxy_url = kube_client.proxy_url('service', service_name, service_port, Gitlab::Kubernetes::Helm::NAMESPACE) + + # ensures headers containing auth data are appended to original k8s client options + options = kube_client.rest_client.options + .merge(prometheus_client_default_options) + .merge(headers: kube_client.headers) + Gitlab::PrometheusClient.new(proxy_url, options) + rescue Kubeclient::HttpError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::ENETUNREACH + # If users have mistakenly set parameters or removed the depended clusters, + # `proxy_url` could raise an exception because gitlab can not communicate with the cluster. + # Since `PrometheusAdapter#can_query?` is eargely loaded on environement pages in gitlab, + # we need to silence the exceptions + end + + def configured? + kube_client.present? && available? + rescue Gitlab::UrlBlocker::BlockedUrlError + false + end + + private + + def kube_client + cluster&.kubeclient&.core_client + end + end + end + end +end diff --git a/app/models/clusters/integrations/prometheus.rb b/app/models/clusters/integrations/prometheus.rb new file mode 100644 index 00000000000..1496d8ff1dd --- /dev/null +++ b/app/models/clusters/integrations/prometheus.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Clusters + module Integrations + class Prometheus < ApplicationRecord + include ::Clusters::Concerns::PrometheusClient + + self.table_name = 'clusters_integration_prometheus' + self.primary_key = :cluster_id + + belongs_to :cluster, class_name: 'Clusters::Cluster', foreign_key: :cluster_id + + validates :cluster, presence: true + validates :enabled, inclusion: { in: [true, false] } + + def available? + enabled? + end + end + end +end diff --git a/app/models/commit.rb b/app/models/commit.rb index bf168aaacc5..5c3e3685c64 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -62,7 +62,8 @@ class Commit collection.sort do |a, b| operands = [a, b].tap { |o| o.reverse! if sort == 'desc' } - attr1, attr2 = operands.first.public_send(order_by), operands.second.public_send(order_by) # rubocop:disable PublicSend + attr1 = operands.first.public_send(order_by) # rubocop:disable GitlabSecurity/PublicSend + attr2 = operands.second.public_send(order_by) # rubocop:disable GitlabSecurity/PublicSend # use case insensitive comparison for string values order_by.in?(%w[email name]) ? attr1.casecmp(attr2) : attr1 <=> attr2 @@ -222,6 +223,14 @@ class Commit end end + def author_full_text + return unless author_name && author_email + + strong_memoize(:author_full_text) do + "#{author_name} <#{author_email}>" + end + end + # Returns full commit message if title is truncated (greater than 99 characters) # otherwise returns commit message without first line def description diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 524429bf12a..e989129209a 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -55,6 +55,8 @@ class CommitStatus < ApplicationRecord scope :for_ref, -> (ref) { where(ref: ref) } scope :by_name, -> (name) { where(name: name) } scope :in_pipelines, ->(pipelines) { where(pipeline: pipelines) } + scope :eager_load_pipeline, -> { eager_load(:pipeline, project: { namespace: :route }) } + scope :with_pipeline, -> { joins(:pipeline) } scope :for_project_paths, -> (paths) do where(project: Project.where_full_path_in(Array(paths))) @@ -179,14 +181,9 @@ class CommitStatus < ApplicationRecord end after_transition any => :failed do |commit_status| - next unless commit_status.project - - # rubocop: disable CodeReuse/ServiceClass commit_status.run_after_commit do - MergeRequests::AddTodoWhenBuildFailsService - .new(project, nil).execute(self) + ::Gitlab::Ci::Pipeline::Metrics.job_failure_reason_counter.increment(reason: commit_status.failure_reason) end - # rubocop: enable CodeReuse/ServiceClass end end @@ -210,26 +207,7 @@ class CommitStatus < ApplicationRecord end def group_name - simplified_commit_status_group_name_feature_flag = Gitlab::SafeRequestStore.fetch("project:#{project_id}:simplified_commit_status_group_name") do - Feature.enabled?(:simplified_commit_status_group_name, project, default_enabled: false) - end - - if simplified_commit_status_group_name_feature_flag - # Only remove one or more [...] "X/Y" "X Y" from the end of build names. - # More about the regular expression logic: https://docs.gitlab.com/ee/ci/jobs/#group-jobs-in-a-pipeline - - name.to_s.sub(%r{([\b\s:]+((\[.*\])|(\d+[\s:\/\\]+\d+)))+\s*\z}, '').strip - else - # Prior implementation, remove [...] "X/Y" "X Y" from the beginning and middle of build names - # 'rspec:linux: 1/10' => 'rspec:linux' - common_name = name.to_s.gsub(%r{\b\d+[\s:\/\\]+\d+\s*}, '') - - # 'rspec:linux: [aws, max memory]' => 'rspec:linux', 'rspec:linux: [aws]' => 'rspec:linux' - common_name.gsub!(%r{: \[.*\]\s*\z}, '') - - common_name.strip! - common_name - end + name.to_s.sub(%r{([\b\s:]+((\[.*\])|(\d+[\s:\/\\]+\d+)))+\s*\z}, '').strip end def failed_but_allowed? @@ -293,7 +271,8 @@ class CommitStatus < ApplicationRecord end def update_older_statuses_retried! - self.class + pipeline + .statuses .latest .where(name: name) .where.not(id: id) diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb index c106c08c04a..fdc418029be 100644 --- a/app/models/concerns/avatarable.rb +++ b/app/models/concerns/avatarable.rb @@ -131,7 +131,6 @@ module Avatarable def clear_avatar_caches return unless respond_to?(:verified_emails) && verified_emails.any? && avatar_changed? - return unless Feature.enabled?(:avatar_cache_for_email, self, type: :development) Gitlab::AvatarCache.delete_by_email(*verified_emails) end diff --git a/app/models/concerns/boards/listable.rb b/app/models/concerns/boards/listable.rb index d6863e87261..b9827a79422 100644 --- a/app/models/concerns/boards/listable.rb +++ b/app/models/concerns/boards/listable.rb @@ -13,6 +13,7 @@ module Boards scope :ordered, -> { order(:list_type, :position) } scope :destroyable, -> { where(list_type: list_types.slice(*destroyable_types).values) } scope :movable, -> { where(list_type: list_types.slice(*movable_types).values) } + scope :without_types, ->(list_types) { where.not(list_type: list_types) } class << self def preload_preferences_for_user(lists, user) diff --git a/app/models/concerns/bulk_member_access_load.rb b/app/models/concerns/bulk_member_access_load.rb index f44ad474cd5..e252ca36629 100644 --- a/app/models/concerns/bulk_member_access_load.rb +++ b/app/models/concerns/bulk_member_access_load.rb @@ -13,13 +13,7 @@ module BulkMemberAccessLoad raise 'Block is mandatory' unless block_given? resource_ids = resource_ids.uniq - key = max_member_access_for_resource_key(resource_klass, memoization_index) - access = {} - - if Gitlab::SafeRequestStore.active? - Gitlab::SafeRequestStore[key] ||= {} - access = Gitlab::SafeRequestStore[key] - end + access = load_access_hash(resource_klass, memoization_index) # Look up only the IDs we need resource_ids -= access.keys @@ -39,10 +33,28 @@ module BulkMemberAccessLoad access end + def merge_value_to_request_store(resource_klass, resource_id, memoization_index, value) + max_member_access_for_resource_ids(resource_klass, [resource_id], memoization_index) do + { resource_id => value } + end + end + private def max_member_access_for_resource_key(klass, memoization_index) "max_member_access_for_#{klass.name.underscore.pluralize}:#{memoization_index}" end + + def load_access_hash(resource_klass, memoization_index) + key = max_member_access_for_resource_key(resource_klass, memoization_index) + + access = {} + if Gitlab::SafeRequestStore.active? + Gitlab::SafeRequestStore[key] ||= {} + access = Gitlab::SafeRequestStore[key] + end + + access + end end end diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index 45944401c2d..34c1b6d25a4 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -56,12 +56,12 @@ module CacheMarkdownField # Update every applicable column in a row if any one is invalidated, as we only store # one version per row def refresh_markdown_cache - updates = cached_markdown_fields.markdown_fields.map do |markdown_field| + updates = cached_markdown_fields.markdown_fields.to_h do |markdown_field| [ cached_markdown_fields.html_field(markdown_field), rendered_field_content(markdown_field) ] - end.to_h + end updates['cached_markdown_version'] = latest_cached_markdown_version diff --git a/app/models/concerns/cascading_namespace_setting_attribute.rb b/app/models/concerns/cascading_namespace_setting_attribute.rb new file mode 100644 index 00000000000..2b4a108a9a0 --- /dev/null +++ b/app/models/concerns/cascading_namespace_setting_attribute.rb @@ -0,0 +1,241 @@ +# frozen_string_literal: true + +# +# Cascading attributes enables managing settings in a flexible way. +# +# - Instance administrator can define an instance-wide default setting, or +# lock the setting to prevent change by group owners. +# - Group maintainers/owners can define a default setting for their group, or +# lock the setting to prevent change by sub-group maintainers/owners. +# +# Behavior: +# +# - When a group does not have a value (value is `nil`), cascade up the +# hierarchy to find the first non-nil value. +# - Settings can be locked at any level to prevent groups/sub-groups from +# overriding. +# - If the setting isn't locked, the default can be overridden. +# - An instance administrator or group maintainer/owner can push settings values +# to groups/sub-groups to override existing values, even when the setting +# is not otherwise locked. +# +module CascadingNamespaceSettingAttribute + extend ActiveSupport::Concern + include Gitlab::Utils::StrongMemoize + + class_methods do + def cascading_settings_feature_enabled? + ::Feature.enabled?(:cascading_namespace_settings, default_enabled: true) + end + + private + + # Facilitates the cascading lookup of values and, + # similar to Rails' `attr_accessor`, defines convenience methods such as + # a reader, writer, and validators. + # + # Example: `cascading_attr :delayed_project_removal` + # + # Public methods defined: + # - `delayed_project_removal` + # - `delayed_project_removal=` + # - `delayed_project_removal_locked?` + # - `delayed_project_removal_locked_by_ancestor?` + # - `delayed_project_removal_locked_by_application_setting?` + # - `delayed_project_removal?` (only defined for boolean attributes) + # - `delayed_project_removal_locked_ancestor` - Returns locked namespace settings object (only namespace_id) + # + # Defined validators ensure attribute value cannot be updated if locked by + # an ancestor or application settings. + # + # Requires database columns be present in both `namespace_settings` and + # `application_settings`. + def cascading_attr(*attributes) + attributes.map(&:to_sym).each do |attribute| + # public methods + define_attr_reader(attribute) + define_attr_writer(attribute) + define_lock_methods(attribute) + alias_boolean(attribute) + + # private methods + define_validator_methods(attribute) + define_after_update(attribute) + + validate :"#{attribute}_changeable?" + validate :"lock_#{attribute}_changeable?" + + after_update :"clear_descendant_#{attribute}_locks", if: -> { saved_change_to_attribute?("lock_#{attribute}", to: true) } + end + end + + # The cascading attribute reader method handles lookups + # with the following criteria: + # + # 1. Returns the dirty value, if the attribute has changed. + # 2. Return locked ancestor value. + # 3. Return locked instance-level application settings value. + # 4. Return this namespace's attribute, if not nil. + # 5. Return value from nearest ancestor where value is not nil. + # 6. Return instance-level application setting. + def define_attr_reader(attribute) + define_method(attribute) do + strong_memoize(attribute) do + next self[attribute] unless self.class.cascading_settings_feature_enabled? + + next self[attribute] if will_save_change_to_attribute?(attribute) + next locked_value(attribute) if cascading_attribute_locked?(attribute) + next self[attribute] unless self[attribute].nil? + + cascaded_value = cascaded_ancestor_value(attribute) + next cascaded_value unless cascaded_value.nil? + + application_setting_value(attribute) + end + end + end + + def define_attr_writer(attribute) + define_method("#{attribute}=") do |value| + clear_memoization(attribute) + + super(value) + end + end + + def define_lock_methods(attribute) + define_method("#{attribute}_locked?") do + cascading_attribute_locked?(attribute) + end + + define_method("#{attribute}_locked_by_ancestor?") do + locked_by_ancestor?(attribute) + end + + define_method("#{attribute}_locked_by_application_setting?") do + locked_by_application_setting?(attribute) + end + + define_method("#{attribute}_locked_ancestor") do + locked_ancestor(attribute) + end + end + + def alias_boolean(attribute) + return unless Gitlab::Database.exists? && type_for_attribute(attribute).type == :boolean + + alias_method :"#{attribute}?", attribute + end + + # Defines two validations - one for the cascadable attribute itself and one + # for the lock attribute. Only allows the respective value to change if + # an ancestor has not already locked the value. + def define_validator_methods(attribute) + define_method("#{attribute}_changeable?") do + return unless cascading_attribute_changed?(attribute) + return unless cascading_attribute_locked?(attribute) + + errors.add(attribute, s_('CascadingSettings|cannot be changed because it is locked by an ancestor')) + end + + define_method("lock_#{attribute}_changeable?") do + return unless cascading_attribute_changed?("lock_#{attribute}") + + if cascading_attribute_locked?(attribute) + return errors.add(:"lock_#{attribute}", s_('CascadingSettings|cannot be changed because it is locked by an ancestor')) + end + + # Don't allow locking a `nil` attribute. + # Even if the value being locked is currently cascaded from an ancestor, + # it should be copied to this record to avoid the ancestor changing the + # value unexpectedly later. + return unless self[attribute].nil? && public_send("lock_#{attribute}?") # rubocop:disable GitlabSecurity/PublicSend + + errors.add(attribute, s_('CascadingSettings|cannot be nil when locking the attribute')) + end + + private :"#{attribute}_changeable?", :"lock_#{attribute}_changeable?" + end + + # When a particular group locks the attribute, clear all sub-group locks + # since the higher lock takes priority. + def define_after_update(attribute) + define_method("clear_descendant_#{attribute}_locks") do + self.class.where(namespace_id: descendants).update_all("lock_#{attribute}" => false) + end + + private :"clear_descendant_#{attribute}_locks" + end + end + + private + + def locked_value(attribute) + ancestor = locked_ancestor(attribute) + return ancestor.read_attribute(attribute) if ancestor + + Gitlab::CurrentSettings.public_send(attribute) # rubocop:disable GitlabSecurity/PublicSend + end + + def locked_ancestor(attribute) + return unless self.class.cascading_settings_feature_enabled? + return unless namespace.has_parent? + + strong_memoize(:"#{attribute}_locked_ancestor") do + self.class + .select(:namespace_id, "lock_#{attribute}", attribute) + .where(namespace_id: namespace_ancestor_ids) + .where(self.class.arel_table["lock_#{attribute}"].eq(true)) + .limit(1).load.first + end + end + + def locked_by_ancestor?(attribute) + return false unless self.class.cascading_settings_feature_enabled? + + locked_ancestor(attribute).present? + end + + def locked_by_application_setting?(attribute) + return false unless self.class.cascading_settings_feature_enabled? + + Gitlab::CurrentSettings.public_send("lock_#{attribute}") # rubocop:disable GitlabSecurity/PublicSend + end + + def cascading_attribute_locked?(attribute) + locked_by_ancestor?(attribute) || locked_by_application_setting?(attribute) + end + + def cascading_attribute_changed?(attribute) + public_send("#{attribute}_changed?") # rubocop:disable GitlabSecurity/PublicSend + end + + def cascaded_ancestor_value(attribute) + return unless namespace.has_parent? + + # rubocop:disable GitlabSecurity/SqlInjection + self.class + .select(attribute) + .joins("join unnest(ARRAY[#{namespace_ancestor_ids.join(',')}]) with ordinality t(namespace_id, ord) USING (namespace_id)") + .where("#{attribute} IS NOT NULL") + .order('t.ord') + .limit(1).first&.read_attribute(attribute) + # rubocop:enable GitlabSecurity/SqlInjection + end + + def application_setting_value(attribute) + Gitlab::CurrentSettings.public_send(attribute) # rubocop:disable GitlabSecurity/PublicSend + end + + def namespace_ancestor_ids + strong_memoize(:namespace_ancestor_ids) do + namespace.self_and_ancestors(hierarchy_order: :asc).pluck(:id).reject { |id| id == namespace_id } + end + end + + def descendants + strong_memoize(:descendants) do + namespace.descendants.pluck(:id) + end + end +end diff --git a/app/models/concerns/ci/artifactable.rb b/app/models/concerns/ci/artifactable.rb index cbe7d3b6abb..0d29955268f 100644 --- a/app/models/concerns/ci/artifactable.rb +++ b/app/models/concerns/ci/artifactable.rb @@ -4,8 +4,10 @@ module Ci module Artifactable extend ActiveSupport::Concern - NotSupportedAdapterError = Class.new(StandardError) + include ObjectStorable + STORE_COLUMN = :file_store + NotSupportedAdapterError = Class.new(StandardError) FILE_FORMAT_ADAPTERS = { gzip: Gitlab::Ci::Build::Artifacts::Adapters::GzipStream, raw: Gitlab::Ci::Build::Artifacts::Adapters::RawStream @@ -20,6 +22,7 @@ module Ci scope :expired_before, -> (timestamp) { where(arel_table[:expire_at].lt(timestamp)) } scope :expired, -> (limit) { expired_before(Time.current).limit(limit) } + scope :project_id_in, ->(ids) { where(project_id: ids) } end def each_blob(&blk) @@ -39,3 +42,5 @@ module Ci end end end + +Ci::Artifactable.prepend_ee_mod diff --git a/app/models/concerns/ci/has_status.rb b/app/models/concerns/ci/has_status.rb index 0412f7a072b..c990da5873a 100644 --- a/app/models/concerns/ci/has_status.rb +++ b/app/models/concerns/ci/has_status.rb @@ -16,6 +16,19 @@ module Ci STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3, failed: 4, canceled: 5, skipped: 6, manual: 7, scheduled: 8, preparing: 9, waiting_for_resource: 10 }.freeze + STATUSES_DESCRIPTION = { + created: 'Pipeline has been created', + waiting_for_resource: 'A resource (for example, a runner) that the pipeline requires to run is unavailable', + preparing: 'Pipeline is preparing to run', + pending: 'Pipeline has not started running yet', + running: 'Pipeline is running', + failed: 'At least one stage of the pipeline failed', + success: 'Pipeline completed successfully', + canceled: 'Pipeline was canceled before completion', + skipped: 'Pipeline was skipped', + manual: 'Pipeline needs to be manually started', + scheduled: 'Pipeline is scheduled to run' + }.freeze UnknownStatusError = Class.new(StandardError) diff --git a/app/models/concerns/counter_attribute.rb b/app/models/concerns/counter_attribute.rb index b468415c4c7..829b2a6ef21 100644 --- a/app/models/concerns/counter_attribute.rb +++ b/app/models/concerns/counter_attribute.rb @@ -33,7 +33,7 @@ module CounterAttribute extend AfterCommitQueue include Gitlab::ExclusiveLeaseHelpers - LUA_STEAL_INCREMENT_SCRIPT = <<~EOS.freeze + LUA_STEAL_INCREMENT_SCRIPT = <<~EOS local increment_key, flushed_key = KEYS[1], KEYS[2] local increment_value = redis.call("get", increment_key) or 0 local flushed_value = redis.call("incrby", flushed_key, increment_value) diff --git a/app/models/concerns/deprecated_assignee.rb b/app/models/concerns/deprecated_assignee.rb index 7f12ce39c96..3f557ee9b48 100644 --- a/app/models/concerns/deprecated_assignee.rb +++ b/app/models/concerns/deprecated_assignee.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -# This module handles backward compatibility for import/export of Merge Requests after +# This module handles backward compatibility for import/export of merge requests after # multiple assignees feature was introduced. Also, it handles the scenarios where # the #26496 background migration hasn't finished yet. # Ideally, most of this code should be removed at #59457. diff --git a/app/models/concerns/enums/ci/commit_status.rb b/app/models/concerns/enums/ci/commit_status.rb index 48b4a402974..de17f50cd29 100644 --- a/app/models/concerns/enums/ci/commit_status.rb +++ b/app/models/concerns/enums/ci/commit_status.rb @@ -20,6 +20,8 @@ module Enums scheduler_failure: 11, data_integrity_failure: 12, forward_deployment_failure: 13, + user_blocked: 14, + project_deleted: 15, insufficient_bridge_permissions: 1_001, downstream_bridge_project_not_found: 1_002, invalid_bridge_trigger: 1_003, diff --git a/app/models/concerns/enums/ci/pipeline.rb b/app/models/concerns/enums/ci/pipeline.rb index f8314d8b429..fdc48d09db2 100644 --- a/app/models/concerns/enums/ci/pipeline.rb +++ b/app/models/concerns/enums/ci/pipeline.rb @@ -13,7 +13,9 @@ module Enums activity_limit_exceeded: 20, size_limit_exceeded: 21, job_activity_limit_exceeded: 22, - deployments_limit_exceeded: 23 + deployments_limit_exceeded: 23, + user_blocked: 24, + project_deleted: 25 } end diff --git a/app/models/concerns/has_repository.rb b/app/models/concerns/has_repository.rb index b9ad78c14fd..774cda2c3e8 100644 --- a/app/models/concerns/has_repository.rb +++ b/app/models/concerns/has_repository.rb @@ -77,9 +77,14 @@ module HasRepository def default_branch_from_preferences return unless empty_repo? - group_branch_default_name = group&.default_branch_name if respond_to?(:group) + (default_branch_from_group_preferences || Gitlab::CurrentSettings.default_branch_name).presence + end + + def default_branch_from_group_preferences + return unless respond_to?(:group) + return unless group - (group_branch_default_name || Gitlab::CurrentSettings.default_branch_name).presence + group.default_branch_name || group.root_ancestor.default_branch_name end def reload_default_branch diff --git a/app/models/concerns/has_timelogs_report.rb b/app/models/concerns/has_timelogs_report.rb new file mode 100644 index 00000000000..90f9876de95 --- /dev/null +++ b/app/models/concerns/has_timelogs_report.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module HasTimelogsReport + extend ActiveSupport::Concern + include Gitlab::Utils::StrongMemoize + + def timelogs(start_time, end_time) + strong_memoize(:timelogs) { timelogs_for(start_time, end_time) } + end + + def user_can_access_group_timelogs?(current_user) + Ability.allowed?(current_user, :read_group_timelogs, self) + end + + private + + def timelogs_for(start_time, end_time) + Timelog.between_times(start_time, end_time).for_issues_in_group(self) + end +end diff --git a/app/models/concerns/integration.rb b/app/models/concerns/integration.rb index 9d446841a9f..5e53f13be95 100644 --- a/app/models/concerns/integration.rb +++ b/app/models/concerns/integration.rb @@ -6,12 +6,12 @@ module Integration class_methods do def with_custom_integration_for(integration, page = nil, per = nil) custom_integration_project_ids = Service + .select(:project_id) .where(type: integration.type) .where(inherit_from_id: nil) - .distinct # Required until https://gitlab.com/gitlab-org/gitlab/-/issues/207385 + .where.not(project_id: nil) .page(page) .per(per) - .pluck(:project_id) Project.where(id: custom_integration_project_ids) end diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index e1be0665452..1e44321e148 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -65,7 +65,7 @@ module Issuable has_many :label_links, as: :target, dependent: :destroy, inverse_of: :target # rubocop:disable Cop/ActiveRecordDependent has_many :labels, through: :label_links - has_many :todos, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :todos, as: :target has_one :metrics, inverse_of: model_name.singular.to_sym, autosave: true @@ -137,6 +137,14 @@ module Issuable scope :references_project, -> { references(:project) } scope :non_archived, -> { join_project.where(projects: { archived: false }) } + scope :includes_for_bulk_update, -> do + associations = %i[author assignees epic group labels metrics project source_project target_project].select do |association| + reflect_on_association(association) + end + + includes(*associations) + end + attr_mentionable :title, pipeline: :single_line attr_mentionable :description @@ -324,7 +332,7 @@ module Issuable # This prevents errors when ignored columns are present in the database. issuable_columns = with_cte ? issue_grouping_columns(use_cte: with_cte) : "#{table_name}.*" - extra_select_columns = extra_select_columns.unshift("(#{highest_priority}) AS highest_priority") + extra_select_columns.unshift("(#{highest_priority}) AS highest_priority") select(issuable_columns) .select(extra_select_columns) @@ -437,7 +445,7 @@ module Issuable end def subscribed_without_subscriptions?(user, project) - participants(user).include?(user) + participant?(user) end def can_assign_epic?(user) diff --git a/app/models/concerns/loaded_in_group_list.rb b/app/models/concerns/loaded_in_group_list.rb index e624b9aa356..59e0ed75d2d 100644 --- a/app/models/concerns/loaded_in_group_list.rb +++ b/app/models/concerns/loaded_in_group_list.rb @@ -73,6 +73,10 @@ module LoadedInGroupList def member_count @member_count ||= try(:preloaded_member_count) || members.count end + + def guest_count + @guest_count ||= members.guests.count + end end LoadedInGroupList::ClassMethods.prepend_if_ee('EE::LoadedInGroupList::ClassMethods') diff --git a/app/models/concerns/milestoneable.rb b/app/models/concerns/milestoneable.rb index ccb334343ff..d42417bb6c1 100644 --- a/app/models/concerns/milestoneable.rb +++ b/app/models/concerns/milestoneable.rb @@ -39,11 +39,13 @@ module Milestoneable private def milestone_is_valid - errors.add(:milestone_id, 'is invalid') if respond_to?(:milestone_id) && milestone_id.present? && !milestone_available? + errors.add(:milestone_id, 'is invalid') if respond_to?(:milestone_id) && !milestone_available? end end def milestone_available? + return true if milestone_id.blank? + project_id == milestone&.project_id || project.ancestors_upto.compact.include?(milestone&.group) end diff --git a/app/models/concerns/milestoneish.rb b/app/models/concerns/milestoneish.rb index 5f24564dc56..eaf64f2541d 100644 --- a/app/models/concerns/milestoneish.rb +++ b/app/models/concerns/milestoneish.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Milestoneish - DISPLAY_ISSUES_LIMIT = 3000 + DISPLAY_ISSUES_LIMIT = 500 def total_issues_count @total_issues_count ||= Milestones::IssuesCountService.new(self).count @@ -15,6 +15,10 @@ module Milestoneish total_issues_count - closed_issues_count end + def total_merge_requests_count + @total_merge_request_count ||= Milestones::MergeRequestsCountService.new(self).count + end + def complete? total_issues_count > 0 && total_issues_count == closed_issues_count end diff --git a/app/models/concerns/object_storable.rb b/app/models/concerns/object_storable.rb new file mode 100644 index 00000000000..c13dddc0b88 --- /dev/null +++ b/app/models/concerns/object_storable.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module ObjectStorable + extend ActiveSupport::Concern + + included do + scope :with_files_stored_locally, -> { where(klass::STORE_COLUMN => ObjectStorage::Store::LOCAL) } + scope :with_files_stored_remotely, -> { where(klass::STORE_COLUMN => ObjectStorage::Store::REMOTE) } + end +end diff --git a/app/models/concerns/participable.rb b/app/models/concerns/participable.rb index af105629398..acd654bd229 100644 --- a/app/models/concerns/participable.rb +++ b/app/models/concerns/participable.rb @@ -56,18 +56,34 @@ module Participable # This method processes attributes of objects in breadth-first order. # # Returns an Array of User instances. - def participants(current_user = nil) - all_participants[current_user] + def participants(user = nil) + filtered_participants_hash[user] + end + + # Checks if the user is a participant in a discussion. + # + # This method processes attributes of objects in breadth-first order. + # + # Returns a Boolean. + def participant?(user) + can_read_participable?(user) && + all_participants_hash[user].include?(user) end private - def all_participants - @all_participants ||= Hash.new do |hash, user| + def all_participants_hash + @all_participants_hash ||= Hash.new do |hash, user| hash[user] = raw_participants(user) end end + def filtered_participants_hash + @filtered_participants_hash ||= Hash.new do |hash, user| + hash[user] = filter_by_ability(all_participants_hash[user]) + end + end + def raw_participants(current_user = nil) current_user ||= author ext = Gitlab::ReferenceExtractor.new(project, current_user) @@ -98,8 +114,6 @@ module Participable end participants.merge(ext.users) - - filter_by_ability(participants) end def filter_by_ability(participants) @@ -110,6 +124,15 @@ module Participable Ability.users_that_can_read_project(participants.to_a, project) end end + + def can_read_participable?(participant) + case self + when PersonalSnippet + participant.can?(:read_snippet, self) + else + participant.can?(:read_project, project) + end + end end Participable.prepend_if_ee('EE::Participable') diff --git a/app/models/concerns/protected_ref.rb b/app/models/concerns/protected_ref.rb index 65195a8d5aa..2828ae4a3a9 100644 --- a/app/models/concerns/protected_ref.rb +++ b/app/models/concerns/protected_ref.rb @@ -4,7 +4,7 @@ module ProtectedRef extend ActiveSupport::Concern included do - belongs_to :project + belongs_to :project, touch: true validates :name, presence: true validates :project, presence: true diff --git a/app/models/concerns/safe_url.rb b/app/models/concerns/safe_url.rb index febca7d241f..7dce05bddba 100644 --- a/app/models/concerns/safe_url.rb +++ b/app/models/concerns/safe_url.rb @@ -3,12 +3,12 @@ module SafeUrl extend ActiveSupport::Concern - def safe_url(usernames_whitelist: []) + def safe_url(allowed_usernames: []) return if url.nil? uri = URI.parse(url) uri.password = '*****' if uri.password - uri.user = '*****' if uri.user && !usernames_whitelist.include?(uri.user) + uri.user = '*****' if uri.user && allowed_usernames.exclude?(uri.user) uri.to_s rescue URI::Error end diff --git a/app/models/concerns/sidebars/container_with_html_options.rb b/app/models/concerns/sidebars/container_with_html_options.rb new file mode 100644 index 00000000000..12ea366c66a --- /dev/null +++ b/app/models/concerns/sidebars/container_with_html_options.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Sidebars + module ContainerWithHtmlOptions + # The attributes returned from this method + # will be applied to helper methods like + # `link_to` or the div containing the container. + def container_html_options + { + aria: { label: title } + }.merge(extra_container_html_options) + end + + # Classes will override mostly this method + # and not `container_html_options`. + def extra_container_html_options + {} + end + + # Attributes to pass to the html_options attribute + # in the helper method that sets the active class + # on each element. + def nav_link_html_options + {} + end + + def title + raise NotImplementedError + end + + # The attributes returned from this method + # will be applied right next to the title, + # for example in the span that renders the title. + def title_html_options + {} + end + + def link + raise NotImplementedError + end + end +end diff --git a/app/models/concerns/sidebars/has_active_routes.rb b/app/models/concerns/sidebars/has_active_routes.rb new file mode 100644 index 00000000000..e7a153f067a --- /dev/null +++ b/app/models/concerns/sidebars/has_active_routes.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Sidebars + module HasActiveRoutes + # This method will indicate for which paths or + # controllers, the menu or menu item should + # be set as active. + # + # The returned values are passed to the `nav_link` helper method, + # so the params can be either `path`, `page`, `controller`. + # Param 'action' is not supported. + def active_routes + {} + end + end +end diff --git a/app/models/concerns/sidebars/has_hint.rb b/app/models/concerns/sidebars/has_hint.rb new file mode 100644 index 00000000000..21dca39dca0 --- /dev/null +++ b/app/models/concerns/sidebars/has_hint.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# This module has the necessary methods to store +# hints for menus. Hints are elements displayed +# when the user hover the menu item. +module Sidebars + module HasHint + def show_hint? + false + end + + def hint_html_options + {} + end + end +end diff --git a/app/models/concerns/sidebars/has_icon.rb b/app/models/concerns/sidebars/has_icon.rb new file mode 100644 index 00000000000..d1a87918285 --- /dev/null +++ b/app/models/concerns/sidebars/has_icon.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +# This module has the necessary methods to show +# sprites or images next to the menu item. +module Sidebars + module HasIcon + def sprite_icon + nil + end + + def sprite_icon_html_options + {} + end + + def image_path + nil + end + + def image_html_options + {} + end + + def icon_or_image? + sprite_icon || image_path + end + end +end diff --git a/app/models/concerns/sidebars/has_pill.rb b/app/models/concerns/sidebars/has_pill.rb new file mode 100644 index 00000000000..ad7064fe63d --- /dev/null +++ b/app/models/concerns/sidebars/has_pill.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# This module introduces the logic to show the "pill" element +# next to the menu item, indicating the a count. +module Sidebars + module HasPill + def has_pill? + false + end + + # In this method we will need to provide the query + # to retrieve the elements count + def pill_count + raise NotImplementedError + end + + def pill_html_options + {} + end + end +end diff --git a/app/models/concerns/sidebars/positionable_list.rb b/app/models/concerns/sidebars/positionable_list.rb new file mode 100644 index 00000000000..30830d547f3 --- /dev/null +++ b/app/models/concerns/sidebars/positionable_list.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# This module handles elements in a list. All elements +# must have a different class +module Sidebars + module PositionableList + def add_element(list, element) + list << element + end + + def insert_element_before(list, before_element, new_element) + index = index_of(list, before_element) + + if index + list.insert(index, new_element) + else + list.unshift(new_element) + end + end + + def insert_element_after(list, after_element, new_element) + index = index_of(list, after_element) + + if index + list.insert(index + 1, new_element) + else + add_element(list, new_element) + end + end + + private + + def index_of(list, element) + list.index { |e| e.is_a?(element) } + end + end +end diff --git a/app/models/concerns/sidebars/renderable.rb b/app/models/concerns/sidebars/renderable.rb new file mode 100644 index 00000000000..a3976af8515 --- /dev/null +++ b/app/models/concerns/sidebars/renderable.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Sidebars + module Renderable + # This method will control whether the menu or menu_item + # should be rendered. It will be overriden by specific + # classes. + def render? + true + end + end +end diff --git a/app/models/concerns/sortable.rb b/app/models/concerns/sortable.rb index 4fe2a0e1827..9f5e9b2bb57 100644 --- a/app/models/concerns/sortable.rb +++ b/app/models/concerns/sortable.rb @@ -9,6 +9,7 @@ module Sortable included do scope :with_order_id_desc, -> { order(self.arel_table['id'].desc) } + scope :with_order_id_asc, -> { order(self.arel_table['id'].asc) } scope :order_id_desc, -> { reorder(self.arel_table['id'].desc) } scope :order_id_asc, -> { reorder(self.arel_table['id'].asc) } scope :order_created_desc, -> { reorder(self.arel_table['created_at'].desc) } diff --git a/app/models/concerns/subscribable.rb b/app/models/concerns/subscribable.rb index 33e9e0e38fb..5a10ea7a248 100644 --- a/app/models/concerns/subscribable.rb +++ b/app/models/concerns/subscribable.rb @@ -17,13 +17,37 @@ module Subscribable def subscribed?(user, project = nil) return false unless user - if subscription = subscriptions.find_by(user: user, project: project) + if (subscription = lazy_subscription(user, project)&.itself) subscription.subscribed else subscribed_without_subscriptions?(user, project) end end + def lazy_subscription(user, project = nil) + return unless user + + # handle project and group labels as well as issuable subscriptions + subscribable_type = self.class.ancestors.include?(Label) ? 'Label' : self.class.name + BatchLoader.for(id: id, subscribable_type: subscribable_type, project_id: project&.id).batch do |items, loader| + values = items.each_with_object({ ids: Set.new, subscribable_types: Set.new, project_ids: Set.new }) do |item, result| + result[:ids] << item[:id] + result[:subscribable_types] << item[:subscribable_type] + result[:project_ids] << item[:project_id] + end + + subscriptions = Subscription.where(subscribable_id: values[:ids], subscribable_type: values[:subscribable_types], project_id: values[:project_ids], user: user) + + subscriptions.each do |subscription| + loader.call({ + id: subscription.subscribable_id, + subscribable_type: subscription.subscribable_type, + project_id: subscription.project_id + }, subscription) + end + end + end + # Override this method to define custom logic to consider a subscribable as # subscribed without an explicit subscription record. def subscribed_without_subscriptions?(user, project) @@ -41,8 +65,10 @@ module Subscribable def toggle_subscription(user, project = nil) unsubscribe_from_other_levels(user, project) + new_value = !subscribed?(user, project) + find_or_initialize_subscription(user, project) - .update(subscribed: !subscribed?(user, project)) + .update(subscribed: new_value) end def subscribe(user, project = nil) @@ -83,6 +109,8 @@ module Subscribable end def find_or_initialize_subscription(user, project) + BatchLoader::Executor.clear_current + subscriptions .find_or_initialize_by(user_id: user.id, project_id: project.try(:id)) end diff --git a/app/models/concerns/taskable.rb b/app/models/concerns/taskable.rb index 5debfa6f834..d8867177059 100644 --- a/app/models/concerns/taskable.rb +++ b/app/models/concerns/taskable.rb @@ -30,7 +30,8 @@ module Taskable end def self.get_updated_tasks(old_content:, new_content:) - old_tasks, new_tasks = get_tasks(old_content), get_tasks(new_content) + old_tasks = get_tasks(old_content) + new_tasks = get_tasks(new_content) new_tasks.select.with_index do |new_task, i| old_task = old_tasks[i] diff --git a/app/models/concerns/token_authenticatable_strategies/encrypted.rb b/app/models/concerns/token_authenticatable_strategies/encrypted.rb index 672402ee4d6..50a2613bb10 100644 --- a/app/models/concerns/token_authenticatable_strategies/encrypted.rb +++ b/app/models/concerns/token_authenticatable_strategies/encrypted.rb @@ -42,14 +42,14 @@ module TokenAuthenticatableStrategies return insecure_strategy.get_token(instance) if migrating? encrypted_token = instance.read_attribute(encrypted_field) - token = Gitlab::CryptoHelper.aes256_gcm_decrypt(encrypted_token) + token = EncryptionHelper.decrypt_token(encrypted_token) token || (insecure_strategy.get_token(instance) if optional?) end def set_token(instance, token) raise ArgumentError unless token.present? - instance[encrypted_field] = Gitlab::CryptoHelper.aes256_gcm_encrypt(token) + instance[encrypted_field] = EncryptionHelper.encrypt_token(token) instance[token_field] = token if migrating? instance[token_field] = nil if optional? token @@ -85,16 +85,9 @@ module TokenAuthenticatableStrategies end def find_by_encrypted_token(token, unscoped) - nonce = Feature.enabled?(:dynamic_nonce_creation) ? find_hashed_iv(token) : Gitlab::CryptoHelper::AES256_GCM_IV_STATIC - encrypted_value = Gitlab::CryptoHelper.aes256_gcm_encrypt(token, nonce: nonce) - - relation(unscoped).find_by(encrypted_field => encrypted_value) - end - - def find_hashed_iv(token) - token_record = TokenWithIv.find_by_plaintext_token(token) - - token_record&.iv || Gitlab::CryptoHelper::AES256_GCM_IV_STATIC + encrypted_value = EncryptionHelper.encrypt_token(token) + token_encrypted_with_static_iv = Gitlab::CryptoHelper.aes256_gcm_encrypt(token) + relation(unscoped).find_by(encrypted_field => [encrypted_value, token_encrypted_with_static_iv]) end def insecure_strategy diff --git a/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb b/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb new file mode 100644 index 00000000000..25c050820d6 --- /dev/null +++ b/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module TokenAuthenticatableStrategies + class EncryptionHelper + DYNAMIC_NONCE_IDENTIFIER = "|" + NONCE_SIZE = 12 + + def self.encrypt_token(plaintext_token) + Gitlab::CryptoHelper.aes256_gcm_encrypt(plaintext_token) + end + + def self.decrypt_token(token) + return unless token + + # The pattern of the token is "#{DYNAMIC_NONCE_IDENTIFIER}#{token}#{iv_of_12_characters}" + if token.start_with?(DYNAMIC_NONCE_IDENTIFIER) && token.size > NONCE_SIZE + DYNAMIC_NONCE_IDENTIFIER.size + token_to_decrypt = token[1...-NONCE_SIZE] + iv = token[-NONCE_SIZE..-1] + + Gitlab::CryptoHelper.aes256_gcm_decrypt(token_to_decrypt, nonce: iv) + else + Gitlab::CryptoHelper.aes256_gcm_decrypt(token) + end + end + end +end diff --git a/app/models/concerns/vulnerability_finding_helpers.rb b/app/models/concerns/vulnerability_finding_helpers.rb new file mode 100644 index 00000000000..cf50305faab --- /dev/null +++ b/app/models/concerns/vulnerability_finding_helpers.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module VulnerabilityFindingHelpers + extend ActiveSupport::Concern +end + +VulnerabilityFindingHelpers.prepend_if_ee('EE::VulnerabilityFindingHelpers') diff --git a/app/models/concerns/vulnerability_finding_signature_helpers.rb b/app/models/concerns/vulnerability_finding_signature_helpers.rb new file mode 100644 index 00000000000..f57e3cb0bfb --- /dev/null +++ b/app/models/concerns/vulnerability_finding_signature_helpers.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module VulnerabilityFindingSignatureHelpers + extend ActiveSupport::Concern +end + +VulnerabilityFindingSignatureHelpers.prepend_if_ee('EE::VulnerabilityFindingSignatureHelpers') diff --git a/app/models/deploy_key.rb b/app/models/deploy_key.rb index db5fd167781..25e3b9fe4f0 100644 --- a/app/models/deploy_key.rb +++ b/app/models/deploy_key.rb @@ -13,8 +13,6 @@ class DeployKey < Key scope :are_public, -> { where(public: true) } scope :with_projects, -> { includes(deploy_keys_projects: { project: [:route, namespace: :route] }) } - ignore_column :can_push, remove_after: '2019-12-15', remove_with: '12.6' - accepts_nested_attributes_for :deploy_keys_projects def private? diff --git a/app/models/deployment.rb b/app/models/deployment.rb index f000e474605..d3280403bfd 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -45,6 +45,7 @@ class Deployment < ApplicationRecord scope :active, -> { where(status: %i[created running]) } scope :older_than, -> (deployment) { where('deployments.id < ?', deployment.id) } scope :with_deployable, -> { joins('INNER JOIN ci_builds ON ci_builds.id = deployments.deployable_id').preload(:deployable) } + scope :with_api_entity_associations, -> { preload({ deployable: { runner: [], tags: [], user: [], job_artifacts_archive: [] } }) } scope :finished_after, ->(date) { where('finished_at >= ?', date) } scope :finished_before, ->(date) { where('finished_at < ?', date) } @@ -93,11 +94,6 @@ class Deployment < ApplicationRecord after_transition any => :success do |deployment| deployment.run_after_commit do Deployments::UpdateEnvironmentWorker.perform_async(id) - end - end - - after_transition any => FINISHED_STATUSES do |deployment| - deployment.run_after_commit do Deployments::LinkMergeRequestWorker.perform_async(id) end end @@ -175,7 +171,7 @@ class Deployment < ApplicationRecord end def commit - project.commit(sha) + @commit ||= project.commit(sha) end def commit_title @@ -225,7 +221,7 @@ class Deployment < ApplicationRecord end def update_merge_request_metrics! - return unless environment.update_merge_request_metrics? && success? + return unless environment.production? && success? merge_requests = project.merge_requests .joins(:metrics) @@ -243,29 +239,18 @@ class Deployment < ApplicationRecord def previous_deployment @previous_deployment ||= - project.deployments.joins(:environment) - .where(environments: { name: self.environment.name }, ref: self.ref) - .where.not(id: self.id) - .order(id: :desc) - .take - end - - def previous_environment_deployment - project - .deployments - .success - .joins(:environment) - .where(environments: { name: environment.name }) - .where.not(id: self.id) - .order(id: :desc) - .take + self.class.for_environment(environment_id) + .success + .where('id < ?', id) + .order(id: :desc) + .take end def stop_action return unless on_stop.present? return unless manual_actions - @stop_action ||= manual_actions.find_by(name: on_stop) + @stop_action ||= manual_actions.find { |action| action.name == self.on_stop } end def finished_at diff --git a/app/models/design_management/design_action.rb b/app/models/design_management/design_action.rb index 22baa916296..43dcce545d2 100644 --- a/app/models/design_management/design_action.rb +++ b/app/models/design_management/design_action.rb @@ -29,7 +29,9 @@ module DesignManagement # - design [DesignManagement::Design]: the design that was changed # - action [Symbol]: the action that gitaly performed def initialize(design, action, content = nil) - @design, @action, @content = design, action, content + @design = design + @action = action + @content = content validate! end diff --git a/app/models/design_management/design_at_version.rb b/app/models/design_management/design_at_version.rb index 211211144f4..2f045358914 100644 --- a/app/models/design_management/design_at_version.rb +++ b/app/models/design_management/design_at_version.rb @@ -18,7 +18,8 @@ module DesignManagement validate :design_and_version_have_issue_id def initialize(design: nil, version: nil) - @design, @version = design, version + @design = design + @version = version end # The ID, needed by GraphQL types and as part of the Lazy-fetch diff --git a/app/models/design_management/repository.rb b/app/models/design_management/repository.rb index 985d6317d5d..2b1e6070e6b 100644 --- a/app/models/design_management/repository.rb +++ b/app/models/design_management/repository.rb @@ -8,7 +8,7 @@ module DesignManagement # repository is entirely GitLab-managed rather than user-facing. # # Enable all uploaded files to be stored in LFS. - MANAGED_GIT_ATTRIBUTES = <<~GA.freeze + MANAGED_GIT_ATTRIBUTES = <<~GA /#{DesignManagement.designs_directory}/* filter=lfs diff=lfs merge=lfs -text GA diff --git a/app/models/design_management/version.rb b/app/models/design_management/version.rb index 49aec8b9720..5cfd8f3ec8e 100644 --- a/app/models/design_management/version.rb +++ b/app/models/design_management/version.rb @@ -14,7 +14,9 @@ module DesignManagement attr_reader :sha, :issue_id, :actions def initialize(sha, issue_id, actions) - @sha, @issue_id, @actions = sha, issue_id, actions + @sha = sha + @issue_id = issue_id + @actions = actions end def message diff --git a/app/models/environment.rb b/app/models/environment.rb index d89909a71a2..4ee93b0ba4a 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -11,8 +11,6 @@ class Environment < ApplicationRecord self.reactive_cache_hard_limit = 10.megabytes self.reactive_cache_work_type = :external_dependency - PRODUCTION_ENVIRONMENT_IDENTIFIERS = %w[prod production].freeze - belongs_to :project, required: true use_fast_destroy :all_deployments @@ -26,13 +24,13 @@ class Environment < ApplicationRecord has_many :self_managed_prometheus_alert_events, inverse_of: :environment has_many :alert_management_alerts, class_name: 'AlertManagement::Alert', inverse_of: :environment - has_one :last_deployment, -> { success.order('deployments.id DESC') }, class_name: 'Deployment' + has_one :last_deployment, -> { success.order('deployments.id DESC') }, class_name: 'Deployment', inverse_of: :environment has_one :last_deployable, through: :last_deployment, source: 'deployable', source_type: 'CommitStatus' has_one :last_pipeline, through: :last_deployable, source: 'pipeline' has_one :last_visible_deployment, -> { visible.distinct_on_environment }, inverse_of: :environment, class_name: 'Deployment' has_one :last_visible_deployable, through: :last_visible_deployment, source: 'deployable', source_type: 'CommitStatus' has_one :last_visible_pipeline, through: :last_visible_deployable, source: 'pipeline' - has_one :upcoming_deployment, -> { running.order('deployments.id DESC') }, class_name: 'Deployment' + has_one :upcoming_deployment, -> { running.order('deployments.id DESC') }, class_name: 'Deployment', inverse_of: :environment has_one :latest_opened_most_severe_alert, -> { order_severity_with_open_prometheus_alert }, class_name: 'AlertManagement::Alert', inverse_of: :environment before_validation :nullify_external_url @@ -88,7 +86,7 @@ class Environment < ApplicationRecord end scope :for_project, -> (project) { where(project_id: project) } - scope :for_tier, -> (tier) { where(tier: tier).where('tier IS NOT NULL') } + scope :for_tier, -> (tier) { where(tier: tier).where.not(tier: nil) } scope :with_deployment, -> (sha) { where('EXISTS (?)', Deployment.select(1).where('deployments.environment_id = environments.id').where(sha: sha)) } scope :unfoldered, -> { where(environment_type: nil) } scope :with_rank, -> do @@ -251,10 +249,6 @@ class Environment < ApplicationRecord last_deployment.try(:created_at) end - def update_merge_request_metrics? - PRODUCTION_ENVIRONMENT_IDENTIFIERS.include?(folder_name.downcase) - end - def ref_path "refs/#{Repository::REF_ENVIRONMENTS}/#{slug}" end diff --git a/app/models/experiment.rb b/app/models/experiment.rb index ac8b6516d02..7ffb321f2b7 100644 --- a/app/models/experiment.rb +++ b/app/models/experiment.rb @@ -21,7 +21,13 @@ class Experiment < ApplicationRecord # Create or update the recorded experiment_user row for the user in this experiment. def record_user_and_group(user, group_type, context = {}) experiment_user = experiment_users.find_or_initialize_by(user: user) - experiment_user.update!(group_type: group_type, context: merged_context(experiment_user, context)) + experiment_user.assign_attributes(group_type: group_type, context: merged_context(experiment_user, context)) + # We only call save when necessary because this causes the request to stick to the primary DB + # even when the save is a no-op + # https://gitlab.com/gitlab-org/gitlab/-/issues/324649 + experiment_user.save! if experiment_user.changed? + + experiment_user end def record_conversion_event_for_user(user, context = {}) @@ -32,7 +38,14 @@ class Experiment < ApplicationRecord end def record_group_and_variant!(group, variant) - experiment_subjects.find_or_initialize_by(group: group).update!(variant: variant) + experiment_subject = experiment_subjects.find_or_initialize_by(group: group) + experiment_subject.assign_attributes(variant: variant) + # We only call save when necessary because this causes the request to stick to the primary DB + # even when the save is a no-op + # https://gitlab.com/gitlab-org/gitlab/-/issues/324649 + experiment_subject.save! if experiment_subject.changed? + + experiment_subject end private diff --git a/app/models/external_issue.rb b/app/models/external_issue.rb index 68b2353556e..36030b80370 100644 --- a/app/models/external_issue.rb +++ b/app/models/external_issue.rb @@ -6,7 +6,8 @@ class ExternalIssue attr_reader :project def initialize(issue_identifier, project) - @issue_identifier, @project = issue_identifier, project + @issue_identifier = issue_identifier + @project = project end def to_s diff --git a/app/models/gpg_key.rb b/app/models/gpg_key.rb index ca6857a14b6..330815ab8c1 100644 --- a/app/models/gpg_key.rb +++ b/app/models/gpg_key.rb @@ -71,12 +71,12 @@ class GpgKey < ApplicationRecord end def emails_with_verified_status - user_infos.map do |user_info| + user_infos.to_h do |user_info| [ user_info[:email], user.verified_email?(user_info[:email]) ] - end.to_h + end end def verified? diff --git a/app/models/group.rb b/app/models/group.rb index 9f8a9996f31..2967c1ffc1d 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -16,6 +16,7 @@ class Group < Namespace include Gitlab::Utils::StrongMemoize include GroupAPICompatibility include EachBatch + include HasTimelogsReport ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT = 10 @@ -70,6 +71,7 @@ class Group < Namespace has_many :group_deploy_keys, through: :group_deploy_keys_groups has_many :group_deploy_tokens has_many :deploy_tokens, through: :group_deploy_tokens + has_many :oauth_applications, class_name: 'Doorkeeper::Application', as: :owner, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_one :dependency_proxy_setting, class_name: 'DependencyProxy::GroupSetting' has_many :dependency_proxy_blobs, class_name: 'DependencyProxy::Blob' @@ -84,7 +86,7 @@ class Group < Namespace validate :visibility_level_allowed_by_sub_groups validate :visibility_level_allowed_by_parent validate :two_factor_authentication_allowed - validates :variables, nested_attributes_duplicates: true + validates :variables, nested_attributes_duplicates: { scope: :environment_scope } validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 } @@ -178,6 +180,25 @@ class Group < Namespace groups.drop(1).each { |group| group.root_ancestor = root } end + # Returns the ids of the passed group models where the `emails_disabled` + # column is set to true anywhere in the ancestor hierarchy. + def ids_with_disabled_email(groups) + innner_query = Gitlab::ObjectHierarchy + .new(Group.where('id = namespaces_with_emails_disabled.id')) + .base_and_ancestors + .where(emails_disabled: true) + .select('1') + .limit(1) + + group_ids = Namespace + .from('(SELECT * FROM namespaces) as namespaces_with_emails_disabled') + .where(namespaces_with_emails_disabled: { id: groups }) + .where('EXISTS (?)', innner_query) + .pluck(:id) + + Set.new(group_ids) + end + private def public_to_user_arel(user) @@ -325,6 +346,10 @@ class Group < Namespace members_with_parents.owners.exists?(user_id: user) end + def blocked_owners + members.blocked.where(access_level: Gitlab::Access::OWNER) + end + def has_maintainer?(user) return false unless user @@ -337,14 +362,29 @@ class Group < Namespace # Check if user is a last owner of the group. def last_owner?(user) - has_owner?(user) && members_with_parents.owners.size == 1 + has_owner?(user) && single_owner? + end + + def member_last_owner?(member) + return member.last_owner unless member.last_owner.nil? + + last_owner?(member.user) + end + + def single_owner? + members_with_parents.owners.size == 1 end - def last_blocked_owner?(user) + def single_blocked_owner? + blocked_owners.size == 1 + end + + def member_last_blocked_owner?(member) + return member.last_blocked_owner unless member.last_blocked_owner.nil? + return false if members_with_parents.owners.any? - blocked_owners = members.blocked.where(access_level: Gitlab::Access::OWNER) - blocked_owners.size == 1 && blocked_owners.exists?(user_id: user) + single_blocked_owner? && blocked_owners.exists?(user_id: member.user) end def ldap_synced? @@ -784,13 +824,11 @@ class Group < Namespace variables = Ci::GroupVariable.where(group: list_of_ids) variables = variables.unprotected unless project.protected_for?(ref) - if Feature.enabled?(:scoped_group_variables, self, default_enabled: :yaml) - variables = if environment - variables.on_environment(environment) - else - variables.where(environment_scope: '*') - end - end + variables = if environment + variables.on_environment(environment) + else + variables.where(environment_scope: '*') + end variables = variables.group_by(&:group_id) list_of_ids.reverse.flat_map { |group| variables[group.id] }.compact diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb index c735e593da7..b56bac58705 100644 --- a/app/models/internal_id.rb +++ b/app/models/internal_id.rb @@ -47,18 +47,10 @@ class InternalId < ApplicationRecord def update_and_save(&block) lock! yield - update_and_save_counter.increment(usage: usage, changed: last_value_changed?) save! last_value end - # Instrumentation to track for-update locks - def update_and_save_counter - strong_memoize(:update_and_save_counter) do - Gitlab::Metrics.counter(:gitlab_internal_id_for_update_lock, 'Number of ROW SHARE (FOR UPDATE) locks on individual records from internal_ids') - end - end - class << self def track_greatest(subject, scope, usage, new_value, init) InternalIdGenerator.new(subject, scope, usage, init) @@ -88,6 +80,8 @@ class InternalId < ApplicationRecord end class InternalIdGenerator + extend Gitlab::Utils::StrongMemoize + # Generate next internal id for a given scope and usage. # # For currently supported usages, see #usage enum. @@ -123,6 +117,8 @@ class InternalId < ApplicationRecord # init: Block that gets called to initialize InternalId record if not present # Make sure to not throw exceptions in the absence of records (if this is expected). def generate + self.class.internal_id_transactions_increment(operation: :generate, usage: usage) + subject.transaction do # Create a record in internal_ids if one does not yet exist # and increment its last value @@ -138,6 +134,8 @@ class InternalId < ApplicationRecord def reset(value) return false unless value + self.class.internal_id_transactions_increment(operation: :reset, usage: usage) + updated = InternalId .where(**scope, usage: usage_value) @@ -152,6 +150,8 @@ class InternalId < ApplicationRecord # # Note this will acquire a ROW SHARE lock on the InternalId record def track_greatest(new_value) + self.class.internal_id_transactions_increment(operation: :track_greatest, usage: usage) + subject.transaction do record.track_greatest_and_save!(new_value) end @@ -162,6 +162,8 @@ class InternalId < ApplicationRecord end def with_lock(&block) + self.class.internal_id_transactions_increment(operation: :with_lock, usage: usage) + record.with_lock(&block) end @@ -197,5 +199,22 @@ class InternalId < ApplicationRecord rescue ActiveRecord::RecordNotUnique lookup end + + def self.internal_id_transactions_increment(operation:, usage:) + self.internal_id_transactions_total.increment( + operation: operation, + usage: usage.to_s, + in_transaction: ActiveRecord::Base.connection.transaction_open?.to_s + ) + end + + def self.internal_id_transactions_total + strong_memoize(:internal_id_transactions_total) do + name = :gitlab_internal_id_transactions_total + comment = 'Counts all the internal ids happening within transaction' + + Gitlab::Metrics.counter(name, comment) + end + end end end diff --git a/app/models/issue.rb b/app/models/issue.rb index 2f2d24cbe93..af78466e6a9 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -24,6 +24,8 @@ class Issue < ApplicationRecord include Todoable include FromUnion + extend ::Gitlab::Utils::Override + DueDateStruct = Struct.new(:title, :name).freeze NoDueDate = DueDateStruct.new('No Due Date', '0').freeze AnyDueDate = DueDateStruct.new('Any Due Date', '').freeze @@ -88,7 +90,6 @@ class Issue < ApplicationRecord test_case: 2 ## EE-only } - alias_attribute :parent_ids, :project_id alias_method :issuing_parent, :project alias_attribute :external_author, :service_desk_reply_to @@ -113,8 +114,8 @@ class Issue < ApplicationRecord scope :order_severity_desc, -> { includes(:issuable_severity).order('issuable_severities.severity DESC NULLS LAST') } scope :preload_associated_models, -> { preload(:assignees, :labels, project: :namespace) } - scope :with_web_entity_associations, -> { preload(:author, :project) } - scope :with_api_entity_associations, -> { preload(:timelogs, :assignees, :author, :notes, :labels, project: [:route, { namespace: :route }] ) } + scope :with_web_entity_associations, -> { preload(:author, project: [:project_feature, :route, namespace: :route]) } + scope :preload_awardable, -> { preload(:award_emoji) } scope :with_label_attributes, ->(label_attributes) { joins(:labels).where(labels: label_attributes) } scope :with_alert_management_alerts, -> { joins(:alert_management_alert) } scope :with_prometheus_alert_events, -> { joins(:issues_prometheus_alert_events) } @@ -191,7 +192,8 @@ class Issue < ApplicationRecord end def self.relative_positioning_query_base(issue) - in_projects(issue.parent_ids) + projects = issue.project.group&.root_ancestor&.all_projects || issue.project + in_projects(projects) end def self.relative_positioning_parent_column @@ -342,6 +344,8 @@ class Issue < ApplicationRecord .preload(preload) .reorder('issue_link_id') + related_issues = yield related_issues if block_given? + cross_project_filter = -> (issues) { issues.where(project: project) } Ability.issues_readable_by_user(related_issues, current_user, @@ -446,10 +450,20 @@ class Issue < ApplicationRecord issue_email_participants.pluck(IssueEmailParticipant.arel_table[:email].lower) end + def issue_assignee_user_ids + issue_assignees.pluck(:user_id) + end + private + # Ensure that the metrics association is safely created and respecting the unique constraint on issue_id + override :ensure_metrics def ensure_metrics - super + if !association(:metrics).loaded? || metrics.blank? + metrics_record = Issue::Metrics.safe_find_or_create_by(issue: self) + self.metrics = metrics_record + end + metrics.record! end diff --git a/app/models/key.rb b/app/models/key.rb index 18fa8aaaa16..131416d1bee 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -43,6 +43,8 @@ class Key < ApplicationRecord scope :preload_users, -> { preload(:user) } scope :for_user, -> (user) { where(user: user) } scope :order_last_used_at_desc, -> { reorder(::Gitlab::Database.nulls_last_order('last_used_at', 'DESC')) } + scope :expired_today_and_not_notified, -> { where(["date(expires_at AT TIME ZONE 'UTC') = CURRENT_DATE AND expiry_notification_delivered_at IS NULL"]) } + scope :expiring_soon_and_not_notified, -> { where(["date(expires_at AT TIME ZONE 'UTC') > CURRENT_DATE AND date(expires_at AT TIME ZONE 'UTC') < ? AND before_expiry_notification_delivered_at IS NULL", DAYS_TO_EXPIRE.days.from_now.to_date]) } def self.regular_keys where(type: ['Key', nil]) diff --git a/app/models/list.rb b/app/models/list.rb index e1954ed72c4..d72afbaee69 100644 --- a/app/models/list.rb +++ b/app/models/list.rb @@ -14,7 +14,6 @@ class List < ApplicationRecord validates :label_id, uniqueness: { scope: :board_id }, if: :label? scope :preload_associated_models, -> { preload(:board, label: :priorities) } - scope :without_types, ->(list_types) { where.not(list_type: list_types) } alias_method :preferences, :list_user_preferences diff --git a/app/models/member.rb b/app/models/member.rb index 38574d67cb6..e978552592d 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -137,6 +137,12 @@ class Member < ApplicationRecord scope :with_source_id, ->(source_id) { where(source_id: source_id) } scope :including_source, -> { includes(:source) } + scope :distinct_on_user_with_max_access_level, -> do + distinct_members = select('DISTINCT ON (user_id, invite_email) *') + .order('user_id, invite_email, access_level DESC, expires_at DESC, created_at ASC') + Member.from(distinct_members, :members) + end + scope :order_name_asc, -> { left_join_users.reorder(Gitlab::Database.nulls_last_order('users.name', 'ASC')) } scope :order_name_desc, -> { left_join_users.reorder(Gitlab::Database.nulls_last_order('users.name', 'DESC')) } scope :order_recent_sign_in, -> { left_join_users.reorder(Gitlab::Database.nulls_last_order('users.last_sign_in_at', 'DESC')) } @@ -278,10 +284,16 @@ class Member < ApplicationRecord Gitlab::Access.sym_options end + def valid_email?(email) + Devise.email_regexp.match?(email) + end + private def parse_users_list(source, list) - emails, user_ids, users = [], [], [] + emails = [] + user_ids = [] + users = [] existing_members = {} list.each do |item| @@ -299,6 +311,7 @@ class Member < ApplicationRecord if user_ids.present? users.concat(User.where(id: user_ids)) + # the below will automatically discard invalid user_ids existing_members = source.members_and_requesters.where(user_id: user_ids).index_by(&:user_id) end diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb index c30f6dc81ee..0f9fdd230ff 100644 --- a/app/models/members/group_member.rb +++ b/app/models/members/group_member.rb @@ -7,7 +7,7 @@ class GroupMember < Member SOURCE_TYPE = 'Namespace' belongs_to :group, foreign_key: 'source_id' - + alias_attribute :namespace_id, :source_id delegate :update_two_factor_requirement, to: :user # Make sure group member points only to group as it source @@ -26,6 +26,8 @@ class GroupMember < Member after_create :update_two_factor_requirement, unless: :invite? after_destroy :update_two_factor_requirement, unless: :invite? + attr_accessor :last_owner, :last_blocked_owner + def self.access_level_roles Gitlab::Access.options_with_owner end diff --git a/app/models/members/last_group_owner_assigner.rb b/app/models/members/last_group_owner_assigner.rb new file mode 100644 index 00000000000..64decb1df36 --- /dev/null +++ b/app/models/members/last_group_owner_assigner.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Members + class LastGroupOwnerAssigner + def initialize(group, members) + @group = group + @members = members + end + + def execute + @last_blocked_owner = no_owners_in_heirarchy? && group.single_blocked_owner? + @group_single_owner = owners.size == 1 + + members.each { |member| set_last_owner(member) } + end + + private + + attr_reader :group, :members, :last_blocked_owner, :group_single_owner + + def no_owners_in_heirarchy? + owners.empty? + end + + def set_last_owner(member) + member.last_owner = member.id.in?(owner_ids) && group_single_owner + member.last_blocked_owner = member.id.in?(blocked_owner_ids) && last_blocked_owner + end + + def owner_ids + @owner_ids ||= owners.where(id: member_ids).ids + end + + def blocked_owner_ids + @blocked_owner_ids ||= group.blocked_owners.where(id: member_ids).ids + end + + def member_ids + @members_ids ||= members.pluck(:id) + end + + def owners + @owners ||= group.members_with_parents.owners.load + end + end +end diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index 833b27756ab..9a86b3a3fd9 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -5,6 +5,8 @@ class ProjectMember < Member belongs_to :project, foreign_key: 'source_id' + delegate :namespace_id, to: :project + # Make sure project member points only to project as it source default_value_for :source_type, SOURCE_TYPE validates :source_type, format: { with: /\AProject\z/ } diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 7efdd79ae1c..e7f3762b9a3 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -37,7 +37,7 @@ class MergeRequest < ApplicationRecord SORTING_PREFERENCE_FIELD = :merge_requests_sort ALLOWED_TO_USE_MERGE_BASE_PIPELINE_FOR_COMPARISON = { - 'Ci::CompareCodequalityReportsService' => ->(project) { ::Gitlab::Ci::Features.display_codequality_backend_comparison?(project) } + 'Ci::CompareCodequalityReportsService' => ->(project) { true } }.freeze belongs_to :target_project, class_name: "Project" @@ -276,6 +276,9 @@ class MergeRequest < ApplicationRecord scope :by_squash_commit_sha, -> (sha) do where(squash_commit_sha: sha) end + scope :by_merge_or_squash_commit_sha, -> (sha) do + from_union([by_squash_commit_sha(sha), by_merge_commit_sha(sha)]) + end scope :by_related_commit_sha, -> (sha) do from_union( [ @@ -285,14 +288,20 @@ class MergeRequest < ApplicationRecord ] ) end - scope :by_cherry_pick_sha, -> (sha) do - joins(:notes).where(notes: { commit_id: sha }) - end scope :join_project, -> { joins(:target_project) } - scope :join_metrics, -> do + scope :join_metrics, -> (target_project_id = nil) do + # Do not join the relation twice + return self if self.arel.join_sources.any? { |join| join.left.try(:name).eql?(MergeRequest::Metrics.table_name) } + query = joins(:metrics) - query = query.where(MergeRequest.arel_table[:target_project_id].eq(MergeRequest::Metrics.arel_table[:target_project_id])) - query + + project_condition = if target_project_id + MergeRequest::Metrics.arel_table[:target_project_id].eq(target_project_id) + else + MergeRequest.arel_table[:target_project_id].eq(MergeRequest::Metrics.arel_table[:target_project_id]) + end + + query.where(project_condition) end scope :references_project, -> { references(:target_project) } scope :with_api_entity_associations, -> { @@ -304,6 +313,7 @@ class MergeRequest < ApplicationRecord } scope :with_csv_entity_associations, -> { preload(:assignees, :approved_by_users, :author, :milestone, metrics: [:merged_by]) } + scope :with_jira_integration_associations, -> { preload_routables.preload(:metrics, :assignees, :author) } scope :by_target_branch_wildcard, ->(wildcard_branch_name) do where("target_branch LIKE ?", ApplicationRecord.sanitize_sql_like(wildcard_branch_name).tr('*', '%')) @@ -346,7 +356,9 @@ class MergeRequest < ApplicationRecord scope :preload_metrics, -> (relation) { preload(metrics: relation) } scope :preload_project_and_latest_diff, -> { preload(:source_project, :latest_merge_request_diff) } scope :preload_latest_diff_commit, -> { preload(latest_merge_request_diff: :merge_request_diff_commits) } - scope :with_web_entity_associations, -> { preload(:author, :target_project) } + scope :preload_milestoneish_associations, -> { preload_routables.preload(:assignees, :labels) } + + scope :with_web_entity_associations, -> { preload(:author, target_project: [:project_feature, group: [:route, :parent], namespace: :route]) } scope :with_auto_merge_enabled, -> do with_state(:opened).where(auto_merge_enabled: true) @@ -1302,11 +1314,8 @@ class MergeRequest < ApplicationRecord message.join("\n\n") end - # Returns the oldest multi-line commit message, or the MR title if none found def default_squash_commit_message - strong_memoize(:default_squash_commit_message) do - first_multiline_commit&.safe_message || title - end + title end # Returns the oldest multi-line commit @@ -1358,11 +1367,11 @@ class MergeRequest < ApplicationRecord def environments_for(current_user, latest: false) return [] unless diff_head_commit - envs = EnvironmentsFinder.new(target_project, current_user, + envs = EnvironmentsByDeploymentsFinder.new(target_project, current_user, ref: target_branch, commit: diff_head_commit, with_tags: true, find_latest: latest).execute if source_project - envs.concat EnvironmentsFinder.new(source_project, current_user, + envs.concat EnvironmentsByDeploymentsFinder.new(source_project, current_user, ref: source_branch, commit: diff_head_commit, find_latest: latest).execute end @@ -1555,8 +1564,6 @@ class MergeRequest < ApplicationRecord end def has_codequality_reports? - return false unless ::Gitlab::Ci::Features.display_codequality_backend_comparison?(project) - actual_head_pipeline&.has_reports?(Ci::JobArtifact.codequality_reports) end diff --git a/app/models/milestone.rb b/app/models/milestone.rb index aa4ddfede99..4cf0e423a15 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -89,6 +89,10 @@ class Milestone < ApplicationRecord .order(:project_id, :group_id, :due_date).select('DISTINCT ON (project_id, group_id) id') end + def self.with_web_entity_associations + preload(:group, project: [:project_feature, group: [:parent], namespace: :route]) + end + def participants User.joins(assigned_issues: :milestone).where("milestones.id = ?", id).distinct end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 3f7ccdb977e..455429608b4 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -13,6 +13,9 @@ class Namespace < ApplicationRecord include Gitlab::Utils::StrongMemoize include IgnorableColumns include Namespaces::Traversal::Recursive + include Namespaces::Traversal::Linear + + ignore_column :delayed_project_removal, remove_with: '14.1', remove_after: '2021-05-22' # Prevent users from creating unreasonably deep level of nesting. # The number 20 was taken based on maximum nesting level of @@ -43,6 +46,9 @@ class Namespace < ApplicationRecord has_one :aggregation_schedule, class_name: 'Namespace::AggregationSchedule' has_one :package_setting_relation, inverse_of: :namespace, class_name: 'PackageSetting' + has_one :admin_note, inverse_of: :namespace + accepts_nested_attributes_for :admin_note, update_only: true + validates :owner, presence: true, unless: ->(n) { n.type == "Group" } validates :name, presence: true, @@ -83,11 +89,11 @@ class Namespace < ApplicationRecord before_destroy(prepend: true) { prepare_for_destroy } after_destroy :rm_dir - before_save :ensure_delayed_project_removal_assigned_to_namespace_settings, if: :delayed_project_removal_changed? - scope :for_user, -> { where('type IS NULL') } scope :sort_by_type, -> { order(Gitlab::Database.nulls_first_order(:type)) } scope :include_route, -> { includes(:route) } + scope :by_parent, -> (parent) { where(parent_id: parent) } + scope :filter_by_path, -> (query) { where('lower(path) = :query', query: query.downcase) } scope :with_statistics, -> do joins('LEFT JOIN project_statistics ps ON ps.namespace_id = namespaces.id') @@ -107,7 +113,7 @@ class Namespace < ApplicationRecord # Make sure that the name is same as strong_memoize name in root_ancestor # method - attr_writer :root_ancestor + attr_writer :root_ancestor, :emails_disabled_memoized class << self def by_path(path) @@ -235,7 +241,7 @@ class Namespace < ApplicationRecord # any ancestor can disable emails for all descendants def emails_disabled? - strong_memoize(:emails_disabled) do + strong_memoize(:emails_disabled_memoized) do if parent_id self_and_ancestors.where(emails_disabled: true).exists? else @@ -260,13 +266,8 @@ class Namespace < ApplicationRecord # Includes projects from this namespace and projects from all subgroups # that belongs to this namespace def all_projects - return Project.where(namespace: self) if user? - - if Feature.enabled?(:recursive_namespace_lookup_as_inner_join, self) - Project.joins("INNER JOIN (#{self_and_descendants.select(:id).to_sql}) namespaces ON namespaces.id=projects.namespace_id") - else - Project.where(namespace: self_and_descendants) - end + namespace = user? ? self : self_and_descendants + Project.where(namespace: namespace) end # Includes pipelines from this namespace and pipelines from all subgroups @@ -288,8 +289,13 @@ class Namespace < ApplicationRecord false end + # Deprecated, use #licensed_feature_available? instead. Remove once Namespace#feature_available? isn't used anymore. + def feature_available?(feature) + licensed_feature_available?(feature) + end + # Overridden in EE::Namespace - def feature_available?(_feature) + def licensed_feature_available?(_feature) false end @@ -347,6 +353,10 @@ class Namespace < ApplicationRecord Plan.default end + def paid? + root? && actual_plan.paid? + end + def actual_limits # We default to PlanLimits.new otherwise a lot of specs would fail # On production each plan should already have associated limits record @@ -412,13 +422,6 @@ class Namespace < ApplicationRecord private - def ensure_delayed_project_removal_assigned_to_namespace_settings - return if Feature.disabled?(:migrate_delayed_project_removal, default_enabled: true) - - self.namespace_settings || build_namespace_settings - namespace_settings.delayed_project_removal = delayed_project_removal - end - def all_projects_with_pages if all_projects.pages_metadata_not_migrated.exists? Gitlab::BackgroundMigration::MigratePagesMetadata.new.perform_on_relation( diff --git a/app/models/namespace/admin_note.rb b/app/models/namespace/admin_note.rb new file mode 100644 index 00000000000..3de809d60be --- /dev/null +++ b/app/models/namespace/admin_note.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class Namespace::AdminNote < ApplicationRecord + belongs_to :namespace, inverse_of: :admin_note + validates :namespace, presence: true + validates :note, length: { maximum: 1000 } +end diff --git a/app/models/namespace/traversal_hierarchy.rb b/app/models/namespace/traversal_hierarchy.rb index cfb6cfdde74..28cf55f7486 100644 --- a/app/models/namespace/traversal_hierarchy.rb +++ b/app/models/namespace/traversal_hierarchy.rb @@ -34,17 +34,20 @@ class Namespace sql = """ UPDATE namespaces SET traversal_ids = cte.traversal_ids - FROM (#{recursive_traversal_ids}) as cte + FROM (#{recursive_traversal_ids(lock: true)}) as cte WHERE namespaces.id = cte.id AND namespaces.traversal_ids <> cte.traversal_ids """ Namespace.connection.exec_query(sql) + rescue ActiveRecord::Deadlocked + db_deadlock_counter.increment(source: 'Namespace#sync_traversal_ids!') + raise end # Identify all incorrect traversal_ids in the current namespace hierarchy. - def incorrect_traversal_ids + def incorrect_traversal_ids(lock: false) Namespace - .joins("INNER JOIN (#{recursive_traversal_ids}) as cte ON namespaces.id = cte.id") + .joins("INNER JOIN (#{recursive_traversal_ids(lock: lock)}) as cte ON namespaces.id = cte.id") .where('namespaces.traversal_ids <> cte.traversal_ids') end @@ -55,10 +58,13 @@ class Namespace # # Note that the traversal_ids represent a calculated traversal path for the # namespace and not the value stored within the traversal_ids attribute. - def recursive_traversal_ids + # + # Optionally locked with FOR UPDATE to ensure isolation between concurrent + # updates of the heirarchy. + def recursive_traversal_ids(lock: false) root_id = Integer(@root.id) - """ + sql = <<~SQL WITH RECURSIVE cte(id, traversal_ids, cycle) AS ( VALUES(#{root_id}, ARRAY[#{root_id}], false) UNION ALL @@ -67,7 +73,11 @@ class Namespace WHERE n.parent_id = cte.id AND NOT cycle ) SELECT id, traversal_ids FROM cte - """ + SQL + + sql += ' FOR UPDATE' if lock + + sql end # This is essentially Namespace#root_ancestor which will soon be rewritten @@ -80,5 +90,9 @@ class Namespace .reorder(nil) .find_by(parent_id: nil) end + + def db_deadlock_counter + Gitlab::Metrics.counter(:db_deadlock, 'Counts the times we have deadlocked in the database') + end end end diff --git a/app/models/namespace_setting.rb b/app/models/namespace_setting.rb index 50844403d7f..d21f9632e18 100644 --- a/app/models/namespace_setting.rb +++ b/app/models/namespace_setting.rb @@ -1,14 +1,20 @@ # frozen_string_literal: true class NamespaceSetting < ApplicationRecord + include CascadingNamespaceSettingAttribute + + cascading_attr :delayed_project_removal + belongs_to :namespace, inverse_of: :namespace_settings validate :default_branch_name_content validate :allow_mfa_for_group + validate :allow_resource_access_token_creation_for_group before_validation :normalize_default_branch_name - NAMESPACE_SETTINGS_PARAMS = [:default_branch_name].freeze + NAMESPACE_SETTINGS_PARAMS = [:default_branch_name, :delayed_project_removal, + :lock_delayed_project_removal, :resource_access_token_creation_allowed].freeze self.primary_key = :namespace_id @@ -31,6 +37,12 @@ class NamespaceSetting < ApplicationRecord errors.add(:allow_mfa_for_subgroups, _('is not allowed since the group is not top-level group.')) end end + + def allow_resource_access_token_creation_for_group + if namespace&.subgroup? && !resource_access_token_creation_allowed + errors.add(:resource_access_token_creation_allowed, _('is not allowed since the group is not top-level group.')) + end + end end NamespaceSetting.prepend_if_ee('EE::NamespaceSetting') diff --git a/app/models/namespaces/traversal/linear.rb b/app/models/namespaces/traversal/linear.rb new file mode 100644 index 00000000000..dd9ca8d9bea --- /dev/null +++ b/app/models/namespaces/traversal/linear.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true +# +# Query a recursively defined namespace hierarchy using linear methods through +# the traversal_ids attribute. +# +# Namespace is a nested hierarchy of one parent to many children. A search +# using only the parent-child relationships is a slow operation. This process +# was previously optimized using Postgresql recursive common table expressions +# (CTE) with acceptable performance. However, it lead to slower than possible +# performance, and resulted in complicated queries that were difficult to make +# performant. +# +# Instead of searching the hierarchy recursively, we store a `traversal_ids` +# attribute on each node. The `traversal_ids` is an ordered array of Namespace +# IDs that define the traversal path from the root Namespace to the current +# Namespace. +# +# For example, suppose we have the following Namespaces: +# +# GitLab (id: 1) > Engineering (id: 2) > Manage (id: 3) > Access (id: 4) +# +# Then `traversal_ids` for group "Access" is [1, 2, 3, 4] +# +# And we can match against other Namespace `traversal_ids` such that: +# +# - Ancestors are [1], [1, 2], [1, 2, 3] +# - Descendants are [1, 2, 3, 4, *] +# - Root is [1] +# - Hierarchy is [1, *] +# +# Note that this search method works so long as the IDs are unique and the +# traversal path is ordered from root to leaf nodes. +# +# We implement this in the database using Postgresql arrays, indexed by a +# generalized inverted index (gin). +module Namespaces + module Traversal + module Linear + extend ActiveSupport::Concern + + UnboundedSearch = Class.new(StandardError) + + included do + after_create :sync_traversal_ids, if: -> { sync_traversal_ids? } + after_update :sync_traversal_ids, if: -> { sync_traversal_ids? && saved_change_to_parent_id? } + + scope :traversal_ids_contains, ->(ids) { where("traversal_ids @> (?)", ids) } + end + + def sync_traversal_ids? + Feature.enabled?(:sync_traversal_ids, root_ancestor, default_enabled: :yaml) + end + + def use_traversal_ids? + Feature.enabled?(:use_traversal_ids, root_ancestor, default_enabled: :yaml) + end + + def self_and_descendants + if use_traversal_ids? + lineage(self) + else + super + end + end + + private + + # Update the traversal_ids for the full hierarchy. + # + # NOTE: self.traversal_ids will be stale. Reload for a fresh record. + def sync_traversal_ids + # Clear any previously memoized root_ancestor as our ancestors have changed. + clear_memoization(:root_ancestor) + + Namespace::TraversalHierarchy.for_namespace(root_ancestor).sync_traversal_ids! + end + + # Make sure we drop the STI `type = 'Group'` condition for better performance. + # Logically equivalent so long as hierarchies remain homogeneous. + def without_sti_condition + self.class.unscope(where: :type) + end + + # Search this namespace's lineage. Bound inclusively by top node. + def lineage(top) + raise UnboundedSearch.new('Must bound search by a top') unless top + + without_sti_condition + .traversal_ids_contains("{#{top.id}}") + end + end + end +end diff --git a/app/models/namespaces/traversal/recursive.rb b/app/models/namespaces/traversal/recursive.rb index d74b7883830..409438f53d2 100644 --- a/app/models/namespaces/traversal/recursive.rb +++ b/app/models/namespaces/traversal/recursive.rb @@ -6,10 +6,14 @@ module Namespaces extend ActiveSupport::Concern def root_ancestor - return self if persisted? && parent_id.nil? + return self if parent.nil? - strong_memoize(:root_ancestor) do - self_and_ancestors.reorder(nil).find_by(parent_id: nil) + if persisted? + strong_memoize(:root_ancestor) do + self_and_ancestors.reorder(nil).find_by(parent_id: nil) + end + else + parent.root_ancestor end end @@ -18,6 +22,7 @@ module Namespaces object_hierarchy(self.class.where(id: id)) .all_objects end + alias_method :recursive_self_and_hierarchy, :self_and_hierarchy # Returns all the ancestors of the current namespaces. def ancestors @@ -26,6 +31,7 @@ module Namespaces object_hierarchy(self.class.where(id: parent_id)) .base_and_ancestors end + alias_method :recursive_ancestors, :ancestors # returns all ancestors upto but excluding the given namespace # when no namespace is given, all ancestors upto the top are returned @@ -40,17 +46,20 @@ module Namespaces object_hierarchy(self.class.where(id: id)) .base_and_ancestors(hierarchy_order: hierarchy_order) end + alias_method :recursive_self_and_ancestors, :self_and_ancestors # Returns all the descendants of the current namespace. def descendants object_hierarchy(self.class.where(parent_id: id)) .base_and_descendants end + alias_method :recursive_descendants, :descendants def self_and_descendants object_hierarchy(self.class.where(id: id)) .base_and_descendants end + alias_method :recursive_self_and_descendants, :self_and_descendants def object_hierarchy(ancestors_base) Gitlab::ObjectHierarchy.new(ancestors_base, options: { use_distinct: Feature.enabled?(:use_distinct_in_object_hierarchy, self) }) diff --git a/app/models/note.rb b/app/models/note.rb index fb540d692d1..3e560a09fbd 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -19,6 +19,7 @@ class Note < ApplicationRecord include Gitlab::SQL::Pattern include ThrottledTouch include FromUnion + include Sortable cache_markdown_field :note, pipeline: :note, issuable_state_filter_enabled: true @@ -103,12 +104,12 @@ class Note < ApplicationRecord scope :system, -> { where(system: true) } scope :user, -> { where(system: false) } scope :common, -> { where(noteable_type: ["", nil]) } - scope :fresh, -> { order(created_at: :asc, id: :asc) } + scope :fresh, -> { order_created_asc.with_order_id_asc } scope :updated_after, ->(time) { where('updated_at > ?', time) } scope :with_updated_at, ->(time) { where(updated_at: time) } - scope :by_updated_at, -> { reorder(:updated_at, :id) } scope :inc_author_project, -> { includes(:project, :author) } scope :inc_author, -> { includes(:author) } + scope :with_api_entity_associations, -> { preload(:note_diff_file, :author) } scope :inc_relations_for_view, -> do includes(:project, { author: :status }, :updated_by, :resolved_by, :award_emoji, { system_note_metadata: :description_version }, :note_diff_file, :diff_note_positions, :suggestions) @@ -135,6 +136,7 @@ class Note < ApplicationRecord project: [:project_members, :namespace, { group: [:group_members] }]) end scope :with_metadata, -> { includes(:system_note_metadata) } + scope :with_web_entity_associations, -> { preload(:project, :author, :noteable) } scope :for_note_or_capitalized_note, ->(text) { where(note: [text, text.capitalize]) } scope :like_note_or_capitalized_note, ->(text) { where('(note LIKE ? OR note LIKE ?)', text, text.capitalize) } @@ -148,6 +150,8 @@ class Note < ApplicationRecord after_commit :notify_after_destroy, on: :destroy class << self + extend Gitlab::Utils::Override + def model_name ActiveModel::Name.new(self, nil, 'note') end @@ -204,6 +208,17 @@ class Note < ApplicationRecord def search(query) fuzzy_search(query, [:note]) end + + # Override the `Sortable` module's `.simple_sorts` to remove name sorting, + # as a `Note` does not have any property that correlates to a "name". + override :simple_sorts + def simple_sorts + super.except('name_asc', 'name_desc') + end + + def cherry_picked_merge_requests(shas) + where(noteable_type: 'MergeRequest', commit_id: shas).select(:noteable_id) + end end # rubocop: disable CodeReuse/ServiceClass diff --git a/app/models/notification_setting.rb b/app/models/notification_setting.rb index 72813b17501..3d049336d44 100644 --- a/app/models/notification_setting.rb +++ b/app/models/notification_setting.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class NotificationSetting < ApplicationRecord + include FromUnion + enum level: { global: 3, watch: 2, participating: 1, mention: 4, disabled: 0, custom: 5 } default_value_for :level, NotificationSetting.levels[:global] @@ -30,6 +32,8 @@ class NotificationSetting < ApplicationRecord scope :preload_source_route, -> { preload(source: [:route]) } + scope :order_by_id_asc, -> { order(id: :asc) } + # NOTE: Applicable unfound_translations.rb also needs to be updated when below events are changed. EMAIL_EVENTS = [ :new_release, diff --git a/app/models/packages/debian/file_entry.rb b/app/models/packages/debian/file_entry.rb new file mode 100644 index 00000000000..eb66f4acfa9 --- /dev/null +++ b/app/models/packages/debian/file_entry.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Packages + module Debian + class FileEntry + include ActiveModel::Model + + DIGESTS = %i[md5 sha1 sha256].freeze + FILENAME_REGEX = %r{\A[a-zA-Z0-9][a-zA-Z0-9_.~+-]*\z}.freeze + + attr_accessor :filename, + :size, + :md5sum, + :section, + :priority, + :sha1sum, + :sha256sum, + :package_file + + validates :filename, :size, :md5sum, :section, :priority, :sha1sum, :sha256sum, :package_file, presence: true + validates :filename, format: { with: FILENAME_REGEX } + validate :valid_package_file_digests, if: -> { md5sum.present? && sha1sum.present? && sha256sum.present? && package_file.present? } + + def component + return 'main' if section.blank? + return 'main' unless section.include?('/') + + section.split('/')[0] + end + + private + + def valid_package_file_digests + DIGESTS.each do |digest| + package_file_digest = package_file["file_#{digest}"] + sum = public_send("#{digest}sum") # rubocop:disable GitlabSecurity/PublicSend + next if package_file_digest == sum + + errors.add("#{digest}sum".to_sym, "mismatch for #{filename}: #{package_file_digest} != #{sum}") + end + end + end + end +end diff --git a/app/models/packages/debian/file_metadatum.rb b/app/models/packages/debian/file_metadatum.rb index 7c9f4f5f3f1..af51f256e18 100644 --- a/app/models/packages/debian/file_metadatum.rb +++ b/app/models/packages/debian/file_metadatum.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Packages::Debian::FileMetadatum < ApplicationRecord + self.primary_key = :package_file_id + belongs_to :package_file, inverse_of: :debian_file_metadatum validates :package_file, presence: true diff --git a/app/models/packages/dependency.rb b/app/models/packages/dependency.rb index a32c3c05bb3..ad3944b5f21 100644 --- a/app/models/packages/dependency.rb +++ b/app/models/packages/dependency.rb @@ -7,8 +7,8 @@ class Packages::Dependency < ApplicationRecord validates :name, uniqueness: { scope: :version_pattern } NAME_VERSION_PATTERN_TUPLE_MATCHING = '(name, version_pattern) = (?, ?)' - MAX_STRING_LENGTH = 255.freeze - MAX_CHUNKED_QUERIES_COUNT = 10.freeze + MAX_STRING_LENGTH = 255 + MAX_CHUNKED_QUERIES_COUNT = 10 def self.ids_for_package_names_and_version_patterns(names_and_version_patterns = {}, chunk_size = 50, max_rows_limit = 200) names_and_version_patterns.reject! { |key, value| key.size > MAX_STRING_LENGTH || value.size > MAX_STRING_LENGTH } diff --git a/app/models/packages/go/module_version.rb b/app/models/packages/go/module_version.rb index a50c78f8e69..fd575e6c96c 100644 --- a/app/models/packages/go/module_version.rb +++ b/app/models/packages/go/module_version.rb @@ -4,6 +4,7 @@ module Packages module Go class ModuleVersion include Gitlab::Utils::StrongMemoize + include Gitlab::Golang VALID_TYPES = %i[ref commit pseudo].freeze @@ -81,6 +82,9 @@ module Packages end def valid? + # assume the module version is valid if a corresponding Package exists + return true if ::Packages::Go::PackageFinder.new(mod.project, mod.name, name).exists? + @mod.path_valid?(major) && @mod.gomod_valid?(gomod) end diff --git a/app/models/packages/maven/metadatum.rb b/app/models/packages/maven/metadatum.rb index 7aed274216b..471c4b3a392 100644 --- a/app/models/packages/maven/metadatum.rb +++ b/app/models/packages/maven/metadatum.rb @@ -19,6 +19,7 @@ class Packages::Maven::Metadatum < ApplicationRecord validate :maven_package_type scope :for_package_ids, -> (package_ids) { where(package_id: package_ids) } + scope :with_path, ->(path) { where(path: path) } scope :order_created, -> { reorder('created_at ASC') } def self.pluck_app_name diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb index 993d1123c86..e510432be8f 100644 --- a/app/models/packages/package.rb +++ b/app/models/packages/package.rb @@ -5,6 +5,8 @@ class Packages::Package < ApplicationRecord include UsageStatistics include Gitlab::Utils::StrongMemoize + DISPLAYABLE_STATUSES = [:default, :error].freeze + belongs_to :project belongs_to :creator, class_name: 'User' @@ -29,6 +31,7 @@ class Packages::Package < ApplicationRecord delegate :recipe, :recipe_path, to: :conan_metadatum, prefix: :conan delegate :codename, :suite, to: :debian_distribution, prefix: :debian_distribution + delegate :target_sha, to: :composer_metadatum, prefix: :composer validates :project, presence: true validates :name, presence: true @@ -69,7 +72,7 @@ class Packages::Package < ApplicationRecord composer: 6, generic: 7, golang: 8, debian: 9, rubygems: 10 } - enum status: { default: 0, hidden: 1, processing: 2 } + enum status: { default: 0, hidden: 1, processing: 2, error: 3 } scope :with_name, ->(name) { where(name: name) } scope :with_name_like, ->(name) { where(arel_table[:name].matches(name)) } @@ -79,7 +82,7 @@ class Packages::Package < ApplicationRecord scope :without_version_like, -> (version) { where.not(arel_table[:version].matches(version)) } scope :with_package_type, ->(package_type) { where(package_type: package_type) } scope :with_status, ->(status) { where(status: status) } - scope :displayable, -> { with_status(:default) } + scope :displayable, -> { with_status(DISPLAYABLE_STATUSES) } scope :including_build_info, -> { includes(pipelines: :user) } scope :including_project_route, -> { includes(project: { namespace: :route }) } scope :including_tags, -> { includes(:tags) } @@ -135,13 +138,26 @@ class Packages::Package < ApplicationRecord after_commit :update_composer_cache, on: :destroy, if: -> { composer? } def self.for_projects(projects) - return none unless projects.any? + unless Feature.enabled?(:maven_packages_group_level_improvements, default_enabled: :yaml) + return none unless projects.any? + end where(project_id: projects) end - def self.only_maven_packages_with_path(path) - joins(:maven_metadatum).where(packages_maven_metadata: { path: path }) + def self.only_maven_packages_with_path(path, use_cte: false) + if use_cte && Feature.enabled?(:maven_metadata_by_path_with_optimization_fence, default_enabled: :yaml) + # This is an optimization fence which assumes that looking up the Metadatum record by path (globally) + # and then filter down the packages (by project or by group and subgroups) will be cheaper than + # looking up all packages within a project or group and filter them by path. + + inner_query = Packages::Maven::Metadatum.where(path: path).select(:id, :package_id) + cte = Gitlab::SQL::CTE.new(:maven_metadata_by_path, inner_query) + with(cte.to_arel) + .joins('INNER JOIN maven_metadata_by_path ON maven_metadata_by_path.package_id=packages_packages.id') + else + joins(:maven_metadatum).where(packages_maven_metadata: { path: path }) + end end def self.by_name_and_file_name(name, file_name) diff --git a/app/models/packages/tag.rb b/app/models/packages/tag.rb index 771d016daed..14a1ae98ed4 100644 --- a/app/models/packages/tag.rb +++ b/app/models/packages/tag.rb @@ -4,7 +4,7 @@ class Packages::Tag < ApplicationRecord validates :package, :name, presence: true - FOR_PACKAGES_TAGS_LIMIT = 200.freeze + FOR_PACKAGES_TAGS_LIMIT = 200 NUGET_TAGS_SEPARATOR = ' ' # https://docs.microsoft.com/en-us/nuget/reference/nuspec#tags scope :preload_package, -> { preload(:package) } diff --git a/app/models/pages/lookup_path.rb b/app/models/pages/lookup_path.rb index 33771580be2..3285a1f7f4c 100644 --- a/app/models/pages/lookup_path.rb +++ b/app/models/pages/lookup_path.rb @@ -50,9 +50,7 @@ module Pages def zip_source return unless deployment&.file - return if deployment.file.file_storage? && !Feature.enabled?(:pages_serve_with_zip_file_protocol, project, default_enabled: true) - - return if deployment.migrated? && !Feature.enabled?(:pages_serve_from_migrated_zip, project, default_enabled: true) + return if deployment.file.file_storage? && !Feature.enabled?(:pages_serve_with_zip_file_protocol, project, default_enabled: :yaml) global_id = ::Gitlab::GlobalId.build(deployment, id: deployment.id).to_s @@ -74,7 +72,7 @@ module Pages path: File.join(project.full_path, 'public/') } rescue LegacyStorageDisabledError => e - Gitlab::ErrorTracking.track_exception(e) + Gitlab::ErrorTracking.track_exception(e, project_id: project.id) nil end diff --git a/app/models/pages_deployment.rb b/app/models/pages_deployment.rb index d67a92af6af..294a4e85d1f 100644 --- a/app/models/pages_deployment.rb +++ b/app/models/pages_deployment.rb @@ -14,6 +14,8 @@ class PagesDeployment < ApplicationRecord scope :older_than, -> (id) { where('id < ?', id) } scope :migrated_from_legacy_storage, -> { where(file: MIGRATED_FILE_NAME) } + scope :with_files_stored_locally, -> { where(file_store: ::ObjectStorage::Store::LOCAL) } + scope :with_files_stored_remotely, -> { where(file_store: ::ObjectStorage::Store::REMOTE) } validates :file, presence: true validates :file_store, presence: true, inclusion: { in: ObjectStorage::SUPPORTED_STORES } diff --git a/app/models/preloaders/labels_preloader.rb b/app/models/preloaders/labels_preloader.rb new file mode 100644 index 00000000000..427f2869aac --- /dev/null +++ b/app/models/preloaders/labels_preloader.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Preloaders + # This class preloads the `project`, `group`, and subscription associations for the given + # labels, user, and project (if provided). A Label can be of type ProjectLabel or GroupLabel + # and the preloader supports both. + # + # Usage: + # labels = Label.where(...) + # Preloaders::LabelsPreloader.new(labels, current_user, @project).preload_all + # labels.first.project # won't fire any query + class LabelsPreloader + attr_reader :labels, :user, :project + + def initialize(labels, user, project = nil) + @labels = labels + @user = user + @project = project + end + + def preload_all + preloader = ActiveRecord::Associations::Preloader.new + + preloader.preload(labels.select {|l| l.is_a? ProjectLabel }, { project: [:project_feature, namespace: :route] }) + preloader.preload(labels.select {|l| l.is_a? GroupLabel }, { group: :route }) + labels.each do |label| + label.lazy_subscription(user) + label.lazy_subscription(user, project) if project.present? + end + end + end +end + +Preloaders::LabelsPreloader.prepend_if_ee('EE::Preloaders::LabelsPreloader') diff --git a/app/models/preloaders/user_max_access_level_in_projects_preloader.rb b/app/models/preloaders/user_max_access_level_in_projects_preloader.rb new file mode 100644 index 00000000000..671091480ee --- /dev/null +++ b/app/models/preloaders/user_max_access_level_in_projects_preloader.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Preloaders + # This class preloads the max access level for the user within the given projects and + # stores the values in requests store via the ProjectTeam class. + class UserMaxAccessLevelInProjectsPreloader + def initialize(projects, user) + @projects = projects + @user = user + end + + def execute + access_levels = @user + .project_authorizations + .where(project_id: @projects) + .group(:project_id) + .maximum(:access_level) + + @projects.each do |project| + access_level = access_levels[project.id] || Gitlab::Access::NO_ACCESS + ProjectTeam.new(project).write_member_access_for_user_id(@user.id, access_level) + end + end + end +end diff --git a/app/models/project.rb b/app/models/project.rb index c52eb95bde8..f03e5293b58 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -36,6 +36,8 @@ class Project < ApplicationRecord include Integration include Repositories::CanHousekeepRepository include EachBatch + include GitlabRoutingHelper + extend Gitlab::Cache::RequestCache extend Gitlab::Utils::Override @@ -219,7 +221,7 @@ class Project < ApplicationRecord has_one :alerting_setting, inverse_of: :project, class_name: 'Alerting::ProjectAlertingSetting' has_one :service_desk_setting, class_name: 'ServiceDeskSetting' - # Merge Requests for target project should be removed with it + # Merge requests for target project should be removed with it has_many :merge_requests, foreign_key: 'target_project_id', inverse_of: :target_project has_many :merge_request_metrics, foreign_key: 'target_project', class_name: 'MergeRequest::Metrics', inverse_of: :target_project has_many :source_of_merge_requests, foreign_key: 'source_project_id', class_name: 'MergeRequest' @@ -517,7 +519,7 @@ class Project < ApplicationRecord scope :with_packages, -> { joins(:packages) } scope :in_namespace, ->(namespace_ids) { where(namespace_id: namespace_ids) } scope :personal, ->(user) { where(namespace_id: user.namespace_id) } - scope :joined, ->(user) { where('namespace_id != ?', user.namespace_id) } + scope :joined, ->(user) { where.not(namespace_id: user.namespace_id) } scope :starred_by, ->(user) { joins(:users_star_projects).where('users_star_projects.user_id': user.id) } scope :visible_to_user, ->(user) { where(id: user.authorized_projects.select(:id).reorder(nil)) } scope :visible_to_user_and_access_level, ->(user, access_level) { where(id: user.authorized_projects.where('project_authorizations.access_level >= ?', access_level).select(:id).reorder(nil)) } @@ -577,7 +579,7 @@ class Project < ApplicationRecord with_issues_available_for_user(user).or(with_merge_requests_available_for_user(user)) end scope :with_merge_requests_enabled, -> { with_feature_enabled(:merge_requests) } - scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct } + scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }) } scope :with_limit, -> (maximum) { limit(maximum) } scope :with_group_runners_enabled, -> do @@ -621,7 +623,7 @@ class Project < ApplicationRecord end def self.with_web_entity_associations - preload(:project_feature, :route, :creator, :group, namespace: [:route, :owner]) + preload(:project_feature, :route, :creator, group: :parent, namespace: [:route, :owner]) end def self.eager_load_namespace_and_owner @@ -1368,15 +1370,15 @@ class Project < ApplicationRecord end def disabled_services - return %w(datadog) unless Feature.enabled?(:datadog_ci_integration, self) + return %w[datadog hipchat] unless Feature.enabled?(:datadog_ci_integration, self) - [] + %w[hipchat] end def find_or_initialize_service(name) return if disabled_services.include?(name) - find_service(services, name) || build_from_instance_or_template(name) || public_send("build_#{name}_service") # rubocop:disable GitlabSecurity/PublicSend + find_service(services, name) || build_from_instance_or_template(name) || build_service(name) end # rubocop: disable CodeReuse/ServiceClass @@ -1713,10 +1715,15 @@ class Project < ApplicationRecord end end + # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/326989 def any_active_runners?(&block) active_runners_with_tags.any?(&block) end + def any_online_runners?(&block) + online_runners_with_tags.any?(&block) + end + def valid_runners_token?(token) self.runners_token && ActiveSupport::SecurityUtils.secure_compare(token, self.runners_token) end @@ -1812,7 +1819,7 @@ class Project < ApplicationRecord # TODO: remove this method https://gitlab.com/gitlab-org/gitlab/-/issues/320775 # rubocop: disable CodeReuse/ServiceClass def legacy_remove_pages - return unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true) + return unless ::Settings.pages.local_store.enabled # Projects with a missing namespace cannot have their pages removed return unless namespace @@ -1848,7 +1855,7 @@ class Project < ApplicationRecord # where().update_all to perform update in the single transaction with check for null ProjectPagesMetadatum .where(project_id: id, pages_deployment_id: nil) - .update_all(pages_deployment_id: deployment.id) + .update_all(deployed: deployment.present?, pages_deployment_id: deployment&.id) end def write_repository_config(gl_full_path: full_path) @@ -2145,8 +2152,8 @@ class Project < ApplicationRecord data = repository.route_map_for(sha) Gitlab::RouteMap.new(data) if data - rescue Gitlab::RouteMap::FormatError - nil + rescue Gitlab::RouteMap::FormatError + nil end end @@ -2165,17 +2172,18 @@ class Project < ApplicationRecord end def default_merge_request_target - return self unless forked_from_project - return self unless forked_from_project.merge_requests_enabled? - - # When our current visibility is more restrictive than the source project, - # (e.g., the fork is `private` but the parent is `public`), target the less - # permissive project - if visibility_level_value < forked_from_project.visibility_level_value - self - else - forked_from_project - end + return self if project_setting.mr_default_target_self + return self unless mr_can_target_upstream? + + forked_from_project + end + + def mr_can_target_upstream? + # When our current visibility is more restrictive than the upstream project, + # (e.g., the fork is `private` but the parent is `public`), don't allow target upstream + forked_from_project && + forked_from_project.merge_requests_enabled? && + forked_from_project.visibility_level_value <= visibility_level_value end def multiple_issue_boards_available? @@ -2322,6 +2330,11 @@ class Project < ApplicationRecord .external_authorization_service_default_label end + # Overridden in EE::Project + def licensed_feature_available?(_feature) + false + end + def licensed_features [] end @@ -2584,6 +2597,10 @@ class Project < ApplicationRecord return Service.build_from_integration(template, project_id: id) if template end + def build_service(name) + "#{name}_service".classify.constantize.new(project_id: id) + end + def services_templates @services_templates ||= Service.for_template end @@ -2734,9 +2751,11 @@ class Project < ApplicationRecord end def active_runners_with_tags - strong_memoize(:active_runners_with_tags) do - active_runners.with_tags - end + @active_runners_with_tags ||= active_runners.with_tags + end + + def online_runners_with_tags + @online_runners_with_tags ||= active_runners_with_tags.online end end diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb index a598bf3f60c..15f6bedfc2e 100644 --- a/app/models/project_feature.rb +++ b/app/models/project_feature.rb @@ -3,6 +3,7 @@ class ProjectFeature < ApplicationRecord include Featurable + # When updating this array, make sure to update rubocop/cop/gitlab/feature_available_usage.rb as well. FEATURES = %i[ issues forking @@ -19,7 +20,7 @@ class ProjectFeature < ApplicationRecord container_registry ].freeze - EXPORTABLE_FEATURES = (FEATURES - [:security_and_compliance]).freeze + EXPORTABLE_FEATURES = (FEATURES - [:security_and_compliance, :pages]).freeze set_available_features(FEATURES) diff --git a/app/models/project_feature_usage.rb b/app/models/project_feature_usage.rb index 4f445758653..02051310af7 100644 --- a/app/models/project_feature_usage.rb +++ b/app/models/project_feature_usage.rb @@ -20,12 +20,29 @@ class ProjectFeatureUsage < ApplicationRecord end def log_jira_dvcs_integration_usage(cloud: true) - transaction(requires_new: true) do - save unless persisted? - touch(self.class.jira_dvcs_integration_field(cloud: cloud)) - end + integration_field = self.class.jira_dvcs_integration_field(cloud: cloud) + + # The feature usage is used only once later to query the feature usage in a + # long date range. Therefore, we just need to update the timestamp once per + # day + return if persisted? && updated_today?(integration_field) + + persist_jira_dvcs_usage(integration_field) + end + + private + + def updated_today?(integration_field) + self[integration_field].present? && self[integration_field].today? + end + + def persist_jira_dvcs_usage(integration_field) + assign_attributes(integration_field => Time.current) + save rescue ActiveRecord::RecordNotUnique reset retry end end + +ProjectFeatureUsage.prepend_if_ee('EE::ProjectFeatureUsage') diff --git a/app/models/project_services/asana_service.rb b/app/models/project_services/asana_service.rb index c4fcdcc05c5..f31bf931a41 100644 --- a/app/models/project_services/asana_service.rb +++ b/app/models/project_services/asana_service.rb @@ -3,6 +3,8 @@ require 'asana' class AsanaService < Service + include ActionView::Helpers::UrlHelper + prop_accessor :api_key, :restrict_to_branch validates :api_key, presence: true, if: :activated? @@ -11,20 +13,12 @@ class AsanaService < Service end def description - s_('AsanaService|Asana - Teamwork without email') + s_('AsanaService|Add commit messages as comments to Asana tasks') end def help - 'This service adds commit messages as comments to Asana tasks. -Once enabled, commit messages are checked for Asana task URLs -(for example, `https://app.asana.com/0/123456/987654`) or task IDs -starting with # (for example, `#987654`). Every task ID found will -get the commit comment added to it. - -You can also close a task with a message containing: `fix #123456`. - -You can create a Personal Access Token here: -https://app.asana.com/0/developer-console' + docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/asana'), target: '_blank', rel: 'noopener noreferrer' + s_('Add commit messages as comments to Asana tasks. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end def self.to_param @@ -36,13 +30,17 @@ https://app.asana.com/0/developer-console' { type: 'text', name: 'api_key', - placeholder: s_('AsanaService|User Personal Access Token. User must have access to task, all comments will be attributed to this user.'), + title: 'API key', + help: s_('AsanaService|User Personal Access Token. User must have access to the task. All comments are attributed to this user.'), + # Example Personal Access Token from Asana docs + placeholder: '0/68a9e79b868c6789e79a124c30b0', required: true }, { type: 'text', name: 'restrict_to_branch', - placeholder: s_('AsanaService|Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches.') + title: 'Restrict to branch (optional)', + help: s_('AsanaService|Comma-separated list of branches to be automatically inspected. Leave blank to include all branches.') } ] end diff --git a/app/models/project_services/assembla_service.rb b/app/models/project_services/assembla_service.rb index 60575e45a90..8845fb99605 100644 --- a/app/models/project_services/assembla_service.rb +++ b/app/models/project_services/assembla_service.rb @@ -9,7 +9,7 @@ class AssemblaService < Service end def description - 'Project Management Software (Source Commits Endpoint)' + _('Manage projects.') end def self.to_param diff --git a/app/models/project_services/bamboo_service.rb b/app/models/project_services/bamboo_service.rb index 8c1f4fef09b..a892d1a4314 100644 --- a/app/models/project_services/bamboo_service.rb +++ b/app/models/project_services/bamboo_service.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true class BambooService < CiService + include ActionView::Helpers::UrlHelper include ReactiveService prop_accessor :bamboo_url, :build_key, :username, :password @@ -31,15 +32,16 @@ class BambooService < CiService end def title - s_('BambooService|Atlassian Bamboo CI') + s_('BambooService|Atlassian Bamboo') end def description - s_('BambooService|A continuous integration and build server') + s_('BambooService|Use the Atlassian Bamboo CI/CD server with GitLab.') end def help - s_('BambooService|You must set up automatic revision labeling and a repository trigger in Bamboo.') + docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/bamboo'), target: '_blank', rel: 'noopener noreferrer' + s_('BambooService|Use Atlassian Bamboo to run CI/CD pipelines. You must set up automatic revision labeling and a repository trigger in Bamboo. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end def self.to_param @@ -48,13 +50,32 @@ class BambooService < CiService def fields [ - { type: 'text', name: 'bamboo_url', - placeholder: s_('BambooService|Bamboo root URL like https://bamboo.example.com'), required: true }, - { type: 'text', name: 'build_key', - placeholder: s_('BambooService|Bamboo build plan key like KEY'), required: true }, - { type: 'text', name: 'username', - placeholder: s_('BambooService|A user with API access, if applicable') }, - { type: 'password', name: 'password' } + { + type: 'text', + name: 'bamboo_url', + title: s_('BambooService|Bamboo URL'), + placeholder: s_('https://bamboo.example.com'), + help: s_('BambooService|Bamboo service root URL.'), + required: true + }, + { + type: 'text', + name: 'build_key', + placeholder: s_('KEY'), + help: s_('BambooService|Bamboo build plan key.'), + required: true + }, + { + type: 'text', + name: 'username', + help: s_('BambooService|The user with API access to the Bamboo server.') + }, + { + type: 'password', + name: 'password', + non_empty_password_title: s_('ProjectService|Enter new password'), + non_empty_password_help: s_('ProjectService|Leave blank to use your current password') + } ] end diff --git a/app/models/project_services/chat_message/merge_message.rb b/app/models/project_services/chat_message/merge_message.rb index b9916a54d75..e45bb9b8ce1 100644 --- a/app/models/project_services/chat_message/merge_message.rb +++ b/app/models/project_services/chat_message/merge_message.rb @@ -28,7 +28,7 @@ module ChatMessage def activity { - title: "Merge Request #{state_or_action_text} by #{user_combined_name}", + title: "Merge request #{state_or_action_text} by #{user_combined_name}", subtitle: "in #{project_link}", text: merge_request_link, image: user_avatar diff --git a/app/models/project_services/chat_notification_service.rb b/app/models/project_services/chat_notification_service.rb index cf7cad09676..4a99842b4d5 100644 --- a/app/models/project_services/chat_notification_service.rb +++ b/app/models/project_services/chat_notification_service.rb @@ -61,11 +61,11 @@ class ChatNotificationService < Service def default_fields [ - { type: 'text', name: 'webhook', placeholder: "e.g. #{webhook_placeholder}", required: true }.freeze, - { type: 'text', name: 'username', placeholder: 'e.g. GitLab' }.freeze, - { type: 'checkbox', name: 'notify_only_broken_pipelines' }.freeze, + { type: 'text', name: 'webhook', placeholder: "#{webhook_placeholder}", required: true }.freeze, + { type: 'text', name: 'username', placeholder: 'GitLab-integration' }.freeze, + { type: 'checkbox', name: 'notify_only_broken_pipelines', help: 'Do not send notifications for successful pipelines.' }.freeze, { type: 'select', name: 'branches_to_be_notified', choices: branch_choices }.freeze, - { type: 'text', name: 'labels_to_be_notified', placeholder: 'e.g. ~backend', help: 'Only supported for issue, merge request and note events.' }.freeze + { type: 'text', name: 'labels_to_be_notified', placeholder: '~backend,~frontend', help: 'Send notifications for issue, merge request, and comment events with the listed labels only. Leave blank to receive notifications for all events.' }.freeze ].freeze end diff --git a/app/models/project_services/ci_service.rb b/app/models/project_services/ci_service.rb index 47106d7bdbb..29edb9ec16f 100644 --- a/app/models/project_services/ci_service.rb +++ b/app/models/project_services/ci_service.rb @@ -2,7 +2,7 @@ # Base class for CI services # List methods you need to implement to get your CI service -# working with GitLab Merge Requests +# working with GitLab merge requests class CiService < Service default_value_for :category, 'ci' diff --git a/app/models/project_services/custom_issue_tracker_service.rb b/app/models/project_services/custom_issue_tracker_service.rb index fc58ba27c3d..aab8661ec55 100644 --- a/app/models/project_services/custom_issue_tracker_service.rb +++ b/app/models/project_services/custom_issue_tracker_service.rb @@ -17,9 +17,9 @@ class CustomIssueTrackerService < IssueTrackerService def fields [ - { type: 'text', name: 'project_url', placeholder: 'Project url', required: true }, - { type: 'text', name: 'issues_url', placeholder: 'Issue url', required: true }, - { type: 'text', name: 'new_issue_url', placeholder: 'New Issue url', required: true } + { type: 'text', name: 'project_url', title: _('Project URL'), required: true }, + { type: 'text', name: 'issues_url', title: s_('ProjectService|Issue URL'), required: true }, + { type: 'text', name: 'new_issue_url', title: s_('ProjectService|New issue URL'), required: true } ] end end diff --git a/app/models/project_services/datadog_service.rb b/app/models/project_services/datadog_service.rb index a48dea71645..9a2d99c46c9 100644 --- a/app/models/project_services/datadog_service.rb +++ b/app/models/project_services/datadog_service.rb @@ -78,7 +78,9 @@ class DatadogService < Service { type: 'password', name: 'api_key', - title: 'API key', + title: _('API key'), + non_empty_password_title: s_('ProjectService|Enter new API key'), + non_empty_password_help: s_('ProjectService|Leave blank to use your current API key'), help: "<a href=\"#{api_keys_url}\" target=\"_blank\">API key</a> used for authentication with Datadog", required: true }, diff --git a/app/models/project_services/discord_service.rb b/app/models/project_services/discord_service.rb index 37bbb9b8752..d7adf63fde4 100644 --- a/app/models/project_services/discord_service.rb +++ b/app/models/project_services/discord_service.rb @@ -3,6 +3,8 @@ require "discordrb/webhooks" class DiscordService < ChatNotificationService + include ActionView::Helpers::UrlHelper + ATTACHMENT_REGEX = /: (?<entry>.*?)\n - (?<name>.*)\n*/.freeze def title @@ -10,7 +12,7 @@ class DiscordService < ChatNotificationService end def description - s_("DiscordService|Receive event notifications in Discord") + s_("DiscordService|Send notifications about project events to a Discord channel.") end def self.to_param @@ -18,13 +20,8 @@ class DiscordService < ChatNotificationService end def help - "This service sends notifications about project events to Discord channels.<br /> - To set up this service: - <ol> - <li><a href='https://support.discordapp.com/hc/en-us/articles/228383668-Intro-to-Webhooks'>Setup a custom Incoming Webhook</a>.</li> - <li>Paste the <strong>Webhook URL</strong> into the field below.</li> - <li>Select events below to enable notifications.</li> - </ol>" + docs_link = link_to _('How do I set up this service?'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/discord_notifications'), target: '_blank', rel: 'noopener noreferrer' + s_('Send notifications about project events to a Discord channel. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end def event_field(event) @@ -36,13 +33,12 @@ class DiscordService < ChatNotificationService end def self.supported_events - %w[push issue confidential_issue merge_request note confidential_note tag_push - pipeline wiki_page] + %w[push issue confidential_issue merge_request note confidential_note tag_push pipeline wiki_page] end def default_fields [ - { type: "text", name: "webhook", placeholder: "e.g. https://discordapp.com/api/webhooks/…" }, + { type: "text", name: "webhook", placeholder: "https://discordapp.com/api/webhooks/…", help: "URL to the webhook for the Discord channel." }, { type: "checkbox", name: "notify_only_broken_pipelines" }, { type: 'select', name: 'branches_to_be_notified', choices: branch_choices } ] diff --git a/app/models/project_services/drone_ci_service.rb b/app/models/project_services/drone_ci_service.rb index 5a49f780d46..ab1ba768a8f 100644 --- a/app/models/project_services/drone_ci_service.rb +++ b/app/models/project_services/drone_ci_service.rb @@ -79,21 +79,25 @@ class DroneCiService < CiService end def title - 'Drone CI' + 'Drone' end def description - 'Drone is a Continuous Integration platform built on Docker, written in Go' + s_('ProjectService|Run CI/CD pipelines with Drone.') end def self.to_param 'drone_ci' end + def help + s_('ProjectService|Run CI/CD pipelines with Drone.') + end + def fields [ - { type: 'text', name: 'token', placeholder: 'Drone CI project specific token', required: true }, - { type: 'text', name: 'drone_url', placeholder: 'http://drone.example.com', required: true }, + { type: 'text', name: 'token', help: s_('ProjectService|Token for the Drone project.'), required: true }, + { type: 'text', name: 'drone_url', title: s_('ProjectService|Drone server URL'), placeholder: 'http://drone.example.com', required: true }, { type: 'checkbox', name: 'enable_ssl_verification', title: "Enable SSL verification" } ] end diff --git a/app/models/project_services/emails_on_push_service.rb b/app/models/project_services/emails_on_push_service.rb index 01d8647d439..cdb69684d16 100644 --- a/app/models/project_services/emails_on_push_service.rb +++ b/app/models/project_services/emails_on_push_service.rb @@ -3,10 +3,19 @@ class EmailsOnPushService < Service include NotificationBranchSelection + RECIPIENTS_LIMIT = 750 + boolean_accessor :send_from_committer_email boolean_accessor :disable_diffs prop_accessor :recipients, :branches_to_be_notified - validates :recipients, presence: true, if: :valid_recipients? + validates :recipients, presence: true, if: :validate_recipients? + validate :number_of_recipients_within_limit, if: :validate_recipients? + + def self.valid_recipients(recipients) + recipients.split.select do |recipient| + recipient.include?('@') + end.uniq(&:downcase) + end def title s_('EmailsOnPushService|Emails on push') @@ -63,11 +72,26 @@ class EmailsOnPushService < Service domains = Notify.allowed_email_domains.map { |domain| "user@#{domain}" }.join(", ") [ { type: 'checkbox', name: 'send_from_committer_email', title: s_("EmailsOnPushService|Send from committer"), - help: s_("EmailsOnPushService|Send notifications from the committer's email address if the domain is part of the domain GitLab is running on (e.g. %{domains}).") % { domains: domains } }, + help: s_("EmailsOnPushService|Send notifications from the committer's email address if the domain matches the domain used by your GitLab instance (such as %{domains}).") % { domains: domains } }, { type: 'checkbox', name: 'disable_diffs', title: s_("EmailsOnPushService|Disable code diffs"), help: s_("EmailsOnPushService|Don't include possibly sensitive code diffs in notification body.") }, { type: 'select', name: 'branches_to_be_notified', choices: branch_choices }, - { type: 'textarea', name: 'recipients', placeholder: s_('EmailsOnPushService|Emails separated by whitespace') } + { + type: 'textarea', + name: 'recipients', + placeholder: s_('EmailsOnPushService|tanuki@example.com gitlab@example.com'), + help: s_('EmailsOnPushService|Emails separated by whitespace.') + } ] end + + private + + def number_of_recipients_within_limit + return if recipients.blank? + + if self.class.valid_recipients(recipients).size > RECIPIENTS_LIMIT + errors.add(:recipients, s_("EmailsOnPushService|can't exceed %{recipients_limit}") % { recipients_limit: RECIPIENTS_LIMIT }) + end + end end diff --git a/app/models/project_services/external_wiki_service.rb b/app/models/project_services/external_wiki_service.rb index 0a09000fff4..c41783d1af4 100644 --- a/app/models/project_services/external_wiki_service.rb +++ b/app/models/project_services/external_wiki_service.rb @@ -1,16 +1,16 @@ # frozen_string_literal: true class ExternalWikiService < Service + include ActionView::Helpers::UrlHelper prop_accessor :external_wiki_url - validates :external_wiki_url, presence: true, public_url: true, if: :activated? def title - s_('ExternalWikiService|External Wiki') + s_('ExternalWikiService|External wiki') end def description - s_('ExternalWikiService|Replaces the link to the internal wiki with a link to an external wiki.') + s_('ExternalWikiService|Link to an external wiki from the sidebar.') end def self.to_param @@ -22,12 +22,20 @@ class ExternalWikiService < Service { type: 'text', name: 'external_wiki_url', - placeholder: s_('ExternalWikiService|The URL of the external Wiki'), + title: s_('ExternalWikiService|External wiki URL'), + placeholder: s_('ExternalWikiService|https://example.com/xxx/wiki/...'), + help: 'Enter the URL to the external wiki.', required: true } ] end + def help + docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/wiki/index', anchor: 'link-an-external-wiki'), target: '_blank', rel: 'noopener noreferrer' + + s_('Link an external wiki from the project\'s sidebar. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } + end + def execute(_data) response = Gitlab::HTTP.get(properties['external_wiki_url'], verify: true) response.body if response.code == 200 diff --git a/app/models/project_services/hipchat_service.rb b/app/models/project_services/hipchat_service.rb index 22c2aebaec3..cd49c6d253d 100644 --- a/app/models/project_services/hipchat_service.rb +++ b/app/models/project_services/hipchat_service.rb @@ -39,7 +39,7 @@ class HipchatService < Service { type: 'text', name: 'room', placeholder: 'Room name or ID' }, { type: 'checkbox', name: 'notify' }, { type: 'select', name: 'color', choices: %w(yellow red green purple gray random) }, - { type: 'text', name: 'api_version', + { type: 'text', name: 'api_version', title: _('API version'), placeholder: 'Leave blank for default (v2)' }, { type: 'text', name: 'server', placeholder: 'Leave blank for default. https://hipchat.example.com' }, diff --git a/app/models/project_services/irker_service.rb b/app/models/project_services/irker_service.rb index 4a6c8339625..4f1ce16ebb2 100644 --- a/app/models/project_services/irker_service.rb +++ b/app/models/project_services/irker_service.rb @@ -6,7 +6,7 @@ class IrkerService < Service prop_accessor :server_host, :server_port, :default_irc_uri prop_accessor :recipients, :channels boolean_accessor :colorize_messages - validates :recipients, presence: true, if: :valid_recipients? + validates :recipients, presence: true, if: :validate_recipients? before_validation :get_channels diff --git a/app/models/project_services/issue_tracker_service.rb b/app/models/project_services/issue_tracker_service.rb index 694374e9548..19a5b4a74bb 100644 --- a/app/models/project_services/issue_tracker_service.rb +++ b/app/models/project_services/issue_tracker_service.rb @@ -73,9 +73,9 @@ class IssueTrackerService < Service def fields [ - { type: 'text', name: 'project_url', placeholder: 'Project url', required: true }, - { type: 'text', name: 'issues_url', placeholder: 'Issue url', required: true }, - { type: 'text', name: 'new_issue_url', placeholder: 'New Issue url', required: true } + { type: 'text', name: 'project_url', title: _('Project URL'), required: true }, + { type: 'text', name: 'issues_url', title: s_('ProjectService|Issue URL'), required: true }, + { type: 'text', name: 'new_issue_url', title: s_('ProjectService|New issue URL'), required: true } ] end diff --git a/app/models/project_services/jenkins_service.rb b/app/models/project_services/jenkins_service.rb index 63ecfc66877..6a123517b84 100644 --- a/app/models/project_services/jenkins_service.rb +++ b/app/models/project_services/jenkins_service.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class JenkinsService < CiService + include ActionView::Helpers::UrlHelper + prop_accessor :jenkins_url, :project_name, :username, :password before_update :reset_password @@ -29,7 +31,6 @@ class JenkinsService < CiService end def execute(data) - return if project.disabled_services.include?(to_param) return unless supported_events.include?(data[:object_kind]) service_hook.execute(data, "#{data[:object_kind]}_hook") @@ -59,15 +60,16 @@ class JenkinsService < CiService end def title - 'Jenkins CI' + 'Jenkins' end def description - 'An extendable open source continuous integration server' + s_('An extendable open source CI/CD server.') end def help - "You must have installed the Git Plugin and GitLab Plugin in Jenkins. [More information](#{Gitlab::Routing.url_helpers.help_page_url('integration/jenkins')})" + docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('integration/jenkins'), target: '_blank', rel: 'noopener noreferrer' + s_('Trigger Jenkins builds when you push to a repository, or when a merge request is created, updated, or merged. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end def self.to_param @@ -77,15 +79,33 @@ class JenkinsService < CiService def fields [ { - type: 'text', name: 'jenkins_url', - placeholder: 'Jenkins URL like http://jenkins.example.com' + type: 'text', + name: 'jenkins_url', + title: s_('ProjectService|Jenkins server URL'), + required: true, + placeholder: 'http://jenkins.example.com', + help: s_('The URL of the Jenkins server.') + }, + { + type: 'text', + name: 'project_name', + required: true, + placeholder: 'my_project_name', + help: s_('The name of the Jenkins project. Copy the name from the end of the URL to the project.') }, { - type: 'text', name: 'project_name', placeholder: 'Project Name', - help: 'The URL-friendly project name. Example: my_project_name' + type: 'text', + name: 'username', + required: true, + help: s_('The username for the Jenkins server.') }, - { type: 'text', name: 'username' }, - { type: 'password', name: 'password' } + { + type: 'password', + name: 'password', + help: s_('The password for the Jenkins server.'), + non_empty_password_title: s_('ProjectService|Enter new password.'), + non_empty_password_help: s_('ProjectService|Leave blank to use your current password.') + } ] end end diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index 5857d86f921..3e14bf44c12 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -31,8 +31,8 @@ class JiraService < IssueTrackerService # TODO: we can probably just delegate as part of # https://gitlab.com/gitlab-org/gitlab/issues/29404 - data_field :username, :password, :url, :api_url, :jira_issue_transition_id, :project_key, :issues_enabled, - :vulnerabilities_enabled, :vulnerabilities_issuetype, :proxy_address, :proxy_port, :proxy_username, :proxy_password + data_field :username, :password, :url, :api_url, :jira_issue_transition_automatic, :jira_issue_transition_id, :project_key, :issues_enabled, + :vulnerabilities_enabled, :vulnerabilities_issuetype before_update :reset_password after_commit :update_deployment_type, on: [:create, :update], if: :update_deployment_type? @@ -116,7 +116,7 @@ class JiraService < IssueTrackerService end def description - s_('JiraService|Jira issue tracker') + s_('JiraService|Track issues in Jira') end def self.to_param @@ -124,15 +124,37 @@ class JiraService < IssueTrackerService end def fields - transition_id_help_path = help_page_path('user/project/integrations/jira', anchor: 'obtaining-a-transition-id') - transition_id_help_link_start = '<a href="%{transition_id_help_path}" target="_blank" rel="noopener noreferrer">'.html_safe % { transition_id_help_path: transition_id_help_path } - [ - { type: 'text', name: 'url', title: s_('JiraService|Web URL'), placeholder: 'https://jira.example.com', required: true }, - { type: 'text', name: 'api_url', title: s_('JiraService|Jira API URL'), placeholder: s_('JiraService|If different from Web URL') }, - { type: 'text', name: 'username', title: s_('JiraService|Username or Email'), placeholder: s_('JiraService|Use a username for server version and an email for cloud version'), required: true }, - { type: 'password', name: 'password', title: s_('JiraService|Password or API token'), placeholder: s_('JiraService|Use a password for server version and an API token for cloud version'), required: true }, - { type: 'text', name: 'jira_issue_transition_id', title: s_('JiraService|Jira workflow transition IDs'), placeholder: s_('JiraService|For example, 12, 24'), help: s_('JiraService|Set transition IDs for Jira workflow transitions. %{link_start}Learn more%{link_end}'.html_safe % { link_start: transition_id_help_link_start, link_end: '</a>'.html_safe }) } + { + type: 'text', + name: 'url', + title: s_('JiraService|Web URL'), + placeholder: 'https://jira.example.com', + help: s_('JiraService|Base URL of the Jira instance.'), + required: true + }, + { + type: 'text', + name: 'api_url', + title: s_('JiraService|Jira API URL'), + help: s_('JiraService|If different from Web URL.') + }, + { + type: 'text', + name: 'username', + title: s_('JiraService|Username or Email'), + help: s_('JiraService|Use a username for server version and an email for cloud version.'), + required: true + }, + { + type: 'password', + name: 'password', + title: s_('JiraService|Password or API token'), + non_empty_password_title: s_('JiraService|Enter new password or API token'), + non_empty_password_help: s_('JiraService|Leave blank to use your current password or API token.'), + help: s_('JiraService|Use a password for server version and an API token for cloud version.'), + required: true + } ] end @@ -159,17 +181,19 @@ class JiraService < IssueTrackerService # support any events. end - def find_issue(issue_key, rendered_fields: false) - options = {} - options = options.merge(expand: 'renderedFields') if rendered_fields + def find_issue(issue_key, rendered_fields: false, transitions: false) + expands = [] + expands << 'renderedFields' if rendered_fields + expands << 'transitions' if transitions + options = { expand: expands.join(',') } if expands.any? - jira_request { client.Issue.find(issue_key, options) } + jira_request { client.Issue.find(issue_key, options || {}) } end def close_issue(entity, external_issue, current_user) - issue = find_issue(external_issue.iid) + issue = find_issue(external_issue.iid, transitions: jira_issue_transition_automatic) - return if issue.nil? || has_resolution?(issue) || !jira_issue_transition_id.present? + return if issue.nil? || has_resolution?(issue) || !issue_transition_enabled? commit_id = case entity when Commit then entity.id @@ -244,6 +268,10 @@ class JiraService < IssueTrackerService true end + def issue_transition_enabled? + jira_issue_transition_automatic || jira_issue_transition_id.present? + end + private def server_info @@ -264,20 +292,44 @@ class JiraService < IssueTrackerService # the issue is transitioned at the order given by the user # if any transition fails it will log the error message and stop the transition sequence def transition_issue(issue) - jira_issue_transition_id.scan(Gitlab::Regex.jira_transition_id_regex).each do |transition_id| - issue.transitions.build.save!(transition: { id: transition_id }) - rescue => error - log_error( - "Issue transition failed", - error: { - exception_class: error.class.name, - exception_message: error.message, - exception_backtrace: Gitlab::BacktraceCleaner.clean_backtrace(error.backtrace) - }, - client_url: client_url - ) - return false + return transition_issue_to_done(issue) if jira_issue_transition_automatic + + jira_issue_transition_id.scan(Gitlab::Regex.jira_transition_id_regex).all? do |transition_id| + transition_issue_to_id(issue, transition_id) + end + end + + def transition_issue_to_id(issue, transition_id) + issue.transitions.build.save!( + transition: { id: transition_id } + ) + + true + rescue => error + log_error( + "Issue transition failed", + error: { + exception_class: error.class.name, + exception_message: error.message, + exception_backtrace: Gitlab::BacktraceCleaner.clean_backtrace(error.backtrace) + }, + client_url: client_url + ) + + false + end + + def transition_issue_to_done(issue) + transitions = issue.transitions rescue [] + + transition = transitions.find do |transition| + status = transition&.to&.statusCategory + status && status['key'] == 'done' end + + return false unless transition + + transition_issue_to_id(issue, transition.id) end def log_usage(action, user) diff --git a/app/models/project_services/jira_tracker_data.rb b/app/models/project_services/jira_tracker_data.rb index 6cbcb1550c1..2c145abf5c9 100644 --- a/app/models/project_services/jira_tracker_data.rb +++ b/app/models/project_services/jira_tracker_data.rb @@ -2,20 +2,23 @@ class JiraTrackerData < ApplicationRecord include Services::DataFields + include IgnorableColumns + + ignore_columns %i[ + encrypted_proxy_address + encrypted_proxy_address_iv + encrypted_proxy_port + encrypted_proxy_port_iv + encrypted_proxy_username + encrypted_proxy_username_iv + encrypted_proxy_password + encrypted_proxy_password_iv + ], remove_with: '14.0', remove_after: '2021-05-22' attr_encrypted :url, encryption_options attr_encrypted :api_url, encryption_options attr_encrypted :username, encryption_options attr_encrypted :password, encryption_options - attr_encrypted :proxy_address, encryption_options - attr_encrypted :proxy_port, encryption_options - attr_encrypted :proxy_username, encryption_options - attr_encrypted :proxy_password, encryption_options - - validates :proxy_address, length: { maximum: 2048 } - validates :proxy_port, length: { maximum: 5 } - validates :proxy_username, length: { maximum: 255 } - validates :proxy_password, length: { maximum: 255 } enum deployment_type: { unknown: 0, server: 1, cloud: 2 }, _prefix: :deployment end diff --git a/app/models/project_services/mattermost_service.rb b/app/models/project_services/mattermost_service.rb index 9cff979fcf2..732a7c32a03 100644 --- a/app/models/project_services/mattermost_service.rb +++ b/app/models/project_services/mattermost_service.rb @@ -2,13 +2,14 @@ class MattermostService < ChatNotificationService include SlackMattermost::Notifier + include ActionView::Helpers::UrlHelper def title - 'Mattermost notifications' + s_('Mattermost notifications') end def description - 'Receive event notifications in Mattermost' + s_('Send notifications about project events to Mattermost channels.') end def self.to_param @@ -16,21 +17,15 @@ class MattermostService < ChatNotificationService end def help - 'This service sends notifications about projects events to Mattermost channels.<br /> - To set up this service: - <ol> - <li><a href="https://docs.mattermost.com/developer/webhooks-incoming.html#enabling-incoming-webhooks">Enable incoming webhooks</a> in your Mattermost installation.</li> - <li><a href="https://docs.mattermost.com/developer/webhooks-incoming.html#creating-integrations-using-incoming-webhooks">Add an incoming webhook</a> in your Mattermost team. The default channel can be overridden for each event.</li> - <li>Paste the webhook <strong>URL</strong> into the field below.</li> - <li>Select events below to enable notifications. The <strong>Channel handle</strong> and <strong>Username</strong> fields are optional.</li> - </ol>' + docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/mattermost'), target: '_blank', rel: 'noopener noreferrer' + s_('Send notifications about project events to Mattermost channels. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end def default_channel_placeholder - "Channel handle (e.g. town-square)" + 'my-channel' end def webhook_placeholder - 'http://mattermost.example.com/hooks/…' + 'http://mattermost.example.com/hooks/' end end diff --git a/app/models/project_services/mattermost_slash_commands_service.rb b/app/models/project_services/mattermost_slash_commands_service.rb index f39d3947e5b..60235a09dcd 100644 --- a/app/models/project_services/mattermost_slash_commands_service.rb +++ b/app/models/project_services/mattermost_slash_commands_service.rb @@ -14,7 +14,7 @@ class MattermostSlashCommandsService < SlashCommandsService end def description - "Perform common operations in Mattermost" + "Perform common tasks with slash commands." end def self.to_param diff --git a/app/models/project_services/microsoft_teams_service.rb b/app/models/project_services/microsoft_teams_service.rb index e8e12a9a206..803c1255195 100644 --- a/app/models/project_services/microsoft_teams_service.rb +++ b/app/models/project_services/microsoft_teams_service.rb @@ -2,7 +2,7 @@ class MicrosoftTeamsService < ChatNotificationService def title - 'Microsoft Teams Notification' + 'Microsoft Teams notifications' end def description @@ -14,13 +14,7 @@ class MicrosoftTeamsService < ChatNotificationService end def help - 'This service sends notifications about projects events to Microsoft Teams channels.<br /> - To set up this service: - <ol> - <li><a href="https://docs.microsoft.com/en-us/microsoftteams/platform/concepts/connectors/connectors-using#setting-up-a-custom-incoming-webhook">Setup a custom Incoming Webhook using Office 365 Connectors For Microsoft Teams</a>.</li> - <li>Paste the <strong>Webhook URL</strong> into the field below.</li> - <li>Select events below to enable notifications.</li> - </ol>' + '<p>Use this service to send notifications about events in GitLab projects to your Microsoft Teams channels. <a href="https://docs.gitlab.com/ee/user/project/integrations/microsoft_teams.html">How do I configure this integration?</a></p>' end def webhook_placeholder @@ -40,8 +34,8 @@ class MicrosoftTeamsService < ChatNotificationService def default_fields [ - { type: 'text', name: 'webhook', placeholder: "e.g. #{webhook_placeholder}" }, - { type: 'checkbox', name: 'notify_only_broken_pipelines' }, + { type: 'text', name: 'webhook', placeholder: "#{webhook_placeholder}" }, + { type: 'checkbox', name: 'notify_only_broken_pipelines', help: 'If selected, successful pipelines do not trigger a notification event.' }, { type: 'select', name: 'branches_to_be_notified', choices: branch_choices } ] end diff --git a/app/models/project_services/mock_ci_service.rb b/app/models/project_services/mock_ci_service.rb index c5e5f4f6400..bd6344c6e1a 100644 --- a/app/models/project_services/mock_ci_service.rb +++ b/app/models/project_services/mock_ci_service.rb @@ -21,10 +21,13 @@ class MockCiService < CiService def fields [ - { type: 'text', + { + type: 'text', name: 'mock_service_url', + title: s_('ProjectService|Mock service URL'), placeholder: 'http://localhost:4004', - required: true } + required: true + } ] end diff --git a/app/models/project_services/pipelines_email_service.rb b/app/models/project_services/pipelines_email_service.rb index 8af4cd952c9..0a0a41c525c 100644 --- a/app/models/project_services/pipelines_email_service.rb +++ b/app/models/project_services/pipelines_email_service.rb @@ -5,7 +5,7 @@ class PipelinesEmailService < Service prop_accessor :recipients, :branches_to_be_notified boolean_accessor :notify_only_broken_pipelines, :notify_only_default_branch - validates :recipients, presence: true, if: :valid_recipients? + validates :recipients, presence: true, if: :validate_recipients? def initialize_properties if properties.nil? @@ -25,11 +25,11 @@ class PipelinesEmailService < Service end def title - _('Pipelines emails') + _('Pipeline status emails') end def description - _('Email the pipelines status to a list of recipients.') + _('Email the pipeline status to a list of recipients.') end def self.to_param @@ -64,7 +64,7 @@ class PipelinesEmailService < Service [ { type: 'textarea', name: 'recipients', - placeholder: _('Emails separated by comma'), + help: _('Comma-separated list of email addresses.'), required: true }, { type: 'checkbox', name: 'notify_only_broken_pipelines' }, diff --git a/app/models/project_services/pushover_service.rb b/app/models/project_services/pushover_service.rb index 7324890551c..1781ec7456d 100644 --- a/app/models/project_services/pushover_service.rb +++ b/app/models/project_services/pushover_service.rb @@ -20,7 +20,7 @@ class PushoverService < Service def fields [ - { type: 'text', name: 'api_key', placeholder: s_('PushoverService|Your application key'), required: true }, + { type: 'text', name: 'api_key', title: _('API key'), placeholder: s_('PushoverService|Your application key'), required: true }, { type: 'text', name: 'user_key', placeholder: s_('PushoverService|Your user key'), required: true }, { type: 'text', name: 'device', placeholder: s_('PushoverService|Leave blank for all active devices') }, { type: 'select', name: 'priority', required: true, choices: diff --git a/app/models/project_services/redmine_service.rb b/app/models/project_services/redmine_service.rb index df78520d65f..26a6cf86bf4 100644 --- a/app/models/project_services/redmine_service.rb +++ b/app/models/project_services/redmine_service.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true class RedmineService < IssueTrackerService + include ActionView::Helpers::UrlHelper validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? def title @@ -8,7 +9,12 @@ class RedmineService < IssueTrackerService end def description - s_('IssueTracker|Redmine issue tracker') + s_('IssueTracker|Use Redmine as the issue tracker.') + end + + def help + docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/redmine'), target: '_blank', rel: 'noopener noreferrer' + s_('IssueTracker|Use Redmine as the issue tracker. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end def self.to_param diff --git a/app/models/project_services/slack_service.rb b/app/models/project_services/slack_service.rb index f42b3de39d5..7badcc24870 100644 --- a/app/models/project_services/slack_service.rb +++ b/app/models/project_services/slack_service.rb @@ -16,7 +16,7 @@ class SlackService < ChatNotificationService end def description - 'Receive event notifications in Slack' + 'Send notifications about project events to Slack.' end def self.to_param @@ -24,7 +24,7 @@ class SlackService < ChatNotificationService end def default_channel_placeholder - _('Slack channels (e.g. general, development)') + _('general, development') end def webhook_placeholder diff --git a/app/models/project_services/teamcity_service.rb b/app/models/project_services/teamcity_service.rb index 209b691ef98..6fc24a4778c 100644 --- a/app/models/project_services/teamcity_service.rb +++ b/app/models/project_services/teamcity_service.rb @@ -51,27 +51,43 @@ class TeamcityService < CiService end def title - 'JetBrains TeamCity CI' + 'JetBrains TeamCity' end def description - 'A continuous integration and build server' + s_('ProjectService|Run CI/CD pipelines with JetBrains TeamCity.') end def help - 'You will want to configure monitoring of all branches so merge '\ - 'requests build, that setting is in the vsc root advanced settings.' + s_('To run CI/CD pipelines with JetBrains TeamCity, input the GitLab project details in the TeamCity project Version Control Settings.') end def fields [ - { type: 'text', name: 'teamcity_url', - placeholder: 'TeamCity root URL like https://teamcity.example.com', required: true }, - { type: 'text', name: 'build_type', - placeholder: 'Build configuration ID', required: true }, - { type: 'text', name: 'username', - placeholder: 'A user with permissions to trigger a manual build' }, - { type: 'password', name: 'password' } + { + type: 'text', + name: 'teamcity_url', + title: s_('ProjectService|TeamCity server URL'), + placeholder: 'https://teamcity.example.com', + required: true + }, + { + type: 'text', + name: 'build_type', + help: s_('ProjectService|The build configuration ID of the TeamCity project.'), + required: true + }, + { + type: 'text', + name: 'username', + help: s_('ProjectService|Must have permission to trigger a manual build in TeamCity.') + }, + { + type: 'password', + name: 'password', + non_empty_password_title: s_('ProjectService|Enter new password'), + non_empty_password_help: s_('ProjectService|Leave blank to use your current password') + } ] end diff --git a/app/models/project_services/youtrack_service.rb b/app/models/project_services/youtrack_service.rb index 7fb3bde44a5..30abd0159b3 100644 --- a/app/models/project_services/youtrack_service.rb +++ b/app/models/project_services/youtrack_service.rb @@ -26,8 +26,8 @@ class YoutrackService < IssueTrackerService def fields [ - { type: 'text', name: 'project_url', title: 'Project URL', placeholder: 'Project URL', required: true }, - { type: 'text', name: 'issues_url', title: 'Issue URL', placeholder: 'Issue URL', required: true } + { type: 'text', name: 'project_url', title: _('Project URL'), required: true }, + { type: 'text', name: 'issues_url', title: s_('ProjectService|Issue URL'), required: true } ] end end diff --git a/app/models/project_team.rb b/app/models/project_team.rb index 5b7eded00cd..1a3f362e6a1 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -174,6 +174,10 @@ class ProjectTeam end end + def write_member_access_for_user_id(user_id, project_access_level) + merge_value_to_request_store(User, user_id, project.id, project_access_level) + end + def max_member_access(user_id) max_member_access_for_user_ids([user_id])[user_id] end diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index cbbdd091feb..963a6b7774a 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -30,7 +30,7 @@ class ProtectedBranch < ApplicationRecord end def self.allow_force_push?(project, ref_name) - return false unless ::Feature.enabled?(:allow_force_push_to_protected_branches, project) + return false unless ::Feature.enabled?(:allow_force_push_to_protected_branches, project, default_enabled: :yaml) project.protected_branches.allowing_force_push.matching(ref_name).any? end diff --git a/app/models/raw_usage_data.rb b/app/models/raw_usage_data.rb index 06cd4ad3f6c..6fe3b26b58b 100644 --- a/app/models/raw_usage_data.rb +++ b/app/models/raw_usage_data.rb @@ -4,7 +4,7 @@ class RawUsageData < ApplicationRecord validates :payload, presence: true validates :recorded_at, presence: true, uniqueness: true - def update_sent_at! - self.update_column(:sent_at, Time.current) + def update_version_metadata!(usage_data_id:) + self.update_columns(sent_at: Time.current, version_usage_data_id_value: usage_data_id) end end diff --git a/app/models/release.rb b/app/models/release.rb index 60c2abcacb3..5ca8f537baa 100644 --- a/app/models/release.rb +++ b/app/models/release.rb @@ -8,7 +8,7 @@ class Release < ApplicationRecord cache_markdown_field :description - belongs_to :project + belongs_to :project, touch: true # releases prior to 11.7 have no author belongs_to :author, class_name: 'User' diff --git a/app/models/release_highlight.rb b/app/models/release_highlight.rb index 1efba6380e9..98d9899a349 100644 --- a/app/models/release_highlight.rb +++ b/app/models/release_highlight.rb @@ -3,17 +3,6 @@ class ReleaseHighlight CACHE_DURATION = 1.hour FILES_PATH = Rails.root.join('data', 'whats_new', '*.yml') - RELEASE_VERSIONS_IN_A_YEAR = 12 - - def self.for_version(version:) - index = self.versions.index(version) - - return if index.nil? - - page = index + 1 - - self.paginated(page: page) - end def self.paginated(page: 1) key = self.cache_key("items:page-#{page}") @@ -82,15 +71,15 @@ class ReleaseHighlight end end - def self.versions - key = self.cache_key('versions') + def self.most_recent_version_digest + key = self.cache_key('most_recent_version_digest') Gitlab::ProcessMemoryCache.cache_backend.fetch(key, expires_in: CACHE_DURATION) do - versions = self.file_paths.first(RELEASE_VERSIONS_IN_A_YEAR).map do |path| - /\d*\_(\d*\_\d*)\.yml$/.match(path).captures[0].gsub(/0(?=\d)/, "").tr("_", ".") - end + version = self.paginated&.items&.first&.[]('release')&.to_s + + next if version.nil? - versions.uniq + Digest::SHA256.hexdigest(version) end end diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb index 880970b72a8..c7387d2197d 100644 --- a/app/models/remote_mirror.rb +++ b/app/models/remote_mirror.rb @@ -84,13 +84,7 @@ class RemoteMirror < ApplicationRecord end after_transition started: :failed do |remote_mirror| - Gitlab::Metrics.add_event(:remote_mirrors_failed) - - remote_mirror.update(last_update_at: Time.current) - - remote_mirror.run_after_commit do - RemoteMirrorNotificationWorker.perform_async(remote_mirror.id) - end + remote_mirror.send_failure_notifications end end @@ -188,6 +182,24 @@ class RemoteMirror < ApplicationRecord update_fail! end + # Force the mrror into the retry state + def hard_retry!(error_message) + update_error_message(error_message) + self.update_status = :to_retry + + save!(validate: false) + end + + # Force the mirror into the failed state + def hard_fail!(error_message) + update_error_message(error_message) + self.update_status = :failed + + save!(validate: false) + + send_failure_notifications + end + def url=(value) super(value) && return unless Gitlab::UrlSanitizer.valid?(value) @@ -207,7 +219,7 @@ class RemoteMirror < ApplicationRecord end def safe_url - super(usernames_whitelist: %w[git]) + super(allowed_usernames: %w[git]) end def bare_url @@ -239,6 +251,17 @@ class RemoteMirror < ApplicationRecord last_update_at.present? ? MAX_INCREMENTAL_RUNTIME : MAX_FIRST_RUNTIME end + def send_failure_notifications + Gitlab::Metrics.add_event(:remote_mirrors_failed) + + run_after_commit do + RemoteMirrorNotificationWorker.perform_async(id) + end + + self.last_update_at = Time.current + save!(validate: false) + end + private def store_credentials diff --git a/app/models/repository.rb b/app/models/repository.rb index 84ca8f0c12a..b2efc9b480b 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -288,6 +288,10 @@ class Repository false end + def search_branch_names(pattern) + redis_set_cache.search('branch_names', pattern) { branch_names } + end + def languages return [] if empty? @@ -829,12 +833,6 @@ class Repository end end - def merge_to_ref(user, source_sha, merge_request, target_ref, message, first_parent_ref, allow_conflicts = false) - branch = merge_request.target_branch - - raw.merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts) - end - def delete_refs(*ref_names) raw.delete_refs(*ref_names) end @@ -995,6 +993,12 @@ class Repository raw_repository.search_files_by_name(query, ref) end + def search_files_by_wildcard_path(path, ref = 'HEAD') + # We need to use RE2 to match Gitaly's regexp engine + regexp_string = RE2::Regexp.escape(path).gsub('\*', '.*?') + raw_repository.search_files_by_regexp("^#{regexp_string}$", ref) + end + def copy_gitattributes(ref) actual_ref = ref || root_ref begin diff --git a/app/models/sent_notification.rb b/app/models/sent_notification.rb index 4165d3b753f..5d7b3879d75 100644 --- a/app/models/sent_notification.rb +++ b/app/models/sent_notification.rb @@ -48,7 +48,7 @@ class SentNotification < ApplicationRecord end def record_note(note, recipient_id, reply_key = self.reply_key, attrs = {}) - attrs[:in_reply_to_discussion_id] = note.discussion_id if note.part_of_discussion? + attrs[:in_reply_to_discussion_id] = note.discussion_id if note.part_of_discussion? || note.can_be_discussion_note? record(note.noteable, recipient_id, reply_key, attrs) end diff --git a/app/models/service.rb b/app/models/service.rb index c49e0869b21..aadc75ae710 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -11,14 +11,14 @@ class Service < ApplicationRecord include EachBatch SERVICE_NAMES = %w[ - asana assembla bamboo bugzilla buildkite campfire confluence custom_issue_tracker datadog discord - drone_ci emails_on_push ewm external_wiki flowdock hangouts_chat hipchat irker jira + asana assembla bamboo bugzilla buildkite campfire confluence custom_issue_tracker discord + drone_ci emails_on_push ewm external_wiki flowdock hangouts_chat irker jira mattermost mattermost_slash_commands microsoft_teams packagist pipelines_email pivotaltracker prometheus pushover redmine slack slack_slash_commands teamcity unify_circuit webex_teams youtrack ].freeze PROJECT_SPECIFIC_SERVICE_NAMES = %w[ - jenkins + datadog jenkins ].freeze # Fake services to help with local development. @@ -413,6 +413,10 @@ class Service < ApplicationRecord !instance? && !group_id end + def project_level? + project_id.present? + end + def parent project || group end @@ -456,7 +460,7 @@ class Service < ApplicationRecord errors.add(:project_id, 'The service cannot belong to both a project and a group') if project_id && group_id end - def valid_recipients? + def validate_recipients? activated? && !importing? end end diff --git a/app/models/sidebars/context.rb b/app/models/sidebars/context.rb new file mode 100644 index 00000000000..d9ac2705aaf --- /dev/null +++ b/app/models/sidebars/context.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# This class stores all the information needed to display and +# render the sidebar and menus. +# It usually stores information regarding the context and calculated +# values where the logic is in helpers. +module Sidebars + class Context + attr_reader :current_user, :container + + def initialize(current_user:, container:, **args) + @current_user = current_user + @container = container + + args.each do |key, value| + singleton_class.public_send(:attr_reader, key) # rubocop:disable GitlabSecurity/PublicSend + instance_variable_set("@#{key}", value) + end + end + end +end diff --git a/app/models/sidebars/menu.rb b/app/models/sidebars/menu.rb new file mode 100644 index 00000000000..a5c8be2bb31 --- /dev/null +++ b/app/models/sidebars/menu.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Sidebars + class Menu + extend ::Gitlab::Utils::Override + include ::Gitlab::Routing + include GitlabRoutingHelper + include Gitlab::Allowable + include ::Sidebars::HasPill + include ::Sidebars::HasIcon + include ::Sidebars::PositionableList + include ::Sidebars::Renderable + include ::Sidebars::ContainerWithHtmlOptions + include ::Sidebars::HasActiveRoutes + + attr_reader :context + delegate :current_user, :container, to: :@context + + def initialize(context) + @context = context + @items = [] + + configure_menu_items + end + + def configure_menu_items + # No-op + end + + override :render? + def render? + @items.empty? || renderable_items.any? + end + + # Menus might have or not a link + override :link + def link + nil + end + + # This method normalizes the information retrieved from the submenus and this menu + # Value from menus is something like: [{ path: 'foo', path: 'bar', controller: :foo }] + # This method filters the information and returns: { path: ['foo', 'bar'], controller: :foo } + def all_active_routes + @all_active_routes ||= begin + ([active_routes] + renderable_items.map(&:active_routes)).flatten.each_with_object({}) do |pairs, hash| + pairs.each do |k, v| + hash[k] ||= [] + hash[k] += Array(v) + hash[k].uniq! + end + + hash + end + end + end + + def has_items? + @items.any? + end + + def add_item(item) + add_element(@items, item) + end + + def insert_item_before(before_item, new_item) + insert_element_before(@items, before_item, new_item) + end + + def insert_item_after(after_item, new_item) + insert_element_after(@items, after_item, new_item) + end + + def has_renderable_items? + renderable_items.any? + end + + def renderable_items + @renderable_items ||= @items.select(&:render?) + end + end +end diff --git a/app/models/sidebars/menu_item.rb b/app/models/sidebars/menu_item.rb new file mode 100644 index 00000000000..7466b31898e --- /dev/null +++ b/app/models/sidebars/menu_item.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Sidebars + class MenuItem + extend ::Gitlab::Utils::Override + include ::Gitlab::Routing + include GitlabRoutingHelper + include Gitlab::Allowable + include ::Sidebars::HasIcon + include ::Sidebars::HasHint + include ::Sidebars::Renderable + include ::Sidebars::ContainerWithHtmlOptions + include ::Sidebars::HasActiveRoutes + + attr_reader :context + + def initialize(context) + @context = context + end + end +end diff --git a/app/models/sidebars/panel.rb b/app/models/sidebars/panel.rb new file mode 100644 index 00000000000..5c8191ebda3 --- /dev/null +++ b/app/models/sidebars/panel.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Sidebars + class Panel + extend ::Gitlab::Utils::Override + include ::Sidebars::PositionableList + + attr_reader :context, :scope_menu, :hidden_menu + + def initialize(context) + @context = context + @scope_menu = nil + @hidden_menu = nil + @menus = [] + + configure_menus + end + + def configure_menus + # No-op + end + + def add_menu(menu) + add_element(@menus, menu) + end + + def insert_menu_before(before_menu, new_menu) + insert_element_before(@menus, before_menu, new_menu) + end + + def insert_menu_after(after_menu, new_menu) + insert_element_after(@menus, after_menu, new_menu) + end + + def set_scope_menu(scope_menu) + @scope_menu = scope_menu + end + + def set_hidden_menu(hidden_menu) + @hidden_menu = hidden_menu + end + + def aria_label + raise NotImplementedError + end + + def has_renderable_menus? + renderable_menus.any? + end + + def renderable_menus + @renderable_menus ||= @menus.select(&:render?) + end + + def container + context.container + end + + # Auxiliar method that helps with the migration from + # regular views to the new logic + def render_raw_scope_menu_partial + # No-op + end + + # Auxiliar method that helps with the migration from + # regular views to the new logic. + # + # Any menu inside this partial will be added after + # all the menus added in the `configure_menus` + # method. + def render_raw_menus_partial + # No-op + end + end +end diff --git a/app/models/sidebars/projects/context.rb b/app/models/sidebars/projects/context.rb new file mode 100644 index 00000000000..4c82309035d --- /dev/null +++ b/app/models/sidebars/projects/context.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + class Context < ::Sidebars::Context + def initialize(current_user:, container:, **args) + super(current_user: current_user, container: container, project: container, **args) + end + end + end +end diff --git a/app/models/sidebars/projects/menus/learn_gitlab/menu.rb b/app/models/sidebars/projects/menus/learn_gitlab/menu.rb new file mode 100644 index 00000000000..4b572846d1a --- /dev/null +++ b/app/models/sidebars/projects/menus/learn_gitlab/menu.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + module LearnGitlab + class Menu < ::Sidebars::Menu + override :link + def link + project_learn_gitlab_path(context.project) + end + + override :active_routes + def active_routes + { controller: :learn_gitlab } + end + + override :title + def title + _('Learn GitLab') + end + + override :extra_container_html_options + def nav_link_html_options + { class: 'home' } + end + + override :sprite_icon + def sprite_icon + 'home' + end + + override :render? + def render? + context.learn_gitlab_experiment_enabled + end + end + end + end + end +end diff --git a/app/models/sidebars/projects/menus/project_overview/menu.rb b/app/models/sidebars/projects/menus/project_overview/menu.rb new file mode 100644 index 00000000000..e6aa8ed159f --- /dev/null +++ b/app/models/sidebars/projects/menus/project_overview/menu.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + module ProjectOverview + class Menu < ::Sidebars::Menu + override :configure_menu_items + def configure_menu_items + add_item(MenuItems::Details.new(context)) + add_item(MenuItems::Activity.new(context)) + add_item(MenuItems::Releases.new(context)) + end + + override :link + def link + project_path(context.project) + end + + override :extra_container_html_options + def extra_container_html_options + { + class: 'shortcuts-project rspec-project-link' + } + end + + override :extra_container_html_options + def nav_link_html_options + { class: 'home' } + end + + override :title + def title + _('Project overview') + end + + override :sprite_icon + def sprite_icon + 'home' + end + end + end + end + end +end diff --git a/app/models/sidebars/projects/menus/project_overview/menu_items/activity.rb b/app/models/sidebars/projects/menus/project_overview/menu_items/activity.rb new file mode 100644 index 00000000000..46d0f0bc43b --- /dev/null +++ b/app/models/sidebars/projects/menus/project_overview/menu_items/activity.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + module ProjectOverview + module MenuItems + class Activity < ::Sidebars::MenuItem + override :link + def link + activity_project_path(context.project) + end + + override :extra_container_html_options + def extra_container_html_options + { + class: 'shortcuts-project-activity' + } + end + + override :active_routes + def active_routes + { path: 'projects#activity' } + end + + override :title + def title + _('Activity') + end + end + end + end + end + end +end diff --git a/app/models/sidebars/projects/menus/project_overview/menu_items/details.rb b/app/models/sidebars/projects/menus/project_overview/menu_items/details.rb new file mode 100644 index 00000000000..c40c2ed8fa2 --- /dev/null +++ b/app/models/sidebars/projects/menus/project_overview/menu_items/details.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + module ProjectOverview + module MenuItems + class Details < ::Sidebars::MenuItem + override :link + def link + project_path(context.project) + end + + override :extra_container_html_options + def extra_container_html_options + { + aria: { label: _('Project details') }, + class: 'shortcuts-project' + } + end + + override :active_routes + def active_routes + { path: 'projects#show' } + end + + override :title + def title + _('Details') + end + end + end + end + end + end +end diff --git a/app/models/sidebars/projects/menus/project_overview/menu_items/releases.rb b/app/models/sidebars/projects/menus/project_overview/menu_items/releases.rb new file mode 100644 index 00000000000..5e8348f4398 --- /dev/null +++ b/app/models/sidebars/projects/menus/project_overview/menu_items/releases.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + module ProjectOverview + module MenuItems + class Releases < ::Sidebars::MenuItem + override :link + def link + project_releases_path(context.project) + end + + override :extra_container_html_options + def extra_container_html_options + { + class: 'shortcuts-project-releases' + } + end + + override :render? + def render? + can?(context.current_user, :read_release, context.project) && !context.project.empty_repo? + end + + override :active_routes + def active_routes + { controller: :releases } + end + + override :title + def title + _('Releases') + end + end + end + end + end + end +end diff --git a/app/models/sidebars/projects/menus/repository/menu.rb b/app/models/sidebars/projects/menus/repository/menu.rb new file mode 100644 index 00000000000..f49a0479521 --- /dev/null +++ b/app/models/sidebars/projects/menus/repository/menu.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + module Repository + class Menu < ::Sidebars::Menu + override :configure_menu_items + def configure_menu_items + add_item(MenuItems::Files.new(context)) + add_item(MenuItems::Commits.new(context)) + add_item(MenuItems::Branches.new(context)) + add_item(MenuItems::Tags.new(context)) + add_item(MenuItems::Contributors.new(context)) + add_item(MenuItems::Graphs.new(context)) + add_item(MenuItems::Compare.new(context)) + end + + override :link + def link + project_tree_path(context.project) + end + + override :extra_container_html_options + def extra_container_html_options + { + class: 'shortcuts-tree' + } + end + + override :title + def title + _('Repository') + end + + override :title_html_options + def title_html_options + { + id: 'js-onboarding-repo-link' + } + end + + override :sprite_icon + def sprite_icon + 'doc-text' + end + + override :render? + def render? + can?(context.current_user, :download_code, context.project) && + !context.project.empty_repo? + end + end + end + end + end +end + +Sidebars::Projects::Menus::Repository::Menu.prepend_if_ee('EE::Sidebars::Projects::Menus::Repository::Menu') diff --git a/app/models/sidebars/projects/menus/repository/menu_items/branches.rb b/app/models/sidebars/projects/menus/repository/menu_items/branches.rb new file mode 100644 index 00000000000..4a62803dd2b --- /dev/null +++ b/app/models/sidebars/projects/menus/repository/menu_items/branches.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + module Repository + module MenuItems + class Branches < ::Sidebars::MenuItem + override :link + def link + project_branches_path(context.project) + end + + override :extra_container_html_options + def extra_container_html_options + { + id: 'js-onboarding-branches-link' + } + end + + override :active_routes + def active_routes + { controller: :branches } + end + + override :title + def title + _('Branches') + end + end + end + end + end + end +end diff --git a/app/models/sidebars/projects/menus/repository/menu_items/commits.rb b/app/models/sidebars/projects/menus/repository/menu_items/commits.rb new file mode 100644 index 00000000000..647cf89133e --- /dev/null +++ b/app/models/sidebars/projects/menus/repository/menu_items/commits.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + module Repository + module MenuItems + class Commits < ::Sidebars::MenuItem + override :link + def link + project_commits_path(context.project, context.current_ref) + end + + override :extra_container_html_options + def extra_container_html_options + { + id: 'js-onboarding-commits-link' + } + end + + override :active_routes + def active_routes + { controller: %w(commit commits) } + end + + override :title + def title + _('Commits') + end + end + end + end + end + end +end diff --git a/app/models/sidebars/projects/menus/repository/menu_items/compare.rb b/app/models/sidebars/projects/menus/repository/menu_items/compare.rb new file mode 100644 index 00000000000..4812636b63f --- /dev/null +++ b/app/models/sidebars/projects/menus/repository/menu_items/compare.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + module Repository + module MenuItems + class Compare < ::Sidebars::MenuItem + override :link + def link + project_compare_index_path(context.project, from: context.project.repository.root_ref, to: context.current_ref) + end + + override :active_routes + def active_routes + { controller: :compare } + end + + override :title + def title + _('Compare') + end + end + end + end + end + end +end diff --git a/app/models/sidebars/projects/menus/repository/menu_items/contributors.rb b/app/models/sidebars/projects/menus/repository/menu_items/contributors.rb new file mode 100644 index 00000000000..d60fd05bb64 --- /dev/null +++ b/app/models/sidebars/projects/menus/repository/menu_items/contributors.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + module Repository + module MenuItems + class Contributors < ::Sidebars::MenuItem + override :link + def link + project_graph_path(context.project, context.current_ref) + end + + override :active_routes + def active_routes + { path: 'graphs#show' } + end + + override :title + def title + _('Contributors') + end + end + end + end + end + end +end diff --git a/app/models/sidebars/projects/menus/repository/menu_items/files.rb b/app/models/sidebars/projects/menus/repository/menu_items/files.rb new file mode 100644 index 00000000000..4989efe9fa5 --- /dev/null +++ b/app/models/sidebars/projects/menus/repository/menu_items/files.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + module Repository + module MenuItems + class Files < ::Sidebars::MenuItem + override :link + def link + project_tree_path(context.project, context.current_ref) + end + + override :active_routes + def active_routes + { controller: %w[tree blob blame edit_tree new_tree find_file] } + end + + override :title + def title + _('Files') + end + end + end + end + end + end +end diff --git a/app/models/sidebars/projects/menus/repository/menu_items/graphs.rb b/app/models/sidebars/projects/menus/repository/menu_items/graphs.rb new file mode 100644 index 00000000000..a57021be4d0 --- /dev/null +++ b/app/models/sidebars/projects/menus/repository/menu_items/graphs.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + module Repository + module MenuItems + class Graphs < ::Sidebars::MenuItem + override :link + def link + project_network_path(context.project, context.current_ref) + end + + override :active_routes + def active_routes + { controller: :network } + end + + override :title + def title + _('Graph') + end + end + end + end + end + end +end diff --git a/app/models/sidebars/projects/menus/repository/menu_items/tags.rb b/app/models/sidebars/projects/menus/repository/menu_items/tags.rb new file mode 100644 index 00000000000..d84bc89b93c --- /dev/null +++ b/app/models/sidebars/projects/menus/repository/menu_items/tags.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + module Repository + module MenuItems + class Tags < ::Sidebars::MenuItem + override :link + def link + project_tags_path(context.project) + end + + override :active_routes + def active_routes + { controller: :tags } + end + + override :title + def title + _('Tags') + end + end + end + end + end + end +end diff --git a/app/models/sidebars/projects/menus/scope/menu.rb b/app/models/sidebars/projects/menus/scope/menu.rb new file mode 100644 index 00000000000..3b699083f75 --- /dev/null +++ b/app/models/sidebars/projects/menus/scope/menu.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + module Scope + class Menu < ::Sidebars::Menu + override :link + def link + project_path(context.project) + end + + override :title + def title + context.project.name + end + end + end + end + end +end diff --git a/app/models/sidebars/projects/panel.rb b/app/models/sidebars/projects/panel.rb new file mode 100644 index 00000000000..ec4fac53a40 --- /dev/null +++ b/app/models/sidebars/projects/panel.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + class Panel < ::Sidebars::Panel + override :configure_menus + def configure_menus + set_scope_menu(Sidebars::Projects::Menus::Scope::Menu.new(context)) + + add_menu(Sidebars::Projects::Menus::ProjectOverview::Menu.new(context)) + add_menu(Sidebars::Projects::Menus::LearnGitlab::Menu.new(context)) + add_menu(Sidebars::Projects::Menus::Repository::Menu.new(context)) + end + + override :render_raw_menus_partial + def render_raw_menus_partial + 'layouts/nav/sidebar/project_menus' + end + + override :aria_label + def aria_label + _('Project navigation') + end + end + end +end diff --git a/app/models/timelog.rb b/app/models/timelog.rb index f4debedb656..c1aa84cbbcd 100644 --- a/app/models/timelog.rb +++ b/app/models/timelog.rb @@ -31,9 +31,9 @@ class Timelog < ApplicationRecord def issuable_id_is_present if issue_id && merge_request_id - errors.add(:base, _('Only Issue ID or Merge Request ID is required')) + errors.add(:base, _('Only Issue ID or merge request ID is required')) elsif issuable.nil? - errors.add(:base, _('Issue or Merge Request ID is required')) + errors.add(:base, _('Issue or merge request ID is required')) end end diff --git a/app/models/todo.rb b/app/models/todo.rb index 176d5e56fc0..c8138587d83 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -148,6 +148,24 @@ class Todo < ApplicationRecord .order(Gitlab::Database.nulls_last_order('highest_priority', 'ASC')) .order('todos.created_at') end + + def pluck_user_id + pluck(:user_id) + end + + # Count todos grouped by user_id and state, using an UNION query + # so we can utilize the partial indexes for each state. + def count_grouped_by_user_id_and_state + grouped_count = select(:user_id, 'count(id) AS count').group(:user_id) + + done = grouped_count.where(state: :done).select("'done' AS state") + pending = grouped_count.where(state: :pending).select("'pending' AS state") + union = unscoped.from_union([done, pending], remove_duplicates: false) + + connection.select_all(union).each_with_object({}) do |row, counts| + counts[[row['user_id'], row['state']]] = row['count'] + end + end end def resource_parent diff --git a/app/models/user.rb b/app/models/user.rb index 11046bdabe4..507e8cc2cf5 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -103,6 +103,8 @@ class User < ApplicationRecord # Profile has_many :keys, -> { regular_keys }, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :expired_today_and_unnotified_keys, -> { expired_today_and_not_notified }, class_name: 'Key' + has_many :expiring_soon_and_unnotified_keys, -> { expiring_soon_and_not_notified }, class_name: 'Key' has_many :deploy_keys, -> { where(type: 'DeployKey') }, dependent: :nullify # rubocop:disable Cop/ActiveRecordDependent has_many :group_deploy_keys has_many :gpg_keys @@ -125,7 +127,7 @@ class User < ApplicationRecord # Groups has_many :members - has_many :group_members, -> { where(requested_at: nil).where("access_level >= ?", Gitlab::Access::GUEST) }, source: 'GroupMember' + has_many :group_members, -> { where(requested_at: nil).where("access_level >= ?", Gitlab::Access::GUEST) }, class_name: 'GroupMember' has_many :groups, through: :group_members has_many :owned_groups, -> { where(members: { access_level: Gitlab::Access::OWNER }) }, through: :group_members, source: :group has_many :maintainers_groups, -> { where(members: { access_level: Gitlab::Access::MAINTAINER }) }, through: :group_members, source: :group @@ -139,7 +141,7 @@ class User < ApplicationRecord -> { where(members: { access_level: [Gitlab::Access::REPORTER, Gitlab::Access::DEVELOPER, Gitlab::Access::MAINTAINER, Gitlab::Access::OWNER] }) }, through: :group_members, source: :group - has_many :minimal_access_group_members, -> { where(access_level: [Gitlab::Access::MINIMAL_ACCESS]) }, source: 'GroupMember', class_name: 'GroupMember' + has_many :minimal_access_group_members, -> { where(access_level: [Gitlab::Access::MINIMAL_ACCESS]) }, class_name: 'GroupMember' has_many :minimal_access_groups, through: :minimal_access_group_members, source: :group # Projects @@ -199,6 +201,8 @@ class User < ApplicationRecord has_many :reviews, foreign_key: :author_id, inverse_of: :author + has_many :in_product_marketing_emails, class_name: '::Users::InProductMarketingEmail' + # # Validations # @@ -350,7 +354,8 @@ class User < ApplicationRecord # this state transition object in order to do a rollback. # For this reason the tradeoff is to disable this cop. after_transition any => :blocked do |user| - Ci::CancelUserPipelinesService.new.execute(user) + Ci::DropPipelineService.new.execute_async_for_all(user.pipelines, :user_blocked, user) + Ci::DisableUserPipelineSchedulesService.new.execute(user) end # rubocop: enable CodeReuse/ServiceClass end @@ -390,6 +395,22 @@ class User < ApplicationRecord .without_impersonation .expired_today_and_not_notified) end + scope :with_ssh_key_expired_today, -> do + includes(:expired_today_and_unnotified_keys) + .where('EXISTS (?)', + ::Key + .select(1) + .where('keys.user_id = users.id') + .expired_today_and_not_notified) + end + scope :with_ssh_key_expiring_soon, -> do + includes(:expiring_soon_and_unnotified_keys) + .where('EXISTS (?)', + ::Key + .select(1) + .where('keys.user_id = users.id') + .expiring_soon_and_not_notified) + end scope :order_recent_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'DESC')) } scope :order_oldest_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'ASC')) } scope :order_recent_last_activity, -> { reorder(Gitlab::Database.nulls_last_order('last_activity_on', 'DESC')) } @@ -743,6 +764,7 @@ class User < ApplicationRecord u.bio = 'The GitLab support bot used for Service Desk' u.name = 'GitLab Support Bot' u.avatar = bot_avatar(image: 'support-bot.png') + u.confirmed_at = Time.zone.now end end @@ -1024,7 +1046,7 @@ class User < ApplicationRecord [ Project.where(namespace: namespace), Project.joins(:project_authorizations) - .where("projects.namespace_id <> ?", namespace.id) + .where.not('projects.namespace_id' => namespace.id) .where(project_authorizations: { user_id: id, access_level: Gitlab::Access::OWNER }) ], remove_duplicates: false @@ -1337,9 +1359,11 @@ class User < ApplicationRecord end def public_verified_emails - emails = verified_emails(include_private_email: false) - emails << email unless temp_oauth_email? - emails.uniq + strong_memoize(:public_verified_emails) do + emails = verified_emails(include_private_email: false) + emails << email unless temp_oauth_email? + emails.uniq + end end def any_email?(check_email) @@ -1595,32 +1619,40 @@ class User < ApplicationRecord @global_notification_setting end + def count_cache_validity_period + if Feature.enabled?(:longer_count_cache_validity, self, default_enabled: :yaml) + 24.hours + else + 20.minutes + end + end + def assigned_open_merge_requests_count(force: false) - Rails.cache.fetch(['users', id, 'assigned_open_merge_requests_count'], force: force, expires_in: 20.minutes) do + Rails.cache.fetch(['users', id, 'assigned_open_merge_requests_count'], force: force, expires_in: count_cache_validity_period) do MergeRequestsFinder.new(self, assignee_id: self.id, state: 'opened', non_archived: true).execute.count end end def review_requested_open_merge_requests_count(force: false) - Rails.cache.fetch(['users', id, 'review_requested_open_merge_requests_count'], force: force, expires_in: 20.minutes) do + Rails.cache.fetch(['users', id, 'review_requested_open_merge_requests_count'], force: force, expires_in: count_cache_validity_period) do MergeRequestsFinder.new(self, reviewer_id: id, state: 'opened', non_archived: true).execute.count end end def assigned_open_issues_count(force: false) - Rails.cache.fetch(['users', id, 'assigned_open_issues_count'], force: force, expires_in: 20.minutes) do + Rails.cache.fetch(['users', id, 'assigned_open_issues_count'], force: force, expires_in: count_cache_validity_period) do IssuesFinder.new(self, assignee_id: self.id, state: 'opened', non_archived: true).execute.count end end def todos_done_count(force: false) - Rails.cache.fetch(['users', id, 'todos_done_count'], force: force, expires_in: 20.minutes) do + Rails.cache.fetch(['users', id, 'todos_done_count'], force: force, expires_in: count_cache_validity_period) do TodosFinder.new(self, state: :done).execute.count end end def todos_pending_count(force: false) - Rails.cache.fetch(['users', id, 'todos_pending_count'], force: force, expires_in: 20.minutes) do + Rails.cache.fetch(['users', id, 'todos_pending_count'], force: force, expires_in: count_cache_validity_period) do TodosFinder.new(self, state: :pending).execute.count end end @@ -1639,8 +1671,7 @@ class User < ApplicationRecord def invalidate_cache_counts invalidate_issue_cache_counts invalidate_merge_request_cache_counts - invalidate_todos_done_count - invalidate_todos_pending_count + invalidate_todos_cache_counts invalidate_personal_projects_count end @@ -1653,11 +1684,8 @@ class User < ApplicationRecord Rails.cache.delete(['users', id, 'review_requested_open_merge_requests_count']) end - def invalidate_todos_done_count + def invalidate_todos_cache_counts Rails.cache.delete(['users', id, 'todos_done_count']) - end - - def invalidate_todos_pending_count Rails.cache.delete(['users', id, 'todos_pending_count']) end @@ -1835,10 +1863,12 @@ class User < ApplicationRecord end def dismissed_callout?(feature_name:, ignore_dismissal_earlier_than: nil) - callouts = self.callouts.with_feature_name(feature_name) - callouts = callouts.with_dismissed_after(ignore_dismissal_earlier_than) if ignore_dismissal_earlier_than + callout = callouts_by_feature_name[feature_name] + + return false unless callout + return callout.dismissed_after?(ignore_dismissal_earlier_than) if ignore_dismissal_earlier_than - callouts.any? + true end # Load the current highest access by looking directly at the user's memberships @@ -1901,6 +1931,10 @@ class User < ApplicationRecord private + def callouts_by_feature_name + @callouts_by_feature_name ||= callouts.index_by(&:feature_name) + end + def authorized_groups_without_shared_membership Group.from_union([ groups, diff --git a/app/models/user_callout.rb b/app/models/user_callout.rb index bb5a9dceaeb..0a4db707be6 100644 --- a/app/models/user_callout.rb +++ b/app/models/user_callout.rb @@ -17,7 +17,7 @@ class UserCallout < ApplicationRecord threat_monitoring_info: 11, # EE-only account_recovery_regular_check: 12, # EE-only webhooks_moved: 13, - service_templates_deprecated: 14, + service_templates_deprecated_callout: 14, admin_integrations_moved: 15, web_ide_alert_dismissed: 16, # no longer in use active_user_count_threshold: 18, # EE-only @@ -29,7 +29,8 @@ class UserCallout < ApplicationRecord registration_enabled_callout: 25, new_user_signups_cap_reached: 26, # EE-only unfinished_tag_cleanup_callout: 27, - eoa_bronze_plan_banner: 28 # EE-only + eoa_bronze_plan_banner: 28, # EE-only + pipeline_needs_banner: 29 } validates :user, presence: true @@ -38,6 +39,7 @@ class UserCallout < ApplicationRecord uniqueness: { scope: :user_id }, inclusion: { in: UserCallout.feature_names.keys } - scope :with_feature_name, -> (feature_name) { where(feature_name: UserCallout.feature_names[feature_name]) } - scope :with_dismissed_after, -> (dismissed_after) { where('dismissed_at > ?', dismissed_after) } + def dismissed_after?(dismissed_after) + dismissed_at > dismissed_after + end end diff --git a/app/models/user_detail.rb b/app/models/user_detail.rb index ef799b01452..6b64f583927 100644 --- a/app/models/user_detail.rb +++ b/app/models/user_detail.rb @@ -19,7 +19,7 @@ class UserDetail < ApplicationRecord # For backward compatibility. # Older migrations (and their tests) reference the `User.migration_bot` where the `bio` attribute is set. - # Here we disable writing the markdown cache when the `bio_html` column does not exists. + # Here we disable writing the markdown cache when the `bio_html` column does not exist. override :invalidated_markdown_cache? def invalidated_markdown_cache? self.class.column_names.include?('bio_html') && super diff --git a/app/models/users/in_product_marketing_email.rb b/app/models/users/in_product_marketing_email.rb new file mode 100644 index 00000000000..195cfe162ac --- /dev/null +++ b/app/models/users/in_product_marketing_email.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Users + class InProductMarketingEmail < ApplicationRecord + include BulkInsertSafe + + belongs_to :user + + validates :user, presence: true + validates :track, presence: true + validates :series, presence: true + validates :user_id, uniqueness: { + scope: [:track, :series], + message: 'has already been sent' + } + + enum track: { + create: 0, + verify: 1, + trial: 2, + team: 3 + }, _suffix: true + + scope :without_track_and_series, -> (track, series) do + users = User.arel_table + product_emails = arel_table + + join_condition = users[:id].eq(product_emails[:user_id]) + .and(product_emails[:track]).eq(tracks[track]) + .and(product_emails[:series]).eq(series) + + arel_join = users.join(product_emails, Arel::Nodes::OuterJoin).on(join_condition) + + joins(arel_join.join_sources) + .where(in_product_marketing_emails: { id: nil }) + .select(Arel.sql("DISTINCT ON(#{users.table_name}.id) #{users.table_name}.*")) + end + + scope :for_user_with_track_and_series, -> (user, track, series) do + where(user: user, track: track, series: series) + end + + def self.save_cta_click(user, track, series) + email = for_user_with_track_and_series(user, track, series).take + + email.update(cta_clicked_at: Time.zone.now) if email && email.cta_clicked_at.blank? + end + end +end diff --git a/app/models/users/merge_request_interaction.rb b/app/models/users/merge_request_interaction.rb new file mode 100644 index 00000000000..35d1d3206b5 --- /dev/null +++ b/app/models/users/merge_request_interaction.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Users + class MergeRequestInteraction + def initialize(user:, merge_request:) + @user = user + @merge_request = merge_request + end + + def declarative_policy_subject + merge_request + end + + def can_merge? + merge_request.can_be_merged_by?(user) + end + + def can_update? + user.can?(:update_merge_request, merge_request) + end + + def review_state + reviewer&.state + end + + def reviewed? + reviewer&.reviewed? == true + end + + def approved? + merge_request.approvals.any? { |app| app.user_id == user.id } + end + + private + + def reviewer + @reviewer ||= merge_request.merge_request_reviewers.find { |r| r.user_id == user.id } + end + + attr_reader :user, :merge_request + end +end + +::Users::MergeRequestInteraction.prepend_if_ee('EE::Users::MergeRequestInteraction') diff --git a/app/models/wiki.rb b/app/models/wiki.rb index df31c54bd0f..47fe40b0e57 100644 --- a/app/models/wiki.rb +++ b/app/models/wiki.rb @@ -160,16 +160,12 @@ class Wiki end def find_file(name, version = 'HEAD', load_content: true) - if Feature.enabled?(:gitaly_find_file, user, default_enabled: :yaml) - data_limit = load_content ? -1 : 0 - blobs = repository.blobs_at([[version, name]], blob_size_limit: data_limit) + data_limit = load_content ? -1 : 0 + blobs = repository.blobs_at([[version, name]], blob_size_limit: data_limit) - return if blobs.empty? + return if blobs.empty? - Gitlab::Git::WikiFile.from_blob(blobs.first) - else - wiki.file(name, version) - end + Gitlab::Git::WikiFile.new(blobs.first) end def create_page(title, content, format = :markdown, message = nil) @@ -196,10 +192,20 @@ class Wiki def delete_page(page, message = nil) return unless page - wiki.delete_page(page.path, commit_details(:deleted, message, page.title)) - after_wiki_activity + if Feature.enabled?(:gitaly_replace_wiki_delete_page, user, default_enabled: :yaml) + capture_git_error(:deleted) do + repository.delete_file(user, page.path, **multi_commit_options(:deleted, message, page.title)) - true + after_wiki_activity + + true + end + else + wiki.delete_page(page.path, commit_details(:deleted, message, page.title)) + after_wiki_activity + + true + end end def page_title_and_dir(title) @@ -276,8 +282,20 @@ class Wiki private + def multi_commit_options(action, message = nil, title = nil) + commit_message = build_commit_message(action, message, title) + git_user = Gitlab::Git::User.from_gitlab(user) + + { + branch_name: repository.root_ref, + message: commit_message, + author_email: git_user.email, + author_name: git_user.name + } + end + def commit_details(action, message = nil, title = nil) - commit_message = message.presence || default_message(action, title) + commit_message = build_commit_message(action, message, title) git_user = Gitlab::Git::User.from_gitlab(user) Gitlab::Git::Wiki::CommitDetails.new(user.id, @@ -287,9 +305,26 @@ class Wiki commit_message) end + def build_commit_message(action, message, title) + message.presence || default_message(action, title) + end + def default_message(action, title) "#{user.username} #{action} page: #{title}" end + + def capture_git_error(action, &block) + yield block + rescue Gitlab::Git::Index::IndexError, + Gitlab::Git::CommitError, + Gitlab::Git::PreReceiveError, + Gitlab::Git::CommandError, + ArgumentError => error + + Gitlab::ErrorTracking.log_exception(error, action: action, wiki_id: id) + + false + end end Wiki.prepend_if_ee('EE::Wiki') |