diff options
Diffstat (limited to 'app/models')
106 files changed, 1368 insertions, 570 deletions
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 6afd8875ad3..6acdc02c799 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -382,6 +382,9 @@ class ApplicationSetting < ApplicationRecord allow_nil: false, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :container_registry_pre_import_tags_rate, + allow_nil: false, + numericality: { greater_than_or_equal_to: 0 } validates :container_registry_import_target_plan, presence: true validates :container_registry_import_created_before, presence: true @@ -502,6 +505,10 @@ class ApplicationSetting < ApplicationRecord length: { maximum: 255, message: _('is too long (maximum is %{count} characters)') }, allow_blank: true + validates :jira_connect_application_key, + length: { maximum: 255, message: _('is too long (maximum is %{count} characters)') }, + allow_blank: true + with_options(presence: true, numericality: { only_integer: true, greater_than: 0 }) do validates :throttle_unauthenticated_api_requests_per_period validates :throttle_unauthenticated_api_period_in_seconds diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index a54dc4f691d..a89ea05fb62 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -102,6 +102,7 @@ module ApplicationSettingImplementation import_sources: Settings.gitlab['import_sources'], invisible_captcha_enabled: false, issues_create_limit: 300, + jira_connect_application_key: nil, local_markdown_version: 0, login_recaptcha_protection_enabled: false, mailgun_signing_key: nil, @@ -224,6 +225,7 @@ module ApplicationSettingImplementation container_registry_import_max_retries: 3, container_registry_import_start_max_retries: 50, container_registry_import_max_step_duration: 5.minutes, + container_registry_pre_import_tags_rate: 0.5, container_registry_pre_import_timeout: 30.minutes, container_registry_import_timeout: 10.minutes, container_registry_import_target_plan: 'free', @@ -508,8 +510,35 @@ module ApplicationSettingImplementation 'https://sandbox-prod.gitlab-static.net' end + def ensure_key_restrictions! + return if Gitlab::Database.read_only? + return unless Gitlab::FIPS.enabled? + + Gitlab::SSHPublicKey.supported_types.each do |key_type| + set_max_key_restriction!(key_type) + end + end + private + def set_max_key_restriction!(key_type) + attr_name = "#{key_type}_key_restriction" + current = self.attributes[attr_name].to_i + + return if current == KeyRestrictionValidator::FORBIDDEN + + min_size = self.class.default_min_key_size(key_type) + + new_value = + if min_size == KeyRestrictionValidator::FORBIDDEN + min_size + else + [min_size, current].max + end + + self.assign_attributes({ attr_name => new_value }) + end + def separate_allowlists(string_array) string_array.reduce([[], []]) do |(ip_allowlist, domain_allowlist), string| address, port = parse_addr_and_port(string) diff --git a/app/models/award_emoji.rb b/app/models/award_emoji.rb index 22e5436dc5c..5430575ace7 100644 --- a/app/models/award_emoji.rb +++ b/app/models/award_emoji.rb @@ -70,7 +70,7 @@ class AwardEmoji < ApplicationRecord def expire_cache awardable.try(:bump_updated_at) - awardable.try(:expire_etag_cache) + awardable.expire_etag_cache if awardable.is_a?(Note) awardable.try(:update_upvotes_count) if upvote? end end diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index dee533944e9..cad2fafe640 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -99,18 +99,7 @@ class BulkImports::Entity < ApplicationRecord end def pipeline_exists?(name) - pipelines.any? { |_, pipeline| pipeline.to_s == name.to_s } - end - - def create_pipeline_trackers! - self.class.transaction do - pipelines.each do |stage, pipeline| - trackers.create!( - stage: stage, - pipeline_name: pipeline - ) - end - end + pipelines.any? { _1[:pipeline].to_s == name.to_s } end def entity_type diff --git a/app/models/bulk_imports/export_status.rb b/app/models/bulk_imports/export_status.rb index a9750a76987..4fea62edb2a 100644 --- a/app/models/bulk_imports/export_status.rb +++ b/app/models/bulk_imports/export_status.rb @@ -13,11 +13,15 @@ module BulkImports end def started? - export_status['status'] == Export::STARTED + !empty? && export_status['status'] == Export::STARTED end def failed? - export_status['status'] == Export::FAILED + !empty? && export_status['status'] == Export::FAILED + end + + def empty? + export_status.nil? end def error @@ -30,14 +34,7 @@ module BulkImports def export_status strong_memoize(:export_status) do - status = fetch_export_status - - relation_export_status = status&.find { |item| item['relation'] == relation } - - # Consider empty response as failed export - raise StandardError, 'Empty relation export status' unless relation_export_status&.present? - - relation_export_status + fetch_export_status&.find { |item| item['relation'] == relation } end rescue StandardError => e { 'status' => Export::FAILED, 'error' => e.message } diff --git a/app/models/bulk_imports/file_transfer/project_config.rb b/app/models/bulk_imports/file_transfer/project_config.rb index 38884df9fcf..8d4c68f7b5a 100644 --- a/app/models/bulk_imports/file_transfer/project_config.rb +++ b/app/models/bulk_imports/file_transfer/project_config.rb @@ -9,6 +9,8 @@ module BulkImports ).freeze LFS_OBJECTS_RELATION = 'lfs_objects' + REPOSITORY_BUNDLE_RELATION = 'repository' + DESIGN_BUNDLE_RELATION = 'design' def import_export_yaml ::Gitlab::ImportExport.config_file @@ -19,7 +21,12 @@ module BulkImports end def file_relations - [UPLOADS_RELATION, LFS_OBJECTS_RELATION] + [ + UPLOADS_RELATION, + LFS_OBJECTS_RELATION, + REPOSITORY_BUNDLE_RELATION, + DESIGN_BUNDLE_RELATION + ] end end end diff --git a/app/models/bulk_imports/tracker.rb b/app/models/bulk_imports/tracker.rb index a994cc3f8ce..fa38b7617d2 100644 --- a/app/models/bulk_imports/tracker.rb +++ b/app/models/bulk_imports/tracker.rb @@ -18,6 +18,8 @@ class BulkImports::Tracker < ApplicationRecord validates :stage, presence: true + delegate :file_extraction_pipeline?, to: :pipeline_class + DEFAULT_PAGE_SIZE = 500 scope :next_pipeline_trackers_for, -> (entity_id) { diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index a06b920342c..13af5b1f8d1 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -215,14 +215,10 @@ module Ci end def downstream_variables - if ::Feature.enabled?(:ci_trigger_forward_variables, project) - calculate_downstream_variables - .reverse # variables priority - .uniq { |var| var[:key] } # only one variable key to pass - .reverse - else - legacy_downstream_variables - end + calculate_downstream_variables + .reverse # variables priority + .uniq { |var| var[:key] } # only one variable key to pass + .reverse end def target_revision_ref @@ -268,16 +264,6 @@ module Ci } end - def legacy_downstream_variables - variables = scoped_variables.concat(pipeline.persisted_variables) - - variables.to_runner_variables.yield_self do |all_variables| - yaml_variables.to_a.map do |hash| - { key: hash[:key], value: ::ExpandVariables.expand(hash[:value], all_variables) } - end - end - end - def calculate_downstream_variables expand_variables = scoped_variables .concat(pipeline.persisted_variables) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index eea8086d71d..e35198ba31f 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -137,13 +137,14 @@ module Ci where('NOT EXISTS (?)', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').trace) end - scope :with_reports, ->(reports_scope) do - with_existing_job_artifacts(reports_scope) + scope :with_artifacts, ->(artifact_scope) do + with_existing_job_artifacts(artifact_scope) .eager_load_job_artifacts end scope :eager_load_job_artifacts, -> { includes(:job_artifacts) } scope :eager_load_tags, -> { includes(:tags) } + scope :eager_load_for_archiving_trace, -> { includes(:project, :pending_state) } scope :eager_load_everything, -> do includes( @@ -424,10 +425,18 @@ module Ci pipeline.manual_actions.reject { |action| action.name == self.name } end + def environment_manual_actions + pipeline.manual_actions.filter { |action| action.expanded_environment_name == self.expanded_environment_name } + end + def other_scheduled_actions pipeline.scheduled_actions.reject { |action| action.name == self.name } end + def environment_scheduled_actions + pipeline.scheduled_actions.filter { |action| action.expanded_environment_name == self.expanded_environment_name } + end + def pages_generator? Gitlab.config.pages.enabled && self.name == 'pages' @@ -559,6 +568,10 @@ module Ci options&.dig(:environment, :on_stop) end + def stop_action_successful? + success? + end + ## # All variables, including persisted environment variables. # @@ -673,7 +686,7 @@ module Ci end def has_live_trace? - trace.live_trace_exist? + trace.live? end def has_archived_trace? @@ -795,6 +808,7 @@ module Ci def execute_hooks return unless project + return if user&.blocked? project.execute_hooks(build_data.dup, :job_hooks) if project.has_active_hooks?(:job_hooks) project.execute_integrations(build_data.dup, :job_hooks) if project.has_active_integrations?(:job_hooks) @@ -826,12 +840,26 @@ module Ci end def erase_erasable_artifacts! + if project.refreshing_build_artifacts_size? + Gitlab::ProjectStatsRefreshConflictsLogger.warn_artifact_deletion_during_stats_refresh( + method: 'Ci::Build#erase_erasable_artifacts!', + project_id: project_id + ) + end + job_artifacts.erasable.destroy_all # rubocop: disable Cop/DestroyAll end def erase(opts = {}) return false unless erasable? + if project.refreshing_build_artifacts_size? + Gitlab::ProjectStatsRefreshConflictsLogger.warn_artifact_deletion_during_stats_refresh( + method: 'Ci::Build#erase', + project_id: project_id + ) + end + job_artifacts.destroy_all # rubocop: disable Cop/DestroyAll erase_trace! update_erased!(opts[:erased_by]) @@ -983,7 +1011,7 @@ module Ci end def collect_test_reports!(test_reports) - test_reports.get_suite(group_name).tap do |test_suite| + test_reports.get_suite(test_suite_name).tap do |test_suite| each_report(Ci::JobArtifact::TEST_REPORT_FILE_TYPES) do |file_type, blob| Gitlab::Ci::Parsers.fabricate!(file_type).parse!( blob, @@ -1002,19 +1030,6 @@ module Ci accessibility_report end - def collect_coverage_reports!(coverage_report) - each_report(Ci::JobArtifact::COVERAGE_REPORT_FILE_TYPES) do |file_type, blob| - Gitlab::Ci::Parsers.fabricate!(file_type).parse!( - blob, - coverage_report, - project_path: project.full_path, - worktree_paths: pipeline.all_worktree_paths - ) - end - - coverage_report - end - def collect_codequality_reports!(codequality_report) each_report(Ci::JobArtifact::CODEQUALITY_REPORT_FILE_TYPES) do |file_type, blob| Gitlab::Ci::Parsers.fabricate!(file_type).parse!(blob, codequality_report) @@ -1032,7 +1047,7 @@ module Ci end def report_artifacts - job_artifacts.with_reports + job_artifacts.all_reports end # Virtual deployment status depending on the environment status. @@ -1056,6 +1071,8 @@ module Ci all_runtime_metadata.delete_all end + deployment&.sync_status_with(self) + Gitlab::AppLogger.info( message: 'Build doomed', class: self.class.name, @@ -1145,6 +1162,14 @@ module Ci Gitlab::Utils::UsageData.track_usage_event('ci_users_executing_deployment_job', user_id) if user_id.present? && count_user_deployment? end + def each_report(report_types) + job_artifacts_for_types(report_types).each do |report_artifact| + report_artifact.each_blob do |blob| + yield report_artifact.file_type, blob, report_artifact + end + end + end + protected def run_status_commit_hooks! @@ -1155,6 +1180,18 @@ module Ci private + def test_suite_name + if matrix_build? + name + else + group_name + end + end + + def matrix_build? + options.dig(:parallel, :matrix).present? + end + def stick_build_if_status_changed return unless saved_change_to_status? return unless running? @@ -1184,14 +1221,6 @@ module Ci end end - def each_report(report_types) - job_artifacts_for_types(report_types).each do |report_artifact| - report_artifact.each_blob do |blob| - yield report_artifact.file_type, blob, report_artifact - end - end - end - def job_artifacts_for_types(report_types) # Use select to leverage cached associations and avoid N+1 queries job_artifacts.select { |artifact| artifact.file_type.in?(report_types) } diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index c831ef12501..81943cfa651 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -124,10 +124,10 @@ module Ci # We will start using this column once we complete https://gitlab.com/gitlab-org/gitlab/-/issues/285597 ignore_column :original_filename, remove_with: '14.7', remove_after: '2022-11-22' - mount_file_store_uploader JobArtifactUploader + mount_file_store_uploader JobArtifactUploader, skip_store_file: true - skip_callback :save, :after, :store_file!, if: :store_after_commit? - after_commit :store_file_after_commit!, on: [:create, :update], if: :store_after_commit? + after_save :store_file_in_transaction!, unless: :store_after_commit? + after_commit :store_file_after_transaction!, on: [:create, :update], if: :store_after_commit? validates :file_format, presence: true, unless: :trace?, on: :create validate :validate_file_format!, unless: :trace?, on: :create @@ -139,6 +139,10 @@ module Ci scope :for_sha, ->(sha, project_id) { joins(job: :pipeline).where(ci_pipelines: { sha: sha, project_id: project_id }) } scope :for_job_ids, ->(job_ids) { where(job_id: job_ids) } scope :for_job_name, ->(name) { joins(:job).where(ci_builds: { name: name }) } + scope :created_at_before, ->(time) { where(arel_table[:created_at].lteq(time)) } + scope :id_before, ->(id) { where(arel_table[:id].lteq(id)) } + scope :id_after, ->(id) { where(arel_table[:id].gt(id)) } + scope :ordered_by_id, -> { order(:id) } scope :with_job, -> { joins(:job).includes(:job) } @@ -148,7 +152,7 @@ module Ci where(file_type: types) end - scope :with_reports, -> do + scope :all_reports, -> do with_file_types(REPORT_TYPES.keys.map(&:to_s)) end @@ -187,7 +191,7 @@ module Ci scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) } scope :unlocked, -> { joins(job: :pipeline).merge(::Ci::Pipeline.unlocked) } scope :order_expired_asc, -> { order(expire_at: :asc) } - scope :with_destroy_preloads, -> { includes(project: [:route, :statistics]) } + scope :with_destroy_preloads, -> { includes(project: [:route, :statistics, :build_artifacts_size_refresh]) } scope :for_project, ->(project) { where(project_id: project) } scope :created_in_time_range, ->(from: nil, to: nil) { where(created_at: from..to) } @@ -358,11 +362,24 @@ module Ci private - def store_file_after_commit! - return unless previous_changes.key?(:file) + def store_file_in_transaction! + store_file_now! if saved_change_to_file? - store_file! - update_file_store + file_stored_in_transaction_hooks + end + + def store_file_after_transaction! + store_file_now! if previous_changes.key?(:file) + + file_stored_after_transaction_hooks + end + + # method overriden in EE + def file_stored_after_transaction_hooks + end + + # method overriden in EE + def file_stored_in_transaction_hooks end def set_size diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index c10069382f2..5d316906bd3 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -81,6 +81,7 @@ module Ci has_many :downloadable_artifacts, -> do not_expired.or(where_exists(::Ci::Pipeline.artifacts_locked.where('ci_pipelines.id = ci_builds.commit_id'))).downloadable.with_job end, through: :latest_builds, source: :job_artifacts + has_many :latest_successful_builds, -> { latest.success.with_project_and_metadata }, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'Ci::Build' has_many :messages, class_name: 'Ci::PipelineMessage', inverse_of: :pipeline @@ -239,7 +240,9 @@ module Ci next if transition.loopback? pipeline.run_after_commit do - PipelineHooksWorker.perform_async(pipeline.id) + unless pipeline.user&.blocked? + PipelineHooksWorker.perform_async(pipeline.id) + end if pipeline.project.jira_subscription_exists? # Passing the seq-id ensures this is idempotent @@ -296,7 +299,12 @@ module Ci ref_status = pipeline.ci_ref&.update_status_by!(pipeline) pipeline.run_after_commit do - PipelineNotificationWorker.perform_async(pipeline.id, ref_status: ref_status) + # We don't send notifications for a pipeline dropped due to the + # user been blocked. + unless pipeline.user&.blocked? + PipelineNotificationWorker + .perform_async(pipeline.id, ref_status: ref_status) + end end end @@ -327,14 +335,14 @@ module Ci scope :created_after, -> (time) { where('ci_pipelines.created_at > ?', time) } scope :created_before_id, -> (id) { where('ci_pipelines.id < ?', id) } scope :before_pipeline, -> (pipeline) { created_before_id(pipeline.id).outside_pipeline_family(pipeline) } - scope :with_pipeline_source, -> (source) { where(source: source)} + scope :with_pipeline_source, -> (source) { where(source: source) } scope :outside_pipeline_family, ->(pipeline) do where.not(id: pipeline.same_family_pipeline_ids) end scope :with_reports, -> (reports_scope) do - where('EXISTS (?)', ::Ci::Build.latest.with_reports(reports_scope).where('ci_pipelines.id=ci_builds.commit_id').select(1)) + where('EXISTS (?)', ::Ci::Build.latest.with_artifacts(reports_scope).where('ci_pipelines.id=ci_builds.commit_id').select(1)) end scope :with_only_interruptible_builds, -> do @@ -688,7 +696,7 @@ module Ci def latest_report_artifacts ::Gitlab::SafeRequestStore.fetch("pipeline:#{self.id}:latest_report_artifacts") do ::Ci::JobArtifact.where( - id: job_artifacts.with_reports + id: job_artifacts.all_reports .select('max(ci_job_artifacts.id) as id') .group(:file_type) ) @@ -1049,12 +1057,16 @@ module Ci @latest_builds_with_artifacts ||= builds.latest.with_artifacts_not_expired.to_a end - def latest_report_builds(reports_scope = ::Ci::JobArtifact.with_reports) - builds.latest.with_reports(reports_scope) + def latest_report_builds(reports_scope = ::Ci::JobArtifact.all_reports) + builds.latest.with_artifacts(reports_scope) end def latest_test_report_builds - latest_report_builds(Ci::JobArtifact.test_reports).preload(:project) + latest_report_builds(Ci::JobArtifact.test_reports).preload(:project, :metadata) + end + + def latest_report_builds_in_self_and_descendants(reports_scope = ::Ci::JobArtifact.all_reports) + builds_in_self_and_descendants.with_artifacts(reports_scope) end def builds_with_coverage @@ -1073,10 +1085,6 @@ module Ci pipeline_artifacts&.report_exists?(:code_coverage) end - def can_generate_coverage_reports? - has_reports?(Ci::JobArtifact.coverage_reports) - end - def has_codequality_mr_diff_report? pipeline_artifacts&.report_exists?(:code_quality_mr_diff) end @@ -1107,14 +1115,6 @@ module Ci end end - def coverage_reports - Gitlab::Ci::Reports::CoverageReports.new.tap do |coverage_reports| - latest_report_builds(Ci::JobArtifact.coverage_reports).includes(:project).find_each do |build| - build.collect_coverage_reports!(coverage_reports) - end - end - end - def codequality_reports Gitlab::Ci::Reports::CodequalityReports.new.tap do |codequality_reports| latest_report_builds(Ci::JobArtifact.codequality_reports).each do |build| @@ -1308,8 +1308,8 @@ module Ci end def has_expired_test_reports? - strong_memoize(:artifacts_expired) do - !has_reports?(::Ci::JobArtifact.test_reports.not_expired) + strong_memoize(:has_expired_test_reports) do + has_reports?(::Ci::JobArtifact.test_reports.expired) end end diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 7a1d52f5aea..61194c9b7d1 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -77,6 +77,7 @@ module Ci has_one :last_build, -> { order('id DESC') }, class_name: 'Ci::Build' before_save :ensure_token + before_save :update_semver, if: -> { version_changed? } scope :active, -> (value = true) { where(active: value) } scope :paused, -> { active(false) } @@ -429,6 +430,7 @@ module Ci values = values&.slice(:version, :revision, :platform, :architecture, :ip_address, :config, :executor) || {} values[:contacted_at] = Time.current values[:executor_type] = EXECUTOR_NAME_TO_TYPES.fetch(values.delete(:executor), :unknown) + values[:semver] = semver_from_version(values[:version]) cache_attributes(values) @@ -449,6 +451,16 @@ module Ci read_attribute(:contacted_at) end + def semver_from_version(version) + parsed_runner_version = ::Gitlab::VersionInfo.parse(version) + + parsed_runner_version.valid? ? parsed_runner_version.to_s : nil + end + + def update_semver + self.semver = semver_from_version(self.version) + end + def namespace_ids strong_memoize(:namespace_ids) do runner_namespaces.pluck(:namespace_id).compact diff --git a/app/models/ci/secure_file.rb b/app/models/ci/secure_file.rb index 9c82e106d6e..078b05ff779 100644 --- a/app/models/ci/secure_file.rb +++ b/app/models/ci/secure_file.rb @@ -23,6 +23,8 @@ module Ci after_initialize :generate_key_data before_validation :assign_checksum + scope :order_by_created_at, -> { order(created_at: :desc) } + default_value_for(:file_store) { Ci::SecureFileUploader.default_store } mount_file_store_uploader Ci::SecureFileUploader diff --git a/app/models/ci/sources/pipeline.rb b/app/models/ci/sources/pipeline.rb index f78caf710a6..2df504cd3de 100644 --- a/app/models/ci/sources/pipeline.rb +++ b/app/models/ci/sources/pipeline.rb @@ -7,10 +7,10 @@ module Ci self.table_name = "ci_sources_pipelines" - belongs_to :project, class_name: "Project" + belongs_to :project, class_name: "::Project" belongs_to :pipeline, class_name: "Ci::Pipeline", inverse_of: :source_pipeline - belongs_to :source_project, class_name: "Project", foreign_key: :source_project_id + belongs_to :source_project, class_name: "::Project", foreign_key: :source_project_id belongs_to :source_job, class_name: "CommitStatus", foreign_key: :source_job_id belongs_to :source_bridge, class_name: "Ci::Bridge", foreign_key: :source_job_id belongs_to :source_pipeline, class_name: "Ci::Pipeline", foreign_key: :source_pipeline_id diff --git a/app/models/clusters/agent.rb b/app/models/clusters/agent.rb index 79fc2b58237..fb12ce7d292 100644 --- a/app/models/clusters/agent.rb +++ b/app/models/clusters/agent.rb @@ -10,8 +10,7 @@ module Clusters belongs_to :created_by_user, class_name: 'User', optional: true belongs_to :project, class_name: '::Project' # Otherwise, it will load ::Clusters::Project - has_many :agent_tokens, class_name: 'Clusters::AgentToken' - has_many :last_used_agent_tokens, -> { order_last_used_at_desc }, class_name: 'Clusters::AgentToken', inverse_of: :agent + has_many :agent_tokens, -> { order_last_used_at_desc }, class_name: 'Clusters::AgentToken', inverse_of: :agent has_many :group_authorizations, class_name: 'Clusters::Agents::GroupAuthorization' has_many :authorized_groups, class_name: '::Group', through: :group_authorizations, source: :group @@ -23,6 +22,7 @@ module Clusters scope :ordered_by_name, -> { order(:name) } scope :with_name, -> (name) { where(name: name) } + scope :has_vulnerabilities, -> (value = true) { where(has_vulnerabilities: value) } validates :name, presence: true, @@ -47,5 +47,9 @@ module Clusters .offset(ACTIVITY_EVENT_LIMIT - 1) .pick(:recorded_at) end + + def to_ability_name + :cluster + end end end diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index e62b6fa5fc5..bed0eab5a58 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Runner < ApplicationRecord - VERSION = '0.39.0' + VERSION = '0.41.0' self.table_name = 'clusters_applications_runners' diff --git a/app/models/clusters/cluster_enabled_grant.rb b/app/models/clusters/cluster_enabled_grant.rb new file mode 100644 index 00000000000..4dca6a78759 --- /dev/null +++ b/app/models/clusters/cluster_enabled_grant.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Clusters + class ClusterEnabledGrant < ApplicationRecord + self.table_name = 'cluster_enabled_grants' + + belongs_to :namespace + end +end diff --git a/app/models/clusters/integrations/prometheus.rb b/app/models/clusters/integrations/prometheus.rb index 8b21fa351a3..0d6177beae7 100644 --- a/app/models/clusters/integrations/prometheus.rb +++ b/app/models/clusters/integrations/prometheus.rb @@ -55,13 +55,23 @@ module Clusters private def activate_project_integrations - ::Clusters::Applications::ActivateServiceWorker - .perform_async(cluster_id, ::Integrations::Prometheus.to_param) + if Feature.enabled?(:rename_integrations_workers) + ::Clusters::Applications::ActivateIntegrationWorker + .perform_async(cluster_id, ::Integrations::Prometheus.to_param) + else + ::Clusters::Applications::ActivateServiceWorker + .perform_async(cluster_id, ::Integrations::Prometheus.to_param) + end end def deactivate_project_integrations - ::Clusters::Applications::DeactivateServiceWorker - .perform_async(cluster_id, ::Integrations::Prometheus.to_param) + if Feature.enabled?(:rename_integrations_workers) + ::Clusters::Applications::DeactivateIntegrationWorker + .perform_async(cluster_id, ::Integrations::Prometheus.to_param) + else + ::Clusters::Applications::DeactivateServiceWorker + .perform_async(cluster_id, ::Integrations::Prometheus.to_param) + end end end end diff --git a/app/models/commit.rb b/app/models/commit.rb index 5293bfcf1ab..ca18cb50e02 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -513,11 +513,16 @@ class Commit # We don't want to do anything for `Commit` model, so this is empty. end + # We are continuing to support `(fixup!|squash!)` here as it is the prefix + # added by `git commit --fixup` which is used by some community members. + # https://gitlab.com/gitlab-org/gitlab/-/issues/342937#note_892065311 + # DRAFT_REGEX = /\A\s*#{Gitlab::Regex.merge_request_draft}|(fixup!|squash!)\s/.freeze - def work_in_progress? + def draft? !!(title =~ DRAFT_REGEX) end + alias_method :work_in_progress?, :draft? def merged_merge_request?(user) !!merged_merge_request(user) diff --git a/app/models/commit_signatures/ssh_signature.rb b/app/models/commit_signatures/ssh_signature.rb new file mode 100644 index 00000000000..dbfbe0c3889 --- /dev/null +++ b/app/models/commit_signatures/ssh_signature.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module CommitSignatures + class SshSignature < ApplicationRecord + include CommitSignature + + belongs_to :key, optional: false + end +end diff --git a/app/models/compare.rb b/app/models/compare.rb index f1b0bf19c11..7f42e1ee491 100644 --- a/app/models/compare.rb +++ b/app/models/compare.rb @@ -40,7 +40,10 @@ class Compare end def commits - @commits ||= Commit.decorate(@compare.commits, project) + @commits ||= begin + decorated_commits = Commit.decorate(@compare.commits, project) + CommitCollection.new(project, decorated_commits) + end end def start_commit diff --git a/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb b/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb index 7cc4bc569d3..1bdb89349aa 100644 --- a/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb +++ b/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb @@ -33,9 +33,14 @@ module Analytics ) duration_in_seconds = Arel::Nodes::Extract.new(duration, :epoch) + # start_event_timestamp and end_event_timestamp do not really influence the order, + # but are included so that they are part of the returned result, for example when + # using Gitlab::Analytics::CycleAnalytics::Aggregated::RecordsFetcher keyset_order( :total_time => { order_expression: arel_order(duration_in_seconds, direction), distinct: false, sql_type: 'double precision' }, - issuable_id_column => { order_expression: arel_order(arel_table[issuable_id_column], direction), distinct: true } + issuable_id_column => { order_expression: arel_order(arel_table[issuable_id_column], direction), distinct: true }, + :end_event_timestamp => { order_expression: arel_order(arel_table[:end_event_timestamp], direction), distinct: true }, + :start_event_timestamp => { order_expression: arel_order(arel_table[:start_event_timestamp], direction), distinct: true } ) end end diff --git a/app/models/concerns/as_cte.rb b/app/models/concerns/as_cte.rb new file mode 100644 index 00000000000..aa38ae3a9c1 --- /dev/null +++ b/app/models/concerns/as_cte.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +# Convert any ActiveRecord::Relation to a Gitlab::SQL::CTE +module AsCte + extend ActiveSupport::Concern + + class_methods do + def as_cte(name, **opts) + Gitlab::SQL::CTE.new(name, all, **opts) + end + end +end diff --git a/app/models/concerns/async_devise_email.rb b/app/models/concerns/async_devise_email.rb index 38c99dc7e71..7cdbed2eef6 100644 --- a/app/models/concerns/async_devise_email.rb +++ b/app/models/concerns/async_devise_email.rb @@ -2,6 +2,7 @@ module AsyncDeviseEmail extend ActiveSupport::Concern + include AfterCommitQueue private @@ -9,6 +10,8 @@ module AsyncDeviseEmail def send_devise_notification(notification, *args) return true unless can?(:receive_notifications) - devise_mailer.__send__(notification, self, *args).deliver_later # rubocop:disable GitlabSecurity/PublicSend + run_after_commit_or_now do + devise_mailer.__send__(notification, self, *args).deliver_later # rubocop:disable GitlabSecurity/PublicSend + end end end diff --git a/app/models/concerns/awardable.rb b/app/models/concerns/awardable.rb index 896f0916d8c..1d0ce594f63 100644 --- a/app/models/concerns/awardable.rb +++ b/app/models/concerns/awardable.rb @@ -18,7 +18,7 @@ module Awardable inner_query = award_emoji_table .project('true') .where(award_emoji_table[:user_id].eq(user.id)) - .where(award_emoji_table[:awardable_type].eq(self.name)) + .where(award_emoji_table[:awardable_type].eq(base_class.name)) .where(award_emoji_table[:awardable_id].eq(self.arel_table[:id])) inner_query = inner_query.where(award_emoji_table[:name].eq(name)) if name.present? @@ -31,7 +31,7 @@ module Awardable inner_query = award_emoji_table .project('true') .where(award_emoji_table[:user_id].eq(user.id)) - .where(award_emoji_table[:awardable_type].eq(self.name)) + .where(award_emoji_table[:awardable_type].eq(base_class.name)) .where(award_emoji_table[:awardable_id].eq(self.arel_table[:id])) inner_query = inner_query.where(award_emoji_table[:name].eq(name)) if name.present? @@ -56,13 +56,11 @@ module Awardable awardable_table = self.arel_table awards_table = AwardEmoji.arel_table - join_clause = awardable_table.join(awards_table, Arel::Nodes::OuterJoin).on( - awards_table[:awardable_id].eq(awardable_table[:id]).and( - awards_table[:awardable_type].eq(self.name).and( - awards_table[:name].eq(emoji_name) - ) - ) - ).join_sources + join_clause = awardable_table + .join(awards_table, Arel::Nodes::OuterJoin) + .on(awards_table[:awardable_id].eq(awardable_table[:id]) + .and(awards_table[:awardable_type].eq(base_class.name).and(awards_table[:name].eq(emoji_name)))) + .join_sources joins(join_clause).group(awardable_table[:id]).reorder( Arel.sql("COUNT(award_emoji.id) #{direction}") diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index 9414d16beef..99dbe464a7c 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -24,6 +24,9 @@ module CacheMarkdownField true end + attr_accessor :skip_markdown_cache_validation + alias_method :skip_markdown_cache_validation?, :skip_markdown_cache_validation + # Returns the default Banzai render context for the cached markdown field. def banzai_render_context(field) raise ArgumentError, "Unknown field: #{field.inspect}" unless @@ -91,7 +94,7 @@ module CacheMarkdownField end def invalidated_markdown_cache? - cached_markdown_fields.html_fields.any? {|html_field| attribute_invalidated?(html_field) } + cached_markdown_fields.html_fields.any? { |html_field| attribute_invalidated?(html_field) } end def attribute_invalidated?(attr) @@ -218,6 +221,8 @@ module CacheMarkdownField # The HTML becomes invalid if any dependent fields change. For now, assume # author and project invalidate the cache in all circumstances. define_method(invalidation_method) do + return false if skip_markdown_cache_validation? + changed_fields = changed_attributes.keys invalidations = changed_fields & [markdown_field.to_s, *INVALIDATED_BY] !invalidations.empty? || !cached_html_up_to_date?(markdown_field) diff --git a/app/models/concerns/ci/artifactable.rb b/app/models/concerns/ci/artifactable.rb index 27040a677ff..78340cf967b 100644 --- a/app/models/concerns/ci/artifactable.rb +++ b/app/models/concerns/ci/artifactable.rb @@ -21,7 +21,7 @@ module Ci }, _suffix: true scope :expired_before, -> (timestamp) { where(arel_table[:expire_at].lt(timestamp)) } - scope :expired, -> (limit) { expired_before(Time.current).limit(limit) } + scope :expired, -> { expired_before(Time.current) } scope :project_id_in, ->(ids) { where(project_id: ids) } end diff --git a/app/models/concerns/enums/ci/pipeline.rb b/app/models/concerns/enums/ci/pipeline.rb index 94d11c871ca..8ed6c54441b 100644 --- a/app/models/concerns/enums/ci/pipeline.rb +++ b/app/models/concerns/enums/ci/pipeline.rb @@ -15,7 +15,7 @@ module Enums size_limit_exceeded: 21, job_activity_limit_exceeded: 22, deployments_limit_exceeded: 23, - user_blocked: 24, + # 24 was previously used by the deprecated `user_blocked` project_deleted: 25 } end diff --git a/app/models/concerns/file_store_mounter.rb b/app/models/concerns/file_store_mounter.rb index bfcf8a1e7b9..f1ac734635d 100644 --- a/app/models/concerns/file_store_mounter.rb +++ b/app/models/concerns/file_store_mounter.rb @@ -4,9 +4,16 @@ module FileStoreMounter extend ActiveSupport::Concern class_methods do - def mount_file_store_uploader(uploader) + # When `skip_store_file: true` is used, the model MUST explicitly call `store_file_now!` + def mount_file_store_uploader(uploader, skip_store_file: false) mount_uploader(:file, uploader) + if skip_store_file + skip_callback :save, :after, :store_file! + + return + end + # This hook is a no-op when the file is uploaded after_commit after_save :update_file_store, if: :saved_change_to_file? end @@ -16,4 +23,9 @@ module FileStoreMounter # The file.object_store is set during `uploader.store!` and `uploader.migrate!` update_column(:file_store, file.object_store) end + + def store_file_now! + store_file! + update_file_store + end end diff --git a/app/models/concerns/integrations/base_data_fields.rb b/app/models/concerns/integrations/base_data_fields.rb index 3cedb90756f..11bdd3aae7b 100644 --- a/app/models/concerns/integrations/base_data_fields.rb +++ b/app/models/concerns/integrations/base_data_fields.rb @@ -4,12 +4,15 @@ module Integrations module BaseDataFields extend ActiveSupport::Concern + LEGACY_FOREIGN_KEY_NAME = %w( + Integrations::IssueTrackerData + Integrations::JiraTrackerData + ).freeze + included do # TODO: Once we rename the tables we can't rely on `table_name` anymore. # https://gitlab.com/gitlab-org/gitlab/-/issues/331953 - belongs_to :integration, inverse_of: self.table_name.to_sym, foreign_key: :service_id - - delegate :activated?, to: :integration, allow_nil: true + belongs_to :integration, inverse_of: self.table_name.to_sym, foreign_key: foreign_key_name validates :integration, presence: true end @@ -23,6 +26,26 @@ module Integrations algorithm: 'aes-256-gcm' } end + + private + + # Older data field models use the `service_id` foreign key for the + # integration association. + def foreign_key_name + return :service_id if self.name.in?(LEGACY_FOREIGN_KEY_NAME) + + :integration_id + end + end + + def activated? + !!integration&.activated? + end + + def to_database_hash + as_json( + only: self.class.column_names + ).except('id', 'service_id', 'integration_id', 'created_at', 'updated_at') end end end diff --git a/app/models/concerns/integrations/has_data_fields.rb b/app/models/concerns/integrations/has_data_fields.rb index 25a1d855119..635147a2f3c 100644 --- a/app/models/concerns/integrations/has_data_fields.rb +++ b/app/models/concerns/integrations/has_data_fields.rb @@ -12,7 +12,8 @@ module Integrations self.class_eval <<-RUBY, __FILE__, __LINE__ + 1 unless method_defined?(arg) def #{arg} - data_fields.send('#{arg}') || (properties && properties['#{arg}']) + value = data_fields.send('#{arg}') + value.nil? ? properties&.dig('#{arg}') : value end end diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 713a4386fee..4dca07132ef 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -106,23 +106,23 @@ module Issuable scope :closed, -> { with_state(:closed) } # rubocop:disable GitlabSecurity/SqlInjection - # The `to_ability_name` method is not an user input. + # The `assignee_association_name` method is not an user input. scope :assigned, -> do - where("EXISTS (SELECT TRUE FROM #{to_ability_name}_assignees WHERE #{to_ability_name}_id = #{to_ability_name}s.id)") + where("EXISTS (SELECT TRUE FROM #{assignee_association_name}_assignees WHERE #{assignee_association_name}_id = #{assignee_association_name}s.id)") end scope :unassigned, -> do - where("NOT EXISTS (SELECT TRUE FROM #{to_ability_name}_assignees WHERE #{to_ability_name}_id = #{to_ability_name}s.id)") + where("NOT EXISTS (SELECT TRUE FROM #{assignee_association_name}_assignees WHERE #{assignee_association_name}_id = #{assignee_association_name}s.id)") end scope :assigned_to, ->(users) do - assignees_class = self.reflect_on_association("#{to_ability_name}_assignees").klass + assignees_class = self.reflect_on_association("#{assignee_association_name}_assignees").klass - condition = assignees_class.where(user_id: users).where(Arel.sql("#{to_ability_name}_id = #{to_ability_name}s.id")) + condition = assignees_class.where(user_id: users).where(Arel.sql("#{assignee_association_name}_id = #{assignee_association_name}s.id")) where(condition.arel.exists) end scope :not_assigned_to, ->(users) do - assignees_class = self.reflect_on_association("#{to_ability_name}_assignees").klass + assignees_class = self.reflect_on_association("#{assignee_association_name}_assignees").klass - condition = assignees_class.where(user_id: users).where(Arel.sql("#{to_ability_name}_id = #{to_ability_name}s.id")) + condition = assignees_class.where(user_id: users).where(Arel.sql("#{assignee_association_name}_id = #{assignee_association_name}s.id")) where(condition.arel.exists.not) end # rubocop:enable GitlabSecurity/SqlInjection @@ -195,8 +195,6 @@ module Issuable end def supports_escalation? - return false unless ::Feature.enabled?(:incident_escalations, project) - incident? end @@ -414,6 +412,10 @@ module Issuable def parent_class ::Project end + + def assignee_association_name + to_ability_name + end end def state diff --git a/app/models/concerns/limitable.rb b/app/models/concerns/limitable.rb index 6ff540b7866..0cccb7b51a8 100644 --- a/app/models/concerns/limitable.rb +++ b/app/models/concerns/limitable.rb @@ -15,17 +15,29 @@ module Limitable validate :validate_plan_limit_not_exceeded, on: :create end + def exceeds_limits? + limits, relation = fetch_plan_limit_data + + limits&.exceeded?(limit_name, relation) + end + private def validate_plan_limit_not_exceeded + limits, relation = fetch_plan_limit_data + + check_plan_limit_not_exceeded(limits, relation) + end + + def fetch_plan_limit_data if GLOBAL_SCOPE == limit_scope - validate_global_plan_limit_not_exceeded + global_plan_limits else - validate_scoped_plan_limit_not_exceeded + scoped_plan_limits end end - def validate_scoped_plan_limit_not_exceeded + def scoped_plan_limits scope_relation = self.public_send(limit_scope) # rubocop:disable GitlabSecurity/PublicSend return unless scope_relation return if limit_feature_flag && ::Feature.disabled?(limit_feature_flag, scope_relation) @@ -34,18 +46,18 @@ module Limitable relation = limit_relation ? self.public_send(limit_relation) : self.class.where(limit_scope => scope_relation) # rubocop:disable GitlabSecurity/PublicSend limits = scope_relation.actual_limits - check_plan_limit_not_exceeded(limits, relation) + [limits, relation] end - def validate_global_plan_limit_not_exceeded + def global_plan_limits relation = self.class.all limits = Plan.default.actual_limits - check_plan_limit_not_exceeded(limits, relation) + [limits, relation] end def check_plan_limit_not_exceeded(limits, relation) - return unless limits.exceeded?(limit_name, relation) + return unless limits&.exceeded?(limit_name, relation) errors.add(:base, _("Maximum number of %{name} (%{count}) exceeded") % { name: limit_name.humanize(capitalize: false), count: limits.public_send(limit_name) }) # rubocop:disable GitlabSecurity/PublicSend diff --git a/app/models/concerns/pg_full_text_searchable.rb b/app/models/concerns/pg_full_text_searchable.rb index bfc539ee392..813827478da 100644 --- a/app/models/concerns/pg_full_text_searchable.rb +++ b/app/models/concerns/pg_full_text_searchable.rb @@ -24,6 +24,7 @@ module PgFullTextSearchable LONG_WORDS_REGEX = %r([A-Za-z0-9+/@]{50,}).freeze TSVECTOR_MAX_LENGTH = 1.megabyte.freeze TEXT_SEARCH_DICTIONARY = 'english' + URL_SCHEME_REGEX = %r{(?<=\A|\W)\w+://(?=\w+)}.freeze def update_search_data! tsvector_sql_nodes = self.class.pg_full_text_searchable_columns.map do |column, weight| @@ -104,6 +105,10 @@ module PgFullTextSearchable def pg_full_text_search(search_term) search_data_table = reflect_on_association(:search_data).klass.arel_table + # This fixes an inconsistency with how to_tsvector and websearch_to_tsquery process URLs + # See https://gitlab.com/gitlab-org/gitlab/-/issues/354784#note_905431920 + search_term = remove_url_scheme(search_term) + joins(:search_data).where( Arel::Nodes::InfixOperation.new( '@@', @@ -115,5 +120,11 @@ module PgFullTextSearchable ) ) end + + private + + def remove_url_scheme(search_term) + search_term.gsub(URL_SCHEME_REGEX, '') + end end end diff --git a/app/models/concerns/project_features_compatibility.rb b/app/models/concerns/project_features_compatibility.rb index 0cab874a240..900e8f7d39b 100644 --- a/app/models/concerns/project_features_compatibility.rb +++ b/app/models/concerns/project_features_compatibility.rb @@ -66,6 +66,10 @@ module ProjectFeaturesCompatibility write_feature_attribute_string(:snippets_access_level, value) end + def package_registry_access_level=(value) + write_feature_attribute_string(:package_registry_access_level, value) + end + def pages_access_level=(value) write_feature_attribute_string(:pages_access_level, value) end diff --git a/app/models/concerns/sensitive_serializable_hash.rb b/app/models/concerns/sensitive_serializable_hash.rb index 94451fcd2c2..4ad8d16fcb9 100644 --- a/app/models/concerns/sensitive_serializable_hash.rb +++ b/app/models/concerns/sensitive_serializable_hash.rb @@ -10,7 +10,7 @@ module SensitiveSerializableHash class_methods do def prevent_from_serialization(*keys) self.attributes_exempt_from_serializable_hash ||= [] - self.attributes_exempt_from_serializable_hash.concat keys + self.attributes_exempt_from_serializable_hash += keys end end diff --git a/app/models/concerns/storage/legacy_namespace.rb b/app/models/concerns/storage/legacy_namespace.rb index 948190dfadf..e418842a30b 100644 --- a/app/models/concerns/storage/legacy_namespace.rb +++ b/app/models/concerns/storage/legacy_namespace.rb @@ -23,22 +23,8 @@ module Storage former_parent_full_path = parent_was&.full_path parent_full_path = parent&.full_path Gitlab::UploadsTransfer.new.move_namespace(path, former_parent_full_path, parent_full_path) - - if any_project_with_pages_deployed? - run_after_commit do - Gitlab::PagesTransfer.new.async.move_namespace(path, former_parent_full_path, parent_full_path) - end - end else Gitlab::UploadsTransfer.new.rename_namespace(full_path_before_last_save, full_path) - - if any_project_with_pages_deployed? - full_path_was = full_path_before_last_save - - run_after_commit do - Gitlab::PagesTransfer.new.async.rename_namespace(full_path_was, full_path) - end - end end # If repositories moved successfully we need to diff --git a/app/models/container_registry/event.rb b/app/models/container_registry/event.rb index 5409bdf5af4..47d21d21afd 100644 --- a/app/models/container_registry/event.rb +++ b/app/models/container_registry/event.rb @@ -76,8 +76,8 @@ module ContainerRegistry return unless supported? return unless target_tag? return unless project - return unless Feature.enabled?(:container_registry_project_statistics, project) + Rails.cache.delete(project.root_ancestor.container_repositories_size_cache_key) ProjectCacheWorker.perform_async(project.id, [], [:container_registry_size]) end end diff --git a/app/models/customer_relations/contact.rb b/app/models/customer_relations/contact.rb index cdb449e00bf..ded6ab8687a 100644 --- a/app/models/customer_relations/contact.rb +++ b/app/models/customer_relations/contact.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class CustomerRelations::Contact < ApplicationRecord + include Gitlab::SQL::Pattern + include Sortable include StripAttribute self.table_name = "customer_relations_contacts" @@ -39,6 +41,25 @@ class CustomerRelations::Contact < ApplicationRecord ']' end + # Searches for contacts with a matching first name, last name, email or description. + # + # This method uses ILIKE on PostgreSQL + # + # query - The search query as a String + # + # Returns an ActiveRecord::Relation. + def self.search(query) + fuzzy_search(query, [:first_name, :last_name, :email, :description], use_minimum_char_limit: false) + end + + def self.search_by_state(state) + where(state: state) + end + + def self.sort_by_name + order("last_name ASC, first_name ASC") + end + def self.find_ids_by_emails(group, emails) raise ArgumentError, "Cannot lookup more than #{MAX_PLUCK} emails" if emails.length > MAX_PLUCK diff --git a/app/models/customer_relations/organization.rb b/app/models/customer_relations/organization.rb index 32adcc7492b..705e84250c9 100644 --- a/app/models/customer_relations/organization.rb +++ b/app/models/customer_relations/organization.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class CustomerRelations::Organization < ApplicationRecord + include Gitlab::SQL::Pattern + include Sortable include StripAttribute self.table_name = "customer_relations_organizations" @@ -21,6 +23,25 @@ class CustomerRelations::Organization < ApplicationRecord validates :description, length: { maximum: 1024 } validate :validate_root_group + # Searches for organizations with a matching name or description. + # + # This method uses ILIKE on PostgreSQL + # + # query - The search query as a String + # + # Returns an ActiveRecord::Relation. + def self.search(query) + fuzzy_search(query, [:name, :description], use_minimum_char_limit: false) + end + + def self.search_by_state(state) + where(state: state) + end + + def self.sort_by_name + order(name: :asc) + end + def self.find_by_name(group_id, name) where(group: group_id) .where('LOWER(name) = LOWER(?)', name) diff --git a/app/models/deployment.rb b/app/models/deployment.rb index 4204ad707b2..fc0dd7e00c7 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -52,6 +52,7 @@ class Deployment < ApplicationRecord scope :upcoming, -> { where(status: %i[blocked running]) } scope :older_than, -> (deployment) { where('deployments.id < ?', deployment.id) } scope :with_api_entity_associations, -> { preload({ deployable: { runner: [], tags: [], user: [], job_artifacts_archive: [] } }) } + scope :with_environment_page_associations, -> { preload(project: [], environment: [], deployable: [:user, :metadata, :project, pipeline: [:manual_actions]]) } scope :finished_after, ->(date) { where('finished_at >= ?', date) } scope :finished_before, ->(date) { where('finished_at < ?', date) } @@ -109,7 +110,11 @@ class Deployment < ApplicationRecord after_transition any => :running do |deployment| deployment.run_after_commit do - Deployments::HooksWorker.perform_async(deployment_id: id, status_changed_at: Time.current) + if Feature.enabled?(:deployment_hooks_skip_worker, deployment.project) + deployment.execute_hooks(Time.current) + else + Deployments::HooksWorker.perform_async(deployment_id: id, status_changed_at: Time.current) + end end end @@ -123,7 +128,11 @@ class Deployment < ApplicationRecord after_transition any => FINISHED_STATUSES do |deployment| deployment.run_after_commit do - Deployments::HooksWorker.perform_async(deployment_id: id, status_changed_at: Time.current) + if Feature.enabled?(:deployment_hooks_skip_worker, deployment.project) + deployment.execute_hooks(Time.current) + else + Deployments::HooksWorker.perform_async(deployment_id: id, status_changed_at: Time.current) + end end end @@ -173,6 +182,38 @@ class Deployment < ApplicationRecord find(ids) end + # This method returns the deployment records of the last deployment pipeline, that successfully executed for the given environment. + # e.g. + # A pipeline contains + # - deploy job A => production environment + # - deploy job B => production environment + # In this case, `last_deployment_group` returns both deployments. + # + # NOTE: Preload environment.last_deployment and pipeline.latest_successful_builds prior to avoid N+1. + def self.last_deployment_group_for_environment(env) + return self.none unless env.last_deployment_pipeline&.latest_successful_builds&.present? + + BatchLoader.for(env).batch do |environments, loader| + latest_successful_build_ids = [] + environments_hash = {} + + environments.each do |environment| + environments_hash[environment.id] = environment + + # Refer comment note above, if not preloaded this can lead to N+1. + latest_successful_build_ids << environment.last_deployment_pipeline.latest_successful_builds.map(&:id) + end + + Deployment + .where(deployable_type: 'CommitStatus', deployable_id: latest_successful_build_ids.flatten) + .preload(last_deployment_group_associations) + .group_by { |deployment| deployment.environment_id } + .each do |env_id, deployment_group| + loader.call(environments_hash[env_id], deployment_group) + end + end + end + def self.distinct_on_environment order('environment_id, deployments.id DESC') .select('DISTINCT ON (environment_id) deployments.*') @@ -247,11 +288,27 @@ class Deployment < ApplicationRecord end def manual_actions - @manual_actions ||= deployable.try(:other_manual_actions) + environment_manual_actions + end + + def other_manual_actions + @other_manual_actions ||= deployable.try(:other_manual_actions) + end + + def environment_manual_actions + @environment_manual_actions ||= deployable.try(:environment_manual_actions) end def scheduled_actions - @scheduled_actions ||= deployable.try(:other_scheduled_actions) + environment_scheduled_actions + end + + def environment_scheduled_actions + @environment_scheduled_actions ||= deployable.try(:environment_scheduled_actions) + end + + def other_scheduled_actions + @other_scheduled_actions ||= deployable.try(:other_scheduled_actions) end def playable_build @@ -414,6 +471,18 @@ class Deployment < ApplicationRecord raise ArgumentError, "The status #{status.inspect} is invalid" end end + + def self.last_deployment_group_associations + { + deployable: { + pipeline: { + manual_actions: [] + } + } + } + end + + private_class_method :last_deployment_group_associations end Deployment.prepend_mod_with('Deployment') diff --git a/app/models/environment.rb b/app/models/environment.rb index 865f5c68af1..da6ab5ed077 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -59,7 +59,7 @@ class Environment < ApplicationRecord allow_nil: true, addressable_url: true - delegate :manual_actions, to: :last_deployment, allow_nil: true + delegate :manual_actions, :other_manual_actions, to: :last_deployment, allow_nil: true delegate :auto_rollback_enabled?, to: :project scope :available, -> { with_state(:available) } @@ -132,10 +132,16 @@ class Environment < ApplicationRecord end event :stop do - transition available: :stopped + transition available: :stopping, if: :wait_for_stop? + transition available: :stopped, unless: :wait_for_stop? + end + + event :stop_complete do + transition %i(available stopping) => :stopped end state :available + state :stopping state :stopped before_transition any => :stopped do |environment| @@ -202,7 +208,7 @@ class Environment < ApplicationRecord # - deploy job A => production environment # - deploy job B => production environment # In this case, `last_deployment_group` returns both deployments, whereas `last_deployable` returns only B. - def last_deployment_group + def legacy_last_deployment_group return Deployment.none unless last_deployment_pipeline successful_deployments.where( @@ -293,6 +299,10 @@ class Environment < ApplicationRecord end end + def wait_for_stop? + stop_actions.present? + end + def stop_with_actions!(current_user) return unless available? @@ -314,20 +324,26 @@ class Environment < ApplicationRecord def stop_actions strong_memoize(:stop_actions) do - # Fix N+1 queries it brings to the serializer. - # Tracked in https://gitlab.com/gitlab-org/gitlab/-/issues/358780 last_deployment_group.map(&:stop_action).compact end end + def last_deployment_group + if ::Feature.enabled?(:batch_load_environment_last_deployment_group, project) + Deployment.last_deployment_group_for_environment(self) + else + legacy_last_deployment_group + end + end + def reset_auto_stop update_column(:auto_stop_at, nil) end def actions_for(environment) - return [] unless manual_actions + return [] unless other_manual_actions - manual_actions.select do |action| + other_manual_actions.select do |action| action.expanded_environment_name == environment end end diff --git a/app/models/error_tracking/client_key.rb b/app/models/error_tracking/client_key.rb index 8e59f6f9ecb..bbc57573aa9 100644 --- a/app/models/error_tracking/client_key.rb +++ b/app/models/error_tracking/client_key.rb @@ -7,6 +7,7 @@ class ErrorTracking::ClientKey < ApplicationRecord validates :public_key, presence: true, length: { maximum: 255 } scope :active, -> { where(active: true) } + scope :enabled_key_for, -> (project_id, public_key) { active.where(project_id: project_id, public_key: public_key) } after_initialize :generate_key diff --git a/app/models/error_tracking/error_event.rb b/app/models/error_tracking/error_event.rb index 18c1467e6f6..3ee82b219dc 100644 --- a/app/models/error_tracking/error_event.rb +++ b/app/models/error_tracking/error_event.rb @@ -15,7 +15,7 @@ class ErrorTracking::ErrorEvent < ApplicationRecord validates :occurred_at, presence: true def stacktrace - @stacktrace ||= build_stacktrace + @stacktrace ||= ErrorTracking::StacktraceBuilder.new(payload).stacktrace end # For compatibility with sentry integration @@ -30,56 +30,4 @@ class ErrorTracking::ErrorEvent < ApplicationRecord def release payload.dig('release') end - - private - - def build_stacktrace - raw_stacktrace = find_stacktrace_from_payload - - return [] unless raw_stacktrace - - raw_stacktrace.map do |entry| - { - 'lineNo' => entry['lineno'], - 'context' => build_stacktrace_context(entry), - 'filename' => entry['filename'], - 'function' => entry['function'], - 'colNo' => 0 # we don't support colNo yet. - } - end - end - - def find_stacktrace_from_payload - exception_entry = payload.dig('exception') - - if exception_entry - exception_values = exception_entry.dig('values') - stack_trace_entry = exception_values&.detect { |h| h['stacktrace'].present? } - stack_trace_entry&.dig('stacktrace', 'frames') - end - end - - def build_stacktrace_context(entry) - context = [] - error_line = entry['context_line'] - error_line_no = entry['lineno'] - pre_context = entry['pre_context'] - post_context = entry['post_context'] - - context += lines_with_position(pre_context, error_line_no - pre_context.size) if pre_context - context += lines_with_position([error_line], error_line_no) - context += lines_with_position(post_context, error_line_no + 1) if post_context - - context.reject(&:blank?) - end - - def lines_with_position(lines, position) - return [] if lines.blank? - - lines.map.with_index do |line, index| - next unless line - - [position + index, line] - end - end end diff --git a/app/models/group.rb b/app/models/group.rb index 86f4b14cb6c..f5aad6e74ff 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -362,7 +362,7 @@ class Group < Namespace end def add_users(users, access_level, current_user: nil, expires_at: nil, tasks_to_be_done: [], tasks_project_id: nil) - Members::Groups::BulkCreatorService.add_users( # rubocop:disable CodeReuse/ServiceClass + Members::Groups::CreatorService.add_users( # rubocop:disable CodeReuse/ServiceClass self, users, access_level, @@ -374,7 +374,7 @@ class Group < Namespace end def add_user(user, access_level, current_user: nil, expires_at: nil, ldap: false, blocking_refresh: true) - Members::Groups::CreatorService.new( # rubocop:disable CodeReuse/ServiceClass + Members::Groups::CreatorService.add_user( # rubocop:disable CodeReuse/ServiceClass self, user, access_level, @@ -382,7 +382,7 @@ class Group < Namespace expires_at: expires_at, ldap: ldap, blocking_refresh: blocking_refresh - ).execute + ) end def add_guest(user, current_user = nil) @@ -432,8 +432,9 @@ class Group < Namespace end # Check if user is a last owner of the group. + # Excludes project_bots def last_owner?(user) - has_owner?(user) && single_owner? + has_owner?(user) && all_owners_excluding_project_bots.size == 1 end def member_last_owner?(member) @@ -442,8 +443,8 @@ class Group < Namespace last_owner?(member.user) end - def single_owner? - members_with_parents.owners.size == 1 + def all_owners_excluding_project_bots + members_with_parents.owners.merge(User.without_project_bot) end def single_blocked_owner? @@ -863,6 +864,12 @@ class Group < Namespace end end + def gitlab_deploy_token + strong_memoize(:gitlab_deploy_token) do + deploy_tokens.gitlab_deploy_token + end + end + private def feature_flag_enabled_for_self_or_ancestor?(feature_flag) diff --git a/app/models/hooks/project_hook.rb b/app/models/hooks/project_hook.rb index 9f45160d3a8..b7ace34141e 100644 --- a/app/models/hooks/project_hook.rb +++ b/app/models/hooks/project_hook.rb @@ -31,11 +31,6 @@ class ProjectHook < WebHook _('Webhooks') end - override :rate_limit - def rate_limit - project.actual_limits.limit_for(:web_hook_calls) - end - override :application_context def application_context super.merge(project: project) diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index 88941df691c..37fd612e652 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -19,6 +19,15 @@ class WebHook < ApplicationRecord algorithm: 'aes-256-gcm', key: Settings.attr_encrypted_db_key_base_32 + attr_encrypted :url_variables, + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_32, + algorithm: 'aes-256-gcm', + marshal: true, + marshaler: ::Gitlab::Json, + encode: false, + encode_iv: false + has_many :web_hook_logs validates :url, presence: true @@ -26,6 +35,9 @@ class WebHook < ApplicationRecord validates :token, format: { without: /\n/ } validates :push_events_branch_filter, branch_filter: true + validates :url_variables, json_schema: { filename: 'web_hooks_url_variables' } + + after_initialize :initialize_url_variables scope :executable, -> do next all unless Feature.enabled?(:web_hooks_disable_failed) @@ -115,19 +127,12 @@ class WebHook < ApplicationRecord # @return [Boolean] Whether or not the WebHook is currently throttled. def rate_limited? - return false unless rate_limit - - Gitlab::ApplicationRateLimiter.peek( - :web_hook_calls, - scope: [self], - threshold: rate_limit - ) + rate_limiter.rate_limited? end - # Threshold for the rate-limit. - # Overridden in ProjectHook and GroupHook, other WebHooks are not rate-limited. + # @return [Integer] The rate limit for the WebHook. `0` for no limit. def rate_limit - nil + rate_limiter.limit end # Returns the associated Project or Group for the WebHook if one exists. @@ -140,9 +145,36 @@ class WebHook < ApplicationRecord { related_class: type } end + def alert_status + if temporarily_disabled? + :temporarily_disabled + elsif permanently_disabled? + :disabled + else + :executable + end + end + + # Exclude binary columns by default - they have no sensible JSON encoding + def serializable_hash(options = nil) + options = options.try(:dup) || {} + options[:except] = Array(options[:except]).dup + options[:except].concat [:encrypted_url_variables, :encrypted_url_variables_iv] + + super(options) + end + private def web_hooks_disable_failed? Feature.enabled?(:web_hooks_disable_failed) end + + def initialize_url_variables + self.url_variables = {} if encrypted_url_variables.nil? + end + + def rate_limiter + @rate_limiter ||= Gitlab::WebHooks::RateLimiter.new(self) + end end diff --git a/app/models/hooks/web_hook_log.rb b/app/models/hooks/web_hook_log.rb index 8c0565e4a38..2f03b3591cf 100644 --- a/app/models/hooks/web_hook_log.rb +++ b/app/models/hooks/web_hook_log.rb @@ -7,6 +7,8 @@ class WebHookLog < ApplicationRecord include CreatedAtFilterable include PartitionedTable + OVERSIZE_REQUEST_DATA = { 'oversize' => true }.freeze + self.primary_key = :id partitioned_by :created_at, strategy: :monthly, retain_for: 3.months @@ -26,6 +28,13 @@ class WebHookLog < ApplicationRecord .order(created_at: :desc) end + # Delete a batch of log records. Returns true if there may be more remaining. + def self.delete_batch_for(web_hook, batch_size:) + raise ArgumentError, 'batch_size is too small' if batch_size < 1 + + where(web_hook: web_hook).limit(batch_size).delete_all == batch_size + end + def success? response_status =~ /^2/ end @@ -34,6 +43,10 @@ class WebHookLog < ApplicationRecord response_status == WebHookService::InternalErrorResponse::ERROR_MESSAGE end + def oversize? + request_data == OVERSIZE_REQUEST_DATA + end + private def obfuscate_basic_auth diff --git a/app/models/integration.rb b/app/models/integration.rb index b5064cfae2d..726e95b7cbf 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -13,7 +13,6 @@ class Integration < ApplicationRecord include IgnorableColumns extend ::Gitlab::Utils::Override - ignore_column :type, remove_with: '15.0', remove_after: '2022-04-22' ignore_column :properties, remove_with: '15.1', remove_after: '2022-05-22' UnknownType = Class.new(StandardError) @@ -47,7 +46,9 @@ class Integration < ApplicationRecord Integrations::BaseSlashCommands ].freeze + SECTION_TYPE_CONFIGURATION = 'configuration' SECTION_TYPE_CONNECTION = 'connection' + SECTION_TYPE_TRIGGER = 'trigger' attr_encrypted :properties, mode: :per_attribute_iv, @@ -143,7 +144,7 @@ class Integration < ApplicationRecord # :nocov: Tested on subclasses. def self.field(name, storage: field_storage, **attrs) - fields << ::Integrations::Field.new(name: name, **attrs) + fields << ::Integrations::Field.new(name: name, integration_class: self, **attrs) case storage when :properties @@ -465,13 +466,14 @@ class Integration < ApplicationRecord super.except('properties') end - # return a hash of columns => values suitable for passing to insert_all - def to_integration_hash + # Returns a hash of attributes (columns => values) used for inserting into the database. + def to_database_hash column = self.class.attribute_aliases.fetch('type', 'type') - as_json(except: %w[id instance project_id group_id]) - .merge(column => type) - .merge(reencrypt_properties) + as_json( + except: %w[id instance project_id group_id created_at updated_at] + ).merge(column => type) + .merge(reencrypt_properties) end def reencrypt_properties @@ -484,10 +486,6 @@ class Integration < ApplicationRecord { 'encrypted_properties' => ep, 'encrypted_properties_iv' => iv } end - def to_data_fields_hash - data_fields.as_json(only: data_fields.class.column_names).except('id', 'service_id', 'integration_id') - end - def event_channel_names [] end @@ -501,10 +499,7 @@ class Integration < ApplicationRecord end def api_field_names - fields - .reject { _1[:type] == 'password' } - .pluck(:name) - .grep_v(/password|token|key/) + fields.reject { _1[:type] == 'password' }.pluck(:name) end def global_fields @@ -579,7 +574,11 @@ class Integration < ApplicationRecord def async_execute(data) return unless supported_events.include?(data[:object_kind]) - ProjectServiceWorker.perform_async(id, data) + if Feature.enabled?(:rename_integrations_workers) + Integrations::ExecuteWorker.perform_async(id, data) + else + ProjectServiceWorker.perform_async(id, data) + end end # override if needed diff --git a/app/models/integrations/bamboo.rb b/app/models/integrations/bamboo.rb index 4e144a688f6..4e30c1ccc69 100644 --- a/app/models/integrations/bamboo.rb +++ b/app/models/integrations/bamboo.rb @@ -6,25 +6,25 @@ module Integrations prepend EnableSslVerification field :bamboo_url, - title: s_('BambooService|Bamboo URL'), - placeholder: s_('https://bamboo.example.com'), - help: s_('BambooService|Bamboo service root URL.'), + title: -> { s_('BambooService|Bamboo URL') }, + placeholder: -> { s_('https://bamboo.example.com') }, + help: -> { s_('BambooService|Bamboo service root URL.') }, required: true field :build_key, - help: s_('BambooService|Bamboo build plan key.'), - non_empty_password_title: s_('BambooService|Enter new build key'), - non_empty_password_help: s_('BambooService|Leave blank to use your current build key.'), - placeholder: s_('KEY'), + help: -> { s_('BambooService|Bamboo build plan key.') }, + non_empty_password_title: -> { s_('BambooService|Enter new build key') }, + non_empty_password_help: -> { s_('BambooService|Leave blank to use your current build key.') }, + placeholder: -> { s_('KEY') }, required: true field :username, - help: s_('BambooService|The user with API access to the Bamboo server.') + help: -> { s_('BambooService|The user with API access to the Bamboo server.') } field :password, type: 'password', - non_empty_password_title: s_('ProjectService|Enter new password'), - non_empty_password_help: s_('ProjectService|Leave blank to use your current password') + non_empty_password_title: -> { s_('ProjectService|Enter new password') }, + non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current password') } validates :bamboo_url, presence: true, public_url: true, if: :activated? validates :build_key, presence: true, if: :activated? diff --git a/app/models/integrations/base_chat_notification.rb b/app/models/integrations/base_chat_notification.rb index 9bf208abcf7..33d4eecbf49 100644 --- a/app/models/integrations/base_chat_notification.rb +++ b/app/models/integrations/base_chat_notification.rb @@ -249,7 +249,7 @@ module Integrations ref = data[:ref] || data.dig(:object_attributes, :ref) return true if ref.blank? # No need to check protected branches when there is no ref - return true if Gitlab::Git.tag_ref?(ref) # Skip protected branch check because it doesn't support tags + return true if Gitlab::Git.tag_ref?(project.repository.expand_ref(ref) || ref) # Skip protected branch check because it doesn't support tags notify_for_branch?(data) end diff --git a/app/models/integrations/buildkite.rb b/app/models/integrations/buildkite.rb index d1e54ce86ee..def646c6d49 100644 --- a/app/models/integrations/buildkite.rb +++ b/app/models/integrations/buildkite.rb @@ -11,16 +11,18 @@ module Integrations ENDPOINT = "https://buildkite.com" field :project_url, - title: _('Pipeline URL'), + title: -> { _('Pipeline URL') }, placeholder: "#{ENDPOINT}/example-org/test-pipeline", required: true field :token, type: 'password', - title: _('Token'), - help: s_('ProjectService|The token you get after you create a Buildkite pipeline with a GitLab repository.'), - non_empty_password_title: s_('ProjectService|Enter new token'), - non_empty_password_help: s_('ProjectService|Leave blank to use your current token.'), + title: -> { _('Token') }, + help: -> do + s_('ProjectService|The token you get after you create a Buildkite pipeline with a GitLab repository.') + end, + non_empty_password_title: -> { s_('ProjectService|Enter new token') }, + non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current token.') }, required: true validates :project_url, presence: true, public_url: true, if: :activated? diff --git a/app/models/integrations/drone_ci.rb b/app/models/integrations/drone_ci.rb index 0c65ed8cd5f..35524503dea 100644 --- a/app/models/integrations/drone_ci.rb +++ b/app/models/integrations/drone_ci.rb @@ -11,15 +11,15 @@ module Integrations DRONE_SAAS_HOSTNAME = 'cloud.drone.io' field :drone_url, - title: s_('ProjectService|Drone server URL'), + title: -> { s_('ProjectService|Drone server URL') }, placeholder: 'http://drone.example.com', required: true field :token, type: 'password', - help: s_('ProjectService|Token for the Drone project.'), - non_empty_password_title: s_('ProjectService|Enter new token'), - non_empty_password_help: s_('ProjectService|Leave blank to use your current token.'), + help: -> { s_('ProjectService|Token for the Drone project.') }, + non_empty_password_title: -> { s_('ProjectService|Enter new token') }, + non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current token.') }, required: true validates :drone_url, presence: true, public_url: true, if: :activated? diff --git a/app/models/integrations/field.rb b/app/models/integrations/field.rb index ca7833c1a56..cbda418755b 100644 --- a/app/models/integrations/field.rb +++ b/app/models/integrations/field.rb @@ -13,10 +13,11 @@ module Integrations exposes_secrets ].freeze - attr_reader :name + attr_reader :name, :integration_class - def initialize(name:, type: 'text', api_only: false, **attributes) + def initialize(name:, integration_class:, type: 'text', api_only: false, **attributes) @name = name.to_s.freeze + @integration_class = integration_class attributes[:type] = SECRET_NAME.match?(@name) ? 'password' : type attributes[:api_only] = api_only @@ -27,7 +28,7 @@ module Integrations return name if key == :name value = @attributes[key] - return value.call if value.respond_to?(:call) + return integration_class.class_exec(&value) if value.respond_to?(:call) value end diff --git a/app/models/integrations/harbor.rb b/app/models/integrations/harbor.rb index 6b561575f30..44813795fc0 100644 --- a/app/models/integrations/harbor.rb +++ b/app/models/integrations/harbor.rb @@ -81,7 +81,7 @@ module Integrations [ { key: 'HARBOR_URL', value: url }, { key: 'HARBOR_PROJECT', value: project_name }, - { key: 'HARBOR_USERNAME', value: username }, + { key: 'HARBOR_USERNAME', value: username.gsub(/^robot\$/, 'robot$$') }, { key: 'HARBOR_PASSWORD', value: password, public: false, masked: true } ] end diff --git a/app/models/integrations/irker.rb b/app/models/integrations/irker.rb index 116d1fb233d..780f4bef0c9 100644 --- a/app/models/integrations/irker.rb +++ b/app/models/integrations/irker.rb @@ -24,14 +24,23 @@ module Integrations end def self.supported_events - %w(push) + %w[push] end def execute(data) return unless supported_events.include?(data[:object_kind]) - IrkerWorker.perform_async(project_id, channels, - colorize_messages, data, settings) + if Feature.enabled?(:rename_integrations_workers) + Integrations::IrkerWorker.perform_async( + project_id, channels, + colorize_messages, data, settings + ) + else + ::IrkerWorker.perform_async( + project_id, channels, + colorize_messages, data, settings + ) + end end def settings @@ -42,7 +51,15 @@ module Integrations end def fields - recipients_docs_link = ActionController::Base.helpers.link_to s_('IrkerService|How to enter channels or users?'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/irker', anchor: 'enter-irker-recipients'), target: '_blank', rel: 'noopener noreferrer' + recipients_docs_link = ActionController::Base.helpers.link_to( + s_('IrkerService|How to enter channels or users?'), + Rails.application.routes.url_helpers.help_page_url( + 'user/project/integrations/irker', + anchor: 'enter-irker-recipients' + ), + target: '_blank', rel: 'noopener noreferrer' + ) + [ { type: 'text', name: 'server_host', placeholder: 'localhost', title: s_('IrkerService|Server host (optional)'), help: s_('IrkerService|irker daemon hostname (defaults to localhost).') }, @@ -53,14 +70,29 @@ module Integrations placeholder: 'irc://irc.network.net:6697/' }, { type: 'textarea', name: 'recipients', title: s_('IrkerService|Recipients'), placeholder: 'irc[s]://irc.network.net[:port]/#channel', required: true, - help: s_('IrkerService|Channels and users separated by whitespaces. %{recipients_docs_link}').html_safe % { recipients_docs_link: recipients_docs_link.html_safe } }, + help: format( + s_('IrkerService|Channels and users separated by whitespaces. %{recipients_docs_link}').html_safe, + recipients_docs_link: recipients_docs_link.html_safe + ) }, { type: 'checkbox', name: 'colorize_messages', title: _('Colorize messages') } ] end def help - docs_link = ActionController::Base.helpers.link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/irker', anchor: 'set-up-an-irker-daemon'), target: '_blank', rel: 'noopener noreferrer' - s_('IrkerService|Send update messages to an irker server. Before you can use this, you need to set up the irker daemon. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } + docs_link = ActionController::Base.helpers.link_to( + _('Learn more.'), + Rails.application.routes.url_helpers.help_page_url( + 'user/project/integrations/irker', + anchor: 'set-up-an-irker-daemon' + ), + target: '_blank', + rel: 'noopener noreferrer' + ) + + format(s_( + 'IrkerService|Send update messages to an irker server. ' \ + 'Before you can use this, you need to set up the irker daemon. %{docs_link}' + ).html_safe, docs_link: docs_link.html_safe) end private @@ -104,12 +136,11 @@ module Integrations end def consider_uri(uri) - return if uri.scheme.nil? - + return unless uri.is_a?(URI) && uri.scheme.present? # Authorize both irc://domain.com/#chan and irc://domain.com/chan - if uri.is_a?(URI) && uri.scheme[/^ircs?\z/] && !uri.path.nil? - uri.to_s - end + return unless uri.scheme =~ /\Aircs?\z/ && !uri.path.nil? + + uri.to_s end end end diff --git a/app/models/integrations/jenkins.rb b/app/models/integrations/jenkins.rb index a1abbce72bc..ab39d1f7b77 100644 --- a/app/models/integrations/jenkins.rb +++ b/app/models/integrations/jenkins.rb @@ -8,24 +8,24 @@ module Integrations extend Gitlab::Utils::Override field :jenkins_url, - title: s_('ProjectService|Jenkins server URL'), + title: -> { s_('ProjectService|Jenkins server URL') }, required: true, placeholder: 'http://jenkins.example.com', - help: s_('The URL of the Jenkins server.') + help: -> { s_('The URL of the Jenkins server.') } field :project_name, required: true, placeholder: 'my_project_name', - help: s_('The name of the Jenkins project. Copy the name from the end of the URL to the project.') + help: -> { s_('The name of the Jenkins project. Copy the name from the end of the URL to the project.') } field :username, - help: s_('The username for the Jenkins server.') + help: -> { s_('The username for the Jenkins server.') } field :password, type: 'password', - help: s_('The password for the Jenkins server.'), - non_empty_password_title: s_('ProjectService|Enter new password.'), - non_empty_password_help: s_('ProjectService|Leave blank to use your current password.') + help: -> { s_('The password for the Jenkins server.') }, + non_empty_password_title: -> { s_('ProjectService|Enter new password.') }, + non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current password.') } before_validation :reset_password diff --git a/app/models/integrations/jira.rb b/app/models/integrations/jira.rb index 992bd01bf5f..125f52104d4 100644 --- a/app/models/integrations/jira.rb +++ b/app/models/integrations/jira.rb @@ -24,7 +24,10 @@ module Integrations validates :password, presence: true, if: :activated? validates :jira_issue_transition_id, - format: { with: Gitlab::Regex.jira_transition_id_regex, message: s_("JiraService|IDs must be a list of numbers that can be split with , or ;") }, + format: { + with: Gitlab::Regex.jira_transition_id_regex, + message: ->(*_) { s_("JiraService|IDs must be a list of numbers that can be split with , or ;") } + }, allow_blank: true # Jira Cloud version is deprecating authentication via username and password. diff --git a/app/models/integrations/microsoft_teams.rb b/app/models/integrations/microsoft_teams.rb index 71cd4ddaf82..625ee0bc522 100644 --- a/app/models/integrations/microsoft_teams.rb +++ b/app/models/integrations/microsoft_teams.rb @@ -35,10 +35,16 @@ module Integrations def default_fields [ - { type: 'text', name: 'webhook', placeholder: "#{webhook_placeholder}" }, - { type: 'checkbox', name: 'notify_only_broken_pipelines', help: 'If selected, successful pipelines do not trigger a notification event.' }, + { type: 'text', section: SECTION_TYPE_CONNECTION, name: 'webhook', required: true, placeholder: "#{webhook_placeholder}" }, + { + type: 'checkbox', + section: SECTION_TYPE_CONFIGURATION, + name: 'notify_only_broken_pipelines', + help: 'If selected, successful pipelines do not trigger a notification event.' + }, { type: 'select', + section: SECTION_TYPE_CONFIGURATION, name: 'branches_to_be_notified', title: s_('Integrations|Branches for which notifications are to be sent'), choices: branch_choices @@ -46,6 +52,26 @@ module Integrations ] end + def sections + [ + { + type: SECTION_TYPE_CONNECTION, + title: s_('Integrations|Connection details'), + description: help + }, + { + type: SECTION_TYPE_TRIGGER, + title: s_('Integrations|Trigger'), + description: s_('Integrations|An event will be triggered when one of the following items happen.') + }, + { + type: SECTION_TYPE_CONFIGURATION, + title: s_('Integrations|Notification settings'), + description: s_('Integrations|Configure the scope of notifications.') + } + ] + end + private def notify(message, opts) diff --git a/app/models/integrations/mock_ci.rb b/app/models/integrations/mock_ci.rb index cd2928136ef..0b3a9bc5405 100644 --- a/app/models/integrations/mock_ci.rb +++ b/app/models/integrations/mock_ci.rb @@ -8,7 +8,7 @@ module Integrations ALLOWED_STATES = %w[failed canceled running pending success success-with-warnings skipped not_found].freeze field :mock_service_url, - title: s_('ProjectService|Mock service URL'), + title: -> { s_('ProjectService|Mock service URL') }, placeholder: 'http://localhost:4004', required: true diff --git a/app/models/integrations/prometheus.rb b/app/models/integrations/prometheus.rb index 427034edb79..36060565317 100644 --- a/app/models/integrations/prometheus.rb +++ b/app/models/integrations/prometheus.rb @@ -84,6 +84,8 @@ module Integrations # Check we can connect to the Prometheus API def test(*args) + return { success: false, result: 'Prometheus configuration error' } unless prometheus_client + prometheus_client.ping { success: true, result: 'Checked API endpoint' } rescue Gitlab::PrometheusClient::Error => err diff --git a/app/models/integrations/teamcity.rb b/app/models/integrations/teamcity.rb index 1205173e40b..a23aa5f783d 100644 --- a/app/models/integrations/teamcity.rb +++ b/app/models/integrations/teamcity.rb @@ -9,21 +9,21 @@ module Integrations TEAMCITY_SAAS_HOSTNAME = /\A[^\.]+\.teamcity\.com\z/i.freeze field :teamcity_url, - title: s_('ProjectService|TeamCity server URL'), + title: -> { s_('ProjectService|TeamCity server URL') }, placeholder: 'https://teamcity.example.com', required: true field :build_type, - help: s_('ProjectService|The build configuration ID of the TeamCity project.'), + help: -> { s_('ProjectService|The build configuration ID of the TeamCity project.') }, required: true field :username, - help: s_('ProjectService|Must have permission to trigger a manual build in TeamCity.') + help: -> { s_('ProjectService|Must have permission to trigger a manual build in TeamCity.') } field :password, type: 'password', - non_empty_password_title: s_('ProjectService|Enter new password'), - non_empty_password_help: s_('ProjectService|Leave blank to use your current password') + non_empty_password_title: -> { s_('ProjectService|Enter new password') }, + non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current password') } validates :teamcity_url, presence: true, public_url: true, if: :activated? validates :build_type, presence: true, if: :activated? diff --git a/app/models/integrations/zentao_tracker_data.rb b/app/models/integrations/zentao_tracker_data.rb index 468e4e5d7d7..e9d63abd66b 100644 --- a/app/models/integrations/zentao_tracker_data.rb +++ b/app/models/integrations/zentao_tracker_data.rb @@ -2,18 +2,7 @@ module Integrations class ZentaoTrackerData < ApplicationRecord - belongs_to :integration, inverse_of: :zentao_tracker_data, foreign_key: :integration_id - delegate :activated?, to: :integration - validates :integration, presence: true - - scope :encryption_options, -> do - { - key: Settings.attr_encrypted_db_key_base_32, - encode: true, - mode: :per_attribute_iv, - algorithm: 'aes-256-gcm' - } - end + include BaseDataFields attr_encrypted :url, encryption_options attr_encrypted :api_url, encryption_options diff --git a/app/models/issue.rb b/app/models/issue.rb index d4eb77ef6de..47aa2b24feb 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -122,12 +122,13 @@ class Issue < ApplicationRecord scope :order_due_date_asc, -> { reorder(arel_table[:due_date].asc.nulls_last) } scope :order_due_date_desc, -> { reorder(arel_table[:due_date].desc.nulls_last) } scope :order_closest_future_date, -> { reorder(Arel.sql('CASE WHEN issues.due_date >= CURRENT_DATE THEN 0 ELSE 1 END ASC, ABS(CURRENT_DATE - issues.due_date) ASC')) } - scope :order_closed_date_desc, -> { reorder(closed_at: :desc) } scope :order_created_at_desc, -> { reorder(created_at: :desc) } scope :order_severity_asc, -> { includes(:issuable_severity).order('issuable_severities.severity ASC NULLS FIRST') } scope :order_severity_desc, -> { includes(:issuable_severity).order('issuable_severities.severity DESC NULLS LAST') } scope :order_escalation_status_asc, -> { includes(:incident_management_issuable_escalation_status).order(IncidentManagement::IssuableEscalationStatus.arel_table[:status].asc.nulls_last).references(:incident_management_issuable_escalation_status) } scope :order_escalation_status_desc, -> { includes(:incident_management_issuable_escalation_status).order(IncidentManagement::IssuableEscalationStatus.arel_table[:status].desc.nulls_last).references(:incident_management_issuable_escalation_status) } + scope :order_closed_at_asc, -> { reorder(arel_table[:closed_at].asc.nulls_last) } + scope :order_closed_at_desc, -> { reorder(arel_table[:closed_at].desc.nulls_last) } scope :preload_associated_models, -> { preload(:assignees, :labels, project: :namespace) } scope :with_web_entity_associations, -> { preload(:author, project: [:project_feature, :route, namespace: :route]) } @@ -138,7 +139,8 @@ class Issue < ApplicationRecord scope :with_api_entity_associations, -> { preload(:timelogs, :closed_by, :assignees, :author, :labels, :issuable_severity, milestone: { project: [:route, { namespace: :route }] }, - project: [:route, { namespace: :route }]) + project: [:route, { namespace: :route }], + duplicated_to: { project: [:project_feature] }) } scope :with_issue_type, ->(types) { where(issue_type: types) } scope :without_issue_type, ->(types) { where.not(issue_type: types) } @@ -149,7 +151,7 @@ class Issue < ApplicationRecord scope :without_hidden, -> { if Feature.enabled?(:ban_user_feature_flag) - where('NOT EXISTS (?)', Users::BannedUser.select(1).where('issues.author_id = banned_users.user_id')) + where.not(author_id: Users::BannedUser.all.select(:user_id)) else all end @@ -295,7 +297,7 @@ class Issue < ApplicationRecord end def self.link_reference_pattern - @link_reference_pattern ||= super("issues", Gitlab::Regex.issue) + @link_reference_pattern ||= super(%r{issues(?:\/incident)?}, Gitlab::Regex.issue) end def self.reference_valid?(reference) @@ -330,6 +332,8 @@ class Issue < ApplicationRecord when 'severity_desc' then order_severity_desc.with_order_id_desc when 'escalation_status_asc' then order_escalation_status_asc.with_order_id_desc when 'escalation_status_desc' then order_escalation_status_desc.with_order_id_desc + when 'closed_at_asc' then order_closed_at_asc + when 'closed_at_desc' then order_closed_at_desc else super end @@ -613,6 +617,11 @@ class Issue < ApplicationRecord super || WorkItems::Type.default_by_type(issue_type) end + def expire_etag_cache + key = Gitlab::Routing.url_helpers.realtime_changes_project_issue_path(project, self) + Gitlab::EtagCaching::Store.new.touch(key) + end + private override :persist_pg_full_text_search_vector @@ -643,11 +652,6 @@ class Issue < ApplicationRecord !confidential? && !hidden? && !::Gitlab::ExternalAuthorization.enabled? end - def expire_etag_cache - key = Gitlab::Routing.url_helpers.realtime_changes_project_issue_path(project, self) - Gitlab::EtagCaching::Store.new.touch(key) - end - def could_not_move(exception) # Symptom of running out of space - schedule rebalancing Issues::RebalancingWorker.perform_async(nil, *project.self_or_root_group_ids) diff --git a/app/models/key.rb b/app/models/key.rb index e093f9faad3..5268ce2e040 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'digest/md5' - class Key < ApplicationRecord include AfterCommitQueue include Sortable @@ -30,6 +28,7 @@ class Key < ApplicationRecord validate :key_meets_restrictions validate :expiration, on: :create + validate :banned_key, if: :should_check_for_banned_key? delegate :name, :email, to: :user, prefix: true @@ -144,6 +143,27 @@ class Key < ApplicationRecord end end + def should_check_for_banned_key? + return false unless user + + key_changed? && Feature.enabled?(:ssh_banned_key, user) + end + + def banned_key + return unless public_key.banned? + + help_page_url = Rails.application.routes.url_helpers.help_page_url( + 'security/ssh_keys_restrictions', + anchor: 'block-banned-or-compromised-keys' + ) + + errors.add( + :key, + _('cannot be used because it belongs to a compromised private key. Stop using this key and generate a new one.'), + help_page_url: help_page_url + ) + end + def forbidden_key_type_message allowed_types = Gitlab::CurrentSettings.allowed_key_types.map(&:upcase) diff --git a/app/models/label.rb b/app/models/label.rb index 7f4556c11c9..6608a0573cb 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -118,7 +118,7 @@ class Label < ApplicationRecord | # Integer-based label ID, or (?<label_name> # String-based single-word label title, or - [A-Za-z0-9_\-\?\.&]+ + #{Gitlab::Regex.sep_by_1(/:{1,2}/, /[A-Za-z0-9_\-\?\.&]+/)} (?<!\.|\?) | # String-based multi-word label surrounded in quotes diff --git a/app/models/member.rb b/app/models/member.rb index 45ad47f56a4..bb5d2b10f8e 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -199,7 +199,6 @@ class Member < ApplicationRecord before_validation :generate_invite_token, on: :create, if: -> (member) { member.invite_email.present? && !member.invite_accepted_at? } after_create :send_invite, if: :invite?, unless: :importing? - after_create :send_request, if: :request?, unless: :importing? after_create :create_notification_setting, unless: [:pending?, :importing?] after_create :post_create_hook, unless: [:pending?, :importing?], if: :hook_prerequisites_met? after_update :post_update_hook, unless: [:pending?, :importing?], if: :hook_prerequisites_met? @@ -207,6 +206,7 @@ class Member < ApplicationRecord after_destroy :post_destroy_hook, unless: :pending?, if: :hook_prerequisites_met? after_save :log_invitation_token_cleanup + after_commit :send_request, if: :request?, unless: :importing?, on: [:create] after_commit on: [:create, :update], unless: :importing? do refresh_member_authorized_projects(blocking: blocking_refresh) end diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb index a8a4fbedc41..87af6a9a7f7 100644 --- a/app/models/members/group_member.rb +++ b/app/models/members/group_member.rb @@ -7,6 +7,7 @@ class GroupMember < Member SOURCE_TYPE = 'Namespace' SOURCE_TYPE_FORMAT = /\ANamespace\z/.freeze + THRESHOLD_FOR_REFRESHING_AUTHORIZATIONS_VIA_PROJECTS = 1000 belongs_to :group, foreign_key: 'source_id' alias_attribute :namespace_id, :source_id @@ -28,6 +29,12 @@ class GroupMember < Member attr_accessor :last_owner, :last_blocked_owner + # For those who get to see a modal with a role dropdown, here are the options presented + def self.permissible_access_level_roles(_, _) + # This method is a stopgap in preparation for https://gitlab.com/gitlab-org/gitlab/-/issues/364087 + access_level_roles + end + def self.access_level_roles Gitlab::Access.options_with_owner end @@ -60,8 +67,28 @@ class GroupMember < Member # its projects are also destroyed, so the removal of project_authorizations # will happen behind the scenes via DB foreign keys anyway. return if destroyed_by_association.present? + return unless user_id + return super if Feature.disabled?(:refresh_authorizations_via_affected_projects_on_group_membership, group) - super + # rubocop:disable CodeReuse/ServiceClass + projects_to_refresh = Groups::ProjectsRequiringAuthorizationsRefresh::OnDirectMembershipFinder.new(group).execute + threshold_exceeded = (projects_to_refresh.size > THRESHOLD_FOR_REFRESHING_AUTHORIZATIONS_VIA_PROJECTS) + + # We want to try the new approach only if the number of affected projects are greater than the set threshold. + return super unless threshold_exceeded + + AuthorizedProjectUpdate::ProjectAccessChangedService + .new(projects_to_refresh) + .execute(blocking: false) + + # Until we compare the inconsistency rates of the new approach + # the old approach, we still run AuthorizedProjectsWorker + # but with some delay and lower urgency as a safety net. + UserProjectAccessChangedService + .new(user_id) + .execute(blocking: false, priority: UserProjectAccessChangedService::LOW_PRIORITY) + + # rubocop:enable CodeReuse/ServiceClass end def send_invite @@ -91,7 +118,10 @@ class GroupMember < Member end def after_accept_invite - notification_service.accept_group_invite(self) + run_after_commit_or_now do + notification_service.accept_group_invite(self) + end + update_two_factor_requirement super diff --git a/app/models/members/last_group_owner_assigner.rb b/app/models/members/last_group_owner_assigner.rb index dcf0a2d0ad3..c85116858c7 100644 --- a/app/models/members/last_group_owner_assigner.rb +++ b/app/models/members/last_group_owner_assigner.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# Optimization class to fix group member n+1 queries class LastGroupOwnerAssigner def initialize(group, members) @group = group @@ -39,6 +40,6 @@ class LastGroupOwnerAssigner end def owners - @owners ||= group.members_with_parents.owners.load + @owners ||= group.all_owners_excluding_project_bots.load end end diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index 995c26d7221..791cb6f0dff 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -44,7 +44,7 @@ class ProjectMember < Member project_ids.each do |project_id| project = Project.find(project_id) - Members::Projects::BulkCreatorService.add_users( # rubocop:disable CodeReuse/ServiceClass + Members::Projects::CreatorService.add_users( # rubocop:disable CodeReuse/ServiceClass project, users, access_level, @@ -73,6 +73,16 @@ class ProjectMember < Member truncate_teams [project.id] end + # For those who get to see a modal with a role dropdown, here are the options presented + def permissible_access_level_roles(current_user, project) + # This method is a stopgap in preparation for https://gitlab.com/gitlab-org/gitlab/-/issues/364087 + if Ability.allowed?(current_user, :manage_owners, project) + Gitlab::Access.options_with_owner + else + ProjectMember.access_level_roles + end + end + def access_level_roles Gitlab::Access.options end @@ -158,7 +168,9 @@ class ProjectMember < Member end def after_accept_invite - notification_service.accept_project_invite(self) + run_after_commit_or_now do + notification_service.accept_project_invite(self) + end super end diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 39b5949ea7a..1a3464d05a2 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -231,7 +231,10 @@ class MergeRequest < ApplicationRecord # rubocop: disable CodeReuse/ServiceClass after_transition [:unchecked, :checking] => :cannot_be_merged do |merge_request, transition| if merge_request.notify_conflict? - NotificationService.new.merge_request_unmergeable(merge_request) + merge_request.run_after_commit do + NotificationService.new.merge_request_unmergeable(merge_request) + end + TodoService.new.merge_request_became_unmergeable(merge_request) end end @@ -1150,6 +1153,19 @@ class MergeRequest < ApplicationRecord can_be_merged? && !should_be_rebased? end + def mergeability_checks + # We want to have the cheapest checks first in the list, that way we can + # fail fast before running the more expensive ones. + # + [ + ::MergeRequests::Mergeability::CheckOpenStatusService, + ::MergeRequests::Mergeability::CheckDraftStatusService, + ::MergeRequests::Mergeability::CheckBrokenStatusService, + ::MergeRequests::Mergeability::CheckDiscussionsStatusService, + ::MergeRequests::Mergeability::CheckCiStatusService + ] + end + # rubocop: disable CodeReuse/ServiceClass def mergeable_state?(skip_ci_check: false, skip_discussions_check: false) if Feature.enabled?(:improved_mergeability_checks, self.project) @@ -1654,9 +1670,9 @@ class MergeRequest < ApplicationRecord # TODO: consider renaming this as with exposed artifacts we generate reports, # not always compare # issue: https://gitlab.com/gitlab-org/gitlab/issues/34224 - def compare_reports(service_class, current_user = nil, report_type = nil ) + def compare_reports(service_class, current_user = nil, report_type = nil, additional_params = {} ) with_reactive_cache(service_class.name, current_user&.id, report_type) do |data| - unless service_class.new(project, current_user, id: id, report_type: report_type) + unless service_class.new(project, current_user, id: id, report_type: report_type, additional_params: additional_params) .latest?(comparison_base_pipeline(service_class.name), actual_head_pipeline, data) raise InvalidateReactiveCache end @@ -1696,7 +1712,12 @@ class MergeRequest < ApplicationRecord service_class.new(project, current_user, id: id, report_type: report_type).execute(comparison_base_pipeline(identifier), actual_head_pipeline) end - def recent_diff_head_shas(limit = 100) + MAX_RECENT_DIFF_HEAD_SHAS = 100 + + def recent_diff_head_shas(limit = MAX_RECENT_DIFF_HEAD_SHAS) + # see MergeRequestDiff.recent + return merge_request_diffs.to_a.sort_by(&:id).reverse.first(limit).pluck(:head_commit_sha) if merge_request_diffs.loaded? + merge_request_diffs.recent(limit).pluck(:head_commit_sha) end @@ -1955,6 +1976,10 @@ class MergeRequest < ApplicationRecord end end + def target_default_branch? + target_branch == project.default_branch + end + private attr_accessor :skip_fetch_ref diff --git a/app/models/merge_request/cleanup_schedule.rb b/app/models/merge_request/cleanup_schedule.rb index 35194b2b318..7f52a110da1 100644 --- a/app/models/merge_request/cleanup_schedule.rb +++ b/app/models/merge_request/cleanup_schedule.rb @@ -8,6 +8,9 @@ class MergeRequest::CleanupSchedule < ApplicationRecord failed: 3 }.freeze + # NOTE: Limit the number of stuck schedule jobs to retry just in case it becomes too big. + STUCK_RETRY_LIMIT = 5 + belongs_to :merge_request, inverse_of: :cleanup_schedule validates :scheduled_at, presence: true @@ -48,6 +51,11 @@ class MergeRequest::CleanupSchedule < ApplicationRecord .order('scheduled_at DESC') } + # NOTE: It is considered stuck as it is unusual to take more than 6 hours to finish the cleanup task. + scope :stuck, -> { + where('updated_at <= NOW() - interval \'6 hours\' AND status = ?', STATUSES[:running]) + } + def self.start_next MergeRequest::CleanupSchedule.transaction do cleanup_schedule = scheduled_and_unstarted.lock('FOR UPDATE SKIP LOCKED').first @@ -58,4 +66,8 @@ class MergeRequest::CleanupSchedule < ApplicationRecord cleanup_schedule end end + + def self.stuck_retry! + self.stuck.limit(STUCK_RETRY_LIMIT).map(&:retry!) + end end diff --git a/app/models/merge_request_diff_file.rb b/app/models/merge_request_diff_file.rb index f3f64971426..f7648937c1d 100644 --- a/app/models/merge_request_diff_file.rb +++ b/app/models/merge_request_diff_file.rb @@ -15,9 +15,11 @@ class MergeRequestDiffFile < ApplicationRecord end def utf8_diff - return '' if diff.blank? + fetched_diff = diff - encode_utf8(diff) if diff.respond_to?(:encoding) + return '' if fetched_diff.blank? + + encode_utf8(fetched_diff) if fetched_diff.respond_to?(:encoding) end def diff diff --git a/app/models/namespace.rb b/app/models/namespace.rb index fcd641671f5..5bb06cdbb4a 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -73,6 +73,8 @@ class Namespace < ApplicationRecord has_one :ci_namespace_mirror, class_name: 'Ci::NamespaceMirror' has_many :sync_events, class_name: 'Namespaces::SyncEvent' + has_one :cluster_enabled_grant, inverse_of: :namespace, class_name: 'Clusters::ClusterEnabledGrant' + validates :owner, presence: true, if: ->(n) { n.owner_required? } validates :name, presence: true, @@ -208,7 +210,7 @@ class Namespace < ApplicationRecord end end - def clean_path(path) + def clean_path(path, limited_to: Namespace.all) path = path.dup # Get the email username by removing everything after an `@` sign. path.gsub!(/@.*\z/, "") @@ -229,7 +231,7 @@ class Namespace < ApplicationRecord path = "blank" if path.blank? uniquify = Uniquify.new - uniquify.string(path) { |s| Namespace.find_by_path_or_name(s) } + uniquify.string(path) { |s| limited_to.find_by_path_or_name(s) } end def clean_name(value) @@ -411,12 +413,10 @@ class Namespace < ApplicationRecord return { scope: :group, status: auto_devops_enabled } unless auto_devops_enabled.nil? strong_memoize(:first_auto_devops_config) do - if has_parent? && cache_first_auto_devops_config? + if has_parent? Rails.cache.fetch(first_auto_devops_config_cache_key_for(id), expires_in: 1.day) do parent.first_auto_devops_config end - elsif has_parent? - parent.first_auto_devops_config else { scope: :instance, status: Gitlab::CurrentSettings.auto_devops_enabled? } end @@ -427,6 +427,28 @@ class Namespace < ApplicationRecord aggregation_schedule.present? end + def container_repositories_size_cache_key + "namespaces:#{id}:container_repositories_size" + end + + def container_repositories_size + strong_memoize(:container_repositories_size) do + next unless Gitlab.com? + next unless root? + next unless ContainerRegistry::GitlabApiClient.supports_gitlab_api? + next 0 if all_container_repositories.empty? + next unless all_container_repositories.all_migrated? + + Rails.cache.fetch(container_repositories_size_cache_key, expires_in: 7.days) do + ContainerRegistry::GitlabApiClient.deduplicated_size(full_path) + end + end + end + + def all_container_repositories + ContainerRepository.for_project_id(all_projects) + end + def pages_virtual_domain Pages::VirtualDomain.new( all_projects_with_pages.includes(:route, :project_feature, pages_metadatum: :pages_deployment), @@ -524,19 +546,35 @@ class Namespace < ApplicationRecord end def storage_enforcement_date + return Date.current if Feature.enabled?(:namespace_storage_limit_bypass_date_check, self) + # should return something like Date.new(2022, 02, 03) # TBD: https://gitlab.com/gitlab-org/gitlab/-/issues/350632 nil end def certificate_based_clusters_enabled? - ::Gitlab::SafeRequestStore.fetch("certificate_based_clusters:ns:#{self.id}") do - Feature.enabled?(:certificate_based_clusters, self, type: :ops) - end + cluster_enabled_granted? || certificate_based_clusters_enabled_ff? + end + + def enabled_git_access_protocol + # If the instance-level setting is enabled, we defer to that + return ::Gitlab::CurrentSettings.enabled_git_access_protocol unless ::Gitlab::CurrentSettings.enabled_git_access_protocol.blank? + + # Otherwise we use the stored setting on the group + namespace_settings&.enabled_git_access_protocol end private + def cluster_enabled_granted? + (Gitlab.com? || Gitlab.dev_or_test_env?) && root_ancestor.cluster_enabled_grant.present? + end + + def certificate_based_clusters_enabled_ff? + Feature.enabled?(:certificate_based_clusters, type: :ops) + end + def expire_child_caches Namespace.where(id: descendants).each_batch do |namespaces| namespaces.touch_all @@ -611,7 +649,7 @@ class Namespace < ApplicationRecord return end - if parent.project_namespace? + if parent&.project_namespace? errors.add(:parent_id, _('project namespace cannot be the parent of another namespace')) end @@ -638,8 +676,6 @@ class Namespace < ApplicationRecord end def expire_first_auto_devops_config_cache - return unless cache_first_auto_devops_config? - descendants_to_expire = self_and_descendants.as_ids return if descendants_to_expire.load.empty? @@ -647,10 +683,6 @@ class Namespace < ApplicationRecord Rails.cache.delete_multi(keys) end - def cache_first_auto_devops_config? - ::Feature.enabled?(:namespaces_cache_first_auto_devops_config) - end - def write_projects_repository_config all_projects.find_each do |project| project.set_full_path @@ -670,8 +702,6 @@ class Namespace < ApplicationRecord end def first_auto_devops_config_cache_key_for(group_id) - return "namespaces:{first_auto_devops_config}:#{group_id}" unless sync_traversal_ids? - # Use SHA2 of `traversal_ids` to account for moving a namespace within the same root ancestor hierarchy. "namespaces:{#{traversal_ids.first}}:first_auto_devops_config:#{group_id}:#{Digest::SHA2.hexdigest(traversal_ids.join(' '))}" end diff --git a/app/models/namespace/root_storage_statistics.rb b/app/models/namespace/root_storage_statistics.rb index 96715863892..77974a0f36b 100644 --- a/app/models/namespace/root_storage_statistics.rb +++ b/app/models/namespace/root_storage_statistics.rb @@ -44,15 +44,26 @@ class Namespace::RootStorageStatistics < ApplicationRecord def merged_attributes attributes_from_project_statistics.merge!( attributes_from_personal_snippets, - attributes_from_namespace_statistics + attributes_from_namespace_statistics, + attributes_for_container_registry_size ) { |key, v1, v2| v1 + v2 } end + def attributes_for_container_registry_size + container_registry_size = namespace.container_repositories_size || 0 + + { + storage_size: container_registry_size, + container_registry_size: container_registry_size + }.with_indifferent_access + end + def attributes_from_project_statistics from_project_statistics - .take - .attributes - .slice(*STATISTICS_ATTRIBUTES) + .take + .attributes + .slice(*STATISTICS_ATTRIBUTES) + .with_indifferent_access end def from_project_statistics @@ -74,7 +85,10 @@ class Namespace::RootStorageStatistics < ApplicationRecord def attributes_from_personal_snippets return {} unless namespace.user_namespace? - from_personal_snippets.take.slice(SNIPPETS_SIZE_STAT_NAME) + from_personal_snippets + .take + .slice(SNIPPETS_SIZE_STAT_NAME) + .with_indifferent_access end def from_personal_snippets @@ -102,7 +116,12 @@ class Namespace::RootStorageStatistics < ApplicationRecord # guard clause. return {} unless namespace.group_namespace? - from_namespace_statistics.take.slice(*self.class.namespace_statistics_attributes) + from_namespace_statistics + .take + .slice( + *self.class.namespace_statistics_attributes + ) + .with_indifferent_access end end diff --git a/app/models/namespace_setting.rb b/app/models/namespace_setting.rb index ef917c8a22e..504daf2662e 100644 --- a/app/models/namespace_setting.rb +++ b/app/models/namespace_setting.rb @@ -9,14 +9,17 @@ class NamespaceSetting < ApplicationRecord belongs_to :namespace, inverse_of: :namespace_settings + enum jobs_to_be_done: { basics: 0, move_repository: 1, code_storage: 2, exploring: 3, ci: 4, other: 5 }, _suffix: true + enum enabled_git_access_protocol: { all: 0, ssh: 1, http: 2 }, _suffix: true + + validates :enabled_git_access_protocol, inclusion: { in: enabled_git_access_protocols.keys } + validate :default_branch_name_content validate :allow_mfa_for_group validate :allow_resource_access_token_creation_for_group before_validation :normalize_default_branch_name - enum jobs_to_be_done: { basics: 0, move_repository: 1, code_storage: 2, exploring: 3, ci: 4, other: 5 }, _suffix: true - chronic_duration_attr :runner_token_expiration_interval_human_readable, :runner_token_expiration_interval chronic_duration_attr :subgroup_runner_token_expiration_interval_human_readable, :subgroup_runner_token_expiration_interval chronic_duration_attr :project_runner_token_expiration_interval_human_readable, :project_runner_token_expiration_interval @@ -24,7 +27,7 @@ class NamespaceSetting < ApplicationRecord NAMESPACE_SETTINGS_PARAMS = [:default_branch_name, :delayed_project_removal, :lock_delayed_project_removal, :resource_access_token_creation_allowed, :prevent_sharing_groups_outside_hierarchy, :new_user_signups_cap, - :setup_for_company, :jobs_to_be_done, :runner_token_expiration_interval, + :setup_for_company, :jobs_to_be_done, :runner_token_expiration_interval, :enabled_git_access_protocol, :subgroup_runner_token_expiration_interval, :project_runner_token_expiration_interval].freeze self.primary_key = :namespace_id diff --git a/app/models/namespaces/project_namespace.rb b/app/models/namespaces/project_namespace.rb index fbd87e3232d..2a2ea11ddc5 100644 --- a/app/models/namespaces/project_namespace.rb +++ b/app/models/namespaces/project_namespace.rb @@ -2,6 +2,13 @@ module Namespaces class ProjectNamespace < Namespace + # These aliases are added to make it easier to sync parent/parent_id attribute with + # project.namespace/project.namespace_id attribute. + # + # TODO: we can remove these attribute aliases when we no longer need to sync these with project model, + # see project#sync_attributes + alias_attribute :namespace, :parent + alias_attribute :namespace_id, :parent_id has_one :project, foreign_key: :project_namespace_id, inverse_of: :project_namespace def self.sti_name diff --git a/app/models/namespaces/traversal/linear.rb b/app/models/namespaces/traversal/linear.rb index b0350b0288f..687fa6a5334 100644 --- a/app/models/namespaces/traversal/linear.rb +++ b/app/models/namespaces/traversal/linear.rb @@ -42,11 +42,11 @@ module Namespaces UnboundedSearch = Class.new(StandardError) included do - before_update :lock_both_roots, if: -> { sync_traversal_ids? && parent_id_changed? } - after_update :sync_traversal_ids, if: -> { sync_traversal_ids? && saved_change_to_parent_id? } + before_update :lock_both_roots, if: -> { parent_id_changed? } + after_update :sync_traversal_ids, if: -> { saved_change_to_parent_id? } # This uses rails internal before_commit API to sync traversal_ids on namespace create, right before transaction is committed. # This helps reduce the time during which the root namespace record is locked to ensure updated traversal_ids are valid - before_commit :sync_traversal_ids, on: [:create], if: -> { sync_traversal_ids? } + before_commit :sync_traversal_ids, on: [:create] end class_methods do @@ -76,10 +76,6 @@ module Namespaces end end - def sync_traversal_ids? - Feature.enabled?(:sync_traversal_ids, root_ancestor) - end - def use_traversal_ids? return false unless Feature.enabled?(:use_traversal_ids) diff --git a/app/models/namespaces/traversal/linear_scopes.rb b/app/models/namespaces/traversal/linear_scopes.rb index f0e9a8feeb2..6f404ec12d0 100644 --- a/app/models/namespaces/traversal/linear_scopes.rb +++ b/app/models/namespaces/traversal/linear_scopes.rb @@ -5,6 +5,8 @@ module Namespaces module LinearScopes extend ActiveSupport::Concern + include AsCte + class_methods do # When filtering namespaces by the traversal_ids column to compile a # list of namespace IDs, it can be faster to reference the ID in @@ -25,25 +27,15 @@ module Namespaces def self_and_ancestors(include_self: true, upto: nil, hierarchy_order: nil) return super unless use_traversal_ids_for_ancestor_scopes? - ancestors_cte, base_cte = ancestor_ctes - namespaces = Arel::Table.new(:namespaces) - - records = unscoped - .with(base_cte.to_arel, ancestors_cte.to_arel) - .distinct - .from([ancestors_cte.table, namespaces]) - .where(namespaces[:id].eq(ancestors_cte.table[:ancestor_id])) - .order_by_depth(hierarchy_order) - - unless include_self - records = records.where(ancestors_cte.table[:base_id].not_eq(ancestors_cte.table[:ancestor_id])) - end - - if upto - records = records.where.not(id: unscoped.where(id: upto).select('unnest(traversal_ids)')) + if Feature.enabled?(:use_traversal_ids_for_ancestor_scopes_with_inner_join) + self_and_ancestors_from_inner_join(include_self: include_self, + upto: upto, hierarchy_order: + hierarchy_order) + else + self_and_ancestors_from_ancestors_cte(include_self: include_self, + upto: upto, + hierarchy_order: hierarchy_order) end - - records end def self_and_ancestor_ids(include_self: true) @@ -87,7 +79,7 @@ module Namespaces depth_order = hierarchy_order == :asc ? :desc : :asc all - .select(Arel.star, 'array_length(traversal_ids, 1) as depth') + .select(Namespace.default_select_columns, 'array_length(traversal_ids, 1) as depth') .order(depth: depth_order, id: :asc) end @@ -125,26 +117,106 @@ module Namespaces use_traversal_ids? end + def self_and_ancestors_from_ancestors_cte(include_self: true, upto: nil, hierarchy_order: nil) + base_cte = all.select('namespaces.id', 'namespaces.traversal_ids').as_cte(:base_ancestors_cte) + + # We have to alias id with 'AS' to avoid ambiguous column references by calling methods. + ancestors_cte = unscoped + .unscope(where: [:type]) + .select('id as base_id', + "#{unnest_func(base_cte.table['traversal_ids']).to_sql} as ancestor_id") + .from(base_cte.table) + .as_cte(:ancestors_cte) + + namespaces = Arel::Table.new(:namespaces) + + records = unscoped + .with(base_cte.to_arel, ancestors_cte.to_arel) + .distinct + .from([ancestors_cte.table, namespaces]) + .where(namespaces[:id].eq(ancestors_cte.table[:ancestor_id])) + .order_by_depth(hierarchy_order) + + unless include_self + records = records.where(ancestors_cte.table[:base_id].not_eq(ancestors_cte.table[:ancestor_id])) + end + + if upto + records = records.where.not(id: unscoped.where(id: upto).select('unnest(traversal_ids)')) + end + + records + end + + def self_and_ancestors_from_inner_join(include_self: true, upto: nil, hierarchy_order: nil) + base_cte = all.reselect('namespaces.traversal_ids').as_cte(:base_ancestors_cte) + + unnest = if include_self + base_cte.table[:traversal_ids] + else + base_cte_traversal_ids = 'base_ancestors_cte.traversal_ids' + traversal_ids_range = "1:array_length(#{base_cte_traversal_ids},1)-1" + Arel.sql("#{base_cte_traversal_ids}[#{traversal_ids_range}]") + end + + ancestor_subselect = "SELECT DISTINCT #{unnest_func(unnest).to_sql} FROM base_ancestors_cte" + ancestors_join = <<~SQL + INNER JOIN (#{ancestor_subselect}) AS ancestors(ancestor_id) ON namespaces.id = ancestors.ancestor_id + SQL + + namespaces = Arel::Table.new(:namespaces) + + records = unscoped + .with(base_cte.to_arel) + .from(namespaces) + .joins(ancestors_join) + .order_by_depth(hierarchy_order) + + if upto + upto_ancestor_ids = unscoped.where(id: upto).select(unnest_func(Arel.sql('traversal_ids'))) + records = records.where.not(id: upto_ancestor_ids) + end + + records + end + def self_and_descendants_with_comparison_operators(include_self: true) base = all.select(:traversal_ids) - base_cte = Gitlab::SQL::CTE.new(:descendants_base_cte, base) + base = base.select(:id) if Feature.enabled?(:linear_scopes_superset) + base_cte = base.as_cte(:descendants_base_cte) namespaces = Arel::Table.new(:namespaces) + withs = [base_cte.to_arel] + froms = [] + + if Feature.enabled?(:linear_scopes_superset) + superset_cte = self.superset_cte(base_cte.table.name) + withs += [superset_cte.to_arel] + froms = [superset_cte.table] + else + froms = [base_cte.table] + end + + # Order is important. namespace should be last to handle future joins. + froms += [namespaces] + + base_ref = froms.first + # Bound the search space to ourselves (optional) and descendants. # # WHERE next_traversal_ids_sibling(base_cte.traversal_ids) > namespaces.traversal_ids records = unscoped .distinct - .with(base_cte.to_arel) - .from([base_cte.table, namespaces]) - .where(next_sibling_func(base_cte.table[:traversal_ids]).gt(namespaces[:traversal_ids])) + .with(*withs) + .from(froms) + .where(next_sibling_func(base_ref[:traversal_ids]).gt(namespaces[:traversal_ids])) # AND base_cte.traversal_ids <= namespaces.traversal_ids if include_self - records.where(base_cte.table[:traversal_ids].lteq(namespaces[:traversal_ids])) + records.where(base_ref[:traversal_ids].lteq(namespaces[:traversal_ids])) else - records.where(base_cte.table[:traversal_ids].lt(namespaces[:traversal_ids])) + records.where(base_ref[:traversal_ids].lt(namespaces[:traversal_ids])) end end @@ -152,6 +224,10 @@ module Namespaces Arel::Nodes::NamedFunction.new('next_traversal_ids_sibling', args) end + def unnest_func(*args) + Arel::Nodes::NamedFunction.new('unnest', args) + end + def self_and_descendants_with_duplicates_with_array_operator(include_self: true) base_ids = select(:id) @@ -166,18 +242,19 @@ module Namespaces end end - def ancestor_ctes - base_scope = all.select('namespaces.id', 'namespaces.traversal_ids') - base_cte = Gitlab::SQL::CTE.new(:base_ancestors_cte, base_scope) - - # We have to alias id with 'AS' to avoid ambiguous column references by calling methods. - ancestors_scope = unscoped - .unscope(where: [:type]) - .select('id as base_id', 'unnest(traversal_ids) as ancestor_id') - .from(base_cte.table) - ancestors_cte = Gitlab::SQL::CTE.new(:ancestors_cte, ancestors_scope) - - [ancestors_cte, base_cte] + def superset_cte(base_name) + superset_sql = <<~SQL + SELECT d1.traversal_ids + FROM #{base_name} d1 + WHERE NOT EXISTS ( + SELECT 1 + FROM #{base_name} d2 + WHERE d2.id = ANY(d1.traversal_ids) + AND d2.id <> d1.id + ) + SQL + + Gitlab::SQL::CTE.new(:superset, superset_sql, materialized: false) end end end diff --git a/app/models/namespaces/traversal/recursive.rb b/app/models/namespaces/traversal/recursive.rb index 53eac27aa54..1c5d395cb3c 100644 --- a/app/models/namespaces/traversal/recursive.rb +++ b/app/models/namespaces/traversal/recursive.rb @@ -63,19 +63,17 @@ module Namespaces # Returns all the descendants of the current namespace. def descendants - object_hierarchy(self.class.where(parent_id: id)) - .base_and_descendants + object_hierarchy(self.class.where(parent_id: id)).base_and_descendants end alias_method :recursive_descendants, :descendants def self_and_descendants - object_hierarchy(self.class.where(id: id)) - .base_and_descendants + object_hierarchy(self.class.where(id: id)).base_and_descendants end alias_method :recursive_self_and_descendants, :self_and_descendants def self_and_descendant_ids - recursive_self_and_descendants.select(:id) + object_hierarchy(self.class.where(id: id)).base_and_descendant_ids end alias_method :recursive_self_and_descendant_ids, :self_and_descendant_ids diff --git a/app/models/note.rb b/app/models/note.rb index 3d2ac69a2ab..41e45a8759f 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -124,7 +124,6 @@ class Note < ApplicationRecord scope :common, -> { where(noteable_type: ["", nil]) } scope :fresh, -> { order_created_asc.with_order_id_asc } scope :updated_after, ->(time) { where('updated_at > ?', time) } - scope :with_updated_at, ->(time) { where(updated_at: time) } scope :with_discussion_ids, ->(discussion_ids) { where(discussion_id: discussion_ids) } scope :with_suggestions, -> { joins(:suggestions) } scope :inc_author, -> { includes(:author) } diff --git a/app/models/packages/cleanup/policy.rb b/app/models/packages/cleanup/policy.rb index 87c101cfb8c..d7df90a4ce0 100644 --- a/app/models/packages/cleanup/policy.rb +++ b/app/models/packages/cleanup/policy.rb @@ -15,7 +15,7 @@ module Packages validates :keep_n_duplicated_package_files, inclusion: { in: KEEP_N_DUPLICATED_PACKAGE_FILES_VALUES, - message: 'keep_n_duplicated_package_files is invalid' + message: 'is invalid' } # used by Schedulable diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb index 7744e578df5..90a1bb4bc69 100644 --- a/app/models/packages/package.rb +++ b/app/models/packages/package.rb @@ -103,7 +103,15 @@ class Packages::Package < ApplicationRecord scope :for_projects, ->(project_ids) { where(project_id: project_ids) } scope :with_name, ->(name) { where(name: name) } scope :with_name_like, ->(name) { where(arel_table[:name].matches(name)) } - scope :with_normalized_pypi_name, ->(name) { where("LOWER(regexp_replace(name, '[-_.]+', '-', 'g')) = ?", name.downcase) } + + scope :with_normalized_pypi_name, ->(name) do + where( + "LOWER(regexp_replace(name, ?, '-', 'g')) = ?", + Gitlab::Regex::Packages::PYPI_NORMALIZED_NAME_REGEX_STRING, + name.downcase + ) + end + scope :search_by_name, ->(query) { fuzzy_search(query, [:name], use_minimum_char_limit: false) } scope :with_version, ->(version) { where(version: version) } scope :without_version_like, -> (version) { where.not(arel_table[:version].matches(version)) } @@ -315,6 +323,13 @@ class Packages::Package < ApplicationRecord ::Packages::MarkPackageFilesForDestructionWorker.perform_async(id) end + # As defined in PEP 503 https://peps.python.org/pep-0503/#normalized-names + def normalized_pypi_name + return name unless pypi? + + name.gsub(/#{Gitlab::Regex::Packages::PYPI_NORMALIZED_NAME_REGEX_STRING}/, '-').downcase + end + private def composer_tag_version? diff --git a/app/models/project.rb b/app/models/project.rb index b66ec28b659..dca47911d20 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -121,6 +121,8 @@ class Project < ApplicationRecord before_save :ensure_runners_token before_validation :ensure_project_namespace_in_sync + before_validation :set_package_registry_access_level, if: :packages_enabled_changed? + after_save :update_project_statistics, if: :saved_change_to_namespace_id? after_save :schedule_sync_event_worker, if: -> { saved_change_to_id? || saved_change_to_namespace_id? } @@ -418,6 +420,8 @@ class Project < ApplicationRecord has_one :ci_project_mirror, class_name: 'Ci::ProjectMirror' has_many :sync_events, class_name: 'Projects::SyncEvent' + has_one :build_artifacts_size_refresh, class_name: 'Projects::BuildArtifactsSizeRefresh' + accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature, update_only: true accepts_nested_attributes_for :project_setting, update_only: true @@ -443,7 +447,7 @@ class Project < ApplicationRecord :pages_enabled?, :analytics_enabled?, :snippets_enabled?, :public_pages?, :private_pages?, :merge_requests_access_level, :forking_access_level, :issues_access_level, :wiki_access_level, :snippets_access_level, :builds_access_level, - :repository_access_level, :pages_access_level, :metrics_dashboard_access_level, :analytics_access_level, + :repository_access_level, :package_registry_access_level, :pages_access_level, :metrics_dashboard_access_level, :analytics_access_level, :operations_enabled?, :operations_access_level, :security_and_compliance_access_level, :container_registry_access_level, :container_registry_enabled?, to: :project_feature, allow_nil: true @@ -598,6 +602,7 @@ class Project < ApplicationRecord scope :inc_routes, -> { includes(:route, namespace: :route) } scope :with_statistics, -> { includes(:statistics) } scope :with_namespace, -> { includes(:namespace) } + scope :with_group, -> { includes(:group) } scope :with_import_state, -> { includes(:import_state) } scope :include_project_feature, -> { includes(:project_feature) } scope :include_integration, -> (integration_association_name) { includes(integration_association_name) } @@ -1167,7 +1172,7 @@ class Project < ApplicationRecord job_type = type.to_s.capitalize if job_id - Gitlab::AppLogger.info("#{job_type} job scheduled for #{full_path} with job ID #{job_id}.") + Gitlab::AppLogger.info("#{job_type} job scheduled for #{full_path} with job ID #{job_id} (primary: #{::Gitlab::Database::LoadBalancing::Session.current.use_primary?}).") else Gitlab::AppLogger.error("#{job_type} job failed to create for #{full_path}.") end @@ -2161,6 +2166,7 @@ class Project < ApplicationRecord .append(key: 'CI_PROJECT_ID', value: id.to_s) .append(key: 'CI_PROJECT_NAME', value: path) .append(key: 'CI_PROJECT_TITLE', value: title) + .append(key: 'CI_PROJECT_DESCRIPTION', value: description) .append(key: 'CI_PROJECT_PATH', value: full_path) .append(key: 'CI_PROJECT_PATH_SLUG', value: full_path_slug) .append(key: 'CI_PROJECT_NAMESPACE', value: namespace.full_path) @@ -2504,7 +2510,13 @@ class Project < ApplicationRecord end def gitlab_deploy_token - @gitlab_deploy_token ||= deploy_tokens.gitlab_deploy_token + strong_memoize(:gitlab_deploy_token) do + if Feature.enabled?(:ci_variable_for_group_gitlab_deploy_token, self) + deploy_tokens.gitlab_deploy_token || group&.gitlab_deploy_token + else + deploy_tokens.gitlab_deploy_token + end + end end def any_lfs_file_locks? @@ -2573,16 +2585,7 @@ class Project < ApplicationRecord end def access_request_approvers_to_be_notified - # For a personal project: - # The creator is added as a member with `Owner` access level, starting from GitLab 14.8 - # The creator was added as a member with `Maintainer` access level, before GitLab 14.8 - # So, to make sure access requests for all personal projects work as expected, - # we need to filter members with the scope `owners_and_maintainers`. - access_request_approvers = if personal? - members.owners_and_maintainers - else - members.maintainers - end + access_request_approvers = members.owners_and_maintainers access_request_approvers.connected_to_user.order_recent_sign_in.limit(Member::ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT) end @@ -2900,6 +2903,14 @@ class Project < ApplicationRecord last_activity_at < ::Gitlab::CurrentSettings.inactive_projects_send_warning_email_after_months.months.ago end + def refreshing_build_artifacts_size? + build_artifacts_size_refresh&.started? + end + + def security_training_available? + licensed_feature_available?(:security_training) + end + private # overridden in EE @@ -3098,7 +3109,6 @@ class Project < ApplicationRecord # create project_namespace when project is created build_project_namespace if project_namespace_creation_enabled? - # we need to keep project and project namespace in sync if there is one sync_attributes(project_namespace) if sync_project_namespace? end @@ -3111,11 +3121,24 @@ class Project < ApplicationRecord end def sync_attributes(project_namespace) - project_namespace.name = name - project_namespace.path = path - project_namespace.parent = namespace - project_namespace.shared_runners_enabled = shared_runners_enabled - project_namespace.visibility_level = visibility_level + attributes_to_sync = changes.slice(*%w(name path namespace_id namespace visibility_level shared_runners_enabled)) + .transform_values { |val| val[1] } + + # if visibility_level is not set explicitly for project, it defaults to 0, + # but for namespace visibility_level defaults to 20, + # so it gets out of sync right away if we do not set it explicitly when creating the project namespace + attributes_to_sync['visibility_level'] ||= visibility_level if new_record? + + # when a project is associated with a group while the group is created we need to ensure we associate the new + # group with the project namespace as well. + # E.g. + # project = create(:project) <- project is saved + # create(:group, projects: [project]) <- associate project with a group that is not yet created. + if attributes_to_sync.has_key?('namespace_id') && attributes_to_sync['namespace_id'].blank? && namespace.present? + attributes_to_sync['parent'] = namespace + end + + project_namespace.assign_attributes(attributes_to_sync) end # SyncEvents are created by PG triggers (with the function `insert_projects_sync_event`) @@ -3132,6 +3155,23 @@ class Project < ApplicationRecord raise ExportLimitExceeded, _('The project size exceeds the export limit.') end end + + def set_package_registry_access_level + return if !project_feature || project_feature.package_registry_access_level_changed? + + self.project_feature.package_registry_access_level = packages_enabled ? enabled_package_registry_access_level_by_project_visibility : ProjectFeature::DISABLED + end + + def enabled_package_registry_access_level_by_project_visibility + case visibility_level + when PUBLIC + ProjectFeature::PUBLIC + when INTERNAL + ProjectFeature::ENABLED + else + ProjectFeature::PRIVATE + end + end end Project.prepend_mod_with('Project') diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb index 27692fe76f0..f478af32788 100644 --- a/app/models/project_feature.rb +++ b/app/models/project_feature.rb @@ -20,6 +20,7 @@ class ProjectFeature < ApplicationRecord operations security_and_compliance container_registry + package_registry ].freeze EXPORTABLE_FEATURES = (FEATURES - [:security_and_compliance, :pages]).freeze @@ -29,7 +30,8 @@ class ProjectFeature < ApplicationRecord PRIVATE_FEATURES_MIN_ACCESS_LEVEL = { merge_requests: Gitlab::Access::REPORTER, metrics_dashboard: Gitlab::Access::REPORTER, - container_registry: Gitlab::Access::REPORTER + container_registry: Gitlab::Access::REPORTER, + package_registry: Gitlab::Access::REPORTER }.freeze PRIVATE_FEATURES_MIN_ACCESS_LEVEL_FOR_PRIVATE_PROJECT = { repository: Gitlab::Access::REPORTER }.freeze @@ -76,6 +78,14 @@ class ProjectFeature < ApplicationRecord end end + default_value_for(:package_registry_access_level) do |feature| + if ::Gitlab.config.packages.enabled + ENABLED + else + DISABLED + end + end + default_value_for(:container_registry_access_level) do |feature| if gitlab_config_features.container_registry ENABLED @@ -142,6 +152,12 @@ class ProjectFeature < ApplicationRecord !public_pages? end + def package_registry_access_level=(value) + super(value).tap do + project.packages_enabled = self.package_registry_access_level != DISABLED if project + end + end + private # Validates builds and merge requests access level @@ -157,7 +173,7 @@ class ProjectFeature < ApplicationRecord end def feature_validation_exclusion - %i(pages) + %i(pages package_registry) end override :resource_member? diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index 95fc135f38f..a0af1b47d01 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -26,7 +26,7 @@ class ProjectStatistics < ApplicationRecord pipeline_artifacts_size: %i[storage_size], snippets_size: %i[storage_size] }.freeze - NAMESPACE_RELATABLE_COLUMNS = [:repository_size, :wiki_size, :lfs_objects_size, :uploads_size].freeze + NAMESPACE_RELATABLE_COLUMNS = [:repository_size, :wiki_size, :lfs_objects_size, :uploads_size, :container_registry_size].freeze scope :for_project_ids, ->(project_ids) { where(project_id: project_ids) } @@ -77,8 +77,6 @@ class ProjectStatistics < ApplicationRecord end def update_container_registry_size - return unless Feature.enabled?(:container_registry_project_statistics, project) - self.container_registry_size = project.container_repositories_size || 0 end diff --git a/app/models/project_team.rb b/app/models/project_team.rb index bb5363598df..97ab5aa2619 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -44,7 +44,7 @@ class ProjectTeam end def add_users(users, access_level, current_user: nil, expires_at: nil, tasks_to_be_done: [], tasks_project_id: nil) - Members::Projects::BulkCreatorService.add_users( # rubocop:disable CodeReuse/ServiceClass + Members::Projects::CreatorService.add_users( # rubocop:disable CodeReuse/ServiceClass project, users, access_level, @@ -56,12 +56,12 @@ class ProjectTeam end def add_user(user, access_level, current_user: nil, expires_at: nil) - Members::Projects::CreatorService.new(project, # rubocop:disable CodeReuse/ServiceClass - user, - access_level, - current_user: current_user, - expires_at: expires_at) - .execute + Members::Projects::CreatorService.add_user( # rubocop:disable CodeReuse/ServiceClass + project, + user, + access_level, + current_user: current_user, + expires_at: expires_at) end # Remove all users from project team diff --git a/app/models/projects/build_artifacts_size_refresh.rb b/app/models/projects/build_artifacts_size_refresh.rb index 959f486a50a..dee4afdefa6 100644 --- a/app/models/projects/build_artifacts_size_refresh.rb +++ b/app/models/projects/build_artifacts_size_refresh.rb @@ -36,6 +36,7 @@ module Projects before_transition created: :running do |refresh| refresh.reset_project_statistics! refresh.refresh_started_at = Time.zone.now + refresh.last_job_artifact_id_on_refresh_start = refresh.project.job_artifacts.last&.id end before_transition running: any do |refresh, transition| @@ -49,6 +50,7 @@ module Projects scope :stale, -> { with_state(:running).where('updated_at < ?', STALE_WINDOW.ago) } scope :remaining, -> { with_state(:created, :pending).or(stale) } + scope :processing_queue, -> { remaining.order(state: :desc) } def self.enqueue_refresh(projects) now = Time.zone.now @@ -64,8 +66,7 @@ module Projects next_refresh = nil transaction do - next_refresh = remaining - .order(:state, :updated_at) + next_refresh = processing_queue .lock('FOR UPDATE SKIP LOCKED') .take @@ -83,9 +84,14 @@ module Projects def next_batch(limit:) project.job_artifacts.select(:id, :size) - .where('created_at <= ? AND id > ?', refresh_started_at, last_job_artifact_id.to_i) - .order(:created_at) + .id_before(last_job_artifact_id_on_refresh_start) + .id_after(last_job_artifact_id.to_i) + .ordered_by_id .limit(limit) end + + def started? + !created? + end end end diff --git a/app/models/protected_tag.rb b/app/models/protected_tag.rb index 6b507429e57..5b2467daddc 100644 --- a/app/models/protected_tag.rb +++ b/app/models/protected_tag.rb @@ -8,7 +8,11 @@ class ProtectedTag < ApplicationRecord protected_ref_access_levels :create def self.protected?(project, ref_name) - refs = project.protected_tags.select(:name) + return false if ref_name.blank? + + refs = Gitlab::SafeRequestStore.fetch("protected-tag:#{project.cache_key}:refs") do + project.protected_tags.select(:name) + end self.matching(ref_name, protected_refs: refs).present? end diff --git a/app/models/release.rb b/app/models/release.rb index c6c0920c4d0..ee5d7bab190 100644 --- a/app/models/release.rb +++ b/app/models/release.rb @@ -31,6 +31,7 @@ class Release < ApplicationRecord validates :description, length: { maximum: Gitlab::Database::MAX_TEXT_SIZE_LIMIT }, if: :description_changed? validates_associated :milestone_releases, message: -> (_, obj) { obj[:value].map(&:errors).map(&:full_messages).join(",") } validates :links, nested_attributes_duplicates: { scope: :release, child_attributes: %i[name url filepath] } + validates :author_id, presence: true, on: :create, if: :validate_release_with_author? scope :sorted, -> { order(released_at: :desc) } scope :preloaded, -> { @@ -54,7 +55,7 @@ class Release < ApplicationRecord MAX_NUMBER_TO_DISPLAY = 3 def to_param - CGI.escape(tag) + tag end def commit @@ -117,6 +118,10 @@ class Release < ApplicationRecord end end + def validate_release_with_author? + Feature.enabled?(:validate_release_with_author, self.project) + end + def set_released_at self.released_at ||= created_at end diff --git a/app/models/repository.rb b/app/models/repository.rb index dc0b5b54fb0..0135020e586 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -13,6 +13,7 @@ class Repository REF_KEEP_AROUND = 'keep-around' REF_ENVIRONMENTS = 'environments' REF_PIPELINES = 'pipelines' + REF_TMP = 'tmp' ARCHIVE_CACHE_TIME = 60 # Cache archives referred to by a (mutable) ref for 1 minute ARCHIVE_CACHE_TIME_IMMUTABLE = 3600 # Cache archives referred to by an immutable reference for 1 hour @@ -175,8 +176,8 @@ class Repository end # Returns a list of commits that are not present in any reference - def new_commits(newrev, allow_quarantine: false) - commits = raw.new_commits(newrev, allow_quarantine: allow_quarantine) + def new_commits(newrev) + commits = raw.new_commits(newrev) ::Commit.decorate(commits, container) end diff --git a/app/models/resource_event.rb b/app/models/resource_event.rb index 54fa4137f73..8b82e0f343c 100644 --- a/app/models/resource_event.rb +++ b/app/models/resource_event.rb @@ -11,7 +11,6 @@ class ResourceEvent < ApplicationRecord belongs_to :user scope :created_after, ->(time) { where('created_at > ?', time) } - scope :created_on_or_before, ->(time) { where('created_at <= ?', time) } def discussion_id strong_memoize(:discussion_id) do diff --git a/app/models/route.rb b/app/models/route.rb index 12b2d5c5bb2..2f6b0a8e8f1 100644 --- a/app/models/route.rb +++ b/app/models/route.rb @@ -13,7 +13,6 @@ class Route < ApplicationRecord presence: true, uniqueness: { case_sensitive: false } - before_validation :delete_conflicting_orphaned_routes after_create :delete_conflicting_redirects after_update :delete_conflicting_redirects, if: :saved_change_to_path? after_update :create_redirect_for_old_path @@ -71,13 +70,4 @@ class Route < ApplicationRecord def create_redirect_for_old_path create_redirect(path_before_last_save) if saved_change_to_path? end - - def delete_conflicting_orphaned_routes - conflicting = self.class.iwhere(path: path) - conflicting_orphaned_routes = conflicting.select do |route| - route.source.nil? - end - - conflicting_orphaned_routes.each(&:destroy) - end end diff --git a/app/models/terraform/state.rb b/app/models/terraform/state.rb index 8c3b85ac4c3..4d17a4d332c 100644 --- a/app/models/terraform/state.rb +++ b/app/models/terraform/state.rb @@ -23,13 +23,10 @@ module Terraform scope :ordered_by_name, -> { order(:name) } scope :with_name, -> (name) { where(name: name) } - validates :name, presence: true, uniqueness: { scope: :project_id } - validates :project_id, presence: true + validates :project_id, :name, presence: true validates :uuid, presence: true, uniqueness: true, length: { is: UUID_LENGTH }, format: { with: HEX_REGEXP, message: 'only allows hex characters' } - before_destroy :ensure_state_is_unlocked - default_value_for(:uuid, allows_nil: false) { SecureRandom.hex(UUID_LENGTH / 2) } def latest_file @@ -90,13 +87,6 @@ module Terraform new_version.save! end - def ensure_state_is_unlocked - return unless locked? - - errors.add(:base, s_("Terraform|You cannot remove the State file because it's locked. Unlock the State file first before removing it.")) - throw :abort # rubocop:disable Cop/BanCatchThrow - end - def parse_serial(file) Gitlab::Json.parse(file)["serial"] rescue JSON::ParserError diff --git a/app/models/terraform/state_version.rb b/app/models/terraform/state_version.rb index 31ff7e4c27d..c50eaa66860 100644 --- a/app/models/terraform/state_version.rb +++ b/app/models/terraform/state_version.rb @@ -2,6 +2,7 @@ module Terraform class StateVersion < ApplicationRecord + include EachBatch include FileStoreMounter belongs_to :terraform_state, class_name: 'Terraform::State', optional: false diff --git a/app/models/time_tracking/timelog_category.rb b/app/models/time_tracking/timelog_category.rb new file mode 100644 index 00000000000..26614f6fc44 --- /dev/null +++ b/app/models/time_tracking/timelog_category.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module TimeTracking + class TimelogCategory < ApplicationRecord + include StripAttribute + include CaseSensitivity + + self.table_name = "timelog_categories" + + belongs_to :namespace, foreign_key: 'namespace_id' + + strip_attributes! :name + + validates :namespace, presence: true + validates :name, presence: true + validates :name, uniqueness: { case_sensitive: false, scope: [:namespace_id] } + validates :name, length: { maximum: 255 } + validates :description, length: { maximum: 1024 } + validates :color, color: true, allow_blank: false, length: { maximum: 7 } + validates :billing_rate, + if: :billable?, + presence: true, + numericality: { greater_than: 0 } + + DEFAULT_COLOR = ::Gitlab::Color.of('#6699cc') + + attribute :color, ::Gitlab::Database::Type::Color.new + default_value_for :color, DEFAULT_COLOR + + def self.find_by_name(namespace_id, name) + where(namespace: namespace_id) + .iwhere(name: name) + end + end +end diff --git a/app/models/user.rb b/app/models/user.rb index b9a8e5855bf..c86fb56795c 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -90,6 +90,7 @@ class User < ApplicationRecord include ForcedEmailConfirmation MINIMUM_INACTIVE_DAYS = 90 + MINIMUM_DAYS_CREATED = 7 # Override Devise::Models::Trackable#update_tracked_fields! # to limit database writes to at most once every hour @@ -338,7 +339,6 @@ class User < ApplicationRecord delegate :path, to: :namespace, allow_nil: true, prefix: true delegate :job_title, :job_title=, to: :user_detail, allow_nil: true - delegate :other_role, :other_role=, to: :user_detail, allow_nil: true delegate :bio, :bio=, to: :user_detail, allow_nil: true delegate :webauthn_xid, :webauthn_xid=, to: :user_detail, allow_nil: true delegate :pronouns, :pronouns=, to: :user_detail, allow_nil: true @@ -414,7 +414,9 @@ class User < ApplicationRecord after_transition any => :deactivated do |user| next unless Gitlab::CurrentSettings.user_deactivation_emails_enabled - NotificationService.new.user_deactivated(user.name, user.notification_email_or_default) + user.run_after_commit do + NotificationService.new.user_deactivated(user.name, user.notification_email_or_default) + end end # rubocop: enable CodeReuse/ServiceClass @@ -478,7 +480,7 @@ class User < ApplicationRecord scope :order_oldest_last_activity, -> { reorder(arel_table[:last_activity_on].asc.nulls_first) } scope :by_id_and_login, ->(id, login) { where(id: id).where('username = LOWER(:login) OR email = LOWER(:login)', login: login) } scope :dormant, -> { with_state(:active).human_or_service_user.where('last_activity_on <= ?', MINIMUM_INACTIVE_DAYS.day.ago.to_date) } - scope :with_no_activity, -> { with_state(:active).human_or_service_user.where(last_activity_on: nil) } + scope :with_no_activity, -> { with_state(:active).human_or_service_user.where(last_activity_on: nil).where('created_at <= ?', MINIMUM_DAYS_CREATED.day.ago.to_date) } scope :by_provider_and_extern_uid, ->(provider, extern_uid) { joins(:identities).merge(Identity.with_extern_uid(provider, extern_uid)) } scope :by_ids_or_usernames, -> (ids, usernames) { where(username: usernames).or(where(id: ids)) } scope :without_forbidden_states, -> { where.not(state: FORBIDDEN_SEARCH_STATES) } @@ -1657,33 +1659,15 @@ class User < ApplicationRecord def ci_owned_runners @ci_owned_runners ||= begin - if ci_owned_runners_cross_joins_fix_enabled? - Ci::Runner - .from_union([ci_owned_project_runners_from_project_members, - ci_owned_project_runners_from_group_members, - ci_owned_group_runners]) - else - Ci::Runner - .from_union([ci_legacy_owned_project_runners, ci_legacy_owned_group_runners]) - .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336436') - end + Ci::Runner + .from_union([ci_owned_project_runners_from_project_members, + ci_owned_project_runners_from_group_members, + ci_owned_group_runners]) end end def owns_runner?(runner) - if ci_owned_runners_cross_joins_fix_enabled? - ci_owned_runners.exists?(runner.id) - else - ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336436') do - ci_owned_runners.exists?(runner.id) - end - end - end - - def ci_owned_runners_cross_joins_fix_enabled? - strong_memoize(:ci_owned_runners_cross_joins_fix_enabled) do - Feature.enabled?(:ci_owned_runners_cross_joins_fix, self) - end + ci_owned_runners.exists?(runner.id) end def notification_email_for(notification_group) @@ -2265,20 +2249,6 @@ class User < ApplicationRecord ::Gitlab::Auth::Ldap::Access.allowed?(self) end - def ci_legacy_owned_project_runners - Ci::RunnerProject - .select('ci_runners.*') - .joins(:runner) - .where(project: authorized_projects(Gitlab::Access::MAINTAINER)) - end - - def ci_legacy_owned_group_runners - Ci::RunnerNamespace - .select('ci_runners.*') - .joins(:runner) - .where(namespace_id: owned_groups.self_and_descendant_ids) - end - def ci_owned_project_runners_from_project_members project_ids = project_members.where('access_level >= ?', Gitlab::Access::MAINTAINER).pluck(:source_id) @@ -2334,12 +2304,7 @@ class User < ApplicationRecord .merge(search_members) .shortest_traversal_ids_prefixes - # Use efficient btree index to perform search - if Feature.enabled?(:ci_owned_runners_unnest_index, self) - Ci::NamespaceMirror.contains_traversal_ids(traversal_ids) - else - Ci::NamespaceMirror.contains_any_of_namespaces(traversal_ids.map(&:last)) - end + Ci::NamespaceMirror.contains_traversal_ids(traversal_ids) end end diff --git a/app/models/user_detail.rb b/app/models/user_detail.rb index 3787ad1c380..b9b69d12729 100644 --- a/app/models/user_detail.rb +++ b/app/models/user_detail.rb @@ -2,6 +2,9 @@ class UserDetail < ApplicationRecord extend ::Gitlab::Utils::Override + include IgnorableColumns + + ignore_columns :other_role, remove_after: '2022-07-22', remove_with: '15.3' REGISTRATION_OBJECTIVE_PAIRS = { basics: 0, move_repository: 1, code_storage: 2, exploring: 3, ci: 4, other: 5, joining_team: 6 }.freeze diff --git a/app/models/users/callout.rb b/app/models/users/callout.rb index b3729c84dd6..0ecae4d148a 100644 --- a/app/models/users/callout.rb +++ b/app/models/users/callout.rb @@ -51,12 +51,16 @@ module Users attention_requests_side_nav: 48, minute_limit_banner: 49, preview_user_over_limit_free_plan_alert: 50, # EE-only - user_reached_limit_free_plan_alert: 51 # EE-only + user_reached_limit_free_plan_alert: 51, # EE-only + submit_license_usage_data_banner: 52, # EE-only + personal_project_limitations_banner: 53 # EE-only } validates :feature_name, presence: true, uniqueness: { scope: :user_id }, inclusion: { in: Users::Callout.feature_names.keys } + + scope :with_feature_name, -> (feature_name) { where(feature_name: feature_name) } end end diff --git a/app/models/wiki.rb b/app/models/wiki.rb index 32d70fcd3b7..c9cb3b0b796 100644 --- a/app/models/wiki.rb +++ b/app/models/wiki.rb @@ -227,32 +227,22 @@ class Wiki end def create_page(title, content, format = :markdown, message = nil) - if Feature.enabled?(:gitaly_replace_wiki_create_page, container, type: :undefined) - with_valid_format(format) do |default_extension| - if file_exists_by_regex?(title) - raise_duplicate_page_error! - end - - capture_git_error(:created) do - create_wiki_repository unless repository_exists? - sanitized_path = sluggified_full_path(title, default_extension) - repository.create_file(user, sanitized_path, content, **multi_commit_options(:created, message, title)) - repository.expire_status_cache if repository.empty? - after_wiki_activity - - true - rescue Gitlab::Git::Index::IndexError - raise_duplicate_page_error! - end + with_valid_format(format) do |default_extension| + if file_exists_by_regex?(title) + raise_duplicate_page_error! end - else - commit = commit_details(:created, message, title) - wiki.write_page(title, format.to_sym, content, commit) - repository.expire_status_cache if repository.empty? - after_wiki_activity + capture_git_error(:created) do + create_wiki_repository unless repository_exists? + sanitized_path = sluggified_full_path(title, default_extension) + repository.create_file(user, sanitized_path, content, **multi_commit_options(:created, message, title)) + repository.expire_status_cache if repository.empty? + after_wiki_activity - true + true + rescue Gitlab::Git::Index::IndexError + raise_duplicate_page_error! + end end rescue Gitlab::Git::Wiki::DuplicatePageError => e @error_message = _("Duplicate page: %{error_message}" % { error_message: e.message }) @@ -395,17 +385,6 @@ class Wiki } end - def commit_details(action, message = nil, title = nil) - commit_message = build_commit_message(action, message, title) - git_user = Gitlab::Git::User.from_gitlab(user) - - Gitlab::Git::Wiki::CommitDetails.new(user.id, - git_user.username, - git_user.name, - git_user.email, - commit_message) - end - def build_commit_message(action, message, title) message.presence || default_message(action, title) end diff --git a/app/models/work_item.rb b/app/models/work_item.rb index 557694da35a..bdd9aae90a4 100644 --- a/app/models/work_item.rb +++ b/app/models/work_item.rb @@ -4,10 +4,29 @@ class WorkItem < Issue self.table_name = 'issues' self.inheritance_column = :_type_disabled + has_one :parent_link, class_name: '::WorkItems::ParentLink', foreign_key: :work_item_id + has_one :work_item_parent, through: :parent_link, class_name: 'WorkItem' + + has_many :child_links, class_name: '::WorkItems::ParentLink', foreign_key: :work_item_parent_id + has_many :work_item_children, through: :child_links, class_name: 'WorkItem', + foreign_key: :work_item_id, source: :work_item + + scope :inc_relations_for_permission_check, -> { includes(:author, project: :project_feature) } + + def self.assignee_association_name + 'issue' + end + def noteable_target_type_name 'issue' end + def widgets + work_item_type.widgets.map do |widget_class| + widget_class.new(self) + end + end + private def record_create_action diff --git a/app/models/work_items/parent_link.rb b/app/models/work_items/parent_link.rb new file mode 100644 index 00000000000..3c405dbce3b --- /dev/null +++ b/app/models/work_items/parent_link.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module WorkItems + class ParentLink < ApplicationRecord + self.table_name = 'work_item_parent_links' + + MAX_CHILDREN = 100 + + belongs_to :work_item + belongs_to :work_item_parent, class_name: 'WorkItem' + + validates :work_item, :work_item_parent, presence: true + validate :validate_child_type + validate :validate_parent_type + validate :validate_same_project + validate :validate_max_children + + private + + def validate_child_type + return unless work_item + + unless work_item.task? + errors.add :work_item, _('Only Task can be assigned as a child in hierarchy.') + end + end + + def validate_parent_type + return unless work_item_parent + + unless work_item_parent.issue? + errors.add :work_item_parent, _('Only Issue can be parent of Task.') + end + end + + def validate_same_project + return if work_item.nil? || work_item_parent.nil? + + if work_item.resource_parent != work_item_parent.resource_parent + errors.add :work_item_parent, _('Parent must be in the same project as child.') + end + end + + def validate_max_children + return unless work_item_parent + + max = persisted? ? MAX_CHILDREN : MAX_CHILDREN - 1 + if work_item_parent.child_links.count > max + errors.add :work_item_parent, _('Parent already has maximum number of children.') + end + end + end +end diff --git a/app/models/work_items/type.rb b/app/models/work_items/type.rb index 0d390fa131d..bf251a3ade5 100644 --- a/app/models/work_items/type.rb +++ b/app/models/work_items/type.rb @@ -20,6 +20,14 @@ module WorkItems task: { name: 'Task', icon_name: 'issue-type-task', enum_value: 4 } }.freeze + WIDGETS_FOR_TYPE = { + issue: [Widgets::Description, Widgets::Hierarchy], + incident: [Widgets::Description], + test_case: [Widgets::Description], + requirement: [Widgets::Description], + task: [Widgets::Description, Widgets::Hierarchy] + }.freeze + cache_markdown_field :description, pipeline: :single_line enum base_type: BASE_TYPES.transform_values { |value| value[:enum_value] } @@ -40,6 +48,10 @@ module WorkItems scope :order_by_name_asc, -> { order(arel_table[:name].lower.asc) } scope :by_type, ->(base_type) { where(base_type: base_type) } + def self.available_widgets + WIDGETS_FOR_TYPE.values.flatten.uniq + end + def self.default_by_type(type) found_type = find_by(namespace_id: nil, base_type: type) return found_type if found_type @@ -60,6 +72,10 @@ module WorkItems namespace.blank? end + def widgets + WIDGETS_FOR_TYPE[base_type.to_sym] + end + private def strip_whitespace diff --git a/app/models/work_items/widgets/base.rb b/app/models/work_items/widgets/base.rb new file mode 100644 index 00000000000..e7075a7a0e8 --- /dev/null +++ b/app/models/work_items/widgets/base.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module WorkItems + module Widgets + class Base + def self.type + name.demodulize.underscore.to_sym + end + + def self.api_symbol + "#{type}_widget".to_sym + end + + def type + self.class.type + end + + def initialize(work_item) + @work_item = work_item + end + + attr_reader :work_item + end + end +end diff --git a/app/models/work_items/widgets/description.rb b/app/models/work_items/widgets/description.rb new file mode 100644 index 00000000000..35b6d295321 --- /dev/null +++ b/app/models/work_items/widgets/description.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module WorkItems + module Widgets + class Description < Base + delegate :description, to: :work_item + + def update(params:) + work_item.description = params[:description] if params&.key?(:description) + end + end + end +end diff --git a/app/models/work_items/widgets/hierarchy.rb b/app/models/work_items/widgets/hierarchy.rb new file mode 100644 index 00000000000..dadd341de83 --- /dev/null +++ b/app/models/work_items/widgets/hierarchy.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module WorkItems + module Widgets + class Hierarchy < Base + def parent + return unless Feature.enabled?(:work_items_hierarchy, work_item.project) + + work_item.work_item_parent + end + + def children + return WorkItem.none unless Feature.enabled?(:work_items_hierarchy, work_item.project) + + work_item.work_item_children + end + end + end +end |