diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-05-19 10:33:21 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-05-19 10:33:21 +0300 |
commit | 36a59d088eca61b834191dacea009677a96c052f (patch) | |
tree | e4f33972dab5d8ef79e3944a9f403035fceea43f /app/models | |
parent | a1761f15ec2cae7c7f7bbda39a75494add0dfd6f (diff) |
Add latest changes from gitlab-org/gitlab@15-0-stable-eev15.0.0-rc42
Diffstat (limited to 'app/models')
100 files changed, 1029 insertions, 603 deletions
diff --git a/app/models/alert_management/alert.rb b/app/models/alert_management/alert.rb index 1ec3cb62c76..9f05c87018d 100644 --- a/app/models/alert_management/alert.rb +++ b/app/models/alert_management/alert.rb @@ -81,7 +81,6 @@ module AlertManagement scope :search, -> (query) { fuzzy_search(query, [:title, :description, :monitoring_tool, :service]) } scope :not_resolved, -> { without_status(:resolved) } scope :with_prometheus_alert, -> { includes(:prometheus_alert) } - scope :with_threat_monitoring_alerts, -> { where(domain: :threat_monitoring ) } scope :with_operations_alerts, -> { where(domain: :operations) } scope :order_start_time, -> (sort_order) { order(started_at: sort_order) } @@ -119,6 +118,10 @@ module AlertManagement end end + def self.find_unresolved_alert(project, fingerprint) + for_fingerprint(project, fingerprint).not_resolved.take + end + def self.last_prometheus_alert_by_project_id ids = select(arel_table[:id].maximum).group(:project_id) with_prometheus_alert.where(id: ids) @@ -143,10 +146,6 @@ module AlertManagement reference.to_i > 0 && reference.to_i <= Gitlab::Database::MAX_INT_VALUE end - def metric_images_available? - ::AlertManagement::MetricImage.available_for?(project) - end - def prometheus? monitoring_tool == Gitlab::AlertManagement::Payload::MONITORING_TOOLS[:prometheus] end diff --git a/app/models/alert_management/metric_image.rb b/app/models/alert_management/metric_image.rb index 8175a31be7a..4ed28c3b1eb 100644 --- a/app/models/alert_management/metric_image.rb +++ b/app/models/alert_management/metric_image.rb @@ -7,10 +7,6 @@ module AlertManagement belongs_to :alert, class_name: 'AlertManagement::Alert', foreign_key: 'alert_id', inverse_of: :metric_images - def self.available_for?(project) - true - end - private def local_path diff --git a/app/models/analytics/cycle_analytics/aggregation.rb b/app/models/analytics/cycle_analytics/aggregation.rb index 2c04e67a04b..2e58d64ae95 100644 --- a/app/models/analytics/cycle_analytics/aggregation.rb +++ b/app/models/analytics/cycle_analytics/aggregation.rb @@ -26,6 +26,14 @@ class Analytics::CycleAnalytics::Aggregation < ApplicationRecord }.compact end + def consistency_check_cursor_for(model) + { + :start_event_timestamp => self["last_consistency_check_#{model.issuable_model.table_name}_start_event_timestamp"], + :end_event_timestamp => self["last_consistency_check_#{model.issuable_model.table_name}_end_event_timestamp"], + model.issuable_id_column => self["last_consistency_check_#{model.issuable_model.table_name}_issuable_id"] + }.compact + end + def refresh_last_run(mode) self["last_#{mode}_run_at"] = Time.current end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 7cd2fe705e3..6afd8875ad3 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -12,6 +12,9 @@ class ApplicationSetting < ApplicationRecord ignore_columns %i[static_objects_external_storage_auth_token], remove_with: '14.9', remove_after: '2022-03-22' ignore_column %i[max_package_files_for_package_destruction], remove_with: '14.9', remove_after: '2022-03-22' ignore_column :user_email_lookup_limit, remove_with: '15.0', remove_after: '2022-04-18' + ignore_column :pseudonymizer_enabled, remove_with: '15.1', remove_after: '2022-06-22' + ignore_column :enforce_ssh_key_expiration, remove_with: '15.2', remove_after: '2022-07-22' + ignore_column :enforce_pat_expiration, remove_with: '15.2', remove_after: '2022-07-22' INSTANCE_REVIEW_MIN_USERS = 50 GRAFANA_URL_ERROR_MESSAGE = 'Please check your Grafana URL setting in ' \ @@ -199,6 +202,10 @@ class ApplicationSetting < ApplicationRecord presence: true, numericality: { only_integer: true, greater_than: 0 } + validates :max_export_size, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :max_import_size, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 } @@ -370,6 +377,8 @@ class ApplicationSetting < ApplicationRecord :container_registry_import_max_retries, :container_registry_import_start_max_retries, :container_registry_import_max_step_duration, + :container_registry_pre_import_timeout, + :container_registry_import_timeout, allow_nil: false, numericality: { only_integer: true, greater_than_or_equal_to: 0 } @@ -480,6 +489,9 @@ class ApplicationSetting < ApplicationRecord validates :raw_blob_request_limit, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :pipeline_limit_per_project_user_sha, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :ci_jwt_signing_key, rsa_key: true, allow_nil: true @@ -613,6 +625,7 @@ class ApplicationSetting < ApplicationRecord attr_encrypted :recaptcha_private_key, encryption_options_base_32_aes_256_gcm attr_encrypted :recaptcha_site_key, encryption_options_base_32_aes_256_gcm attr_encrypted :slack_app_secret, encryption_options_base_32_aes_256_gcm + attr_encrypted :slack_app_signing_secret, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) attr_encrypted :slack_app_verification_token, encryption_options_base_32_aes_256_gcm attr_encrypted :ci_jwt_signing_key, encryption_options_base_32_aes_256_gcm attr_encrypted :customers_dot_jwt_signing_key, encryption_options_base_32_aes_256_gcm @@ -638,6 +651,7 @@ class ApplicationSetting < ApplicationRecord reset_memoized_terms end after_commit :expire_performance_bar_allowed_user_ids_cache, if: -> { previous_changes.key?('performance_bar_allowed_group_id') } + after_commit :reset_deletion_warning_redis_key, if: :saved_change_to_inactive_projects_delete_after_months? def validate_grafana_url validate_url(parsed_grafana_url, :grafana_url, GRAFANA_URL_ERROR_MESSAGE) @@ -768,6 +782,10 @@ class ApplicationSetting < ApplicationRecord ) end end + + def reset_deletion_warning_redis_key + Gitlab::InactiveProjectsDeletionWarningTracker.reset_all + end end ApplicationSetting.prepend_mod_with('ApplicationSetting') diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 194356acc51..a54dc4f691d 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -108,6 +108,7 @@ module ApplicationSettingImplementation mailgun_events_enabled: false, max_artifacts_size: Settings.artifacts['max_size'], max_attachment_size: Settings.gitlab['max_attachment_size'], + max_export_size: 0, max_import_size: 0, max_yaml_size_bytes: 1.megabyte, max_yaml_depth: 100, @@ -223,6 +224,8 @@ module ApplicationSettingImplementation container_registry_import_max_retries: 3, container_registry_import_start_max_retries: 50, container_registry_import_max_step_duration: 5.minutes, + container_registry_pre_import_timeout: 30.minutes, + container_registry_import_timeout: 10.minutes, container_registry_import_target_plan: 'free', container_registry_import_created_before: '2022-01-23 00:00:00', kroki_enabled: false, diff --git a/app/models/broadcast_message.rb b/app/models/broadcast_message.rb index b255c774347..1f921c71984 100644 --- a/app/models/broadcast_message.rb +++ b/app/models/broadcast_message.rb @@ -131,7 +131,7 @@ class BroadcastMessage < ApplicationRecord end def matches_current_user_access_level?(user_access_level) - return false if target_access_levels.present? && Feature.disabled?(:role_targeted_broadcast_messages, default_enabled: :yaml) + return false if target_access_levels.present? && Feature.disabled?(:role_targeted_broadcast_messages) return true unless target_access_levels.present? target_access_levels.include? user_access_level diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index ff444ddefa3..a06b920342c 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -57,6 +57,14 @@ module Ci end end + def retryable? + return false unless Feature.enabled?(:ci_recreate_downstream_pipeline, project) + + return false if failed? && (pipeline_loop_detected? || reached_max_descendant_pipelines_depth?) + + super + end + def self.with_preloads preload( :metadata, @@ -65,8 +73,11 @@ module Ci ) end - def retryable? - false + def self.clone_accessors + %i[pipeline project ref tag options name + allow_failure stage stage_id stage_idx + yaml_variables when description needs_attributes + scheduling_type].freeze end def inherit_status_from_downstream!(pipeline) @@ -204,7 +215,7 @@ module Ci end def downstream_variables - if ::Feature.enabled?(:ci_trigger_forward_variables, project, default_enabled: :yaml) + if ::Feature.enabled?(:ci_trigger_forward_variables, project) calculate_downstream_variables .reverse # variables priority .uniq { |var| var[:key] } # only one variable key to pass diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index a8ad55fd5a4..eea8086d71d 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -45,6 +45,7 @@ module Ci has_one :runtime_metadata, class_name: 'Ci::RunningBuild', foreign_key: :build_id has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id, inverse_of: :build has_many :report_results, class_name: 'Ci::BuildReportResult', inverse_of: :build + has_one :namespace, through: :project # Projects::DestroyService destroys Ci::Pipelines, which use_fast_destroy on :job_artifacts # before we delete builds. By doing this, the relation should be empty and not fire any @@ -74,6 +75,7 @@ module Ci delegate :gitlab_deploy_token, to: :project delegate :harbor_integration, to: :project delegate :trigger_short_token, to: :trigger_request, allow_nil: true + delegate :ensure_persistent_ref, to: :pipeline ## # Since Gitlab 11.5, deployments records started being created right after @@ -325,7 +327,7 @@ module Ci after_transition pending: :running do |build| build.run_after_commit do - build.pipeline.persistent_ref.create + build.ensure_persistent_ref BuildHooksWorker.perform_async(id) end @@ -335,7 +337,7 @@ module Ci build.run_after_commit do build.run_status_commit_hooks! - if Feature.enabled?(:ci_build_finished_worker_namespace_changed, build.project, default_enabled: :yaml) + if Feature.enabled?(:ci_build_finished_worker_namespace_changed, build.project) Ci::BuildFinishedWorker.perform_async(id) else ::BuildFinishedWorker.perform_async(id) @@ -504,7 +506,7 @@ module Ci if metadata&.expanded_environment_name.present? metadata.expanded_environment_name else - if ::Feature.enabled?(:ci_expand_environment_name_and_url, project, default_enabled: :yaml) + if ::Feature.enabled?(:ci_expand_environment_name_and_url, project) ExpandVariables.expand(environment, -> { simple_variables.sort_and_expand_all }) else ExpandVariables.expand(environment, -> { simple_variables }) @@ -675,7 +677,7 @@ module Ci end def has_archived_trace? - trace.archived_trace_exist? + trace.archived? end def artifacts_file @@ -752,7 +754,7 @@ module Ci end def valid_token?(token) - self.token && ActiveSupport::SecurityUtils.secure_compare(token, self.token) + self.token && token.present? && ActiveSupport::SecurityUtils.secure_compare(token, self.token) end # acts_as_taggable uses this method create/remove tags with contexts @@ -823,7 +825,6 @@ module Ci end end - # and use that for `ExpireBuildInstanceArtifactsWorker`? def erase_erasable_artifacts! job_artifacts.erasable.destroy_all # rubocop: disable Cop/DestroyAll end @@ -884,10 +885,6 @@ module Ci job_artifacts.find_by(file_type: file_types_ids)&.file end - def coverage_regex - super || project.try(:build_coverage_regex) - end - def steps [Gitlab::Ci::Build::Step.from_commands(self), Gitlab::Ci::Build::Step.from_release(self), @@ -911,6 +908,8 @@ module Ci end end + return cache unless project.ci_separated_caches + type_suffix = pipeline.protected_ref? ? 'protected' : 'non_protected' cache.map do |entry| entry.merge(key: "#{entry[:key]}-#{type_suffix}") @@ -1224,7 +1223,7 @@ module Ci def job_jwt_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - break variables unless Feature.enabled?(:ci_job_jwt, project, default_enabled: true) + break variables unless Feature.enabled?(:ci_job_jwt, project) jwt = Gitlab::Ci::Jwt.for_build(self) jwt_v2 = Gitlab::Ci::JwtV2.for_build(self) diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index ca68989002c..4ee661d89f4 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -38,7 +38,7 @@ module Ci job_timeout_source: 4 } - ignore_columns :runner_features, remove_with: '14.7', remove_after: '2021-11-22' + ignore_columns :runner_features, remove_with: '15.1', remove_after: '2022-05-22' def update_timeout_state timeout = timeout_with_highest_precedence diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index dff8bb89021..c831ef12501 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -45,7 +45,7 @@ module Ci dotenv: '.env', cobertura: 'cobertura-coverage.xml', terraform: 'tfplan.json', - cluster_applications: 'gl-cluster-applications.json', # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/333441 + cluster_applications: 'gl-cluster-applications.json', # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/361094 requirements: 'requirements.json', coverage_fuzzing: 'gl-coverage-fuzzing.json', api_fuzzing: 'gl-api-fuzzing-report.json' @@ -64,7 +64,7 @@ module Ci network_referee: :gzip, dotenv: :gzip, cobertura: :gzip, - cluster_applications: :gzip, + cluster_applications: :gzip, # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/361094 lsif: :zip, # Security reports and license scanning reports are raw artifacts @@ -187,7 +187,6 @@ module Ci scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) } scope :unlocked, -> { joins(job: :pipeline).merge(::Ci::Pipeline.unlocked) } scope :order_expired_asc, -> { order(expire_at: :asc) } - scope :order_expired_desc, -> { order(expire_at: :desc) } scope :with_destroy_preloads, -> { includes(project: [:route, :statistics]) } scope :for_project, ->(project) { where(project_id: project) } @@ -323,12 +322,12 @@ module Ci end end - def archived_trace_exists? + def stored? file&.file&.exists? end def self.archived_trace_exists_for?(job_id) - where(job_id: job_id).trace.take&.archived_trace_exists? + where(job_id: job_id).trace.take&.stored? end def self.max_artifact_size(type:, project:) diff --git a/app/models/ci/namespace_settings.rb b/app/models/ci/namespace_settings.rb new file mode 100644 index 00000000000..d519a48311f --- /dev/null +++ b/app/models/ci/namespace_settings.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# CI::NamespaceSettings mixin +# +# This module is intended to encapsulate CI/CD settings-specific logic +# and be prepended in the `Namespace` model +module Ci + module NamespaceSettings + # Overridden in EE::Namespace + def allow_stale_runner_pruning? + false + end + + # Overridden in EE::Namespace + def allow_stale_runner_pruning=(_value) + raise NotImplementedError + end + end +end diff --git a/app/models/ci/pending_build.rb b/app/models/ci/pending_build.rb index 41dc74ef050..d900a056242 100644 --- a/app/models/ci/pending_build.rb +++ b/app/models/ci/pending_build.rb @@ -31,7 +31,7 @@ module Ci end def maintain_denormalized_data? - ::Feature.enabled?(:ci_pending_builds_maintain_denormalized_data, default_enabled: :yaml) + ::Feature.enabled?(:ci_pending_builds_maintain_denormalized_data) end private diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 2d0479e02a3..c10069382f2 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -36,10 +36,10 @@ module Ci # Ci::CreatePipelineService returns Ci::Pipeline so this is the only place # where we can pass additional information from the service. This accessor - # is used for storing the processed CI YAML contents for linting purposes. + # is used for storing the processed metadata for linting purposes. # There is an open issue to address this: # https://gitlab.com/gitlab-org/gitlab/-/issues/259010 - attr_accessor :merged_yaml + attr_accessor :config_metadata # This is used to retain access to the method defined by `Ci::HasRef` # before being overridden in this class. @@ -198,7 +198,7 @@ module Ci # Create a separate worker for each new operation before_transition [:created, :waiting_for_resource, :preparing, :pending] => :running do |pipeline| - pipeline.started_at = Time.current + pipeline.started_at ||= Time.current end before_transition any => [:success, :failed, :canceled] do |pipeline| @@ -253,8 +253,6 @@ module Ci after_transition any => ::Ci::Pipeline.completed_statuses do |pipeline| pipeline.run_after_commit do - pipeline.persistent_ref.delete - pipeline.all_merge_requests.each do |merge_request| next unless merge_request.auto_merge_enabled? @@ -288,6 +286,12 @@ module Ci end end + after_transition any => ::Ci::Pipeline.stopped_statuses do |pipeline| + pipeline.run_after_commit do + pipeline.persistent_ref.delete + end + end + after_transition any => [:success, :failed] do |pipeline| ref_status = pipeline.ci_ref&.update_status_by!(pipeline) @@ -336,7 +340,7 @@ module Ci scope :with_only_interruptible_builds, -> do where('NOT EXISTS (?)', Ci::Build.where('ci_builds.commit_id = ci_pipelines.id') - .with_status(:running, :success, :failed) + .with_status(STARTED_STATUSES) .not_interruptible ) end @@ -978,7 +982,7 @@ module Ci end # With multi-project and parent-child pipelines - def self_with_upstreams_and_downstreams + def all_pipelines_in_hierarchy object_hierarchy.all_objects end @@ -992,6 +996,7 @@ module Ci object_hierarchy(project_condition: :same).base_and_descendants end + # Follow the parent-child relationships and return the top-level parent def root_ancestor return self unless child? @@ -1000,6 +1005,12 @@ module Ci .first end + # Follow the upstream pipeline relationships, regardless of multi-project or + # parent-child, and return the top-level ancestor. + def upstream_root + object_hierarchy.base_and_ancestors(hierarchy_order: :desc).first + end + def bridge_triggered? source_bridge.present? end @@ -1257,6 +1268,12 @@ module Ci self.ci_ref = Ci::Ref.ensure_for(self) end + def ensure_persistent_ref + return if persistent_ref.exist? + + persistent_ref.create + end + def reset_source_bridge!(current_user) return unless bridge_waiting? @@ -1271,10 +1288,11 @@ module Ci def security_reports(report_types: []) reports_scope = report_types.empty? ? ::Ci::JobArtifact.security_reports : ::Ci::JobArtifact.security_reports(file_types: report_types) + types_to_collect = report_types.empty? ? ::Ci::JobArtifact::SECURITY_REPORT_FILE_TYPES : report_types ::Gitlab::Ci::Reports::Security::Reports.new(self).tap do |security_reports| latest_report_builds(reports_scope).each do |build| - build.collect_security_reports!(security_reports) + build.collect_security_reports!(security_reports, report_types: types_to_collect) end end end diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb index d79ff74753a..f666629c8fd 100644 --- a/app/models/ci/processable.rb +++ b/app/models/ci/processable.rb @@ -101,6 +101,21 @@ module Ci :merge_train_pipeline?, to: :pipeline + def clone(current_user:) + new_attributes = self.class.clone_accessors.to_h do |attribute| + [attribute, public_send(attribute)] # rubocop:disable GitlabSecurity/PublicSend + end + + if persisted_environment.present? + new_attributes[:metadata_attributes] ||= {} + new_attributes[:metadata_attributes][:expanded_environment_name] = expanded_environment_name + end + + new_attributes[:user] = current_user + + self.class.new(new_attributes) + end + def retryable? return false if retried? || archived? || deployment_rejected? diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index b9ba9d8e1b0..7a1d52f5aea 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -12,6 +12,7 @@ module Ci include Gitlab::Utils::StrongMemoize include TaggableQueries include Presentable + include EachBatch add_authentication_token_field :token, encrypted: :optional, expires_at: :compute_token_expiration, expiration_enforced?: :token_expiration_enforced? @@ -59,7 +60,7 @@ module Ci AVAILABLE_TYPES_LEGACY = %w[specific shared].freeze AVAILABLE_TYPES = runner_types.keys.freeze - AVAILABLE_STATUSES = %w[active paused online offline not_connected never_contacted stale].freeze # TODO: Remove in %15.0: not_connected. In %16.0: active, paused. Relevant issues: https://gitlab.com/gitlab-org/gitlab/-/issues/347303, https://gitlab.com/gitlab-org/gitlab/-/issues/347305, https://gitlab.com/gitlab-org/gitlab/-/issues/344648 + AVAILABLE_STATUSES = %w[active paused online offline never_contacted stale].freeze # TODO: Remove in %16.0: active, paused. Relevant issue: https://gitlab.com/gitlab-org/gitlab/-/issues/344648 AVAILABLE_SCOPES = (AVAILABLE_TYPES_LEGACY + AVAILABLE_TYPES + AVAILABLE_STATUSES).freeze FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level maximum_timeout_human_readable].freeze @@ -83,7 +84,6 @@ module Ci scope :recent, -> { where('ci_runners.created_at >= :date OR ci_runners.contacted_at >= :date', date: stale_deadline) } scope :stale, -> { where('ci_runners.created_at < :date AND (ci_runners.contacted_at IS NULL OR ci_runners.contacted_at < :date)', date: stale_deadline) } scope :offline, -> { where(arel_table[:contacted_at].lteq(online_contact_time_deadline)) } - scope :not_connected, -> { where(contacted_at: nil) } # TODO: Remove in 15.0 scope :never_contacted, -> { where(contacted_at: nil) } scope :ordered, -> { order(id: :desc) } @@ -289,7 +289,7 @@ module Ci def assign_to(project, current_user = nil) if instance_type? - self.runner_type = :project_type + raise ArgumentError, 'Transitioning an instance runner to a project runner is not supported' elsif group_type? raise ArgumentError, 'Transitioning a group runner to a project runner is not supported' end @@ -322,6 +322,9 @@ module Ci end def status(legacy_mode = nil) + # TODO Deprecate legacy_mode in %16.0 and make it a no-op + # (see https://gitlab.com/gitlab-org/gitlab/-/issues/360545) + # TODO Remove legacy_mode in %17.0 return deprecated_rest_status if legacy_mode == '14.5' return :stale if stale? @@ -331,10 +334,12 @@ module Ci end # DEPRECATED - # TODO Remove in %16.0 in favor of `status` for REST calls + # TODO Remove in %16.0 in favor of `status` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/344648 def deprecated_rest_status - if contacted_at.nil? # TODO Remove in %15.0, see https://gitlab.com/gitlab-org/gitlab/-/issues/344648 - :not_connected + return :stale if stale? + + if contacted_at.nil? + :never_contacted elsif active? online? ? :online : :offline else @@ -462,7 +467,7 @@ module Ci end def self.token_expiration_enforced? - Feature.enabled?(:enforce_runner_token_expires_at, default_enabled: :yaml) + Feature.enabled?(:enforce_runner_token_expires_at) end private diff --git a/app/models/ci/secure_file.rb b/app/models/ci/secure_file.rb index 6a26a5341aa..9c82e106d6e 100644 --- a/app/models/ci/secure_file.rb +++ b/app/models/ci/secure_file.rb @@ -3,8 +3,11 @@ module Ci class SecureFile < Ci::ApplicationRecord include FileStoreMounter + include IgnorableColumns include Limitable + ignore_column :permissions, remove_with: '15.2', remove_after: '2022-06-22' + FILE_SIZE_LIMIT = 5.megabytes.freeze CHECKSUM_ALGORITHM = 'sha256' @@ -14,14 +17,12 @@ module Ci belongs_to :project, optional: false validates :file, presence: true, file_size: { maximum: FILE_SIZE_LIMIT } - validates :checksum, :file_store, :name, :permissions, :project_id, presence: true + validates :checksum, :file_store, :name, :project_id, presence: true validates :name, uniqueness: { scope: :project } after_initialize :generate_key_data before_validation :assign_checksum - enum permissions: { read_only: 0, read_write: 1, execute: 2 } - default_value_for(:file_store) { Ci::SecureFileUploader.default_store } mount_file_store_uploader Ci::SecureFileUploader diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb index 21f7e410843..d1e169a1f78 100644 --- a/app/models/clusters/applications/prometheus.rb +++ b/app/models/clusters/applications/prometheus.rb @@ -18,7 +18,6 @@ module Clusters default_value_for :version, VERSION scope :preload_cluster_platform, -> { preload(cluster: [:platform_kubernetes]) } - scope :with_clusters_with_cilium, -> { joins(:cluster).merge(Clusters::Cluster.with_available_cilium) } attr_encrypted :alert_manager_token, mode: :per_attribute_iv, diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 87afa9f9491..014f7530357 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -137,7 +137,6 @@ module Clusters scope :aws_installed, -> { aws_provided.joins(:provider_aws).merge(Clusters::Providers::Aws.with_status(:created)) } scope :with_available_elasticstack, -> { joins(:application_elastic_stack).merge(::Clusters::Applications::ElasticStack.available) } - scope :with_available_cilium, -> { joins(:application_cilium).merge(::Clusters::Applications::Cilium.available) } scope :distinct_with_deployed_environments, -> { joins(:environments).merge(::Deployment.success).distinct } scope :managed, -> { where(managed: true) } diff --git a/app/models/clusters/instance.rb b/app/models/clusters/instance.rb index 2a09ba11564..b5acc6a68f9 100644 --- a/app/models/clusters/instance.rb +++ b/app/models/clusters/instance.rb @@ -9,5 +9,11 @@ module Clusters def flipper_id self.class.to_s end + + def certificate_based_clusters_enabled? + ::Gitlab::SafeRequestStore.fetch("certificate_based_clusters:") do + Feature.enabled?(:certificate_based_clusters, type: :ops) + end + end end end diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index 1bd8e8b44cb..9d4f0a89403 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -11,6 +11,16 @@ module Clusters RESERVED_NAMESPACES = %w(gitlab-managed-apps).freeze + IGNORED_CONNECTION_EXCEPTIONS = [ + Gitlab::UrlBlocker::BlockedUrlError, + Kubeclient::HttpError, + Errno::ECONNREFUSED, + URI::InvalidURIError, + Errno::EHOSTUNREACH, + OpenSSL::X509::StoreError, + OpenSSL::SSL::SSLError + ].freeze + self.table_name = 'cluster_platforms_kubernetes' self.reactive_cache_work_type = :external_dependency @@ -102,10 +112,23 @@ module Clusters def calculate_reactive_cache_for(environment) return unless enabled? - pods = read_pods(environment.deployment_namespace) - deployments = read_deployments(environment.deployment_namespace) + pods = [] + deployments = [] + ingresses = [] + + begin + pods = read_pods(environment.deployment_namespace) + deployments = read_deployments(environment.deployment_namespace) + + ingresses = read_ingresses(environment.deployment_namespace) + rescue *IGNORED_CONNECTION_EXCEPTIONS => e + log_kube_connection_error(e) - ingresses = read_ingresses(environment.deployment_namespace) + # Return hash with default values so that it is cached. + return { + pods: pods, deployments: deployments, ingresses: ingresses + } + end # extract only the data required for display to avoid unnecessary caching { @@ -292,6 +315,23 @@ module Clusters } end end + + def log_kube_connection_error(error) + logger.error({ + exception: { + class: error.class.name, + message: error.message + }, + status_code: error.try(:error_code), + namespace: self.namespace, + class_name: self.class.name, + event: :kube_connection_error + }) + end + + def logger + @logger ||= Gitlab::Kubernetes::Logger.build + end end end end diff --git a/app/models/concerns/bulk_member_access_load.rb b/app/models/concerns/bulk_member_access_load.rb index efc65e55e40..c3aa3019abb 100644 --- a/app/models/concerns/bulk_member_access_load.rb +++ b/app/models/concerns/bulk_member_access_load.rb @@ -12,6 +12,11 @@ module BulkMemberAccessLoad end end + def purge_resource_id_from_request_store(resource_klass, resource_id) + Gitlab::SafeRequestPurger.execute(resource_key: max_member_access_for_resource_key(resource_klass), + resource_ids: [resource_id]) + end + def max_member_access_for_resource_key(klass) "max_member_access_for_#{klass.name.underscore.pluralize}:#{self.class}:#{self.id}" end diff --git a/app/models/concerns/ci/has_deployment_name.rb b/app/models/concerns/ci/has_deployment_name.rb index fe288134872..887653e846e 100644 --- a/app/models/concerns/ci/has_deployment_name.rb +++ b/app/models/concerns/ci/has_deployment_name.rb @@ -5,7 +5,7 @@ module Ci extend ActiveSupport::Concern def count_user_deployment? - Feature.enabled?(:job_deployment_count) && deployment_name? + deployment_name? end def deployment_name? diff --git a/app/models/concerns/ci/has_status.rb b/app/models/concerns/ci/has_status.rb index 313c767e59f..cca66c3ec94 100644 --- a/app/models/concerns/ci/has_status.rb +++ b/app/models/concerns/ci/has_status.rb @@ -7,16 +7,15 @@ module Ci DEFAULT_STATUS = 'created' BLOCKED_STATUS = %w[manual scheduled].freeze AVAILABLE_STATUSES = %w[created waiting_for_resource preparing pending running success failed canceled skipped manual scheduled].freeze - # TODO: replace STARTED_STATUSES with data from BUILD_STARTED_RUNNING_STATUSES in https://gitlab.com/gitlab-org/gitlab/-/issues/273378 - # see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82149#note_865508501 - BUILD_STARTED_RUNNING_STATUSES = %w[running success failed].freeze - STARTED_STATUSES = %w[running success failed skipped manual scheduled].freeze + STARTED_STATUSES = %w[running success failed].freeze ACTIVE_STATUSES = %w[waiting_for_resource preparing pending running].freeze COMPLETED_STATUSES = %w[success failed canceled skipped].freeze + STOPPED_STATUSES = COMPLETED_STATUSES + BLOCKED_STATUS ORDERED_STATUSES = %w[failed preparing pending running waiting_for_resource manual scheduled canceled success skipped created].freeze PASSED_WITH_WARNINGS_STATUSES = %w[failed canceled].to_set.freeze EXCLUDE_IGNORED_STATUSES = %w[manual failed canceled].to_set.freeze - CANCELABLE_STATUSES = %w[running waiting_for_resource preparing pending created scheduled].freeze + ALIVE_STATUSES = (ACTIVE_STATUSES + ['created']).freeze + CANCELABLE_STATUSES = (ALIVE_STATUSES + ['scheduled']).freeze STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3, failed: 4, canceled: 5, skipped: 6, manual: 7, scheduled: 8, preparing: 9, waiting_for_resource: 10 }.freeze @@ -47,6 +46,10 @@ module Ci def completed_statuses COMPLETED_STATUSES.map(&:to_sym) end + + def stopped_statuses + STOPPED_STATUSES.map(&:to_sym) + end end included do @@ -78,8 +81,8 @@ module Ci scope :skipped, -> { with_status(:skipped) } scope :manual, -> { with_status(:manual) } scope :scheduled, -> { with_status(:scheduled) } - scope :alive, -> { with_status(:created, :waiting_for_resource, :preparing, :pending, :running) } - scope :alive_or_scheduled, -> { with_status(:created, :waiting_for_resource, :preparing, :pending, :running, :scheduled) } + scope :alive, -> { with_status(*ALIVE_STATUSES) } + scope :alive_or_scheduled, -> { with_status(*klass::CANCELABLE_STATUSES) } scope :created_or_pending, -> { with_status(:created, :pending) } scope :running_or_pending, -> { with_status(:running, :pending) } scope :finished, -> { with_status(:success, :failed, :canceled) } @@ -98,7 +101,7 @@ module Ci end def started? - STARTED_STATUSES.include?(status) && started_at + STARTED_STATUSES.include?(status) && !!started_at end def active? diff --git a/app/models/concerns/cross_database_modification.rb b/app/models/concerns/cross_database_modification.rb index 85645e482f6..dea62f03f91 100644 --- a/app/models/concerns/cross_database_modification.rb +++ b/app/models/concerns/cross_database_modification.rb @@ -103,7 +103,7 @@ module CrossDatabaseModification def track_gitlab_schema_in_current_transaction? return false unless Feature::FlipperFeature.table_exists? - Feature.enabled?(:track_gitlab_schema_in_current_transaction, default_enabled: :yaml) + Feature.enabled?(:track_gitlab_schema_in_current_transaction) rescue ActiveRecord::NoDatabaseError, PG::ConnectionBad false end diff --git a/app/models/concerns/deployment_platform.rb b/app/models/concerns/deployment_platform.rb index d9c622f247a..2b5e1a204cb 100644 --- a/app/models/concerns/deployment_platform.rb +++ b/app/models/concerns/deployment_platform.rb @@ -3,7 +3,7 @@ module DeploymentPlatform # rubocop:disable Gitlab/ModuleWithInstanceVariables def deployment_platform(environment: nil) - return if Feature.disabled?(:certificate_based_clusters, default_enabled: :yaml, type: :ops) + return unless self.namespace.certificate_based_clusters_enabled? @deployment_platform ||= {} diff --git a/app/models/concerns/integrations/loggable.rb b/app/models/concerns/integrations/loggable.rb new file mode 100644 index 00000000000..57847ea335c --- /dev/null +++ b/app/models/concerns/integrations/loggable.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Integrations + module Loggable + def log_info(message, params = {}) + message = build_message(message, params) + + logger.info(message) + end + + def log_error(message, params = {}) + message = build_message(message, params) + + logger.error(message) + end + + def log_exception(error, params = {}) + Gitlab::ExceptionLogFormatter.format!(error, params) + + log_error(params[:message] || error.message, params) + end + + def build_message(message, params = {}) + { + integration_class: self.class.name, + integration_id: id, + project_id: project&.id, + project_path: project&.full_path, + message: message + }.merge(params) + end + + def logger + Gitlab::IntegrationsLogger + end + end +end diff --git a/app/models/concerns/integrations/reset_secret_fields.rb b/app/models/concerns/integrations/reset_secret_fields.rb new file mode 100644 index 00000000000..f79c4392f19 --- /dev/null +++ b/app/models/concerns/integrations/reset_secret_fields.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# Integrations should reset their "secret" fields (type: 'password') when certain "exposing" +# fields are changed (e.g. URLs), to avoid leaking secrets to unauthorized parties. +# The result of this is that users have to reenter the secrets to confirm the change. +module Integrations + module ResetSecretFields + extend ActiveSupport::Concern + + included do + before_validation :reset_secret_fields!, if: :reset_secret_fields? + end + + def exposing_secrets_fields + # TODO: Once all integrations use `Integrations::Field` we can remove the `.try` here. + # See: https://gitlab.com/groups/gitlab-org/-/epics/7652 + fields.select { _1.try(:exposes_secrets) }.pluck(:name) + end + + private + + def reset_secret_fields? + exposing_secrets_fields.any? do |field| + public_send("#{field}_changed?") # rubocop:disable GitlabSecurity/PublicSend + end + end + + def reset_secret_fields! + secret_fields.each do |field| + next if public_send("#{field}_touched?") # rubocop:disable GitlabSecurity/PublicSend + + public_send("#{field}=", nil) # rubocop:disable GitlabSecurity/PublicSend + + # NOTE: Some of our specs also write to properties in addition to data fields, + # in order to test backwards compatibility. So in those cases we also need to + # clear the field in properties, since the setter above will only affect the data field. + self.properties = properties.except(field) if properties.present? + end + end + end +end diff --git a/app/models/concerns/integrations/slack_mattermost_notifier.rb b/app/models/concerns/integrations/slack_mattermost_notifier.rb index be13701289a..3bdaa852ddf 100644 --- a/app/models/concerns/integrations/slack_mattermost_notifier.rb +++ b/app/models/concerns/integrations/slack_mattermost_notifier.rb @@ -14,11 +14,21 @@ module Integrations # - https://gitlab.com/gitlab-org/slack-notifier#middleware # - https://gitlab.com/gitlab-org/gitlab/-/issues/347048 notifier = ::Slack::Messenger.new(webhook, opts.merge(http_client: HTTPClient)) - notifier.ping( + responses = notifier.ping( message.pretext, attachments: message.attachments, fallback: message.fallback ) + + responses.each do |response| + unless response.success? + log_error('SlackMattermostNotifier HTTP error response', + request_host: response.request.uri.host, + response_code: response.code, + response_body: response.body + ) + end + end end class HTTPClient diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index dbd760a9c45..713a4386fee 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -195,7 +195,7 @@ module Issuable end def supports_escalation? - return false unless ::Feature.enabled?(:incident_escalations, project, default_enabled: :yaml) + return false unless ::Feature.enabled?(:incident_escalations, project) incident? end @@ -520,7 +520,7 @@ module Issuable changes.merge!(hook_association_changes(old_associations)) end - Gitlab::HookData::IssuableBuilder.new(self).build(user: user, changes: changes) + Gitlab::DataBuilder::Issuable.new(self).build(user: user, changes: changes) end def labels_array diff --git a/app/models/concerns/limitable.rb b/app/models/concerns/limitable.rb index fab1aa21634..6ff540b7866 100644 --- a/app/models/concerns/limitable.rb +++ b/app/models/concerns/limitable.rb @@ -28,8 +28,8 @@ module Limitable def validate_scoped_plan_limit_not_exceeded scope_relation = self.public_send(limit_scope) # rubocop:disable GitlabSecurity/PublicSend return unless scope_relation - return if limit_feature_flag && ::Feature.disabled?(limit_feature_flag, scope_relation, default_enabled: :yaml) - return if limit_feature_flag_for_override && ::Feature.enabled?(limit_feature_flag_for_override, scope_relation, default_enabled: :yaml) + return if limit_feature_flag && ::Feature.disabled?(limit_feature_flag, scope_relation) + return if limit_feature_flag_for_override && ::Feature.enabled?(limit_feature_flag_for_override, scope_relation) relation = limit_relation ? self.public_send(limit_relation) : self.class.where(limit_scope => scope_relation) # rubocop:disable GitlabSecurity/PublicSend limits = scope_relation.actual_limits diff --git a/app/models/concerns/merge_request_reviewer_state.rb b/app/models/concerns/merge_request_reviewer_state.rb index 893d06b4da8..18ec1c253e1 100644 --- a/app/models/concerns/merge_request_reviewer_state.rb +++ b/app/models/concerns/merge_request_reviewer_state.rb @@ -16,8 +16,6 @@ module MergeRequestReviewerState belongs_to :updated_state_by, class_name: 'User', foreign_key: :updated_state_by_user_id - after_initialize :set_state, unless: :persisted? - def attention_requested_by return unless attention_requested? diff --git a/app/models/concerns/packages/destructible.rb b/app/models/concerns/packages/destructible.rb index a3b7d8580c1..647c63b7f60 100644 --- a/app/models/concerns/packages/destructible.rb +++ b/app/models/concerns/packages/destructible.rb @@ -5,7 +5,7 @@ module Packages extend ActiveSupport::Concern class_methods do - def next_pending_destruction(order_by: nil) + def next_pending_destruction(order_by:) set = pending_destruction.limit(1).lock('FOR UPDATE SKIP LOCKED') set = set.order(order_by) if order_by set.take diff --git a/app/models/concerns/pg_full_text_searchable.rb b/app/models/concerns/pg_full_text_searchable.rb index 68357c44300..bfc539ee392 100644 --- a/app/models/concerns/pg_full_text_searchable.rb +++ b/app/models/concerns/pg_full_text_searchable.rb @@ -80,6 +80,15 @@ module PgFullTextSearchable pg_full_text_searchable_columns[column[:name]] = column[:weight] end + # When multiple updates are done in a transaction, `saved_changes` will only report the latest save + # and we may miss an update to the searchable columns. + # As a workaround, we set a dirty flag here and update the search data in `after_save_commit`. + after_save do + next unless pg_full_text_searchable_columns.keys.any? { |f| saved_changes.has_key?(f) } + + @update_pg_full_text_search_data = true + end + # We update this outside the transaction because this could raise an error if the resulting tsvector # is too long. When that happens, we still persist the create / update but the model will not have a # search data record. This is fine in most cases because this is a very rare occurrence and only happens @@ -87,9 +96,8 @@ module PgFullTextSearchable # # We also do not want to use a subtransaction here due to: https://gitlab.com/groups/gitlab-org/-/epics/6540 after_save_commit do - next unless pg_full_text_searchable_columns.keys.any? { |f| saved_changes.has_key?(f) } - - update_search_data! + update_search_data! if @update_pg_full_text_search_data + @update_pg_full_text_search_data = nil end end diff --git a/app/models/concerns/project_services_loggable.rb b/app/models/concerns/project_services_loggable.rb deleted file mode 100644 index e5385435138..00000000000 --- a/app/models/concerns/project_services_loggable.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module ProjectServicesLoggable - def log_info(message, params = {}) - message = build_message(message, params) - - logger.info(message) - end - - def log_error(message, params = {}) - message = build_message(message, params) - - logger.error(message) - end - - def build_message(message, params = {}) - { - service_class: self.class.name, - project_id: project&.id, - project_path: project&.full_path, - message: message - }.merge(params) - end - - def logger - Gitlab::ProjectServiceLogger - end -end diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index 2cf95ac0dae..5b759dedb26 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -97,7 +97,7 @@ module Routable def full_name # We have to test for persistence as the cache key uses #updated_at - return (route&.name || build_full_name) unless persisted? && Feature.enabled?(:cached_route_lookups, self, type: :ops, default_enabled: :yaml) + return (route&.name || build_full_name) unless persisted? && Feature.enabled?(:cached_route_lookups, self, type: :ops) # Return the name as-is if the parent is missing return name if route.nil? && parent.nil? && name.present? @@ -115,7 +115,7 @@ module Routable def full_path # We have to test for persistence as the cache key uses #updated_at - return (route&.path || build_full_path) unless persisted? && Feature.enabled?(:cached_route_lookups, self, type: :ops, default_enabled: :yaml) + return (route&.path || build_full_path) unless persisted? && Feature.enabled?(:cached_route_lookups, self, type: :ops) # Return the path as-is if the parent is missing return path if route.nil? && parent.nil? && path.present? diff --git a/app/models/concerns/sha256_attribute.rb b/app/models/concerns/sha256_attribute.rb deleted file mode 100644 index 3c906642b1a..00000000000 --- a/app/models/concerns/sha256_attribute.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -module Sha256Attribute - extend ActiveSupport::Concern - - class_methods do - def sha256_attribute(name) - return if ENV['STATIC_VERIFICATION'] - - validate_binary_column_exists!(name) unless Rails.env.production? - - attribute(name, Gitlab::Database::Sha256Attribute.new) - end - - # This only gets executed in non-production environments as an additional check to ensure - # the column is the correct type. In production it should behave like any other attribute. - # See https://gitlab.com/gitlab-org/gitlab/merge_requests/5502 for more discussion - def validate_binary_column_exists!(name) - return unless database_exists? - - unless table_exists? - warn "WARNING: sha256_attribute #{name.inspect} is invalid since the table doesn't exist - you may need to run database migrations" - return - end - - column = columns.find { |c| c.name == name.to_s } - - unless column - warn "WARNING: sha256_attribute #{name.inspect} is invalid since the column doesn't exist - you may need to run database migrations" - return - end - - unless column.type == :binary - raise ArgumentError, "sha256_attribute #{name.inspect} is invalid since the column type is not :binary" - end - rescue StandardError => error - Gitlab::AppLogger.error "Sha256Attribute initialization: #{error.message}" - raise - end - - def database_exists? - database.exists? - end - end -end diff --git a/app/models/concerns/sha_attribute.rb b/app/models/concerns/sha_attribute.rb index e49f4d03bda..701d2fda5c5 100644 --- a/app/models/concerns/sha_attribute.rb +++ b/app/models/concerns/sha_attribute.rb @@ -3,39 +3,71 @@ module ShaAttribute extend ActiveSupport::Concern - # Needed for the database method - include DatabaseReflection + class ShaAttributeTypeMismatchError < StandardError + def initialize(column_name, column_type) + @column_name = column_name + @column_type = column_type + end + + def message + "sha_attribute :#{@column_name} should be a :binary column but it is :#{@column_type}" + end + end + + class Sha256AttributeTypeMismatchError < ShaAttributeTypeMismatchError + def message + "sha256_attribute :#{@column_name} should be a :binary column but it is :#{@column_type}" + end + end class_methods do def sha_attribute(name) return if ENV['STATIC_VERIFICATION'] - validate_binary_column_exists!(name) if Rails.env.development? || Rails.env.test? + sha_attribute_fields << name attribute(name, Gitlab::Database::ShaAttribute.new) end + def sha_attribute_fields + @sha_attribute_fields ||= [] + end + + def sha256_attribute(name) + return if ENV['STATIC_VERIFICATION'] + + sha256_attribute_fields << name + + attribute(name, Gitlab::Database::Sha256Attribute.new) + end + + def sha256_attribute_fields + @sha256_attribute_fields ||= [] + end + # This only gets executed in non-production environments as an additional check to ensure # the column is the correct type. In production it should behave like any other attribute. # See https://gitlab.com/gitlab-org/gitlab/merge_requests/5502 for more discussion - def validate_binary_column_exists!(name) - return unless database_exists? - return unless table_exists? + def load_schema! + super - column = columns.find { |c| c.name == name.to_s } + return if Rails.env.production? - return unless column + sha_attribute_fields.each do |field| + column = columns_hash[field.to_s] - unless column.type == :binary - raise ArgumentError, "sha_attribute #{name.inspect} is invalid since the column type is not :binary" + if column && column.type != :binary + raise ShaAttributeTypeMismatchError.new(column.name, column.type) + end end - rescue StandardError => error - Gitlab::AppLogger.error "ShaAttribute initialization: #{error.message}" - raise - end - def database_exists? - database.exists? + sha256_attribute_fields.each do |field| + column = columns_hash[field.to_s] + + if column && column.type != :binary + raise Sha256AttributeTypeMismatchError.new(column.name, column.type) + end + end end end end diff --git a/app/models/container_registry/event.rb b/app/models/container_registry/event.rb index c1b865ae578..5409bdf5af4 100644 --- a/app/models/container_registry/event.rb +++ b/app/models/container_registry/event.rb @@ -2,6 +2,8 @@ module ContainerRegistry class Event + include Gitlab::Utils::StrongMemoize + ALLOWED_ACTIONS = %w(push delete).freeze PUSH_ACTION = 'push' EVENT_TRACKING_CATEGORY = 'container_registry:notification' @@ -17,7 +19,7 @@ module ContainerRegistry end def handle! - # no op + update_project_statistics end def track! @@ -58,10 +60,25 @@ module ContainerRegistry end def container_registry_path - path = event.dig('target', 'repository') - return unless path + strong_memoize(:container_registry_path) do + path = event.dig('target', 'repository') + next unless path + + ContainerRegistry::Path.new(path) + end + end + + def project + container_registry_path&.repository_project + end + + def update_project_statistics + return unless supported? + return unless target_tag? + return unless project + return unless Feature.enabled?(:container_registry_project_statistics, project) - ContainerRegistry::Path.new(path) + ProjectCacheWorker.perform_async(project.id, [], [:container_registry_size]) end end end diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 78bd520d5d5..c965d7cffe1 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -43,7 +43,8 @@ class ContainerRepository < ApplicationRecord migration_canceled: 4, not_found: 5, native_import: 6, - migration_forced_canceled: 7 + migration_forced_canceled: 7, + migration_canceled_by_registry: 8 } delegate :client, :gitlab_api_client, to: :registry @@ -214,9 +215,9 @@ class ContainerRepository < ApplicationRecord container_repository.migration_skipped_at = Time.zone.now end - before_transition any => %i[import_done import_aborted] do |container_repository| + before_transition any => %i[import_done import_aborted import_skipped] do |container_repository| container_repository.run_after_commit do - ::ContainerRegistry::Migration::EnqueuerWorker.perform_async + ::ContainerRegistry::Migration::EnqueuerWorker.enqueue_a_job end end end @@ -325,17 +326,13 @@ class ContainerRepository < ApplicationRecord return if importing? start_import(forced: true) - when 'import_canceled', 'pre_import_canceled' - return if import_skipped? - - skip_import(reason: :migration_canceled) when 'import_complete' finish_import - when 'import_failed' + when 'import_failed', 'import_canceled' retry_import when 'pre_import_complete' finish_pre_import_and_start_import - when 'pre_import_failed' + when 'pre_import_failed', 'pre_import_canceled' retry_pre_import else yield @@ -376,6 +373,10 @@ class ContainerRepository < ApplicationRecord migration_retries_count >= ContainerRegistry::Migration.max_retries end + def nearing_or_exceeded_retry_limit? + migration_retries_count >= ContainerRegistry::Migration.max_retries - 1 + end + def last_import_step_done_at [migration_pre_import_done_at, migration_import_done_at, migration_aborted_at, migration_skipped_at].compact.max end @@ -460,12 +461,8 @@ class ContainerRepository < ApplicationRecord client.delete_repository_tag_by_name(self.path, name) end - def reset_expiration_policy_started_at! - update!(expiration_policy_started_at: nil) - end - def start_expiration_policy! - update!(expiration_policy_started_at: Time.zone.now) + update!(expiration_policy_started_at: Time.zone.now, last_cleanup_deleted_tags_count: nil) end def size diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index 360a9ffbc53..3c0f7d91a03 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -5,7 +5,11 @@ class DeployToken < ApplicationRecord include TokenAuthenticatable include PolicyActor include Gitlab::Utils::StrongMemoize - add_authentication_token_field :token, encrypted: :optional + include IgnorableColumns + + ignore_column :token, remove_with: '15.2', remove_after: '2022-07-22' + + add_authentication_token_field :token, encrypted: :required AVAILABLE_SCOPES = %i(read_repository read_registry write_registry read_package_registry write_package_registry).freeze @@ -126,6 +130,10 @@ class DeployToken < ApplicationRecord end end + def impersonated? + false + end + def expires_at expires_at = read_attribute(:expires_at) expires_at != Forever.date ? expires_at : nil diff --git a/app/models/deployment.rb b/app/models/deployment.rb index 63d531d82c3..4204ad707b2 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -70,6 +70,11 @@ class Deployment < ApplicationRecord transition created: :blocked end + # This transition is possible when we have manual jobs. + event :create do + transition skipped: :created + end + event :unblock do transition blocked: :created end @@ -348,7 +353,7 @@ class Deployment < ApplicationRecord def sync_status_with(build) return false unless ::Deployment.statuses.include?(build.status) - return false if build.created? || build.status == self.status + return false if build.status == self.status update_status!(build.status) rescue StandardError => e @@ -403,6 +408,8 @@ class Deployment < ApplicationRecord skip! when 'blocked' block! + when 'created' + create! else raise ArgumentError, "The status #{status.inspect} is invalid" end diff --git a/app/models/design_management/action.rb b/app/models/design_management/action.rb index b9df2873a73..5f407a5867d 100644 --- a/app/models/design_management/action.rb +++ b/app/models/design_management/action.rb @@ -19,6 +19,7 @@ module DesignManagement scope :ordered, -> { order(version_id: :asc) } scope :by_design, -> (design) { where(design: design) } scope :by_event, -> (event) { where(event: event) } + scope :with_version, -> { preload(:version) } # For each design, only select the most recent action scope :most_recent, -> do diff --git a/app/models/environment.rb b/app/models/environment.rb index 9e663b2ee74..865f5c68af1 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -26,7 +26,7 @@ class Environment < ApplicationRecord has_many :self_managed_prometheus_alert_events, inverse_of: :environment has_many :alert_management_alerts, class_name: 'AlertManagement::Alert', inverse_of: :environment - has_one :last_deployment, -> { Feature.enabled?(:env_last_deployment_by_finished_at, default_enabled: :yaml) ? success.ordered : success.distinct_on_environment }, class_name: 'Deployment', inverse_of: :environment + has_one :last_deployment, -> { success.ordered }, class_name: 'Deployment', inverse_of: :environment has_one :last_visible_deployment, -> { visible.distinct_on_environment }, inverse_of: :environment, class_name: 'Deployment' has_one :last_visible_deployable, through: :last_visible_deployment, source: 'deployable', source_type: 'CommitStatus', disable_joins: true has_one :last_visible_pipeline, through: :last_visible_deployable, source: 'pipeline', disable_joins: true @@ -314,13 +314,9 @@ class Environment < ApplicationRecord def stop_actions strong_memoize(:stop_actions) do - if ::Feature.enabled?(:environment_multiple_stop_actions, project, default_enabled: :yaml) - # Fix N+1 queries it brings to the serializer. - # Tracked in https://gitlab.com/gitlab-org/gitlab/-/issues/358780 - last_deployment_group.map(&:stop_action).compact - else - [last_deployment&.stop_action].compact - end + # Fix N+1 queries it brings to the serializer. + # Tracked in https://gitlab.com/gitlab-org/gitlab/-/issues/358780 + last_deployment_group.map(&:stop_action).compact end end diff --git a/app/models/event.rb b/app/models/event.rb index e9a98c06b59..7760be3e817 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -357,6 +357,8 @@ class Event < ApplicationRecord Project.unscoped.where(id: project_id) .where('last_activity_at <= ?', RESET_PROJECT_ACTIVITY_INTERVAL.ago) .touch_all(:last_activity_at, time: created_at) # rubocop: disable Rails/SkipsModelValidations + + Gitlab::InactiveProjectsDeletionWarningTracker.new(project.id).reset end def authored_by?(user) @@ -369,6 +371,10 @@ class Event < ApplicationRecord Event._to_partial_path end + def has_no_project_and_group? + project_id.nil? && group_id.nil? + end + protected def capability diff --git a/app/models/event_collection.rb b/app/models/event_collection.rb index fc093894847..4258027aa56 100644 --- a/app/models/event_collection.rb +++ b/app/models/event_collection.rb @@ -8,6 +8,8 @@ class EventCollection include Gitlab::Utils::StrongMemoize + attr_reader :filter + # To prevent users from putting too much pressure on the database by cycling # through thousands of events we put a limit on the number of pages. MAX_PAGE = 10 @@ -19,7 +21,7 @@ class EventCollection @projects = projects @limit = limit @offset = offset - @filter = filter + @filter = filter || EventFilter.new(EventFilter::ALL) @groups = groups end @@ -44,35 +46,46 @@ class EventCollection private def project_events - in_operator_optimized_relation('project_id', projects) + in_operator_optimized_relation('project_id', projects, Project) end def group_events - in_operator_optimized_relation('group_id', groups) + in_operator_optimized_relation('group_id', groups, Namespace) end def project_and_group_events - Event.from_union([project_events, group_events]).recent + if EventFilter::PROJECT_ONLY_EVENT_TYPES.include?(filter.filter) + project_events + else + Event.from_union([project_events, group_events]).recent + end end - def in_operator_optimized_relation(parent_column, parents) - scope = filtered_events - array_scope = parents.select(:id) - array_mapping_scope = -> (parent_id_expression) { Event.where(Event.arel_table[parent_column].eq(parent_id_expression)).reorder(id: :desc) } - finder_query = -> (id_expression) { Event.where(Event.arel_table[:id].eq(id_expression)) } + def in_operator_optimized_relation(parent_column, parents, parent_model) + query_builder_params = if Feature.enabled?(:optimized_project_and_group_activity_queries) + array_data = { + scope_ids: parents.pluck(:id), + scope_model: parent_model, + mapping_column: parent_column + } + filter.in_operator_query_builder_params(array_data) + else + { + scope: filtered_events, + array_scope: parents.select(:id), + array_mapping_scope: -> (parent_id_expression) { Event.where(Event.arel_table[parent_column].eq(parent_id_expression)).reorder(id: :desc) }, + finder_query: -> (id_expression) { Event.where(Event.arel_table[:id].eq(id_expression)) } + } + end Gitlab::Pagination::Keyset::InOperatorOptimization::QueryBuilder - .new( - scope: scope, - array_scope: array_scope, - array_mapping_scope: array_mapping_scope, - finder_query: finder_query - ) + .new(**query_builder_params) .execute + .limit(@limit + @offset) end def filtered_events - @filter ? @filter.apply_filter(base_relation) : base_relation + filter.apply_filter(base_relation) end def paginate_events(events) @@ -99,3 +112,5 @@ class EventCollection end end end + +EventCollection.prepend_mod diff --git a/app/models/group.rb b/app/models/group.rb index 990c06fdc41..86f4b14cb6c 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -146,7 +146,7 @@ class Group < Namespace validates :group_feature, presence: true add_authentication_token_field :runners_token, - encrypted: -> { Feature.enabled?(:groups_tokens_optional_encryption, default_enabled: true) ? :optional : :required }, + encrypted: -> { Feature.enabled?(:groups_tokens_optional_encryption) ? :optional : :required }, prefix: RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX after_create :post_create_hook @@ -870,7 +870,7 @@ class Group < Namespace actors << self if root_ancestor != self actors.any? do |actor| - ::Feature.enabled?(feature_flag, actor, default_enabled: :yaml) + ::Feature.enabled?(feature_flag, actor) end end diff --git a/app/models/group_group_link.rb b/app/models/group_group_link.rb index b0020f097b5..a70110c4076 100644 --- a/app/models/group_group_link.rb +++ b/app/models/group_group_link.rb @@ -41,3 +41,5 @@ class GroupGroupLink < ApplicationRecord Gitlab::Access.human_access(self.group_access) end end + +GroupGroupLink.prepend_mod_with('GroupGroupLink') diff --git a/app/models/incident_management/timeline_event.rb b/app/models/incident_management/timeline_event.rb new file mode 100644 index 00000000000..d30d6906e14 --- /dev/null +++ b/app/models/incident_management/timeline_event.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module IncidentManagement + class TimelineEvent < ApplicationRecord + include CacheMarkdownField + + self.table_name = 'incident_management_timeline_events' + + cache_markdown_field :note, + pipeline: :'incident_management/timeline_event', + issuable_reference_expansion_enabled: true + + belongs_to :project + belongs_to :author, class_name: 'User', foreign_key: :author_id + belongs_to :incident, class_name: 'Issue', foreign_key: :issue_id, inverse_of: :incident_management_timeline_events + belongs_to :updated_by_user, class_name: 'User', foreign_key: :updated_by_user_id + belongs_to :promoted_from_note, class_name: 'Note', foreign_key: :promoted_from_note_id + + validates :project, :incident, :occurred_at, presence: true + validates :action, presence: true, length: { maximum: 128 } + validates :note, :note_html, presence: true, length: { maximum: 10_000 } + + scope :order_occurred_at_asc, -> { reorder(occurred_at: :asc) } + end +end diff --git a/app/models/instance_configuration.rb b/app/models/instance_configuration.rb index 00e55d0fd89..8a8c1a29375 100644 --- a/app/models/instance_configuration.rb +++ b/app/models/instance_configuration.rb @@ -13,6 +13,7 @@ class InstanceConfiguration { ssh_algorithms_hashes: ssh_algorithms_hashes, host: host, gitlab_pages: gitlab_pages, + ci_cd_limits: ci_cd_limits, size_limits: size_limits, package_file_size_limits: package_file_size_limits, rate_limits: rate_limits }.deep_symbolize_keys @@ -47,6 +48,7 @@ class InstanceConfiguration { max_attachment_size: application_settings[:max_attachment_size].megabytes, receive_max_input_size: application_settings[:receive_max_input_size]&.megabytes, + max_export_size: application_settings[:max_export_size] > 0 ? application_settings[:max_export_size].megabytes : nil, max_import_size: application_settings[:max_import_size] > 0 ? application_settings[:max_import_size].megabytes : nil, diff_max_patch_bytes: application_settings[:diff_max_patch_bytes].bytes, max_artifacts_size: application_settings[:max_artifacts_size].megabytes, @@ -128,6 +130,23 @@ class InstanceConfiguration } end + def ci_cd_limits + Plan.all.to_h { |plan| [plan.name.capitalize, plan_ci_cd_limits(plan)] } + end + + def plan_ci_cd_limits(plan) + plan.actual_limits.slice( + :ci_pipeline_size, + :ci_active_jobs, + :ci_active_pipelines, + :ci_project_subscriptions, + :ci_pipeline_schedules, + :ci_needs_size_limit, + :ci_registered_group_runners, + :ci_registered_project_runners + ) + end + def ssh_algorithm_file(algorithm) File.join(SSH_ALGORITHMS_PATH, "ssh_host_#{algorithm.downcase}_key.pub") end diff --git a/app/models/integration.rb b/app/models/integration.rb index c0e244e38b6..b5064cfae2d 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -5,14 +5,14 @@ class Integration < ApplicationRecord include Sortable include Importable - include ProjectServicesLoggable + include Integrations::Loggable include Integrations::HasDataFields + include Integrations::ResetSecretFields include FromUnion include EachBatch include IgnorableColumns extend ::Gitlab::Utils::Override - ignore_column :template, remove_with: '15.0', remove_after: '2022-04-22' ignore_column :type, remove_with: '15.0', remove_after: '2022-04-22' ignore_column :properties, remove_with: '15.1', remove_after: '2022-05-22' @@ -161,7 +161,7 @@ class Integration < ApplicationRecord end def fields - self.class.fields + self.class.fields.dup end # Provide convenient accessor methods for each serialized property. @@ -279,7 +279,7 @@ class Integration < ApplicationRecord end def self.dev_integration_names - return [] unless Rails.env.development? + return [] unless Gitlab.dev_or_test_env? DEV_INTEGRATION_NAMES end @@ -447,6 +447,7 @@ class Integration < ApplicationRecord # TODO: Once all integrations use `Integrations::Field` we can # use `#secret?` here. + # See: https://gitlab.com/groups/gitlab-org/-/epics/7652 def secret_fields fields.select { |f| f[:type] == 'password' }.pluck(:name) end diff --git a/app/models/integrations/bamboo.rb b/app/models/integrations/bamboo.rb index b384a94d713..4e144a688f6 100644 --- a/app/models/integrations/bamboo.rb +++ b/app/models/integrations/bamboo.rb @@ -5,7 +5,26 @@ module Integrations include ReactivelyCached prepend EnableSslVerification - prop_accessor :bamboo_url, :build_key, :username, :password + field :bamboo_url, + title: s_('BambooService|Bamboo URL'), + placeholder: s_('https://bamboo.example.com'), + help: s_('BambooService|Bamboo service root URL.'), + required: true + + field :build_key, + help: s_('BambooService|Bamboo build plan key.'), + non_empty_password_title: s_('BambooService|Enter new build key'), + non_empty_password_help: s_('BambooService|Leave blank to use your current build key.'), + placeholder: s_('KEY'), + required: true + + field :username, + help: s_('BambooService|The user with API access to the Bamboo server.') + + field :password, + type: 'password', + non_empty_password_title: s_('ProjectService|Enter new password'), + non_empty_password_help: s_('ProjectService|Leave blank to use your current password') validates :bamboo_url, presence: true, public_url: true, if: :activated? validates :build_key, presence: true, if: :activated? @@ -43,39 +62,6 @@ module Integrations 'bamboo' end - def fields - [ - { - type: 'text', - name: 'bamboo_url', - title: s_('BambooService|Bamboo URL'), - placeholder: s_('https://bamboo.example.com'), - help: s_('BambooService|Bamboo service root URL.'), - required: true - }, - { - type: 'password', - name: 'build_key', - help: s_('BambooService|Bamboo build plan key.'), - non_empty_password_title: s_('BambooService|Enter new build key'), - non_empty_password_help: s_('BambooService|Leave blank to use your current build key.'), - placeholder: s_('KEY'), - required: true - }, - { - type: 'text', - name: 'username', - help: s_('BambooService|The user with API access to the Bamboo server.') - }, - { - type: 'password', - name: 'password', - non_empty_password_title: s_('ProjectService|Enter new password'), - non_empty_password_help: s_('ProjectService|Leave blank to use your current password') - } - ] - end - def build_page(sha, ref) with_reactive_cache(sha, ref) {|cached| cached[:build_page] } end diff --git a/app/models/integrations/base_chat_notification.rb b/app/models/integrations/base_chat_notification.rb index 54bd595892f..9bf208abcf7 100644 --- a/app/models/integrations/base_chat_notification.rb +++ b/app/models/integrations/base_chat_notification.rb @@ -149,6 +149,10 @@ module Integrations raise NotImplementedError end + def webhook_placeholder + raise NotImplementedError + end + private def log_usage(_, _) diff --git a/app/models/integrations/base_ci.rb b/app/models/integrations/base_ci.rb index b2e269b1b50..4f8732da703 100644 --- a/app/models/integrations/base_ci.rb +++ b/app/models/integrations/base_ci.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -# Base class for CI services -# List methods you need to implement to get your CI service +# Base class for CI integrations +# List methods you need to implement to get your CI integration # working with GitLab merge requests module Integrations class BaseCi < Integration @@ -12,7 +12,7 @@ module Integrations end def self.supported_events - %w(push) + %w[push] end # Return complete url to build page @@ -30,10 +30,10 @@ module Integrations # # # Ex. - # @service.commit_status('13be4ac', 'master') + # @integration.commit_status('13be4ac', 'master') # # => 'success' # - # @service.commit_status('2abe4ac', 'dev') + # @integration.commit_status('2abe4ac', 'dev') # # => 'running' # # diff --git a/app/models/integrations/buildkite.rb b/app/models/integrations/buildkite.rb index 3b802271a36..d1e54ce86ee 100644 --- a/app/models/integrations/buildkite.rb +++ b/app/models/integrations/buildkite.rb @@ -10,7 +10,18 @@ module Integrations ENDPOINT = "https://buildkite.com" - prop_accessor :project_url, :token + field :project_url, + title: _('Pipeline URL'), + placeholder: "#{ENDPOINT}/example-org/test-pipeline", + required: true + + field :token, + type: 'password', + title: _('Token'), + help: s_('ProjectService|The token you get after you create a Buildkite pipeline with a GitLab repository.'), + non_empty_password_title: s_('ProjectService|Enter new token'), + non_empty_password_help: s_('ProjectService|Leave blank to use your current token.'), + required: true validates :project_url, presence: true, public_url: true, if: :activated? validates :token, presence: true, if: :activated? @@ -74,24 +85,6 @@ module Integrations s_('ProjectService|Run CI/CD pipelines with Buildkite.') end - def fields - [ - { type: 'password', - name: 'token', - title: _('Token'), - help: s_('ProjectService|The token you get after you create a Buildkite pipeline with a GitLab repository.'), - non_empty_password_title: s_('ProjectService|Enter new token'), - non_empty_password_help: s_('ProjectService|Leave blank to use your current token.'), - required: true }, - - { type: 'text', - name: 'project_url', - title: _('Pipeline URL'), - placeholder: "#{ENDPOINT}/example-org/test-pipeline", - required: true } - ] - end - def calculate_reactive_cache(sha, ref) response = Gitlab::HTTP.try_get(commit_status_path(sha), request_options) diff --git a/app/models/integrations/drone_ci.rb b/app/models/integrations/drone_ci.rb index 73f78bec381..0c65ed8cd5f 100644 --- a/app/models/integrations/drone_ci.rb +++ b/app/models/integrations/drone_ci.rb @@ -10,7 +10,17 @@ module Integrations DRONE_SAAS_HOSTNAME = 'cloud.drone.io' - prop_accessor :drone_url, :token + field :drone_url, + title: s_('ProjectService|Drone server URL'), + placeholder: 'http://drone.example.com', + required: true + + field :token, + type: 'password', + help: s_('ProjectService|Token for the Drone project.'), + non_empty_password_title: s_('ProjectService|Enter new token'), + non_empty_password_help: s_('ProjectService|Leave blank to use your current token.'), + required: true validates :drone_url, presence: true, public_url: true, if: :activated? validates :token, presence: true, if: :activated? @@ -94,26 +104,6 @@ module Integrations s_('ProjectService|Run CI/CD pipelines with Drone.') end - def fields - [ - { - type: 'password', - name: 'token', - help: s_('ProjectService|Token for the Drone project.'), - non_empty_password_title: s_('ProjectService|Enter new token'), - non_empty_password_help: s_('ProjectService|Leave blank to use your current token.'), - required: true - }, - { - type: 'text', - name: 'drone_url', - title: s_('ProjectService|Drone server URL'), - placeholder: 'http://drone.example.com', - required: true - } - ] - end - override :hook_url def hook_url [drone_url, "/hook", "?owner=#{project.namespace.full_path}", "&name=#{project.path}", "&access_token=#{token}"].join diff --git a/app/models/integrations/field.rb b/app/models/integrations/field.rb index f00c4236a92..ca7833c1a56 100644 --- a/app/models/integrations/field.rb +++ b/app/models/integrations/field.rb @@ -10,6 +10,7 @@ module Integrations non_empty_password_help non_empty_password_title api_only + exposes_secrets ].freeze attr_reader :name diff --git a/app/models/integrations/jenkins.rb b/app/models/integrations/jenkins.rb index 32f11ee23eb..a1abbce72bc 100644 --- a/app/models/integrations/jenkins.rb +++ b/app/models/integrations/jenkins.rb @@ -7,7 +7,25 @@ module Integrations prepend EnableSslVerification extend Gitlab::Utils::Override - prop_accessor :jenkins_url, :project_name, :username, :password + field :jenkins_url, + title: s_('ProjectService|Jenkins server URL'), + required: true, + placeholder: 'http://jenkins.example.com', + help: s_('The URL of the Jenkins server.') + + field :project_name, + required: true, + placeholder: 'my_project_name', + help: s_('The name of the Jenkins project. Copy the name from the end of the URL to the project.') + + field :username, + help: s_('The username for the Jenkins server.') + + field :password, + type: 'password', + help: s_('The password for the Jenkins server.'), + non_empty_password_title: s_('ProjectService|Enter new password.'), + non_empty_password_help: s_('ProjectService|Leave blank to use your current password.') before_validation :reset_password @@ -15,7 +33,6 @@ module Integrations validates :project_name, presence: true, if: :activated? validates :username, presence: true, if: ->(service) { service.activated? && service.password_touched? && service.password.present? } - default_value_for :push_events, true default_value_for :merge_requests_events, false default_value_for :tag_push_events, false @@ -72,37 +89,5 @@ module Integrations def self.to_param 'jenkins' end - - def fields - [ - { - type: 'text', - name: 'jenkins_url', - title: s_('ProjectService|Jenkins server URL'), - required: true, - placeholder: 'http://jenkins.example.com', - help: s_('The URL of the Jenkins server.') - }, - { - type: 'text', - name: 'project_name', - required: true, - placeholder: 'my_project_name', - help: s_('The name of the Jenkins project. Copy the name from the end of the URL to the project.') - }, - { - type: 'text', - name: 'username', - help: s_('The username for the Jenkins server.') - }, - { - type: 'password', - name: 'password', - help: s_('The password for the Jenkins server.'), - non_empty_password_title: s_('ProjectService|Enter new password.'), - non_empty_password_help: s_('ProjectService|Leave blank to use your current password.') - } - ] - end end end diff --git a/app/models/integrations/jira.rb b/app/models/integrations/jira.rb index a800b9e5baa..992bd01bf5f 100644 --- a/app/models/integrations/jira.rb +++ b/app/models/integrations/jira.rb @@ -31,7 +31,6 @@ module Integrations # We should use username/password for Jira Server and email/api_token for Jira Cloud, # for more information check: https://gitlab.com/gitlab-org/gitlab-foss/issues/49936. - before_validation :reset_password after_commit :update_deployment_type, on: [:create, :update], if: :update_deployment_type? enum comment_detail: { @@ -46,12 +45,14 @@ module Integrations required: true, title: -> { s_('JiraService|Web URL') }, help: -> { s_('JiraService|Base URL of the Jira instance.') }, - placeholder: 'https://jira.example.com' + placeholder: 'https://jira.example.com', + exposes_secrets: true field :api_url, section: SECTION_TYPE_CONNECTION, title: -> { s_('JiraService|Jira API URL') }, - help: -> { s_('JiraService|If different from Web URL.') } + help: -> { s_('JiraService|If different from Web URL.') }, + exposes_secrets: true field :username, section: SECTION_TYPE_CONNECTION, @@ -98,13 +99,6 @@ module Integrations jira_tracker_data || self.build_jira_tracker_data end - def reset_password - return unless reset_password? - - data_fields.password = nil - self.properties = properties.except('password') - end - def set_default_data return unless issues_tracker.present? @@ -174,7 +168,8 @@ module Integrations sections.push({ type: SECTION_TYPE_JIRA_ISSUES, title: _('Issues'), - description: jira_issues_section_description + description: jira_issues_section_description, + plan: 'premium' }) end @@ -358,16 +353,7 @@ module Integrations true rescue StandardError => error - log_error( - "Issue transition failed", - error: { - exception_class: error.class.name, - exception_message: error.message, - exception_backtrace: Gitlab::BacktraceCleaner.clean_backtrace(error.backtrace) - }, - client_url: client_url - ) - + log_exception(error, message: 'Issue transition failed', client_url: client_url) false end @@ -544,9 +530,7 @@ module Integrations yield rescue StandardError => error @error = error - payload = { client_url: client_url } - Gitlab::ExceptionLogFormatter.format!(error, payload) - log_error("Error sending message", payload) + log_exception(error, message: 'Error sending message', client_url: client_url) nil end @@ -554,15 +538,6 @@ module Integrations api_url.presence || url end - def reset_password? - # don't reset the password if a new one is provided - return false if password_touched? - return true if api_url_changed? - return false if api_url.present? - - url_changed? - end - def update_deployment_type? api_url_changed? || url_changed? || username_changed? || password_changed? end diff --git a/app/models/integrations/mock_ci.rb b/app/models/integrations/mock_ci.rb index 568fb609a44..cd2928136ef 100644 --- a/app/models/integrations/mock_ci.rb +++ b/app/models/integrations/mock_ci.rb @@ -7,7 +7,11 @@ module Integrations ALLOWED_STATES = %w[failed canceled running pending success success-with-warnings skipped not_found].freeze - prop_accessor :mock_service_url + field :mock_service_url, + title: s_('ProjectService|Mock service URL'), + placeholder: 'http://localhost:4004', + required: true + validates :mock_service_url, presence: true, public_url: true, if: :activated? def title @@ -22,18 +26,6 @@ module Integrations 'mock_ci' end - def fields - [ - { - type: 'text', - name: 'mock_service_url', - title: s_('ProjectService|Mock service URL'), - placeholder: 'http://localhost:4004', - required: true - } - ] - end - # Return complete url to build page # # Ex. diff --git a/app/models/integrations/packagist.rb b/app/models/integrations/packagist.rb index 738319ce835..758c9e4761b 100644 --- a/app/models/integrations/packagist.rb +++ b/app/models/integrations/packagist.rb @@ -10,9 +10,6 @@ module Integrations validates :username, presence: true, if: :activated? validates :token, presence: true, if: :activated? - default_value_for :push_events, true - default_value_for :tag_push_events, true - def title 'Packagist' end diff --git a/app/models/integrations/prometheus.rb b/app/models/integrations/prometheus.rb index d6aafe45ae9..427034edb79 100644 --- a/app/models/integrations/prometheus.rb +++ b/app/models/integrations/prometheus.rb @@ -27,10 +27,7 @@ module Integrations after_commit :track_events - after_create_commit :create_default_alerts - scope :preload_project, -> { preload(:project) } - scope :with_clusters_with_cilium, -> { joins(project: [:clusters]).merge(Clusters::Cluster.with_available_cilium) } def show_active_box? false @@ -169,12 +166,6 @@ module Integrations manual_configuration_changed? && !manual_configuration? end - def create_default_alerts - return unless project_id - - ::Prometheus::CreateDefaultAlertsWorker.perform_async(project_id) - end - def behind_iap? manual_configuration? && google_iap_audience_client_id.present? && google_iap_service_account_json.present? end diff --git a/app/models/integrations/teamcity.rb b/app/models/integrations/teamcity.rb index f0f83f118d7..1205173e40b 100644 --- a/app/models/integrations/teamcity.rb +++ b/app/models/integrations/teamcity.rb @@ -8,7 +8,22 @@ module Integrations TEAMCITY_SAAS_HOSTNAME = /\A[^\.]+\.teamcity\.com\z/i.freeze - prop_accessor :teamcity_url, :build_type, :username, :password + field :teamcity_url, + title: s_('ProjectService|TeamCity server URL'), + placeholder: 'https://teamcity.example.com', + required: true + + field :build_type, + help: s_('ProjectService|The build configuration ID of the TeamCity project.'), + required: true + + field :username, + help: s_('ProjectService|Must have permission to trigger a manual build in TeamCity.') + + field :password, + type: 'password', + non_empty_password_title: s_('ProjectService|Enter new password'), + non_empty_password_help: s_('ProjectService|Leave blank to use your current password') validates :teamcity_url, presence: true, public_url: true, if: :activated? validates :build_type, presence: true, if: :activated? @@ -51,35 +66,6 @@ module Integrations s_('To run CI/CD pipelines with JetBrains TeamCity, input the GitLab project details in the TeamCity project Version Control Settings.') end - def fields - [ - { - type: 'text', - name: 'teamcity_url', - title: s_('ProjectService|TeamCity server URL'), - placeholder: 'https://teamcity.example.com', - required: true - }, - { - type: 'text', - name: 'build_type', - help: s_('ProjectService|The build configuration ID of the TeamCity project.'), - required: true - }, - { - type: 'text', - name: 'username', - help: s_('ProjectService|Must have permission to trigger a manual build in TeamCity.') - }, - { - type: 'password', - name: 'password', - non_empty_password_title: s_('ProjectService|Enter new password'), - non_empty_password_help: s_('ProjectService|Leave blank to use your current password') - } - ] - end - def build_page(sha, ref) with_reactive_cache(sha, ref) {|cached| cached[:build_page] } end diff --git a/app/models/issue.rb b/app/models/issue.rb index 484cceb9129..d4eb77ef6de 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -88,6 +88,7 @@ class Issue < ApplicationRecord has_many :prometheus_alerts, through: :prometheus_alert_events has_many :issue_customer_relations_contacts, class_name: 'CustomerRelations::IssueContact', inverse_of: :issue has_many :customer_relations_contacts, through: :issue_customer_relations_contacts, source: :contact, class_name: 'CustomerRelations::Contact', inverse_of: :issues + has_many :incident_management_timeline_events, class_name: 'IncidentManagement::TimelineEvent', foreign_key: :issue_id, inverse_of: :incident alias_attribute :escalation_status, :incident_management_issuable_escalation_status @@ -142,14 +143,12 @@ class Issue < ApplicationRecord scope :with_issue_type, ->(types) { where(issue_type: types) } scope :without_issue_type, ->(types) { where.not(issue_type: types) } - scope :public_only, -> { - without_hidden.where(confidential: false) - } + scope :public_only, -> { where(confidential: false) } scope :confidential_only, -> { where(confidential: true) } scope :without_hidden, -> { - if Feature.enabled?(:ban_user_feature_flag, default_enabled: :yaml) + if Feature.enabled?(:ban_user_feature_flag) where('NOT EXISTS (?)', Users::BannedUser.select(1).where('issues.author_id = banned_users.user_id')) else all diff --git a/app/models/key.rb b/app/models/key.rb index 42ea0f29171..e093f9faad3 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -5,7 +5,7 @@ require 'digest/md5' class Key < ApplicationRecord include AfterCommitQueue include Sortable - include Sha256Attribute + include ShaAttribute include Expirable include FromUnion @@ -24,17 +24,12 @@ class Key < ApplicationRecord length: { maximum: 5000 }, format: { with: /\A(#{Gitlab::SSHPublicKey.supported_algorithms.join('|')})/ } - validates :fingerprint, - uniqueness: true, - presence: { message: 'cannot be generated' }, - unless: -> { Gitlab::FIPS.enabled? } - validates :fingerprint_sha256, uniqueness: true, - presence: { message: 'cannot be generated' }, - if: -> { Gitlab::FIPS.enabled? } + presence: { message: 'cannot be generated' } validate :key_meets_restrictions + validate :expiration, on: :create delegate :name, :email, to: :user, prefix: true @@ -154,6 +149,10 @@ class Key < ApplicationRecord "type is forbidden. Must be #{Gitlab::Utils.to_exclusive_sentence(allowed_types)}" end + + def expiration + errors.add(:key, message: 'has expired') if expired? + end end Key.prepend_mod_with('Key') diff --git a/app/models/label.rb b/app/models/label.rb index 4c9f071f43a..7f4556c11c9 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -160,11 +160,6 @@ class Label < ApplicationRecord on_project_boards(project_id).where(id: label_id).exists? end - # Generate a hex color based on hex-encoded value - def self.color_for(value) - "##{Digest::MD5.hexdigest(value)[0..5]}" - end - def open_issues_count(user = nil) issues_count(user, state: 'opened') end diff --git a/app/models/loose_foreign_keys/deleted_record.rb b/app/models/loose_foreign_keys/deleted_record.rb index ebda5872f1c..6dfd6ea2aae 100644 --- a/app/models/loose_foreign_keys/deleted_record.rb +++ b/app/models/loose_foreign_keys/deleted_record.rb @@ -10,7 +10,7 @@ class LooseForeignKeys::DeletedRecord < Gitlab::Database::SharedModel partitioned_by :partition, strategy: :sliding_list, next_partition_if: -> (active_partition) do - return false if Feature.disabled?(:lfk_automatic_partition_creation, default_enabled: :yaml) + return false if Feature.disabled?(:lfk_automatic_partition_creation) oldest_record_in_partition = LooseForeignKeys::DeletedRecord .select(:id, :created_at) @@ -22,7 +22,7 @@ class LooseForeignKeys::DeletedRecord < Gitlab::Database::SharedModel oldest_record_in_partition.present? && oldest_record_in_partition.created_at < PARTITION_DURATION.ago end, detach_partition_if: -> (partition) do - return false if Feature.disabled?(:lfk_automatic_partition_dropping, default_enabled: :yaml) + return false if Feature.disabled?(:lfk_automatic_partition_dropping) !LooseForeignKeys::DeletedRecord .for_partition(partition) diff --git a/app/models/member.rb b/app/models/member.rb index 18ad2785d6e..a5084c8a60c 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -77,6 +77,10 @@ class Member < ApplicationRecord ]).merge(self) end + scope :excluding_users, ->(user_ids) do + where.not(user_id: user_ids) + end + # This scope encapsulates (most of) the conditions a row in the member table # must satisfy if it is a valid permission. Of particular note: # @@ -165,6 +169,7 @@ class Member < ApplicationRecord scope :owners, -> { active.where(access_level: OWNER) } scope :owners_and_maintainers, -> { active.where(access_level: [OWNER, MAINTAINER]) } scope :with_user, -> (user) { where(user: user) } + scope :by_access_level, -> (access_level) { active.where(access_level: access_level) } scope :preload_user_and_notification_settings, -> { preload(user: :notification_settings) } @@ -516,7 +521,7 @@ class Member < ApplicationRecord end def blocking_refresh - return true unless Feature.enabled?(:allow_non_blocking_member_refresh, default_enabled: :yaml) + return true unless Feature.enabled?(:allow_non_blocking_member_refresh) return true if @blocking_refresh.nil? @blocking_refresh diff --git a/app/models/members_preloader.rb b/app/models/members_preloader.rb index 8b8eca54550..ba7e4b39989 100644 --- a/app/models/members_preloader.rb +++ b/app/models/members_preloader.rb @@ -13,7 +13,7 @@ class MembersPreloader ActiveRecord::Associations::Preloader.new.preload(members, :created_by) ActiveRecord::Associations::Preloader.new.preload(members, user: :status) ActiveRecord::Associations::Preloader.new.preload(members, user: :u2f_registrations) - ActiveRecord::Associations::Preloader.new.preload(members, user: :webauthn_registrations) if Feature.enabled?(:webauthn, default_enabled: :yaml) + ActiveRecord::Associations::Preloader.new.preload(members, user: :webauthn_registrations) if Feature.enabled?(:webauthn) end end diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 4c6ed399bf9..39b5949ea7a 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -1152,7 +1152,7 @@ class MergeRequest < ApplicationRecord # rubocop: disable CodeReuse/ServiceClass def mergeable_state?(skip_ci_check: false, skip_discussions_check: false) - if Feature.enabled?(:improved_mergeability_checks, self.project, default_enabled: :yaml) + if Feature.enabled?(:improved_mergeability_checks, self.project) additional_checks = MergeRequests::Mergeability::RunChecksService.new( merge_request: self, params: { @@ -1438,30 +1438,8 @@ class MergeRequest < ApplicationRecord actual_head_pipeline.success? end - ## - # This method is for looking for active environments which created via pipelines for merge requests. - # Since deployments run on a merge request ref (e.g. `refs/merge-requests/:iid/head`), - # we cannot look up environments with source branch name. - def legacy_environments - return Environment.none unless actual_head_pipeline&.merge_request? - - build_for_actual_head_pipeline = Ci::Build.latest.where(pipeline: actual_head_pipeline) - - environments = build_for_actual_head_pipeline.joins(:metadata) - .where.not('ci_builds_metadata.expanded_environment_name' => nil) - .distinct('ci_builds_metadata.expanded_environment_name') - .limit(100) - .pluck(:expanded_environment_name) - - Environment.where(project: project, name: environments) - end - def environments_in_head_pipeline(deployment_status: nil) - if ::Feature.enabled?(:fix_related_environments_for_merge_requests, target_project, default_enabled: :yaml) - actual_head_pipeline&.environments_in_self_and_descendants(deployment_status: deployment_status) || Environment.none - else - legacy_environments - end + actual_head_pipeline&.environments_in_self_and_descendants(deployment_status: deployment_status) || Environment.none end def fetch_ref! @@ -1977,10 +1955,6 @@ class MergeRequest < ApplicationRecord end end - def attention_requested_enabled? - Feature.enabled?(:mr_attention_requests, project, default_enabled: :yaml) - end - private attr_accessor :skip_fetch_ref diff --git a/app/models/merge_request_assignee.rb b/app/models/merge_request_assignee.rb index 77b46fa50f4..fd8e5860040 100644 --- a/app/models/merge_request_assignee.rb +++ b/app/models/merge_request_assignee.rb @@ -10,12 +10,6 @@ class MergeRequestAssignee < ApplicationRecord scope :in_projects, ->(project_ids) { joins(:merge_request).where(merge_requests: { target_project_id: project_ids }) } - def set_state - if Feature.enabled?(:mr_attention_requests, self.merge_request&.project, default_enabled: :yaml) - self.state = MergeRequestReviewer.find_by(user_id: self.user_id, merge_request_id: self.merge_request_id)&.state || :attention_requested - end - end - def cache_key [model_name.cache_key, id, state, assignee.cache_key] end diff --git a/app/models/merge_request_reviewer.rb b/app/models/merge_request_reviewer.rb index 8c75fb2e4e6..4abf0fa09f0 100644 --- a/app/models/merge_request_reviewer.rb +++ b/app/models/merge_request_reviewer.rb @@ -6,12 +6,6 @@ class MergeRequestReviewer < ApplicationRecord belongs_to :merge_request belongs_to :reviewer, class_name: 'User', foreign_key: :user_id, inverse_of: :merge_request_reviewers - def set_state - if Feature.enabled?(:mr_attention_requests, self.merge_request&.project, default_enabled: :yaml) - self.state = MergeRequestAssignee.find_by(user_id: self.user_id, merge_request_id: self.merge_request_id)&.state || :attention_requested - end - end - def cache_key [model_name.cache_key, id, state, reviewer.cache_key] end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 3b75b6d163a..fcd641671f5 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -16,6 +16,7 @@ class Namespace < ApplicationRecord include Namespaces::Traversal::Linear include EachBatch include BlocksUnsafeSerialization + include Ci::NamespaceSettings # Temporary column used for back-filling project namespaces. # Remove it once the back-filling of all project namespaces is done. @@ -44,6 +45,7 @@ class Namespace < ApplicationRecord has_many :projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :project_statistics has_one :namespace_settings, inverse_of: :namespace, class_name: 'NamespaceSetting', autosave: true + has_one :ci_cd_settings, inverse_of: :namespace, class_name: 'NamespaceCiCdSetting', autosave: true has_one :namespace_statistics has_one :namespace_route, foreign_key: :namespace_id, autosave: false, inverse_of: :namespace, class_name: 'Route' has_many :namespace_members, foreign_key: :member_namespace_id, inverse_of: :member_namespace, class_name: 'Member' @@ -110,6 +112,8 @@ class Namespace < ApplicationRecord delegate :name, to: :owner, allow_nil: true, prefix: true delegate :avatar_url, to: :owner, allow_nil: true + delegate :prevent_sharing_groups_outside_hierarchy, :prevent_sharing_groups_outside_hierarchy=, + to: :namespace_settings, allow_nil: true after_save :schedule_sync_event_worker, if: -> { saved_change_to_id? || saved_change_to_parent_id? } @@ -126,7 +130,7 @@ class Namespace < ApplicationRecord before_destroy(prepend: true) { prepare_for_destroy } after_destroy :rm_dir after_commit :expire_child_caches, on: :update, if: -> { - Feature.enabled?(:cached_route_lookups, self, type: :ops, default_enabled: :yaml) && + Feature.enabled?(:cached_route_lookups, self, type: :ops) && saved_change_to_name? || saved_change_to_path? || saved_change_to_parent_id? } @@ -238,11 +242,11 @@ class Namespace < ApplicationRecord return unless host.ends_with?(gitlab_host) name = host.delete_suffix(gitlab_host) - Namespace.where(parent_id: nil).by_path(name) + Namespace.top_most.by_path(name) end def top_most - where(parent_id: nil) + by_parent(nil) end end @@ -351,7 +355,7 @@ class Namespace < ApplicationRecord # Includes projects from this namespace and projects from all subgroups # that belongs to this namespace def all_projects - if Feature.enabled?(:recursive_approach_for_all_projects, default_enabled: :yaml) + if Feature.enabled?(:recursive_approach_for_all_projects) namespace = user_namespace? ? self : self_and_descendant_ids Project.where(namespace: namespace) else @@ -372,6 +376,10 @@ class Namespace < ApplicationRecord false end + def all_project_ids_except(ids) + all_projects.where.not(id: ids).pluck(:id) + end + # Deprecated, use #licensed_feature_available? instead. Remove once Namespace#feature_available? isn't used anymore. def feature_available?(feature, _user = nil) licensed_feature_available?(feature) @@ -512,7 +520,7 @@ class Namespace < ApplicationRecord end def issue_repositioning_disabled? - Feature.enabled?(:block_issue_repositioning, self, type: :ops, default_enabled: :yaml) + Feature.enabled?(:block_issue_repositioning, self, type: :ops) end def storage_enforcement_date @@ -521,6 +529,12 @@ class Namespace < ApplicationRecord nil end + def certificate_based_clusters_enabled? + ::Gitlab::SafeRequestStore.fetch("certificate_based_clusters:ns:#{self.id}") do + Feature.enabled?(:certificate_based_clusters, self, type: :ops) + end + end + private def expire_child_caches @@ -634,7 +648,7 @@ class Namespace < ApplicationRecord end def cache_first_auto_devops_config? - ::Feature.enabled?(:namespaces_cache_first_auto_devops_config, default_enabled: :yaml) + ::Feature.enabled?(:namespaces_cache_first_auto_devops_config) end def write_projects_repository_config diff --git a/app/models/namespace_ci_cd_setting.rb b/app/models/namespace_ci_cd_setting.rb new file mode 100644 index 00000000000..c9c3a3206b6 --- /dev/null +++ b/app/models/namespace_ci_cd_setting.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class NamespaceCiCdSetting < ApplicationRecord # rubocop:disable Gitlab/NamespacedClass + belongs_to :namespace, inverse_of: :ci_cd_settings + + self.primary_key = :namespace_id +end + +NamespaceCiCdSetting.prepend_mod diff --git a/app/models/namespaces/traversal/linear.rb b/app/models/namespaces/traversal/linear.rb index 6320e0bc39d..b0350b0288f 100644 --- a/app/models/namespaces/traversal/linear.rb +++ b/app/models/namespaces/traversal/linear.rb @@ -77,38 +77,38 @@ module Namespaces end def sync_traversal_ids? - Feature.enabled?(:sync_traversal_ids, root_ancestor, default_enabled: :yaml) + Feature.enabled?(:sync_traversal_ids, root_ancestor) end def use_traversal_ids? - return false unless Feature.enabled?(:use_traversal_ids, default_enabled: :yaml) + return false unless Feature.enabled?(:use_traversal_ids) traversal_ids.present? end def use_traversal_ids_for_self_and_hierarchy? return false unless use_traversal_ids? - return false unless Feature.enabled?(:use_traversal_ids_for_self_and_hierarchy, root_ancestor, default_enabled: :yaml) + return false unless Feature.enabled?(:use_traversal_ids_for_self_and_hierarchy, root_ancestor) traversal_ids.present? end def use_traversal_ids_for_ancestors? return false unless use_traversal_ids? - return false unless Feature.enabled?(:use_traversal_ids_for_ancestors, root_ancestor, default_enabled: :yaml) + return false unless Feature.enabled?(:use_traversal_ids_for_ancestors, root_ancestor) traversal_ids.present? end def use_traversal_ids_for_ancestors_upto? return false unless use_traversal_ids? - return false unless Feature.enabled?(:use_traversal_ids_for_ancestors_upto, root_ancestor, default_enabled: :yaml) + return false unless Feature.enabled?(:use_traversal_ids_for_ancestors_upto, root_ancestor) traversal_ids.present? end def use_traversal_ids_for_root_ancestor? - return false unless Feature.enabled?(:use_traversal_ids_for_root_ancestor, default_enabled: :yaml) + return false unless Feature.enabled?(:use_traversal_ids_for_root_ancestor) traversal_ids.present? end diff --git a/app/models/namespaces/traversal/linear_scopes.rb b/app/models/namespaces/traversal/linear_scopes.rb index 0cac4c9143a..f0e9a8feeb2 100644 --- a/app/models/namespaces/traversal/linear_scopes.rb +++ b/app/models/namespaces/traversal/linear_scopes.rb @@ -55,7 +55,7 @@ module Namespaces def self_and_descendants(include_self: true) return super unless use_traversal_ids_for_descendants_scopes? - if Feature.enabled?(:traversal_ids_btree, default_enabled: :yaml) + if Feature.enabled?(:traversal_ids_btree) self_and_descendants_with_comparison_operators(include_self: include_self) else records = self_and_descendants_with_duplicates_with_array_operator(include_self: include_self) @@ -67,7 +67,7 @@ module Namespaces def self_and_descendant_ids(include_self: true) return super unless use_traversal_ids_for_descendants_scopes? - if Feature.enabled?(:traversal_ids_btree, default_enabled: :yaml) + if Feature.enabled?(:traversal_ids_btree) self_and_descendants_with_comparison_operators(include_self: include_self).as_ids else self_and_descendants_with_duplicates_with_array_operator(include_self: include_self) @@ -102,26 +102,26 @@ module Namespaces private def use_traversal_ids? - Feature.enabled?(:use_traversal_ids, default_enabled: :yaml) + Feature.enabled?(:use_traversal_ids) end def use_traversal_ids_roots? - Feature.enabled?(:use_traversal_ids_roots, default_enabled: :yaml) && + Feature.enabled?(:use_traversal_ids_roots) && use_traversal_ids? end def use_traversal_ids_for_ancestor_scopes? - Feature.enabled?(:use_traversal_ids_for_ancestor_scopes, default_enabled: :yaml) && + Feature.enabled?(:use_traversal_ids_for_ancestor_scopes) && use_traversal_ids? end def use_traversal_ids_for_descendants_scopes? - Feature.enabled?(:use_traversal_ids_for_descendants_scopes, default_enabled: :yaml) && + Feature.enabled?(:use_traversal_ids_for_descendants_scopes) && use_traversal_ids? end def use_traversal_ids_for_self_and_hierarchy_scopes? - Feature.enabled?(:use_traversal_ids_for_self_and_hierarchy_scopes, default_enabled: :yaml) && + Feature.enabled?(:use_traversal_ids_for_self_and_hierarchy_scopes) && use_traversal_ids? end diff --git a/app/models/packages/build_info.rb b/app/models/packages/build_info.rb index 38245bef7a5..61e2194006b 100644 --- a/app/models/packages/build_info.rb +++ b/app/models/packages/build_info.rb @@ -7,6 +7,6 @@ class Packages::BuildInfo < ApplicationRecord scope :pluck_pipeline_ids, -> { pluck(:pipeline_id) } scope :without_empty_pipelines, -> { where.not(pipeline_id: nil) } scope :order_by_pipeline_id, -> (direction) { order(pipeline_id: direction) } - scope :with_pipeline_id_less_than, -> (pipeline_id) { where("pipeline_id < ?", pipeline_id) } - scope :with_pipeline_id_greater_than, -> (pipeline_id) { where("pipeline_id > ?", pipeline_id) } + scope :with_pipeline_id_less_than, -> (pipeline_id) { where("#{table_name}.pipeline_id < ?", pipeline_id) } + scope :with_pipeline_id_greater_than, -> (pipeline_id) { where("#{table_name}.pipeline_id > ?", pipeline_id) } end diff --git a/app/models/packages/cleanup.rb b/app/models/packages/cleanup.rb new file mode 100644 index 00000000000..16bba4f445d --- /dev/null +++ b/app/models/packages/cleanup.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true +module Packages + module Cleanup + def self.table_name_prefix + 'packages_cleanup_' + end + end +end diff --git a/app/models/packages/cleanup/policy.rb b/app/models/packages/cleanup/policy.rb new file mode 100644 index 00000000000..87c101cfb8c --- /dev/null +++ b/app/models/packages/cleanup/policy.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Packages + module Cleanup + class Policy < ApplicationRecord + include Schedulable + + KEEP_N_DUPLICATED_PACKAGE_FILES_VALUES = %w[all 1 10 20 30 40 50].freeze + + self.primary_key = :project_id + + belongs_to :project + + validates :project, presence: true + validates :keep_n_duplicated_package_files, + inclusion: { + in: KEEP_N_DUPLICATED_PACKAGE_FILES_VALUES, + message: 'keep_n_duplicated_package_files is invalid' + } + + # used by Schedulable + def self.active + where.not(keep_n_duplicated_package_files: 'all') + end + + def set_next_run_at + # fixed cadence of 12 hours + self.next_run_at = Time.zone.now + 12.hours + end + end + end +end diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 2804588be85..93119bbff1f 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -245,8 +245,9 @@ class PagesDomain < ApplicationRecord def validate_pages_domain return unless domain - if domain.downcase.ends_with?(".#{Settings.pages.host.downcase}") || domain.casecmp(Settings.pages.host) == 0 - self.errors.add(:domain, "#{Settings.pages.host} and its subdomains cannot be used as custom pages domains. Please compare our documentation at https://docs.gitlab.com/ee/administration/pages/#advanced-configuration against your configuration.") + if domain.downcase.ends_with?(".#{Settings.pages.host.downcase}") + error_template = _("Subdomains of the Pages root domain %{root_domain} are reserved and cannot be used as custom Pages domains.") + self.errors.add(:domain, error_template % { root_domain: Settings.pages.host }) end end diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index 021ff789b13..68ba3d6eab4 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -49,10 +49,6 @@ class PersonalAccessToken < ApplicationRecord !revoked? && !expired? end - def expired_but_not_enforced? - false - end - def self.redis_getdel(user_id) Gitlab::Redis::SharedState.with do |redis| redis_key = redis_shared_state_key(user_id) diff --git a/app/models/preloaders/group_root_ancestor_preloader.rb b/app/models/preloaders/group_root_ancestor_preloader.rb index 3ca713d9635..29c60e90964 100644 --- a/app/models/preloaders/group_root_ancestor_preloader.rb +++ b/app/models/preloaders/group_root_ancestor_preloader.rb @@ -8,7 +8,7 @@ module Preloaders end def execute - return unless ::Feature.enabled?(:use_traversal_ids, default_enabled: :yaml) + return unless ::Feature.enabled?(:use_traversal_ids) # type == 'Group' condition located on subquery to prevent a filter in the query root_query = Namespace.joins("INNER JOIN (#{join_sql}) as root_query ON root_query.root_id = namespaces.id") diff --git a/app/models/preloaders/user_max_access_level_in_groups_preloader.rb b/app/models/preloaders/user_max_access_level_in_groups_preloader.rb index 2cd54b975f3..8df986b47a2 100644 --- a/app/models/preloaders/user_max_access_level_in_groups_preloader.rb +++ b/app/models/preloaders/user_max_access_level_in_groups_preloader.rb @@ -10,7 +10,7 @@ module Preloaders end def execute - if ::Feature.enabled?(:use_traversal_ids, default_enabled: :yaml) + if ::Feature.enabled?(:use_traversal_ids) preload_with_traversal_ids else preload_direct_memberships diff --git a/app/models/preloaders/user_max_access_level_in_projects_preloader.rb b/app/models/preloaders/user_max_access_level_in_projects_preloader.rb index 3764e9dcb16..2e2272a2ef5 100644 --- a/app/models/preloaders/user_max_access_level_in_projects_preloader.rb +++ b/app/models/preloaders/user_max_access_level_in_projects_preloader.rb @@ -5,25 +5,50 @@ module Preloaders # stores the values in requests store via the ProjectTeam class. class UserMaxAccessLevelInProjectsPreloader def initialize(projects, user) - @projects = projects + @projects = if projects.is_a?(Array) + Project.where(id: projects) + else + # Push projects base query in to a sub-select to avoid + # table name clashes. Performs better than aliasing. + Project.where(id: projects.reselect(:id)) + end + @user = user end def execute - # Use reselect to override the existing select to prevent - # the error `subquery has too many columns` - # NotificationsController passes in an Array so we need to check the type - project_ids = @projects.is_a?(ActiveRecord::Relation) ? @projects.reselect(:id) : @projects - access_levels = @user - .project_authorizations - .where(project_id: project_ids) - .group(:project_id) - .maximum(:access_level) - - @projects.each do |project| - access_level = access_levels[project.id] || Gitlab::Access::NO_ACCESS + project_authorizations = ProjectAuthorization.arel_table + + auths = @projects + .select( + Project.default_select_columns, + project_authorizations[:user_id], + project_authorizations[:access_level] + ) + .joins(project_auth_join) + + auths.each do |project| + access_level = project.access_level || Gitlab::Access::NO_ACCESS ProjectTeam.new(project).write_member_access_for_user_id(@user.id, access_level) end end + + private + + def project_auth_join + project_authorizations = ProjectAuthorization.arel_table + projects = Project.arel_table + + projects + .join( + project_authorizations.as(project_authorizations.name), + Arel::Nodes::OuterJoin + ) + .on( + project_authorizations[:project_id].eq(projects[:id]) + .and(project_authorizations[:user_id].eq(@user.id)) + ) + .join_sources + end end end diff --git a/app/models/project.rb b/app/models/project.rb index f7182d1645c..f4e39524e47 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -49,6 +49,7 @@ class Project < ApplicationRecord ignore_columns :container_registry_enabled, remove_after: '2021-09-22', remove_with: '14.4' BoardLimitExceeded = Class.new(StandardError) + ExportLimitExceeded = Class.new(StandardError) ignore_columns :mirror_last_update_at, :mirror_last_successful_update_at, remove_after: '2021-09-22', remove_with: '14.4' ignore_columns :pull_mirror_branch_prefix, remove_after: '2021-09-22', remove_with: '14.4' @@ -112,7 +113,7 @@ class Project < ApplicationRecord default_value_for(:ci_config_path) { Gitlab::CurrentSettings.default_ci_config_path } add_authentication_token_field :runners_token, - encrypted: -> { Feature.enabled?(:projects_tokens_optional_encryption, default_enabled: true) ? :optional : :required }, + encrypted: -> { Feature.enabled?(:projects_tokens_optional_encryption) ? :optional : :required }, prefix: RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX before_validation :mark_remote_mirrors_for_removal, if: -> { RemoteMirror.table_exists? } @@ -237,6 +238,7 @@ class Project < ApplicationRecord has_many :package_files, through: :packages, class_name: 'Packages::PackageFile' # debian_distributions and associated component_files must be destroyed by ruby code in order to properly remove carrierwave uploads has_many :debian_distributions, class_name: 'Packages::Debian::ProjectDistribution', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_one :packages_cleanup_policy, class_name: 'Packages::Cleanup::Policy', inverse_of: :project has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project has_one :import_export_upload, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -409,7 +411,6 @@ class Project < ApplicationRecord has_one :operations_feature_flags_client, class_name: 'Operations::FeatureFlagsClient' has_many :operations_feature_flags_user_lists, class_name: 'Operations::FeatureFlags::UserList' - has_many :error_tracking_errors, inverse_of: :project, class_name: 'ErrorTracking::Error' has_many :error_tracking_client_keys, inverse_of: :project, class_name: 'ErrorTracking::ClientKey' has_many :timelogs @@ -448,6 +449,7 @@ class Project < ApplicationRecord to: :project_feature, allow_nil: true alias_method :container_registry_enabled, :container_registry_enabled? delegate :show_default_award_emojis, :show_default_award_emojis=, :show_default_award_emojis?, + :enforce_auth_checks_on_uploads, :enforce_auth_checks_on_uploads=, :enforce_auth_checks_on_uploads?, :warn_about_potentially_unwanted_characters, :warn_about_potentially_unwanted_characters=, :warn_about_potentially_unwanted_characters?, to: :project_setting, allow_nil: true delegate :scheduled?, :started?, :in_progress?, :failed?, :finished?, @@ -462,7 +464,7 @@ class Project < ApplicationRecord delegate :add_user, :add_users, to: :team delegate :add_guest, :add_reporter, :add_developer, :add_maintainer, :add_owner, :add_role, to: :team delegate :group_runners_enabled, :group_runners_enabled=, to: :ci_cd_settings, allow_nil: true - delegate :root_ancestor, to: :namespace, allow_nil: true + delegate :root_ancestor, :certificate_based_clusters_enabled?, to: :namespace, allow_nil: true delegate :last_pipeline, to: :commit, allow_nil: true delegate :external_dashboard_url, to: :metrics_setting, allow_nil: true, prefix: true delegate :dashboard_timezone, to: :metrics_setting, allow_nil: true, prefix: true @@ -471,6 +473,7 @@ class Project < ApplicationRecord delegate :job_token_scope_enabled, :job_token_scope_enabled=, to: :ci_cd_settings, prefix: :ci, allow_nil: true delegate :keep_latest_artifact, :keep_latest_artifact=, to: :ci_cd_settings, allow_nil: true delegate :restrict_user_defined_variables, :restrict_user_defined_variables=, to: :ci_cd_settings, allow_nil: true + delegate :separated_caches, :separated_caches=, to: :ci_cd_settings, prefix: :ci, allow_nil: true delegate :runner_token_expiration_interval, :runner_token_expiration_interval=, :runner_token_expiration_interval_human_readable, :runner_token_expiration_interval_human_readable=, to: :ci_cd_settings, allow_nil: true delegate :actual_limits, :actual_plan_name, :actual_plan, to: :namespace, allow_nil: true delegate :allow_merge_on_skipped_pipeline, :allow_merge_on_skipped_pipeline?, @@ -742,6 +745,16 @@ class Project < ApplicationRecord Project.with(cte.to_arel).from(cte.alias_to(Project.arel_table)) end + def self.inactive + project_statistics = ::ProjectStatistics.arel_table + minimum_size_mb = ::Gitlab::CurrentSettings.inactive_projects_min_size_mb.megabytes + last_activity_cutoff = ::Gitlab::CurrentSettings.inactive_projects_send_warning_email_after_months.months.ago + + joins(:statistics) + .where((project_statistics[:storage_size]).gt(minimum_size_mb)) + .where('last_activity_at < ?', last_activity_cutoff) + end + scope :active, -> { joins(:issues, :notes, :merge_requests).order('issues.created_at, notes.created_at, merge_requests.created_at DESC') } scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) } @@ -962,7 +975,7 @@ class Project < ApplicationRecord end def ancestors(hierarchy_order: nil) - if Feature.enabled?(:linear_project_ancestors, self, default_enabled: :yaml) + if Feature.enabled?(:linear_project_ancestors, self) group&.self_and_ancestors(hierarchy_order: hierarchy_order) || Group.none else ancestors_upto(hierarchy_order: hierarchy_order) @@ -1013,6 +1026,10 @@ class Project < ApplicationRecord packages.where(package_type: package_type).exists? end + def packages_cleanup_policy + super || build_packages_cleanup_policy + end + def first_auto_devops_config return namespace.first_auto_devops_config if auto_devops&.enabled.nil? @@ -1020,7 +1037,7 @@ class Project < ApplicationRecord end def unlink_forks_upon_visibility_decrease_enabled? - Feature.enabled?(:unlink_fork_network_upon_visibility_decrease, self, default_enabled: true) + Feature.enabled?(:unlink_fork_network_upon_visibility_decrease, self) end # LFS and hashed repository storage are required for using Design Management. @@ -2047,6 +2064,8 @@ class Project < ApplicationRecord end def add_export_job(current_user:, after_export_strategy: nil, params: {}) + check_project_export_limit! + job_id = ProjectExportWorker.perform_async(current_user.id, self.id, after_export_strategy, params) if job_id @@ -2866,12 +2885,12 @@ class Project < ApplicationRecord end def work_items_feature_flag_enabled? - group&.work_items_feature_flag_enabled? || Feature.enabled?(:work_items, self, default_enabled: :yaml) + group&.work_items_feature_flag_enabled? || Feature.enabled?(:work_items, self) end def enqueue_record_project_target_platforms return unless Gitlab.com? - return unless Feature.enabled?(:record_projects_target_platforms, self, default_enabled: :yaml) + return unless Feature.enabled?(:record_projects_target_platforms, self) Projects::RecordTargetPlatformsWorker.perform_async(id) end @@ -2903,7 +2922,7 @@ class Project < ApplicationRecord if @topic_list != self.topic_list self.topics.delete_all self.topics = @topic_list.map do |topic| - Projects::Topic.where('lower(name) = ?', topic.downcase).order(total_projects_count: :desc).first_or_create(name: topic) + Projects::Topic.where('lower(name) = ?', topic.downcase).order(total_projects_count: :desc).first_or_create(name: topic, title: topic) end end @@ -3105,6 +3124,14 @@ class Project < ApplicationRecord Projects::SyncEvent.enqueue_worker end end + + def check_project_export_limit! + return if Gitlab::CurrentSettings.current_application_settings.max_export_size == 0 + + if self.statistics.storage_size > Gitlab::CurrentSettings.current_application_settings.max_export_size.megabytes + raise ExportLimitExceeded, _('The project size exceeds the export limit.') + end + end end Project.prepend_mod_with('Project') diff --git a/app/models/project_ci_cd_setting.rb b/app/models/project_ci_cd_setting.rb index 28a493cae33..38740aa20dd 100644 --- a/app/models/project_ci_cd_setting.rb +++ b/app/models/project_ci_cd_setting.rb @@ -18,11 +18,12 @@ class ProjectCiCdSetting < ApplicationRecord allow_nil: true default_value_for :forward_deployment_enabled, true + default_value_for :separated_caches, true chronic_duration_attr :runner_token_expiration_interval_human_readable, :runner_token_expiration_interval def forward_deployment_enabled? - super && ::Feature.enabled?(:forward_deployment_enabled, project, default_enabled: true) + super && ::Feature.enabled?(:forward_deployment_enabled, project) end def keep_latest_artifacts_available? diff --git a/app/models/project_import_state.rb b/app/models/project_import_state.rb index fabbd5b49cb..b1c1a5b6697 100644 --- a/app/models/project_import_state.rb +++ b/app/models/project_import_state.rb @@ -69,11 +69,9 @@ class ProjectImportState < ApplicationRecord project.reset_cache_and_import_attrs if Gitlab::ImportSources.importer_names.include?(project.import_type) && project.repo_exists? - # rubocop: disable CodeReuse/ServiceClass state.run_after_commit do - Projects::AfterImportService.new(project).execute + Projects::AfterImportWorker.perform_async(project.id) end - # rubocop: enable CodeReuse/ServiceClass end end end diff --git a/app/models/project_pages_metadatum.rb b/app/models/project_pages_metadatum.rb index dc1e9319340..7a3ece4bc92 100644 --- a/app/models/project_pages_metadatum.rb +++ b/app/models/project_pages_metadatum.rb @@ -8,8 +8,6 @@ class ProjectPagesMetadatum < ApplicationRecord self.primary_key = :project_id - ignore_columns :artifacts_archive_id, remove_with: '15.0', remove_after: '2022-04-22' - belongs_to :project, inverse_of: :pages_metadatum belongs_to :pages_deployment diff --git a/app/models/project_setting.rb b/app/models/project_setting.rb index 6cd6eee2616..e9fd7e4446c 100644 --- a/app/models/project_setting.rb +++ b/app/models/project_setting.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class ProjectSetting < ApplicationRecord - ALLOWED_TARGET_PLATFORMS = %w(ios osx tvos watchos).freeze + ALLOWED_TARGET_PLATFORMS = %w(ios osx tvos watchos android).freeze belongs_to :project, inverse_of: :project_setting @@ -21,7 +21,7 @@ class ProjectSetting < ApplicationRecord validate :validates_mr_default_target_self default_value_for(:legacy_open_source_license_available) do - Feature.enabled?(:legacy_open_source_license_available, default_enabled: :yaml, type: :ops) + Feature.enabled?(:legacy_open_source_license_available, type: :ops) end def squash_enabled_by_default? @@ -36,6 +36,15 @@ class ProjectSetting < ApplicationRecord super(val&.map(&:to_s)&.sort) end + def human_squash_option + case squash_option + when 'never' then 'Do not allow' + when 'always' then 'Require' + when 'default_on' then 'Encourage' + when 'default_off' then 'Allow' + end + end + private def validates_mr_default_target_self diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index 99cec647a98..95fc135f38f 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -19,7 +19,7 @@ class ProjectStatistics < ApplicationRecord before_save :update_storage_size - COLUMNS_TO_REFRESH = [:repository_size, :wiki_size, :lfs_objects_size, :commit_count, :snippets_size, :uploads_size].freeze + COLUMNS_TO_REFRESH = [:repository_size, :wiki_size, :lfs_objects_size, :commit_count, :snippets_size, :uploads_size, :container_registry_size].freeze INCREMENTABLE_COLUMNS = { build_artifacts_size: %i[storage_size], packages_size: %i[storage_size], @@ -76,6 +76,12 @@ class ProjectStatistics < ApplicationRecord self.uploads_size = project.uploads.sum(:size) end + def update_container_registry_size + return unless Feature.enabled?(:container_registry_project_statistics, project) + + self.container_registry_size = project.container_repositories_size || 0 + end + # `wiki_size` and `snippets_size` have no default value in the database # and the column can be nil. # This means that, when the columns were added, all rows had nil diff --git a/app/models/project_team.rb b/app/models/project_team.rb index 4b89d95c1a3..bb5363598df 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -193,6 +193,10 @@ class ProjectTeam project.merge_value_to_request_store(User, user_id, project_access_level) end + def purge_member_access_cache_for_user_id(user_id) + project.purge_resource_id_from_request_store(User, user_id) + end + def max_member_access(user_id) max_member_access_for_user_ids([user_id])[user_id] end diff --git a/app/models/projects/topic.rb b/app/models/projects/topic.rb index 9214a23e259..bc7f94e4374 100644 --- a/app/models/projects/topic.rb +++ b/app/models/projects/topic.rb @@ -9,6 +9,7 @@ module Projects validates :name, presence: true, length: { maximum: 255 } validates :name, uniqueness: { case_sensitive: false }, if: :name_changed? + validates :title, presence: true, length: { maximum: 255 }, on: :create validates :description, length: { maximum: 1024 } has_many :project_topics, class_name: 'Projects::ProjectTopic' @@ -22,13 +23,17 @@ module Projects reorder(order_expression.desc, arel_table['non_private_projects_count'].desc, arel_table['id']) end + def title_or_name + title || name + end + class << self def find_by_name_case_insensitive(name) find_by('LOWER(name) = ?', name.downcase) end def search(query) - fuzzy_search(query, [:name]) + fuzzy_search(query, [:name, :title]) end def update_non_private_projects_counter(ids_before, ids_after, project_visibility_level_before, project_visibility_level_after) diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index 96002c8668a..77038d52efe 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -68,6 +68,10 @@ class ProtectedBranch < ApplicationRecord def allow_multiple?(type) type == :push end + + def self.downcase_humanized_name + name.underscore.humanize.downcase + end end ProtectedBranch.prepend_mod_with('ProtectedBranch') diff --git a/app/models/raw_usage_data.rb b/app/models/raw_usage_data.rb index 6fe3b26b58b..a6844eb8616 100644 --- a/app/models/raw_usage_data.rb +++ b/app/models/raw_usage_data.rb @@ -1,9 +1,16 @@ # frozen_string_literal: true class RawUsageData < ApplicationRecord + REPORTING_CADENCE = 7.days.freeze + validates :payload, presence: true validates :recorded_at, presence: true, uniqueness: true + scope :for_current_reporting_cycle, -> do + where('created_at >= ?', REPORTING_CADENCE.ago.beginning_of_day) + .order(created_at: :desc) + end + def update_version_metadata!(usage_data_id:) self.update_columns(sent_at: Time.current, version_usage_data_id_value: usage_data_id) end diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb index 0be56d8b4a4..2643ef272d8 100644 --- a/app/models/system_note_metadata.rb +++ b/app/models/system_note_metadata.rb @@ -24,7 +24,7 @@ class SystemNoteMetadata < ApplicationRecord opened closed merged duplicate locked unlocked outdated reviewer tag due_date pinned_embed cherry_pick health_status approved unapproved status alert_issue_added relate unrelate new_alert_added severity - attention_requested attention_request_removed contact + attention_requested attention_request_removed contact timeline_event ].freeze validates :note, presence: true, unless: :importing? diff --git a/app/models/user.rb b/app/models/user.rb index 26d47de4f00..8aae4441852 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -239,6 +239,8 @@ class User < ApplicationRecord has_many :timelogs + has_many :resource_label_events, dependent: :nullify # rubocop:disable Cop/ActiveRecordDependent + # # Validations # @@ -941,7 +943,7 @@ class User < ApplicationRecord end def two_factor_u2f_enabled? - return false if Feature.enabled?(:webauthn, default_enabled: :yaml) + return false if Feature.enabled?(:webauthn) if u2f_registrations.loaded? u2f_registrations.any? @@ -955,7 +957,7 @@ class User < ApplicationRecord end def two_factor_webauthn_enabled? - return false unless Feature.enabled?(:webauthn, default_enabled: :yaml) + return false unless Feature.enabled?(:webauthn) (webauthn_registrations.loaded? && webauthn_registrations.any?) || (!webauthn_registrations.loaded? && webauthn_registrations.exists?) end @@ -1583,7 +1585,7 @@ class User < ApplicationRecord end def manageable_groups(include_groups_with_developer_maintainer_access: false) - owned_and_maintainer_group_hierarchy = if Feature.enabled?(:linear_user_manageable_groups, self, default_enabled: :yaml) + owned_and_maintainer_group_hierarchy = if Feature.enabled?(:linear_user_manageable_groups, self) owned_or_maintainers_groups.self_and_descendants else Gitlab::ObjectHierarchy.new(owned_or_maintainers_groups).base_and_descendants @@ -1673,7 +1675,7 @@ class User < ApplicationRecord def ci_owned_runners_cross_joins_fix_enabled? strong_memoize(:ci_owned_runners_cross_joins_fix_enabled) do - Feature.enabled?(:ci_owned_runners_cross_joins_fix, self, default_enabled: :yaml) + Feature.enabled?(:ci_owned_runners_cross_joins_fix, self) end end @@ -1735,7 +1737,7 @@ class User < ApplicationRecord end def attention_requested_open_merge_requests_count(force: false) - if Feature.enabled?(:uncached_mr_attention_requests_count, self, default_enabled: :yaml) + if Feature.enabled?(:uncached_mr_attention_requests_count, self) MergeRequestsFinder.new(self, attention: self.username, state: 'opened', non_archived: true).execute.count else Rails.cache.fetch(attention_request_cache_key, force: force, expires_in: COUNT_CACHE_VALIDITY_PERIOD) do @@ -2070,6 +2072,10 @@ class User < ApplicationRecord end end + def mr_attention_requests_enabled? + Feature.enabled?(:mr_attention_requests, self) + end + protected # override, from Devise::Validatable @@ -2313,7 +2319,7 @@ class User < ApplicationRecord # to avoid querying descendants since they are already covered # by ancestor namespaces. If the FF is not available fallback to # inefficient search: https://gitlab.com/gitlab-org/gitlab/-/issues/336436 - unless Feature.enabled?(:use_traversal_ids, default_enabled: :yaml) + unless Feature.enabled?(:use_traversal_ids) return Ci::NamespaceMirror.contains_any_of_namespaces(search_members.pluck(:source_id)) end @@ -2322,7 +2328,7 @@ class User < ApplicationRecord .shortest_traversal_ids_prefixes # Use efficient btree index to perform search - if Feature.enabled?(:ci_owned_runners_unnest_index, self, default_enabled: :yaml) + if Feature.enabled?(:ci_owned_runners_unnest_index, self) Ci::NamespaceMirror.contains_traversal_ids(traversal_ids) else Ci::NamespaceMirror.contains_any_of_namespaces(traversal_ids.map(&:last)) diff --git a/app/models/user_custom_attribute.rb b/app/models/user_custom_attribute.rb index 62614a851c1..559e93be360 100644 --- a/app/models/user_custom_attribute.rb +++ b/app/models/user_custom_attribute.rb @@ -6,13 +6,34 @@ class UserCustomAttribute < ApplicationRecord validates :user_id, :key, :value, presence: true validates :key, uniqueness: { scope: [:user_id] } - def self.upsert_custom_attributes(custom_attributes) - created_at = DateTime.now - updated_at = DateTime.now + scope :by_key, ->(key) { where(key: key) } + scope :by_user_id, ->(user_id) { where(user_id: user_id) } + scope :by_updated_at, ->(updated_at) { where(updated_at: updated_at) } + scope :arkose_sessions, -> { by_key('arkose_session') } - custom_attributes.map! do |custom_attribute| - custom_attribute.merge({ created_at: created_at, updated_at: updated_at }) + class << self + def upsert_custom_attributes(custom_attributes) + created_at = DateTime.now + updated_at = DateTime.now + + custom_attributes.map! do |custom_attribute| + custom_attribute.merge({ created_at: created_at, updated_at: updated_at }) + end + upsert_all(custom_attributes, unique_by: [:user_id, :key]) + end + + def sessions + return none if blocked_users.empty? + + arkose_sessions + .by_user_id(blocked_users.map(&:user_id)) + .select(:value) + end + + private + + def blocked_users + by_key('blocked_at').by_updated_at(Date.yesterday.all_day) end - upsert_all(custom_attributes, unique_by: [:user_id, :key]) end end diff --git a/app/models/users/callout.rb b/app/models/users/callout.rb index a91a3406b22..b3729c84dd6 100644 --- a/app/models/users/callout.rb +++ b/app/models/users/callout.rb @@ -49,7 +49,9 @@ module Users storage_enforcement_banner_fourth_enforcement_threshold: 46, attention_requests_top_nav: 47, attention_requests_side_nav: 48, - minute_limit_banner: 49 + minute_limit_banner: 49, + preview_user_over_limit_free_plan_alert: 50, # EE-only + user_reached_limit_free_plan_alert: 51 # EE-only } validates :feature_name, diff --git a/app/models/users/in_product_marketing_email.rb b/app/models/users/in_product_marketing_email.rb index f2f1d18339e..82c2e336a09 100644 --- a/app/models/users/in_product_marketing_email.rb +++ b/app/models/users/in_product_marketing_email.rb @@ -4,15 +4,28 @@ module Users class InProductMarketingEmail < ApplicationRecord include BulkInsertSafe + BUILD_IOS_APP_GUIDE = 'build_ios_app_guide' + CAMPAIGNS = [BUILD_IOS_APP_GUIDE].freeze + belongs_to :user validates :user, presence: true - validates :track, presence: true - validates :series, presence: true + + validates :track, :series, presence: true, if: -> { campaign.blank? } + validates :campaign, presence: true, if: -> { track.blank? && series.blank? } + validates :campaign, inclusion: { in: CAMPAIGNS }, allow_nil: true + validates :user_id, uniqueness: { scope: [:track, :series], - message: 'has already been sent' - } + message: 'track series email has already been sent' + }, if: -> { track.present? } + + validates :user_id, uniqueness: { + scope: :campaign, + message: 'campaign email has already been sent' + }, if: -> { campaign.present? } + + validate :campaign_or_track_series enum track: { create: 0, @@ -31,23 +44,47 @@ module Users INACTIVE_TRACK_NAMES = %w(invite_team).freeze ACTIVE_TRACKS = tracks.except(*INACTIVE_TRACK_NAMES) + scope :for_user_with_track_and_series, -> (user, track, series) do + where(user: user, track: track, series: series) + end + scope :without_track_and_series, -> (track, series) do - users = User.arel_table - product_emails = arel_table + join_condition = for_user.and(for_track_and_series(track, series)) + users_without_records(join_condition) + end + + scope :without_campaign, -> (campaign) do + join_condition = for_user.and(for_campaign(campaign)) + users_without_records(join_condition) + end - join_condition = users[:id].eq(product_emails[:user_id]) - .and(product_emails[:track]).eq(ACTIVE_TRACKS[track]) - .and(product_emails[:series]).eq(series) + def self.users_table + User.arel_table + end - arel_join = users.join(product_emails, Arel::Nodes::OuterJoin).on(join_condition) + def self.distinct_users_sql + name = users_table.table_name + Arel.sql("DISTINCT ON(#{name}.id) #{name}.*") + end + def self.users_without_records(condition) + arel_join = users_table.join(arel_table, Arel::Nodes::OuterJoin).on(condition) joins(arel_join.join_sources) .where(in_product_marketing_emails: { id: nil }) - .select(Arel.sql("DISTINCT ON(#{users.table_name}.id) #{users.table_name}.*")) + .select(distinct_users_sql) end - scope :for_user_with_track_and_series, -> (user, track, series) do - where(user: user, track: track, series: series) + def self.for_user + arel_table[:user_id].eq(users_table[:id]) + end + + def self.for_campaign(campaign) + arel_table[:campaign].eq(campaign) + end + + def self.for_track_and_series(track, series) + arel_table[:track].eq(ACTIVE_TRACKS[track]) + .and(arel_table[:series]).eq(series) end def self.save_cta_click(user, track, series) @@ -55,5 +92,13 @@ module Users email.update(cta_clicked_at: Time.zone.now) if email && email.cta_clicked_at.blank? end + + private + + def campaign_or_track_series + if campaign.present? && (track.present? || series.present?) + errors.add(:campaign, 'should be a campaign or a track and series but not both') + end + end end end diff --git a/app/models/wiki.rb b/app/models/wiki.rb index b3f09b20463..32d70fcd3b7 100644 --- a/app/models/wiki.rb +++ b/app/models/wiki.rb @@ -13,43 +13,65 @@ class Wiki markdown: { name: 'Markdown', default_extension: :md, + extension_regex: Regexp.new('md|mkdn?|mdown|markdown', 'i'), created_by_user: true }, rdoc: { name: 'RDoc', default_extension: :rdoc, + extension_regex: Regexp.new('rdoc', 'i'), created_by_user: true }, asciidoc: { name: 'AsciiDoc', default_extension: :asciidoc, + extension_regex: Regexp.new('adoc|asciidoc', 'i'), created_by_user: true }, org: { name: 'Org', default_extension: :org, + extension_regex: Regexp.new('org', 'i'), created_by_user: true }, textile: { name: 'Textile', - default_extension: :textile + default_extension: :textile, + extension_regex: Regexp.new('textile', 'i') }, creole: { name: 'Creole', - default_extension: :creole + default_extension: :creole, + extension_regex: Regexp.new('creole', 'i') }, rest: { name: 'reStructuredText', - default_extension: :rst + default_extension: :rst, + extension_regex: Regexp.new('re?st(\.txt)?', 'i') }, mediawiki: { name: 'MediaWiki', - default_extension: :mediawiki + default_extension: :mediawiki, + extension_regex: Regexp.new('(media)?wiki', 'i') + }, + pod: { + name: 'Pod', + default_extension: :pod, + extension_regex: Regexp.new('pod', 'i') + }, + plaintext: { + name: 'Plain Text', + default_extension: :txt, + extension_regex: Regexp.new('txt', 'i') } }.freeze unless defined?(MARKUPS) VALID_USER_MARKUPS = MARKUPS.select { |_, v| v[:created_by_user] }.freeze unless defined?(VALID_USER_MARKUPS) + unless defined?(ALLOWED_EXTENSIONS_REGEX) + ALLOWED_EXTENSIONS_REGEX = Regexp.union(MARKUPS.map { |key, value| value[:extension_regex] }).freeze + end + CouldNotCreateWikiError = Class.new(StandardError) HOMEPAGE = 'home' @@ -205,50 +227,61 @@ class Wiki end def create_page(title, content, format = :markdown, message = nil) - commit = commit_details(:created, message, title) - - wiki.write_page(title, format.to_sym, content, commit) - repository.expire_status_cache if repository.empty? - after_wiki_activity - - true - rescue Gitlab::Git::Wiki::DuplicatePageError => e - @error_message = "Duplicate page: #{e.message}" - false - end - - def update_page(page, content:, title: nil, format: :markdown, message: nil) - if Feature.enabled?(:gitaly_replace_wiki_update_page, container, default_enabled: :yaml) + if Feature.enabled?(:gitaly_replace_wiki_create_page, container, type: :undefined) with_valid_format(format) do |default_extension| - title = title.presence || Pathname(page.path).sub_ext('').to_s - - # If the format is the same we keep the former extension. This check is for formats - # that can have more than one extension like Markdown (.md, .markdown) - # If we don't do this we will override the existing extension. - extension = page.format != format.to_sym ? default_extension : File.extname(page.path).downcase[1..] - - capture_git_error(:updated) do - repository.update_file( - user, - sluggified_full_path(title, extension), - content, - previous_path: page.path, - **multi_commit_options(:updated, message, title)) + if file_exists_by_regex?(title) + raise_duplicate_page_error! + end + capture_git_error(:created) do + create_wiki_repository unless repository_exists? + sanitized_path = sluggified_full_path(title, default_extension) + repository.create_file(user, sanitized_path, content, **multi_commit_options(:created, message, title)) + repository.expire_status_cache if repository.empty? after_wiki_activity true + rescue Gitlab::Git::Index::IndexError + raise_duplicate_page_error! end end else - commit = commit_details(:updated, message, page.title) - - wiki.update_page(page.path, title || page.name, format.to_sym, content, commit) + commit = commit_details(:created, message, title) + wiki.write_page(title, format.to_sym, content, commit) + repository.expire_status_cache if repository.empty? after_wiki_activity true end + rescue Gitlab::Git::Wiki::DuplicatePageError => e + @error_message = _("Duplicate page: %{error_message}" % { error_message: e.message }) + + false + end + + def update_page(page, content:, title: nil, format: :markdown, message: nil) + with_valid_format(format) do |default_extension| + title = title.presence || Pathname(page.path).sub_ext('').to_s + + # If the format is the same we keep the former extension. This check is for formats + # that can have more than one extension like Markdown (.md, .markdown) + # If we don't do this we will override the existing extension. + extension = page.format != format.to_sym ? default_extension : File.extname(page.path).downcase[1..] + + capture_git_error(:updated) do + repository.update_file( + user, + sluggified_full_path(title, extension), + content, + previous_path: page.path, + **multi_commit_options(:updated, message, title)) + + after_wiki_activity + + true + end + end end def delete_page(page, message = nil) @@ -393,12 +426,33 @@ class Wiki yield default_extension end + def file_exists_by_regex?(title) + return false unless repository_exists? + + escaped_title = Regexp.escape(sluggified_title(title)) + regex = Regexp.new("^#{escaped_title}\.#{ALLOWED_EXTENSIONS_REGEX}$", 'i') + + repository.ls_files('HEAD').any? { |s| s =~ regex } + end + + def raise_duplicate_page_error! + raise Gitlab::Git::Wiki::DuplicatePageError, _('A page with that title already exists') + end + def sluggified_full_path(title, extension) sluggified_title(title) + '.' + extension end def sluggified_title(title) - Gitlab::EncodingHelper.encode_utf8_no_detect(title).tr(' ', '-') + utf8_encoded_title = Gitlab::EncodingHelper.encode_utf8_no_detect(title) + + sanitized_title(utf8_encoded_title).tr(' ', '-') + end + + def sanitized_title(title) + clean_absolute_path = File.expand_path(title, '/') + + Pathname.new(clean_absolute_path).relative_path_from('/').to_s end end diff --git a/app/models/work_items/type.rb b/app/models/work_items/type.rb index e2d38dc9903..0d390fa131d 100644 --- a/app/models/work_items/type.rb +++ b/app/models/work_items/type.rb @@ -41,6 +41,10 @@ module WorkItems scope :by_type, ->(base_type) { where(base_type: base_type) } def self.default_by_type(type) + found_type = find_by(namespace_id: nil, base_type: type) + return found_type if found_type + + Gitlab::DatabaseImporters::WorkItems::BaseTypeImporter.upsert_types find_by(namespace_id: nil, base_type: type) end |