diff options
Diffstat (limited to 'app/services/projects')
11 files changed, 324 insertions, 177 deletions
diff --git a/app/services/projects/alerting/notify_service.rb b/app/services/projects/alerting/notify_service.rb index c21a61bcb52..9403c7bcfed 100644 --- a/app/services/projects/alerting/notify_service.rb +++ b/app/services/projects/alerting/notify_service.rb @@ -2,14 +2,13 @@ module Projects module Alerting - class NotifyService + class NotifyService < ::BaseProjectService extend ::Gitlab::Utils::Override include ::AlertManagement::AlertProcessing include ::AlertManagement::Responses - def initialize(project, payload) - @project = project - @payload = payload + def initialize(project, params) + super(project: project, params: params.to_h) end def execute(token, integration = nil) @@ -29,15 +28,11 @@ module Projects private - attr_reader :project, :payload, :integration + attr_reader :integration + alias_method :payload, :params def valid_payload_size? - Gitlab::Utils::DeepSize.new(payload.to_h).valid? - end - - override :alert_source - def alert_source - super || integration&.name || 'Generic Alert Endpoint' + Gitlab::Utils::DeepSize.new(params).valid? end def active_integration? diff --git a/app/services/projects/blame_service.rb b/app/services/projects/blame_service.rb index b324ea27360..57b913b04e6 100644 --- a/app/services/projects/blame_service.rb +++ b/app/services/projects/blame_service.rb @@ -10,6 +10,7 @@ module Projects @blob = blob @commit = commit @page = extract_page(params) + @pagination_enabled = pagination_state(params) end attr_reader :page @@ -19,7 +20,7 @@ module Projects end def pagination - return unless pagination_enabled? + return unless pagination_enabled Kaminari.paginate_array([], total_count: blob_lines_count, limit: per_page) .tap { |pagination| pagination.max_paginates_per(per_page) } @@ -28,10 +29,10 @@ module Projects private - attr_reader :blob, :commit + attr_reader :blob, :commit, :pagination_enabled def blame_range - return unless pagination_enabled? + return unless pagination_enabled first_line = (page - 1) * per_page + 1 last_line = (first_line + per_page).to_i - 1 @@ -51,6 +52,12 @@ module Projects PER_PAGE end + def pagination_state(params) + return false if Gitlab::Utils.to_boolean(params[:no_pagination], default: false) + + Feature.enabled?(:blame_page_pagination, commit.project) + end + def overlimit?(page) page * per_page >= blob_lines_count + per_page end @@ -58,9 +65,5 @@ module Projects def blob_lines_count @blob_lines_count ||= blob.data.lines.count end - - def pagination_enabled? - Feature.enabled?(:blame_page_pagination, commit.project) - end end end diff --git a/app/services/projects/container_repository/base_container_repository_service.rb b/app/services/projects/container_repository/base_container_repository_service.rb new file mode 100644 index 00000000000..d7539737e78 --- /dev/null +++ b/app/services/projects/container_repository/base_container_repository_service.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Projects + module ContainerRepository + class BaseContainerRepositoryService < ::BaseContainerService + include ::Gitlab::Utils::StrongMemoize + + alias_method :container_repository, :container + + def initialize(container_repository:, current_user: nil, params: {}) + super(container: container_repository, current_user: current_user, params: params) + end + + delegate :project, to: :container_repository + end + end +end diff --git a/app/services/projects/container_repository/cleanup_tags_base_service.rb b/app/services/projects/container_repository/cleanup_tags_base_service.rb new file mode 100644 index 00000000000..8ea4ae4830a --- /dev/null +++ b/app/services/projects/container_repository/cleanup_tags_base_service.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +module Projects + module ContainerRepository + class CleanupTagsBaseService < BaseContainerRepositoryService + private + + def filter_out_latest!(tags) + tags.reject!(&:latest?) + end + + def filter_by_name!(tags) + regex_delete = ::Gitlab::UntrustedRegexp.new("\\A#{name_regex_delete || name_regex}\\z") + regex_retain = ::Gitlab::UntrustedRegexp.new("\\A#{name_regex_keep}\\z") + + tags.select! do |tag| + # regex_retain will override any overlapping matches by regex_delete + regex_delete.match?(tag.name) && !regex_retain.match?(tag.name) + end + end + + # Should return [tags_to_delete, tags_to_keep] + def partition_by_keep_n(tags) + return [tags, []] unless keep_n + + tags = order_by_date_desc(tags) + + tags.partition.with_index { |_, index| index >= keep_n_as_integer } + end + + # Should return [tags_to_delete, tags_to_keep] + def partition_by_older_than(tags) + return [tags, []] unless older_than + + older_than_timestamp = older_than_in_seconds.ago + + tags.partition do |tag| + timestamp = pushed_at(tag) + + timestamp && timestamp < older_than_timestamp + end + end + + def order_by_date_desc(tags) + now = DateTime.current + tags.sort_by! { |tag| pushed_at(tag) || now } + .reverse! + end + + def delete_tags(tags) + return success(deleted: []) unless tags.any? + + service = Projects::ContainerRepository::DeleteTagsService.new( + project, + current_user, + tags: tags.map(&:name), + container_expiration_policy: container_expiration_policy + ) + + service.execute(container_repository) + end + + def can_destroy? + return true if container_expiration_policy + + can?(current_user, :destroy_container_image, project) + end + + def valid_regex? + %w[name_regex_delete name_regex name_regex_keep].each do |param_name| + regex = params[param_name] + ::Gitlab::UntrustedRegexp.new(regex) unless regex.blank? + end + true + rescue RegexpError => e + ::Gitlab::ErrorTracking.log_exception(e, project_id: project.id) + false + end + + def older_than + params['older_than'] + end + + def name_regex_delete + params['name_regex_delete'] + end + + def name_regex + params['name_regex'] + end + + def name_regex_keep + params['name_regex_keep'] + end + + def container_expiration_policy + params['container_expiration_policy'] + end + + def keep_n + params['keep_n'] + end + + def project + container_repository.project + end + + def keep_n_as_integer + keep_n.to_i + end + + def older_than_in_seconds + strong_memoize(:older_than_in_seconds) do + ChronicDuration.parse(older_than).seconds + end + end + end + end +end diff --git a/app/services/projects/container_repository/cleanup_tags_service.rb b/app/services/projects/container_repository/cleanup_tags_service.rb index 0a8e8e72766..285c3e252ef 100644 --- a/app/services/projects/container_repository/cleanup_tags_service.rb +++ b/app/services/projects/container_repository/cleanup_tags_service.rb @@ -2,39 +2,33 @@ module Projects module ContainerRepository - class CleanupTagsService - include BaseServiceUtility - include ::Gitlab::Utils::StrongMemoize + class CleanupTagsService < CleanupTagsBaseService + def initialize(container_repository:, current_user: nil, params: {}) + super - def initialize(container_repository, user = nil, params = {}) - @container_repository = container_repository - @current_user = user @params = params.dup - - @project = container_repository.project - @tags = container_repository.tags - tags_size = @tags.size - @counts = { - original_size: tags_size, - cached_tags_count: 0 - } + @counts = { cached_tags_count: 0 } end def execute return error('access denied') unless can_destroy? return error('invalid regex') unless valid_regex? - filter_out_latest - filter_by_name + tags = container_repository.tags + @counts[:original_size] = tags.size + + filter_out_latest!(tags) + filter_by_name!(tags) + + tags = truncate(tags) + populate_from_cache(tags) - truncate - populate_from_cache + tags = filter_keep_n(tags) + tags = filter_by_older_than(tags) - filter_keep_n - filter_by_older_than + @counts[:before_delete_size] = tags.size - delete_tags.merge(@counts).tap do |result| - result[:before_delete_size] = @tags.size + delete_tags(tags).merge(@counts).tap do |result| result[:deleted_size] = result[:deleted]&.size result[:status] = :error if @counts[:before_truncate_size] != @counts[:after_truncate_size] @@ -43,94 +37,45 @@ module Projects private - def delete_tags - return success(deleted: []) unless @tags.any? - - service = Projects::ContainerRepository::DeleteTagsService.new( - @project, - @current_user, - tags: @tags.map(&:name), - container_expiration_policy: container_expiration_policy - ) - - service.execute(@container_repository) - end - - def filter_out_latest - @tags.reject!(&:latest?) - end - - def order_by_date - now = DateTime.current - @tags.sort_by! { |tag| tag.created_at || now } - .reverse! - end + def filter_keep_n(tags) + tags, tags_to_keep = partition_by_keep_n(tags) - def filter_by_name - regex_delete = ::Gitlab::UntrustedRegexp.new("\\A#{name_regex_delete || name_regex}\\z") - regex_retain = ::Gitlab::UntrustedRegexp.new("\\A#{name_regex_keep}\\z") - - @tags.select! do |tag| - # regex_retain will override any overlapping matches by regex_delete - regex_delete.match?(tag.name) && !regex_retain.match?(tag.name) - end - end - - def filter_keep_n - return unless keep_n + cache_tags(tags_to_keep) - order_by_date - cache_tags(@tags.first(keep_n_as_integer)) - @tags = @tags.drop(keep_n_as_integer) + tags end - def filter_by_older_than - return unless older_than - - older_than_timestamp = older_than_in_seconds.ago - - @tags, tags_to_keep = @tags.partition do |tag| - tag.created_at && tag.created_at < older_than_timestamp - end + def filter_by_older_than(tags) + tags, tags_to_keep = partition_by_older_than(tags) cache_tags(tags_to_keep) - end - def can_destroy? - return true if container_expiration_policy - - can?(@current_user, :destroy_container_image, @project) + tags end - def valid_regex? - %w(name_regex_delete name_regex name_regex_keep).each do |param_name| - regex = @params[param_name] - ::Gitlab::UntrustedRegexp.new(regex) unless regex.blank? - end - true - rescue RegexpError => e - ::Gitlab::ErrorTracking.log_exception(e, project_id: @project.id) - false + def pushed_at(tag) + tag.created_at end - def truncate - @counts[:before_truncate_size] = @tags.size - @counts[:after_truncate_size] = @tags.size + def truncate(tags) + @counts[:before_truncate_size] = tags.size + @counts[:after_truncate_size] = tags.size - return if max_list_size == 0 + return tags if max_list_size == 0 # truncate the list to make sure that after the #filter_keep_n # execution, the resulting list will be max_list_size truncated_size = max_list_size + keep_n_as_integer - return if @tags.size <= truncated_size + return tags if tags.size <= truncated_size - @tags = @tags.sample(truncated_size) - @counts[:after_truncate_size] = @tags.size + tags = tags.sample(truncated_size) + @counts[:after_truncate_size] = tags.size + tags end - def populate_from_cache - @counts[:cached_tags_count] = cache.populate(@tags) if caching_enabled? + def populate_from_cache(tags) + @counts[:cached_tags_count] = cache.populate(tags) if caching_enabled? end def cache_tags(tags) @@ -139,7 +84,7 @@ module Projects def cache strong_memoize(:cache) do - ::Gitlab::ContainerRepository::Tags::Cache.new(@container_repository) + ::Gitlab::ContainerRepository::Tags::Cache.new(container_repository) end end @@ -153,40 +98,6 @@ module Projects def max_list_size ::Gitlab::CurrentSettings.current_application_settings.container_registry_cleanup_tags_service_max_list_size.to_i end - - def keep_n - @params['keep_n'] - end - - def keep_n_as_integer - keep_n.to_i - end - - def older_than_in_seconds - strong_memoize(:older_than_in_seconds) do - ChronicDuration.parse(older_than).seconds - end - end - - def older_than - @params['older_than'] - end - - def name_regex_delete - @params['name_regex_delete'] - end - - def name_regex - @params['name_regex'] - end - - def name_regex_keep - @params['name_regex_keep'] - end - - def container_expiration_policy - @params['container_expiration_policy'] - end end end end diff --git a/app/services/projects/container_repository/gitlab/cleanup_tags_service.rb b/app/services/projects/container_repository/gitlab/cleanup_tags_service.rb new file mode 100644 index 00000000000..81bb94c867a --- /dev/null +++ b/app/services/projects/container_repository/gitlab/cleanup_tags_service.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Projects + module ContainerRepository + module Gitlab + class CleanupTagsService < CleanupTagsBaseService + include ::Projects::ContainerRepository::Gitlab::Timeoutable + + TAGS_PAGE_SIZE = 1000 + + def initialize(container_repository:, current_user: nil, params: {}) + super + @params = params.dup + end + + def execute + return error('access denied') unless can_destroy? + return error('invalid regex') unless valid_regex? + + with_timeout do |start_time, result| + container_repository.each_tags_page(page_size: TAGS_PAGE_SIZE) do |tags| + execute_for_tags(tags, result) + + raise TimeoutError if timeout?(start_time) + end + end + end + + private + + def execute_for_tags(tags, overall_result) + original_size = tags.size + + filter_out_latest!(tags) + filter_by_name!(tags) + + tags = filter_by_keep_n(tags) + tags = filter_by_older_than(tags) + + overall_result[:before_delete_size] += tags.size + overall_result[:original_size] += original_size + + result = delete_tags(tags) + + overall_result[:deleted_size] += result[:deleted]&.size + overall_result[:deleted] += result[:deleted] + overall_result[:status] = result[:status] unless overall_result[:status] == :error + end + + def with_timeout + result = { + original_size: 0, + before_delete_size: 0, + deleted_size: 0, + deleted: [] + } + + yield Time.zone.now, result + + result + rescue TimeoutError + result[:status] = :error + + result + end + + def filter_by_keep_n(tags) + partition_by_keep_n(tags).first + end + + def filter_by_older_than(tags) + partition_by_older_than(tags).first + end + + def pushed_at(tag) + tag.updated_at || tag.created_at + end + end + end + end +end diff --git a/app/services/projects/container_repository/gitlab/delete_tags_service.rb b/app/services/projects/container_repository/gitlab/delete_tags_service.rb index 81cef554dec..530cf87c338 100644 --- a/app/services/projects/container_repository/gitlab/delete_tags_service.rb +++ b/app/services/projects/container_repository/gitlab/delete_tags_service.rb @@ -6,10 +6,7 @@ module Projects class DeleteTagsService include BaseServiceUtility include ::Gitlab::Utils::StrongMemoize - - DISABLED_TIMEOUTS = [nil, 0].freeze - - TimeoutError = Class.new(StandardError) + include ::Projects::ContainerRepository::Gitlab::Timeoutable def initialize(container_repository, tag_names) @container_repository = container_repository @@ -44,16 +41,6 @@ module Projects @deleted_tags.any? ? success(deleted: @deleted_tags) : error('could not delete tags') end - - def timeout?(start_time) - return false if service_timeout.in?(DISABLED_TIMEOUTS) - - (Time.zone.now - start_time) > service_timeout - end - - def service_timeout - ::Gitlab::CurrentSettings.current_application_settings.container_registry_delete_tags_service_timeout - end end end end diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index 6381ee67ce7..c72f9b4b602 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -96,7 +96,7 @@ module Projects log_info("#{current_user.name} created a new project \"#{@project.full_name}\"") if @project.import? - experiment(:combined_registration, user: current_user).track(:import_project) + Gitlab::Tracking.event(self.class.name, 'import_project', user: current_user) else # Skip writing the config for project imports/forks because it # will always fail since the Git directory doesn't exist until @@ -158,14 +158,25 @@ module Projects priority: UserProjectAccessChangedService::LOW_PRIORITY ) else - @project.add_owner(@project.namespace.owner, current_user: current_user) + owner_user = @project.namespace.owner + owner_member = @project.add_owner(owner_user, current_user: current_user) + + # There is a possibility that the sidekiq job to refresh the authorizations of the owner_user in this project + # isn't picked up (or finished) by the time the user is redirected to the newly created project's page. + # If that happens, the user will hit a 404. To avoid that scenario, we manually create a `project_authorizations` record for the user here. + if owner_member.persisted? + owner_user.project_authorizations.safe_find_or_create_by( + project: @project, + access_level: ProjectMember::OWNER + ) + end # During the process of adding a project owner, a check on permissions is made on the user which caches # the max member access for that user on this project. # Since that is `0` before the member is created - and we are still inside the request # cycle when we need to do other operations that might check those permissions (e.g. write a commit) # we need to purge that cache so that the updated permissions is fetched instead of using the outdated cached value of 0 # from before member creation - @project.team.purge_member_access_cache_for_user_id(@project.namespace.owner.id) + @project.team.purge_member_access_cache_for_user_id(owner_user.id) end end diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index 06a44b07f9f..f1525ed9763 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -67,9 +67,9 @@ module Projects end def remove_snippets - # We're setting the hard_delete param because we dont need to perform the access checks within the service since + # We're setting the skip_authorization param because we dont need to perform the access checks within the service since # the user has enough access rights to remove the project and its resources. - response = ::Snippets::BulkDestroyService.new(current_user, project.snippets).execute(hard_delete: true) + response = ::Snippets::BulkDestroyService.new(current_user, project.snippets).execute(skip_authorization: true) if response.error? log_error("Snippet deletion failed on #{project.full_path} with the following message: #{response.message}") @@ -134,6 +134,8 @@ module Projects destroy_ci_records! destroy_mr_diff_relations! + destroy_merge_request_diffs! if ::Feature.enabled?(:extract_mr_diff_deletions) + # Rails attempts to load all related records into memory before # destroying: https://github.com/rails/rails/issues/22510 # This ensures we delete records in batches. @@ -158,10 +160,9 @@ module Projects # # rubocop: disable CodeReuse/ActiveRecord def destroy_mr_diff_relations! - mr_batch_size = 100 delete_batch_size = 1000 - project.merge_requests.each_batch(column: :iid, of: mr_batch_size) do |relation_ids| + project.merge_requests.each_batch(column: :iid, of: BATCH_SIZE) do |relation_ids| [MergeRequestDiffCommit, MergeRequestDiffFile].each do |model| loop do inner_query = model @@ -180,6 +181,23 @@ module Projects end # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord + def destroy_merge_request_diffs! + delete_batch_size = 1000 + + project.merge_requests.each_batch(column: :iid, of: BATCH_SIZE) do |relation| + loop do + deleted_rows = MergeRequestDiff + .where(merge_request: relation) + .limit(delete_batch_size) + .delete_all + + break if deleted_rows == 0 + end + end + end + # rubocop: enable CodeReuse/ActiveRecord + def destroy_ci_records! # Make sure to destroy this first just in case the project is undergoing stats refresh. # This is to avoid logging the artifact deletion in Ci::JobArtifacts::DestroyBatchService. diff --git a/app/services/projects/prometheus/alerts/notify_service.rb b/app/services/projects/prometheus/alerts/notify_service.rb index 6265a74fad2..9f260345937 100644 --- a/app/services/projects/prometheus/alerts/notify_service.rb +++ b/app/services/projects/prometheus/alerts/notify_service.rb @@ -3,9 +3,8 @@ module Projects module Prometheus module Alerts - class NotifyService + class NotifyService < ::BaseProjectService include Gitlab::Utils::StrongMemoize - include ::IncidentManagement::Settings include ::AlertManagement::Responses # This set of keys identifies a payload as a valid Prometheus @@ -26,14 +25,13 @@ module Projects # https://gitlab.com/gitlab-com/gl-infra/production/-/issues/6086 PROCESS_MAX_ALERTS = 100 - def initialize(project, payload) - @project = project - @payload = payload + def initialize(project, params) + super(project: project, params: params.to_h) end def execute(token, integration = nil) return bad_request unless valid_payload_size? - return unprocessable_entity unless self.class.processable?(payload) + return unprocessable_entity unless self.class.processable?(params) return unauthorized unless valid_alert_manager_token?(token, integration) truncate_alerts! if max_alerts_exceeded? @@ -53,10 +51,8 @@ module Projects private - attr_reader :project, :payload - def valid_payload_size? - Gitlab::Utils::DeepSize.new(payload.to_h).valid? + Gitlab::Utils::DeepSize.new(params).valid? end def max_alerts_exceeded? @@ -75,11 +71,11 @@ module Projects } ) - payload['alerts'] = alerts.first(PROCESS_MAX_ALERTS) + params['alerts'] = alerts.first(PROCESS_MAX_ALERTS) end def alerts - payload['alerts'] + params['alerts'] end def valid_alert_manager_token?(token, integration) @@ -152,7 +148,7 @@ module Projects def process_prometheus_alerts alerts.map do |alert| AlertManagement::ProcessPrometheusAlertService - .new(project, alert.to_h) + .new(project, alert) .execute end end diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb index dd1c2b94e18..bf90783fcbe 100644 --- a/app/services/projects/update_pages_service.rb +++ b/app/services/projects/update_pages_service.rb @@ -65,11 +65,20 @@ module Projects def build_commit_status GenericCommitStatus.new( user: build.user, - stage: 'deploy', + ci_stage: stage, name: 'pages:deploy' ) end + # rubocop: disable Performance/ActiveRecordSubtransactionMethods + def stage + build.pipeline.stages.safe_find_or_create_by(name: 'deploy', pipeline_id: build.pipeline.id) do |stage| + stage.position = GenericCommitStatus::EXTERNAL_STAGE_IDX + stage.project = build.project + end + end + # rubocop: enable Performance/ActiveRecordSubtransactionMethods + def create_pages_deployment(artifacts_path, build) sha256 = build.job_artifacts_archive.file_sha256 File.open(artifacts_path) do |file| |