Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2021-10-07 15:12:09 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2021-10-07 15:12:09 +0300
commit43c3400c67f6470d4a19f143008990ee142dd828 (patch)
tree2268d48d59023e8353c6ab55290e8031e224dacc
parentbc935f05bc8d7dd89c3e7c88f90264e90b636e07 (diff)
Add latest changes from gitlab-org/gitlab@master
-rw-r--r--app/assets/javascripts/vue_merge_request_widget/components/extensions/base.vue2
-rw-r--r--app/assets/javascripts/vue_merge_request_widget/components/extensions/container.js48
-rw-r--r--app/controllers/search_controller.rb5
-rw-r--r--app/models/protected_branch.rb2
-rw-r--r--app/models/user.rb1
-rw-r--r--app/services/container_expiration_policies/cleanup_service.rb4
-rw-r--r--app/services/issues/close_service.rb4
-rw-r--r--app/services/projects/container_repository/cleanup_tags_service.rb168
-rw-r--r--app/workers/ci/stuck_builds/drop_running_worker.rb17
-rw-r--r--app/workers/cleanup_container_repository_worker.rb4
-rw-r--r--app/workers/stuck_ci_jobs_worker.rb17
-rw-r--r--db/post_migrate/20211004110500_add_temporary_index_to_issue_metrics.rb15
-rw-r--r--db/post_migrate/20211004110927_schedule_fix_first_mentioned_in_commit_at_job.rb27
-rw-r--r--db/schema_migrations/202110041105001
-rw-r--r--db/schema_migrations/202110041109271
-rw-r--r--db/structure.sql2
-rw-r--r--doc/user/compliance/compliance_report/index.md6
-rw-r--r--lib/gitlab/background_migration/fix_first_mentioned_in_commit_at.rb82
-rw-r--r--lib/gitlab/database/migrations/background_migration_helpers.rb2
-rw-r--r--locale/gitlab.pot10
-rw-r--r--qa/qa/specs/features/api/1_manage/bulk_import_project_spec.rb5
-rw-r--r--spec/controllers/search_controller_spec.rb10
-rw-r--r--spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb131
-rw-r--r--spec/models/protected_branch_spec.rb11
-rw-r--r--spec/models/user_spec.rb10
-rw-r--r--spec/services/container_expiration_policies/cleanup_service_spec.rb4
-rw-r--r--spec/services/issues/close_service_spec.rb2
-rw-r--r--spec/services/projects/container_repository/cleanup_tags_service_spec.rb4
-rw-r--r--spec/workers/ci/stuck_builds/drop_running_worker_spec.rb58
-rw-r--r--spec/workers/cleanup_container_repository_worker_spec.rb6
-rw-r--r--spec/workers/stuck_ci_jobs_worker_spec.rb62
31 files changed, 482 insertions, 239 deletions
diff --git a/app/assets/javascripts/vue_merge_request_widget/components/extensions/base.vue b/app/assets/javascripts/vue_merge_request_widget/components/extensions/base.vue
index 7ea24171aa8..b2c9d28a88b 100644
--- a/app/assets/javascripts/vue_merge_request_widget/components/extensions/base.vue
+++ b/app/assets/javascripts/vue_merge_request_widget/components/extensions/base.vue
@@ -117,7 +117,7 @@ export default {
</script>
<template>
- <section class="media-section mr-widget-border-top" data-testid="widget-extension">
+ <section class="media-section" data-testid="widget-extension">
<div class="media gl-p-5">
<status-icon
:name="$options.name"
diff --git a/app/assets/javascripts/vue_merge_request_widget/components/extensions/container.js b/app/assets/javascripts/vue_merge_request_widget/components/extensions/container.js
index 46046d16fcf..b9dfd3bd41e 100644
--- a/app/assets/javascripts/vue_merge_request_widget/components/extensions/container.js
+++ b/app/assets/javascripts/vue_merge_request_widget/components/extensions/container.js
@@ -1,3 +1,4 @@
+import { __ } from '~/locale';
import { registeredExtensions } from './index';
export default {
@@ -12,23 +13,42 @@ export default {
if (extensions.length === 0) return null;
- return h('div', {}, [
- ...extensions.map((extension) =>
+ return h(
+ 'div',
+ {
+ attrs: {
+ role: 'region',
+ 'aria-label': __('Merge request reports'),
+ },
+ },
+ [
h(
- { ...extension },
+ 'ul',
{
- props: {
- ...extension.props.reduce(
- (acc, key) => ({
- ...acc,
- [key]: this.mr[key],
- }),
- {},
- ),
- },
+ class: 'gl-p-0 gl-m-0 gl-list-style-none',
},
+ [
+ ...extensions.map((extension, index) =>
+ h('li', { attrs: { class: index > 0 && 'mr-widget-border-top' } }, [
+ h(
+ { ...extension },
+ {
+ props: {
+ ...extension.props.reduce(
+ (acc, key) => ({
+ ...acc,
+ [key]: this.mr[key],
+ }),
+ {},
+ ),
+ },
+ },
+ ),
+ ]),
+ ),
+ ],
),
- ),
- ]);
+ ],
+ );
},
};
diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb
index 5fd4d21b260..0a18559fc81 100644
--- a/app/controllers/search_controller.rb
+++ b/app/controllers/search_controller.rb
@@ -12,6 +12,7 @@ class SearchController < ApplicationController
around_action :allow_gitaly_ref_name_caching
before_action :block_anonymous_global_searches, :check_scope_global_search_enabled, except: :opensearch
+ before_action :strip_surrounding_whitespace_from_search, except: :opensearch
skip_before_action :authenticate_user!
requires_cross_project_access if: -> do
search_term_present = params[:search].present? || params[:term].present?
@@ -197,6 +198,10 @@ class SearchController < ApplicationController
def count_action_name?
action_name.to_sym == :count
end
+
+ def strip_surrounding_whitespace_from_search
+ %i(term search).each { |param| params[param]&.strip! }
+ end
end
SearchController.prepend_mod_with('SearchController')
diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb
index 3d32144e0f8..b4e2d17c3e5 100644
--- a/app/models/protected_branch.rb
+++ b/app/models/protected_branch.rb
@@ -10,6 +10,8 @@ class ProtectedBranch < ApplicationRecord
scope :allowing_force_push,
-> { where(allow_force_push: true) }
+ scope :get_ids_by_name, -> (name) { where(name: name).pluck(:id) }
+
protected_ref_access_levels :merge, :push
def self.protected_ref_accessible_to?(ref, user, project:, action:, protected_refs: nil)
diff --git a/app/models/user.rb b/app/models/user.rb
index f4e6069ac9a..25a2588a6a7 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -457,6 +457,7 @@ class User < ApplicationRecord
scope :dormant, -> { active.where('last_activity_on <= ?', MINIMUM_INACTIVE_DAYS.day.ago.to_date) }
scope :with_no_activity, -> { active.where(last_activity_on: nil) }
scope :by_provider_and_extern_uid, ->(provider, extern_uid) { joins(:identities).merge(Identity.with_extern_uid(provider, extern_uid)) }
+ scope :get_ids_by_username, -> (username) { where(username: username).pluck(:id) }
def preferred_language
read_attribute('preferred_language') ||
diff --git a/app/services/container_expiration_policies/cleanup_service.rb b/app/services/container_expiration_policies/cleanup_service.rb
index 52530b91578..0da5e552c48 100644
--- a/app/services/container_expiration_policies/cleanup_service.rb
+++ b/app/services/container_expiration_policies/cleanup_service.rb
@@ -24,8 +24,8 @@ module ContainerExpirationPolicies
begin
service_result = Projects::ContainerRepository::CleanupTagsService
- .new(project, nil, policy_params.merge('container_expiration_policy' => true))
- .execute(repository)
+ .new(repository, nil, policy_params.merge('container_expiration_policy' => true))
+ .execute
rescue StandardError
repository.cleanup_unfinished!
diff --git a/app/services/issues/close_service.rb b/app/services/issues/close_service.rb
index 856f6932d61..ac846c769a3 100644
--- a/app/services/issues/close_service.rb
+++ b/app/services/issues/close_service.rb
@@ -91,11 +91,11 @@ module Issues
end
end
- def store_first_mentioned_in_commit_at(issue, merge_request)
+ def store_first_mentioned_in_commit_at(issue, merge_request, max_commit_lookup: 100)
metrics = issue.metrics
return if metrics.nil? || metrics.first_mentioned_in_commit_at
- first_commit_timestamp = merge_request.commits(limit: 1).first.try(:authored_date)
+ first_commit_timestamp = merge_request.commits(limit: max_commit_lookup).last.try(:authored_date)
return unless first_commit_timestamp
metrics.update!(first_mentioned_in_commit_at: first_commit_timestamp)
diff --git a/app/services/projects/container_repository/cleanup_tags_service.rb b/app/services/projects/container_repository/cleanup_tags_service.rb
index 064cdce6f10..3a60de0f1ee 100644
--- a/app/services/projects/container_repository/cleanup_tags_service.rb
+++ b/app/services/projects/container_repository/cleanup_tags_service.rb
@@ -2,148 +2,152 @@
module Projects
module ContainerRepository
- class CleanupTagsService < BaseService
+ class CleanupTagsService
+ include BaseServiceUtility
include ::Gitlab::Utils::StrongMemoize
- def execute(container_repository)
- return error('access denied') unless can_destroy?
- return error('invalid regex') unless valid_regex?
+ def initialize(container_repository, user = nil, params = {})
+ @container_repository = container_repository
+ @current_user = user
+ @params = params.dup
- tags = container_repository.tags
- original_size = tags.size
+ @project = container_repository.project
+ @tags = container_repository.tags
+ tags_size = @tags.size
+ @counts = {
+ original_size: tags_size,
+ cached_tags_count: 0
+ }
+ end
- tags = without_latest(tags)
- tags = filter_by_name(tags)
+ def execute
+ return error('access denied') unless can_destroy?
+ return error('invalid regex') unless valid_regex?
- before_truncate_size = tags.size
- tags = truncate(tags)
- after_truncate_size = tags.size
+ filter_out_latest
+ filter_by_name
- cached_tags_count = populate_tags_from_cache(container_repository, tags) || 0
+ truncate
+ populate_from_cache
- tags = filter_keep_n(container_repository, tags)
- tags = filter_by_older_than(container_repository, tags)
+ filter_keep_n
+ filter_by_older_than
- delete_tags(container_repository, tags).tap do |result|
- result[:original_size] = original_size
- result[:before_truncate_size] = before_truncate_size
- result[:after_truncate_size] = after_truncate_size
- result[:cached_tags_count] = cached_tags_count
- result[:before_delete_size] = tags.size
+ delete_tags.merge(@counts).tap do |result|
+ result[:before_delete_size] = @tags.size
result[:deleted_size] = result[:deleted]&.size
- result[:status] = :error if before_truncate_size != after_truncate_size
+ result[:status] = :error if @counts[:before_truncate_size] != @counts[:after_truncate_size]
end
end
private
- def delete_tags(container_repository, tags)
- return success(deleted: []) unless tags.any?
-
- tag_names = tags.map(&:name)
+ def delete_tags
+ return success(deleted: []) unless @tags.any?
service = Projects::ContainerRepository::DeleteTagsService.new(
- container_repository.project,
- current_user,
- tags: tag_names,
- container_expiration_policy: params['container_expiration_policy']
+ @project,
+ @current_user,
+ tags: @tags.map(&:name),
+ container_expiration_policy: container_expiration_policy
)
- service.execute(container_repository)
+ service.execute(@container_repository)
end
- def without_latest(tags)
- tags.reject(&:latest?)
+ def filter_out_latest
+ @tags.reject!(&:latest?)
end
- def order_by_date(tags)
+ def order_by_date
now = DateTime.current
- tags.sort_by { |tag| tag.created_at || now }.reverse
+ @tags.sort_by! { |tag| tag.created_at || now }
+ .reverse!
end
- def filter_by_name(tags)
- regex_delete = ::Gitlab::UntrustedRegexp.new("\\A#{params['name_regex_delete'] || params['name_regex']}\\z")
- regex_retain = ::Gitlab::UntrustedRegexp.new("\\A#{params['name_regex_keep']}\\z")
+ def filter_by_name
+ regex_delete = ::Gitlab::UntrustedRegexp.new("\\A#{name_regex_delete || name_regex}\\z")
+ regex_retain = ::Gitlab::UntrustedRegexp.new("\\A#{name_regex_keep}\\z")
- tags.select do |tag|
+ @tags.select! do |tag|
# regex_retain will override any overlapping matches by regex_delete
regex_delete.match?(tag.name) && !regex_retain.match?(tag.name)
end
end
- def filter_keep_n(container_repository, tags)
- return tags unless params['keep_n']
+ def filter_keep_n
+ return unless keep_n
- tags = order_by_date(tags)
- cache_tags(container_repository, tags.first(keep_n))
- tags.drop(keep_n)
+ order_by_date
+ cache_tags(@tags.first(keep_n_as_integer))
+ @tags = @tags.drop(keep_n_as_integer)
end
- def filter_by_older_than(container_repository, tags)
- return tags unless older_than
+ def filter_by_older_than
+ return unless older_than
older_than_timestamp = older_than_in_seconds.ago
- tags, tags_to_keep = tags.partition do |tag|
+ @tags, tags_to_keep = @tags.partition do |tag|
tag.created_at && tag.created_at < older_than_timestamp
end
- cache_tags(container_repository, tags_to_keep)
-
- tags
+ cache_tags(tags_to_keep)
end
def can_destroy?
- return true if params['container_expiration_policy']
+ return true if container_expiration_policy
- can?(current_user, :destroy_container_image, project)
+ can?(@current_user, :destroy_container_image, @project)
end
def valid_regex?
%w(name_regex_delete name_regex name_regex_keep).each do |param_name|
- regex = params[param_name]
+ regex = @params[param_name]
::Gitlab::UntrustedRegexp.new(regex) unless regex.blank?
end
true
rescue RegexpError => e
- ::Gitlab::ErrorTracking.log_exception(e, project_id: project.id)
+ ::Gitlab::ErrorTracking.log_exception(e, project_id: @project.id)
false
end
- def truncate(tags)
- return tags unless throttling_enabled?
- return tags if max_list_size == 0
+ def truncate
+ @counts[:before_truncate_size] = @tags.size
+ @counts[:after_truncate_size] = @tags.size
+
+ return unless throttling_enabled?
+ return if max_list_size == 0
# truncate the list to make sure that after the #filter_keep_n
# execution, the resulting list will be max_list_size
- truncated_size = max_list_size + keep_n
+ truncated_size = max_list_size + keep_n_as_integer
- return tags if tags.size <= truncated_size
+ return if @tags.size <= truncated_size
- tags.sample(truncated_size)
+ @tags = @tags.sample(truncated_size)
+ @counts[:after_truncate_size] = @tags.size
end
- def populate_tags_from_cache(container_repository, tags)
- cache(container_repository).populate(tags) if caching_enabled?(container_repository)
+ def populate_from_cache
+ @counts[:cached_tags_count] = cache.populate(@tags) if caching_enabled?
end
- def cache_tags(container_repository, tags)
- cache(container_repository).insert(tags, older_than_in_seconds) if caching_enabled?(container_repository)
+ def cache_tags(tags)
+ cache.insert(tags, older_than_in_seconds) if caching_enabled?
end
- def cache(container_repository)
- # TODO Implement https://gitlab.com/gitlab-org/gitlab/-/issues/340277 to avoid passing
- # the container repository parameter which is bad for a memoized function
+ def cache
strong_memoize(:cache) do
- ::Projects::ContainerRepository::CacheTagsCreatedAtService.new(container_repository)
+ ::Projects::ContainerRepository::CacheTagsCreatedAtService.new(@container_repository)
end
end
- def caching_enabled?(container_repository)
- params['container_expiration_policy'] &&
+ def caching_enabled?
+ container_expiration_policy &&
older_than.present? &&
- Feature.enabled?(:container_registry_expiration_policies_caching, container_repository.project)
+ Feature.enabled?(:container_registry_expiration_policies_caching, @project)
end
def throttling_enabled?
@@ -155,7 +159,11 @@ module Projects
end
def keep_n
- params['keep_n'].to_i
+ @params['keep_n']
+ end
+
+ def keep_n_as_integer
+ keep_n.to_i
end
def older_than_in_seconds
@@ -165,7 +173,23 @@ module Projects
end
def older_than
- params['older_than']
+ @params['older_than']
+ end
+
+ def name_regex_delete
+ @params['name_regex_delete']
+ end
+
+ def name_regex
+ @params['name_regex']
+ end
+
+ def name_regex_keep
+ @params['name_regex_keep']
+ end
+
+ def container_expiration_policy
+ @params['container_expiration_policy']
end
end
end
diff --git a/app/workers/ci/stuck_builds/drop_running_worker.rb b/app/workers/ci/stuck_builds/drop_running_worker.rb
index 35326bf74c9..db571fdc38d 100644
--- a/app/workers/ci/stuck_builds/drop_running_worker.rb
+++ b/app/workers/ci/stuck_builds/drop_running_worker.rb
@@ -4,6 +4,7 @@ module Ci
module StuckBuilds
class DropRunningWorker
include ApplicationWorker
+ include ExclusiveLeaseGuard
idempotent!
@@ -17,26 +18,16 @@ module Ci
feature_category :continuous_integration
- EXCLUSIVE_LEASE_KEY = 'ci_stuck_builds_drop_running_worker_lease'
-
def perform
- return unless try_obtain_lease
-
- begin
+ try_obtain_lease do
Ci::StuckBuilds::DropRunningService.new.execute
- ensure
- remove_lease
end
end
private
- def try_obtain_lease
- @uuid = Gitlab::ExclusiveLease.new(EXCLUSIVE_LEASE_KEY, timeout: 30.minutes).try_obtain
- end
-
- def remove_lease
- Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid)
+ def lease_timeout
+ 30.minutes
end
end
end
diff --git a/app/workers/cleanup_container_repository_worker.rb b/app/workers/cleanup_container_repository_worker.rb
index 9adc026ced2..7274ecf62f9 100644
--- a/app/workers/cleanup_container_repository_worker.rb
+++ b/app/workers/cleanup_container_repository_worker.rb
@@ -28,8 +28,8 @@ class CleanupContainerRepositoryWorker
end
result = Projects::ContainerRepository::CleanupTagsService
- .new(project, current_user, params)
- .execute(container_repository)
+ .new(container_repository, current_user, params)
+ .execute
if run_by_container_expiration_policy? && result[:status] == :success
container_repository.reset_expiration_policy_started_at!
diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb
index 6282c9200b1..72fcf06dce1 100644
--- a/app/workers/stuck_ci_jobs_worker.rb
+++ b/app/workers/stuck_ci_jobs_worker.rb
@@ -2,6 +2,7 @@
class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker
include ApplicationWorker
+ include ExclusiveLeaseGuard
# rubocop:disable Scalability/CronWorkerContext
# This is an instance-wide cleanup query, so there's no meaningful
@@ -14,28 +15,18 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker
feature_category :continuous_integration
worker_resource_boundary :cpu
- EXCLUSIVE_LEASE_KEY = 'stuck_ci_builds_worker_lease'
-
def perform
Ci::StuckBuilds::DropRunningWorker.perform_in(20.minutes)
Ci::StuckBuilds::DropScheduledWorker.perform_in(40.minutes)
- return unless try_obtain_lease
-
- begin
+ try_obtain_lease do
Ci::StuckBuilds::DropService.new.execute
- ensure
- remove_lease
end
end
private
- def try_obtain_lease
- @uuid = Gitlab::ExclusiveLease.new(EXCLUSIVE_LEASE_KEY, timeout: 30.minutes).try_obtain
- end
-
- def remove_lease
- Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid)
+ def lease_timeout
+ 30.minutes
end
end
diff --git a/db/post_migrate/20211004110500_add_temporary_index_to_issue_metrics.rb b/db/post_migrate/20211004110500_add_temporary_index_to_issue_metrics.rb
new file mode 100644
index 00000000000..cfc37c55121
--- /dev/null
+++ b/db/post_migrate/20211004110500_add_temporary_index_to_issue_metrics.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class AddTemporaryIndexToIssueMetrics < Gitlab::Database::Migration[1.0]
+ disable_ddl_transaction!
+
+ INDEX_NAME = 'index_issue_metrics_first_mentioned_in_commit'
+
+ def up
+ add_concurrent_index :issue_metrics, :issue_id, where: 'EXTRACT(YEAR FROM first_mentioned_in_commit_at) > 2019', name: INDEX_NAME
+ end
+
+ def down
+ remove_concurrent_index_by_name :issue_metrics, name: INDEX_NAME
+ end
+end
diff --git a/db/post_migrate/20211004110927_schedule_fix_first_mentioned_in_commit_at_job.rb b/db/post_migrate/20211004110927_schedule_fix_first_mentioned_in_commit_at_job.rb
new file mode 100644
index 00000000000..c7612db3aaf
--- /dev/null
+++ b/db/post_migrate/20211004110927_schedule_fix_first_mentioned_in_commit_at_job.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class ScheduleFixFirstMentionedInCommitAtJob < Gitlab::Database::Migration[1.0]
+ MIGRATION = 'FixFirstMentionedInCommitAt'
+ BATCH_SIZE = 10_000
+ INTERVAL = 2.minutes.to_i
+
+ disable_ddl_transaction!
+
+ def up
+ scope = define_batchable_model('issue_metrics')
+ .where('EXTRACT(YEAR FROM first_mentioned_in_commit_at) > 2019')
+
+ queue_background_migration_jobs_by_range_at_intervals(
+ scope,
+ MIGRATION,
+ INTERVAL,
+ batch_size: BATCH_SIZE,
+ track_jobs: true,
+ primary_column_name: :issue_id
+ )
+ end
+
+ def down
+ # noop
+ end
+end
diff --git a/db/schema_migrations/20211004110500 b/db/schema_migrations/20211004110500
new file mode 100644
index 00000000000..e22ed05de83
--- /dev/null
+++ b/db/schema_migrations/20211004110500
@@ -0,0 +1 @@
+1b0b562aefb724afe24b8640a22013cea6fddd0e594d6723f6819f69804ba9f7 \ No newline at end of file
diff --git a/db/schema_migrations/20211004110927 b/db/schema_migrations/20211004110927
new file mode 100644
index 00000000000..aa70a4aa0d8
--- /dev/null
+++ b/db/schema_migrations/20211004110927
@@ -0,0 +1 @@
+50c937f979c83f6937364d92bf65ed42ef963f2d241eadcee6355c1b256c3ec9 \ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index 5fd10356a24..189e4e8efaf 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -25448,6 +25448,8 @@ CREATE UNIQUE INDEX index_issue_links_on_source_id_and_target_id ON issue_links
CREATE INDEX index_issue_links_on_target_id ON issue_links USING btree (target_id);
+CREATE INDEX index_issue_metrics_first_mentioned_in_commit ON issue_metrics USING btree (issue_id) WHERE (date_part('year'::text, first_mentioned_in_commit_at) > (2019)::double precision);
+
CREATE INDEX index_issue_metrics_on_issue_id_and_timestamps ON issue_metrics USING btree (issue_id, first_mentioned_in_commit_at, first_associated_with_milestone_at, first_added_to_board_at);
CREATE INDEX index_issue_on_project_id_state_id_and_blocking_issues_count ON issues USING btree (project_id, state_id, blocking_issues_count);
diff --git a/doc/user/compliance/compliance_report/index.md b/doc/user/compliance/compliance_report/index.md
index d30cedfb3cd..63dd06bcfdb 100644
--- a/doc/user/compliance/compliance_report/index.md
+++ b/doc/user/compliance/compliance_report/index.md
@@ -30,7 +30,7 @@ Prerequisites:
To view the compliance report:
1. On the top bar, select **Menu > Groups** and find your group.
-1. On the left sidebar, select **Security & Compliance > Compliance**.
+1. On the left sidebar, select **Security & Compliance > Compliance report**.
NOTE:
The compliance report shows only the latest merge request on each project.
@@ -87,7 +87,7 @@ Depending on the merge strategy, the merge commit SHA can be a merge commit, squ
To download the Chain of Custody report:
1. On the top bar, select **Menu > Groups** and find your group.
-1. On the left sidebar, select **Security & Compliance > Compliance**.
+1. On the left sidebar, select **Security & Compliance > Compliance report**.
1. Select **List of all merge commits**.
### Commit-specific Chain of Custody Report **(ULTIMATE)**
@@ -97,7 +97,7 @@ To download the Chain of Custody report:
You can generate a commit-specific Chain of Custody report for a given commit SHA.
1. On the top bar, select **Menu > Groups** and find your group.
-1. On the left sidebar, select **Security & Compliance > Compliance**.
+1. On the left sidebar, select **Security & Compliance > Compliance report**.
1. At the top of the compliance report, to the right of **List of all merge commits**, select the down arrow (**{angle-down}**).
1. Enter the merge commit SHA, and then select **Export commit custody report**.
SHA and then select **Export commit custody report**.
diff --git a/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at.rb b/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at.rb
new file mode 100644
index 00000000000..0dc1e36ff19
--- /dev/null
+++ b/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at.rb
@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Class that fixes the incorrectly set authored_date within
+ # issue_metrics table
+ class FixFirstMentionedInCommitAt
+ SUB_BATCH_SIZE = 500
+
+ # rubocop: disable Style/Documentation
+ class TmpIssueMetrics < ActiveRecord::Base
+ include EachBatch
+
+ self.table_name = 'issue_metrics'
+
+ def self.from_2020
+ where('EXTRACT(YEAR FROM first_mentioned_in_commit_at) > 2019')
+ end
+ end
+ # rubocop: enable Style/Documentation
+
+ def perform(start_id, end_id)
+ scope(start_id, end_id).each_batch(of: SUB_BATCH_SIZE, column: :issue_id) do |sub_batch|
+ first, last = sub_batch.pluck(Arel.sql('min(issue_id), max(issue_id)')).first
+
+ # The query need to be reconstructed because .each_batch modifies the default scope
+ # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510
+ inner_query = TmpIssueMetrics
+ .unscoped
+ .merge(scope(first, last))
+ .from("issue_metrics, #{lateral_query}")
+ .select('issue_metrics.issue_id', 'first_authored_date.authored_date')
+ .where('issue_metrics.first_mentioned_in_commit_at > first_authored_date.authored_date')
+
+ TmpIssueMetrics.connection.execute <<~UPDATE_METRICS
+ WITH cte AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
+ #{inner_query.to_sql}
+ )
+ UPDATE issue_metrics
+ SET
+ first_mentioned_in_commit_at = cte.authored_date
+ FROM
+ cte
+ WHERE
+ cte.issue_id = issue_metrics.issue_id
+ UPDATE_METRICS
+ end
+ end
+
+ private
+
+ def scope(start_id, end_id)
+ TmpIssueMetrics.from_2020.where(issue_id: start_id..end_id)
+ end
+
+ def lateral_query
+ <<~SQL
+ LATERAL (
+ SELECT MIN(first_authored_date.authored_date) as authored_date
+ FROM merge_requests_closing_issues,
+ LATERAL (
+ SELECT id
+ FROM merge_request_diffs
+ WHERE merge_request_id = merge_requests_closing_issues.merge_request_id
+ ORDER BY id DESC
+ LIMIT 1
+ ) last_diff_id,
+ LATERAL (
+ SELECT authored_date
+ FROM merge_request_diff_commits
+ WHERE
+ merge_request_diff_id = last_diff_id.id
+ ORDER BY relative_order DESC
+ LIMIT 1
+ ) first_authored_date
+ WHERE merge_requests_closing_issues.issue_id = issue_metrics.issue_id
+ ) first_authored_date
+ SQL
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb
index 19d80ba1d64..bdaf0d35a83 100644
--- a/lib/gitlab/database/migrations/background_migration_helpers.rb
+++ b/lib/gitlab/database/migrations/background_migration_helpers.rb
@@ -106,7 +106,7 @@ module Gitlab
final_delay = 0
batch_counter = 0
- model_class.each_batch(of: batch_size) do |relation, index|
+ model_class.each_batch(of: batch_size, column: primary_column_name) do |relation, index|
max = relation.arel_table[primary_column_name].maximum
min = relation.arel_table[primary_column_name].minimum
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index 1b19945b7ea..8087f8563c8 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -8460,13 +8460,10 @@ msgstr ""
msgid "Completed"
msgstr ""
-msgid "Compliance"
-msgstr ""
-
-msgid "Compliance Dashboard"
+msgid "Compliance framework"
msgstr ""
-msgid "Compliance framework"
+msgid "Compliance report"
msgstr ""
msgid "ComplianceDashboard|created by:"
@@ -21350,6 +21347,9 @@ msgstr ""
msgid "Merge request events"
msgstr ""
+msgid "Merge request reports"
+msgstr ""
+
msgid "Merge request was scheduled to merge after pipeline succeeds"
msgstr ""
diff --git a/qa/qa/specs/features/api/1_manage/bulk_import_project_spec.rb b/qa/qa/specs/features/api/1_manage/bulk_import_project_spec.rb
index b88fdfdc757..9935908d55e 100644
--- a/qa/qa/specs/features/api/1_manage/bulk_import_project_spec.rb
+++ b/qa/qa/specs/features/api/1_manage/bulk_import_project_spec.rb
@@ -61,7 +61,10 @@ module QA
end
context 'with project' do
- it 'successfully imports project' do
+ it(
+ 'successfully imports project',
+ testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/2297'
+ ) do
expect { imported_group.import_status }.to eventually_eq('finished').within(import_wait_duration)
imported_projects = imported_group.reload!.projects
diff --git a/spec/controllers/search_controller_spec.rb b/spec/controllers/search_controller_spec.rb
index 6bcb88278a0..73e8e0c7dd4 100644
--- a/spec/controllers/search_controller_spec.rb
+++ b/spec/controllers/search_controller_spec.rb
@@ -215,6 +215,16 @@ RSpec.describe SearchController do
end
end
+ it 'strips surrounding whitespace from search query' do
+ get :show, params: { scope: 'notes', search: ' foobar ' }
+ expect(assigns[:search_term]).to eq 'foobar'
+ end
+
+ it 'strips surrounding whitespace from autocomplete term' do
+ expect(controller).to receive(:search_autocomplete_opts).with('youcompleteme')
+ get :autocomplete, params: { term: ' youcompleteme ' }
+ end
+
it 'finds issue comments' do
project = create(:project, :public)
note = create(:note_on_issue, project: project)
diff --git a/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb b/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb
new file mode 100644
index 00000000000..3e6de5390a7
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb
@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::FixFirstMentionedInCommitAt, :migration, schema: 20211004110500 do
+ let(:namespaces) { table(:namespaces) }
+ let(:projects) { table(:projects) }
+ let(:users) { table(:users) }
+ let(:merge_requests) { table(:merge_requests) }
+ let(:issues) { table(:issues) }
+ let(:issue_metrics) { table(:issue_metrics) }
+ let(:merge_requests_closing_issues) { table(:merge_requests_closing_issues) }
+ let(:diffs) { table(:merge_request_diffs) }
+ let(:ten_days_ago) { 10.days.ago }
+ let(:commits) do
+ table(:merge_request_diff_commits).tap do |t|
+ t.extend(SuppressCompositePrimaryKeyWarning)
+ end
+ end
+
+ let(:namespace) { namespaces.create!(name: 'ns', path: 'ns') }
+ let(:project) { projects.create!(namespace_id: namespace.id) }
+
+ let!(:issue1) do
+ issues.create!(
+ title: 'issue',
+ description: 'description',
+ project_id: project.id
+ )
+ end
+
+ let!(:issue2) do
+ issues.create!(
+ title: 'issue',
+ description: 'description',
+ project_id: project.id
+ )
+ end
+
+ let!(:merge_request1) do
+ merge_requests.create!(
+ source_branch: 'a',
+ target_branch: 'master',
+ target_project_id: project.id
+ )
+ end
+
+ let!(:merge_request2) do
+ merge_requests.create!(
+ source_branch: 'b',
+ target_branch: 'master',
+ target_project_id: project.id
+ )
+ end
+
+ let!(:merge_request_closing_issue1) do
+ merge_requests_closing_issues.create!(issue_id: issue1.id, merge_request_id: merge_request1.id)
+ end
+
+ let!(:merge_request_closing_issue2) do
+ merge_requests_closing_issues.create!(issue_id: issue2.id, merge_request_id: merge_request2.id)
+ end
+
+ let!(:diff1) { diffs.create!(merge_request_id: merge_request1.id) }
+ let!(:diff2) { diffs.create!(merge_request_id: merge_request1.id) }
+
+ let!(:other_diff) { diffs.create!(merge_request_id: merge_request2.id) }
+
+ let!(:commit1) do
+ commits.create!(
+ merge_request_diff_id: diff2.id,
+ relative_order: 0,
+ sha: Gitlab::Database::ShaAttribute.serialize('aaa'),
+ authored_date: 5.days.ago
+ )
+ end
+
+ let!(:commit2) do
+ commits.create!(
+ merge_request_diff_id: diff2.id,
+ relative_order: 1,
+ sha: Gitlab::Database::ShaAttribute.serialize('aaa'),
+ authored_date: 10.days.ago
+ )
+ end
+
+ let!(:commit3) do
+ commits.create!(
+ merge_request_diff_id: other_diff.id,
+ relative_order: 1,
+ sha: Gitlab::Database::ShaAttribute.serialize('aaa'),
+ authored_date: 5.days.ago
+ )
+ end
+
+ def run_migration
+ described_class
+ .new
+ .perform(issue_metrics.minimum(:issue_id), issue_metrics.maximum(:issue_id))
+ end
+
+ context 'when the persisted first_mentioned_in_commit_at is later than the first commit authored_date' do
+ it 'updates the issue_metrics record' do
+ record1 = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: Time.current)
+ record2 = issue_metrics.create!(issue_id: issue2.id, first_mentioned_in_commit_at: Time.current)
+
+ run_migration
+ record1.reload
+ record2.reload
+
+ expect(record1.first_mentioned_in_commit_at).to be_within(2.seconds).of(commit2.authored_date)
+ expect(record2.first_mentioned_in_commit_at).to be_within(2.seconds).of(commit3.authored_date)
+ end
+ end
+
+ context 'when the persisted first_mentioned_in_commit_at is earlier than the first commit authored_date' do
+ it 'does not update the issue_metrics record' do
+ record = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: 20.days.ago)
+
+ expect { run_migration }.not_to change { record.reload.first_mentioned_in_commit_at }
+ end
+ end
+
+ context 'when the first_mentioned_in_commit_at is null' do
+ it 'does nothing' do
+ record = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: nil)
+
+ expect { run_migration }.not_to change { record.reload.first_mentioned_in_commit_at }
+ end
+ end
+end
diff --git a/spec/models/protected_branch_spec.rb b/spec/models/protected_branch_spec.rb
index 019c01af672..587a9683a8e 100644
--- a/spec/models/protected_branch_spec.rb
+++ b/spec/models/protected_branch_spec.rb
@@ -308,4 +308,15 @@ RSpec.describe ProtectedBranch do
expect(described_class.by_name('')).to be_empty
end
end
+
+ describe '.get_ids_by_name' do
+ let(:branch_name) { 'branch_name' }
+ let!(:protected_branch) { create(:protected_branch, name: branch_name) }
+ let(:branch_id) { protected_branch.id }
+
+ it 'returns the id for each protected branch matching name' do
+ expect(described_class.get_ids_by_name([branch_name]))
+ .to match_array([branch_id])
+ end
+ end
end
diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb
index b763ea066be..db805a804c8 100644
--- a/spec/models/user_spec.rb
+++ b/spec/models/user_spec.rb
@@ -6180,4 +6180,14 @@ RSpec.describe User do
it_behaves_like 'groups_with_developer_maintainer_project_access examples'
end
end
+
+ describe '.get_ids_by_username' do
+ let(:user_name) { 'user_name' }
+ let!(:user) { create(:user, username: user_name) }
+ let(:user_id) { user.id }
+
+ it 'returns the id of each record matching username' do
+ expect(described_class.get_ids_by_username([user_name])).to match_array([user_id])
+ end
+ end
end
diff --git a/spec/services/container_expiration_policies/cleanup_service_spec.rb b/spec/services/container_expiration_policies/cleanup_service_spec.rb
index c8bb8bfe06d..a1f76e5e5dd 100644
--- a/spec/services/container_expiration_policies/cleanup_service_spec.rb
+++ b/spec/services/container_expiration_policies/cleanup_service_spec.rb
@@ -24,8 +24,8 @@ RSpec.describe ContainerExpirationPolicies::CleanupService do
it 'completely clean up the repository' do
expect(Projects::ContainerRepository::CleanupTagsService)
- .to receive(:new).with(project, nil, cleanup_tags_service_params).and_return(cleanup_tags_service)
- expect(cleanup_tags_service).to receive(:execute).with(repository).and_return(status: :success)
+ .to receive(:new).with(repository, nil, cleanup_tags_service_params).and_return(cleanup_tags_service)
+ expect(cleanup_tags_service).to receive(:execute).and_return(status: :success)
response = subject
diff --git a/spec/services/issues/close_service_spec.rb b/spec/services/issues/close_service_spec.rb
index 7a54647fb07..93ef046a632 100644
--- a/spec/services/issues/close_service_spec.rb
+++ b/spec/services/issues/close_service_spec.rb
@@ -168,7 +168,7 @@ RSpec.describe Issues::CloseService do
context 'updating `metrics.first_mentioned_in_commit_at`' do
context 'when `metrics.first_mentioned_in_commit_at` is not set' do
it 'uses the first commit authored timestamp' do
- expected = closing_merge_request.commits.first.authored_date
+ expected = closing_merge_request.commits.take(100).last.authored_date
close_issue
diff --git a/spec/services/projects/container_repository/cleanup_tags_service_spec.rb b/spec/services/projects/container_repository/cleanup_tags_service_spec.rb
index 63a4d717df2..289bbf4540e 100644
--- a/spec/services/projects/container_repository/cleanup_tags_service_spec.rb
+++ b/spec/services/projects/container_repository/cleanup_tags_service_spec.rb
@@ -9,7 +9,7 @@ RSpec.describe Projects::ContainerRepository::CleanupTagsService, :clean_gitlab_
let_it_be(:project, reload: true) { create(:project, :private) }
let(:repository) { create(:container_repository, :root, project: project) }
- let(:service) { described_class.new(project, user, params) }
+ let(:service) { described_class.new(repository, user, params) }
let(:tags) { %w[latest A Ba Bb C D E] }
before do
@@ -39,7 +39,7 @@ RSpec.describe Projects::ContainerRepository::CleanupTagsService, :clean_gitlab_
end
describe '#execute' do
- subject { service.execute(repository) }
+ subject { service.execute }
shared_examples 'reading and removing tags' do |caching_enabled: true|
context 'when no params are specified' do
diff --git a/spec/workers/ci/stuck_builds/drop_running_worker_spec.rb b/spec/workers/ci/stuck_builds/drop_running_worker_spec.rb
index 9837902c449..6d3aa71fe81 100644
--- a/spec/workers/ci/stuck_builds/drop_running_worker_spec.rb
+++ b/spec/workers/ci/stuck_builds/drop_running_worker_spec.rb
@@ -5,66 +5,24 @@ require 'spec_helper'
RSpec.describe Ci::StuckBuilds::DropRunningWorker do
include ExclusiveLeaseHelpers
- let(:worker_lease_key) { Ci::StuckBuilds::DropRunningWorker::EXCLUSIVE_LEASE_KEY }
- let(:worker_lease_uuid) { SecureRandom.uuid }
- let(:worker2) { described_class.new }
-
- subject(:worker) { described_class.new }
-
- before do
- stub_exclusive_lease(worker_lease_key, worker_lease_uuid)
- end
+ let(:worker) { described_class.new }
+ let(:lease_uuid) { SecureRandom.uuid }
describe '#perform' do
+ subject { worker.perform }
+
it_behaves_like 'an idempotent worker'
it 'executes an instance of Ci::StuckBuilds::DropRunningService' do
+ expect_to_obtain_exclusive_lease(worker.lease_key, lease_uuid)
+
expect_next_instance_of(Ci::StuckBuilds::DropRunningService) do |service|
expect(service).to receive(:execute).exactly(:once)
end
- worker.perform
- end
-
- context 'with an exclusive lease' do
- it 'does not execute concurrently' do
- expect(worker).to receive(:remove_lease).exactly(:once)
- expect(worker2).not_to receive(:remove_lease)
-
- worker.perform
-
- stub_exclusive_lease_taken(worker_lease_key)
-
- worker2.perform
- end
-
- it 'can execute in sequence' do
- expect(worker).to receive(:remove_lease).at_least(:once)
- expect(worker2).to receive(:remove_lease).at_least(:once)
-
- worker.perform
- worker2.perform
- end
-
- it 'cancels exclusive leases after worker perform' do
- expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid)
+ expect_to_cancel_exclusive_lease(worker.lease_key, lease_uuid)
- worker.perform
- end
-
- context 'when the DropRunningService fails' do
- it 'ensures cancellation of the exclusive lease' do
- expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid)
-
- allow_next_instance_of(Ci::StuckBuilds::DropRunningService) do |service|
- expect(service).to receive(:execute) do
- raise 'The query timed out'
- end
- end
-
- expect { worker.perform }.to raise_error(/The query timed out/)
- end
- end
+ subject
end
end
end
diff --git a/spec/workers/cleanup_container_repository_worker_spec.rb b/spec/workers/cleanup_container_repository_worker_spec.rb
index 9cf8974a2a1..6ae4308bd46 100644
--- a/spec/workers/cleanup_container_repository_worker_spec.rb
+++ b/spec/workers/cleanup_container_repository_worker_spec.rb
@@ -17,7 +17,7 @@ RSpec.describe CleanupContainerRepositoryWorker, :clean_gitlab_redis_shared_stat
it 'executes the destroy service' do
expect(Projects::ContainerRepository::CleanupTagsService).to receive(:new)
- .with(project, user, params.merge('container_expiration_policy' => false))
+ .with(repository, user, params.merge('container_expiration_policy' => false))
.and_return(service)
expect(service).to receive(:execute)
@@ -49,7 +49,7 @@ RSpec.describe CleanupContainerRepositoryWorker, :clean_gitlab_redis_shared_stat
expect(repository).to receive(:start_expiration_policy!).and_call_original
expect(repository).to receive(:reset_expiration_policy_started_at!).and_call_original
expect(Projects::ContainerRepository::CleanupTagsService).to receive(:new)
- .with(project, nil, params.merge('container_expiration_policy' => true))
+ .with(repository, nil, params.merge('container_expiration_policy' => true))
.and_return(service)
expect(service).to receive(:execute).and_return(status: :success)
@@ -62,7 +62,7 @@ RSpec.describe CleanupContainerRepositoryWorker, :clean_gitlab_redis_shared_stat
expect(repository).to receive(:start_expiration_policy!).and_call_original
expect(repository).not_to receive(:reset_expiration_policy_started_at!)
expect(Projects::ContainerRepository::CleanupTagsService).to receive(:new)
- .with(project, nil, params.merge('container_expiration_policy' => true))
+ .with(repository, nil, params.merge('container_expiration_policy' => true))
.and_return(service)
expect(service).to receive(:execute).and_return(status: :error, message: 'timeout while deleting tags')
diff --git a/spec/workers/stuck_ci_jobs_worker_spec.rb b/spec/workers/stuck_ci_jobs_worker_spec.rb
index 1eee3535f71..aedf7f1c0b5 100644
--- a/spec/workers/stuck_ci_jobs_worker_spec.rb
+++ b/spec/workers/stuck_ci_jobs_worker_spec.rb
@@ -5,76 +5,34 @@ require 'spec_helper'
RSpec.describe StuckCiJobsWorker do
include ExclusiveLeaseHelpers
- let(:worker_lease_key) { StuckCiJobsWorker::EXCLUSIVE_LEASE_KEY }
- let(:worker_lease_uuid) { SecureRandom.uuid }
- let(:worker2) { described_class.new }
-
- subject(:worker) { described_class.new }
-
- before do
- stub_exclusive_lease(worker_lease_key, worker_lease_uuid)
- end
+ let(:worker) { described_class.new }
+ let(:lease_uuid) { SecureRandom.uuid }
describe '#perform' do
+ subject { worker.perform }
+
it 'enqueues a Ci::StuckBuilds::DropRunningWorker job' do
expect(Ci::StuckBuilds::DropRunningWorker).to receive(:perform_in).with(20.minutes).exactly(:once)
- worker.perform
+ subject
end
it 'enqueues a Ci::StuckBuilds::DropScheduledWorker job' do
expect(Ci::StuckBuilds::DropScheduledWorker).to receive(:perform_in).with(40.minutes).exactly(:once)
- worker.perform
+ subject
end
it 'executes an instance of Ci::StuckBuilds::DropService' do
+ expect_to_obtain_exclusive_lease(worker.lease_key, lease_uuid)
+
expect_next_instance_of(Ci::StuckBuilds::DropService) do |service|
expect(service).to receive(:execute).exactly(:once)
end
- worker.perform
- end
-
- context 'with an exclusive lease' do
- it 'does not execute concurrently' do
- expect(worker).to receive(:remove_lease).exactly(:once)
- expect(worker2).not_to receive(:remove_lease)
-
- worker.perform
-
- stub_exclusive_lease_taken(worker_lease_key)
-
- worker2.perform
- end
-
- it 'can execute in sequence' do
- expect(worker).to receive(:remove_lease).at_least(:once)
- expect(worker2).to receive(:remove_lease).at_least(:once)
-
- worker.perform
- worker2.perform
- end
-
- it 'cancels exclusive leases after worker perform' do
- expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid)
+ expect_to_cancel_exclusive_lease(worker.lease_key, lease_uuid)
- worker.perform
- end
-
- context 'when the DropService fails' do
- it 'ensures cancellation of the exclusive lease' do
- expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid)
-
- allow_next_instance_of(Ci::StuckBuilds::DropService) do |service|
- expect(service).to receive(:execute) do
- raise 'The query timed out'
- end
- end
-
- expect { worker.perform }.to raise_error(/The query timed out/)
- end
- end
+ subject
end
end
end