diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-10-07 15:12:09 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-10-07 15:12:09 +0300 |
commit | 43c3400c67f6470d4a19f143008990ee142dd828 (patch) | |
tree | 2268d48d59023e8353c6ab55290e8031e224dacc | |
parent | bc935f05bc8d7dd89c3e7c88f90264e90b636e07 (diff) |
Add latest changes from gitlab-org/gitlab@master
31 files changed, 482 insertions, 239 deletions
diff --git a/app/assets/javascripts/vue_merge_request_widget/components/extensions/base.vue b/app/assets/javascripts/vue_merge_request_widget/components/extensions/base.vue index 7ea24171aa8..b2c9d28a88b 100644 --- a/app/assets/javascripts/vue_merge_request_widget/components/extensions/base.vue +++ b/app/assets/javascripts/vue_merge_request_widget/components/extensions/base.vue @@ -117,7 +117,7 @@ export default { </script> <template> - <section class="media-section mr-widget-border-top" data-testid="widget-extension"> + <section class="media-section" data-testid="widget-extension"> <div class="media gl-p-5"> <status-icon :name="$options.name" diff --git a/app/assets/javascripts/vue_merge_request_widget/components/extensions/container.js b/app/assets/javascripts/vue_merge_request_widget/components/extensions/container.js index 46046d16fcf..b9dfd3bd41e 100644 --- a/app/assets/javascripts/vue_merge_request_widget/components/extensions/container.js +++ b/app/assets/javascripts/vue_merge_request_widget/components/extensions/container.js @@ -1,3 +1,4 @@ +import { __ } from '~/locale'; import { registeredExtensions } from './index'; export default { @@ -12,23 +13,42 @@ export default { if (extensions.length === 0) return null; - return h('div', {}, [ - ...extensions.map((extension) => + return h( + 'div', + { + attrs: { + role: 'region', + 'aria-label': __('Merge request reports'), + }, + }, + [ h( - { ...extension }, + 'ul', { - props: { - ...extension.props.reduce( - (acc, key) => ({ - ...acc, - [key]: this.mr[key], - }), - {}, - ), - }, + class: 'gl-p-0 gl-m-0 gl-list-style-none', }, + [ + ...extensions.map((extension, index) => + h('li', { attrs: { class: index > 0 && 'mr-widget-border-top' } }, [ + h( + { ...extension }, + { + props: { + ...extension.props.reduce( + (acc, key) => ({ + ...acc, + [key]: this.mr[key], + }), + {}, + ), + }, + }, + ), + ]), + ), + ], ), - ), - ]); + ], + ); }, }; diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb index 5fd4d21b260..0a18559fc81 100644 --- a/app/controllers/search_controller.rb +++ b/app/controllers/search_controller.rb @@ -12,6 +12,7 @@ class SearchController < ApplicationController around_action :allow_gitaly_ref_name_caching before_action :block_anonymous_global_searches, :check_scope_global_search_enabled, except: :opensearch + before_action :strip_surrounding_whitespace_from_search, except: :opensearch skip_before_action :authenticate_user! requires_cross_project_access if: -> do search_term_present = params[:search].present? || params[:term].present? @@ -197,6 +198,10 @@ class SearchController < ApplicationController def count_action_name? action_name.to_sym == :count end + + def strip_surrounding_whitespace_from_search + %i(term search).each { |param| params[param]&.strip! } + end end SearchController.prepend_mod_with('SearchController') diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index 3d32144e0f8..b4e2d17c3e5 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -10,6 +10,8 @@ class ProtectedBranch < ApplicationRecord scope :allowing_force_push, -> { where(allow_force_push: true) } + scope :get_ids_by_name, -> (name) { where(name: name).pluck(:id) } + protected_ref_access_levels :merge, :push def self.protected_ref_accessible_to?(ref, user, project:, action:, protected_refs: nil) diff --git a/app/models/user.rb b/app/models/user.rb index f4e6069ac9a..25a2588a6a7 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -457,6 +457,7 @@ class User < ApplicationRecord scope :dormant, -> { active.where('last_activity_on <= ?', MINIMUM_INACTIVE_DAYS.day.ago.to_date) } scope :with_no_activity, -> { active.where(last_activity_on: nil) } scope :by_provider_and_extern_uid, ->(provider, extern_uid) { joins(:identities).merge(Identity.with_extern_uid(provider, extern_uid)) } + scope :get_ids_by_username, -> (username) { where(username: username).pluck(:id) } def preferred_language read_attribute('preferred_language') || diff --git a/app/services/container_expiration_policies/cleanup_service.rb b/app/services/container_expiration_policies/cleanup_service.rb index 52530b91578..0da5e552c48 100644 --- a/app/services/container_expiration_policies/cleanup_service.rb +++ b/app/services/container_expiration_policies/cleanup_service.rb @@ -24,8 +24,8 @@ module ContainerExpirationPolicies begin service_result = Projects::ContainerRepository::CleanupTagsService - .new(project, nil, policy_params.merge('container_expiration_policy' => true)) - .execute(repository) + .new(repository, nil, policy_params.merge('container_expiration_policy' => true)) + .execute rescue StandardError repository.cleanup_unfinished! diff --git a/app/services/issues/close_service.rb b/app/services/issues/close_service.rb index 856f6932d61..ac846c769a3 100644 --- a/app/services/issues/close_service.rb +++ b/app/services/issues/close_service.rb @@ -91,11 +91,11 @@ module Issues end end - def store_first_mentioned_in_commit_at(issue, merge_request) + def store_first_mentioned_in_commit_at(issue, merge_request, max_commit_lookup: 100) metrics = issue.metrics return if metrics.nil? || metrics.first_mentioned_in_commit_at - first_commit_timestamp = merge_request.commits(limit: 1).first.try(:authored_date) + first_commit_timestamp = merge_request.commits(limit: max_commit_lookup).last.try(:authored_date) return unless first_commit_timestamp metrics.update!(first_mentioned_in_commit_at: first_commit_timestamp) diff --git a/app/services/projects/container_repository/cleanup_tags_service.rb b/app/services/projects/container_repository/cleanup_tags_service.rb index 064cdce6f10..3a60de0f1ee 100644 --- a/app/services/projects/container_repository/cleanup_tags_service.rb +++ b/app/services/projects/container_repository/cleanup_tags_service.rb @@ -2,148 +2,152 @@ module Projects module ContainerRepository - class CleanupTagsService < BaseService + class CleanupTagsService + include BaseServiceUtility include ::Gitlab::Utils::StrongMemoize - def execute(container_repository) - return error('access denied') unless can_destroy? - return error('invalid regex') unless valid_regex? + def initialize(container_repository, user = nil, params = {}) + @container_repository = container_repository + @current_user = user + @params = params.dup - tags = container_repository.tags - original_size = tags.size + @project = container_repository.project + @tags = container_repository.tags + tags_size = @tags.size + @counts = { + original_size: tags_size, + cached_tags_count: 0 + } + end - tags = without_latest(tags) - tags = filter_by_name(tags) + def execute + return error('access denied') unless can_destroy? + return error('invalid regex') unless valid_regex? - before_truncate_size = tags.size - tags = truncate(tags) - after_truncate_size = tags.size + filter_out_latest + filter_by_name - cached_tags_count = populate_tags_from_cache(container_repository, tags) || 0 + truncate + populate_from_cache - tags = filter_keep_n(container_repository, tags) - tags = filter_by_older_than(container_repository, tags) + filter_keep_n + filter_by_older_than - delete_tags(container_repository, tags).tap do |result| - result[:original_size] = original_size - result[:before_truncate_size] = before_truncate_size - result[:after_truncate_size] = after_truncate_size - result[:cached_tags_count] = cached_tags_count - result[:before_delete_size] = tags.size + delete_tags.merge(@counts).tap do |result| + result[:before_delete_size] = @tags.size result[:deleted_size] = result[:deleted]&.size - result[:status] = :error if before_truncate_size != after_truncate_size + result[:status] = :error if @counts[:before_truncate_size] != @counts[:after_truncate_size] end end private - def delete_tags(container_repository, tags) - return success(deleted: []) unless tags.any? - - tag_names = tags.map(&:name) + def delete_tags + return success(deleted: []) unless @tags.any? service = Projects::ContainerRepository::DeleteTagsService.new( - container_repository.project, - current_user, - tags: tag_names, - container_expiration_policy: params['container_expiration_policy'] + @project, + @current_user, + tags: @tags.map(&:name), + container_expiration_policy: container_expiration_policy ) - service.execute(container_repository) + service.execute(@container_repository) end - def without_latest(tags) - tags.reject(&:latest?) + def filter_out_latest + @tags.reject!(&:latest?) end - def order_by_date(tags) + def order_by_date now = DateTime.current - tags.sort_by { |tag| tag.created_at || now }.reverse + @tags.sort_by! { |tag| tag.created_at || now } + .reverse! end - def filter_by_name(tags) - regex_delete = ::Gitlab::UntrustedRegexp.new("\\A#{params['name_regex_delete'] || params['name_regex']}\\z") - regex_retain = ::Gitlab::UntrustedRegexp.new("\\A#{params['name_regex_keep']}\\z") + def filter_by_name + regex_delete = ::Gitlab::UntrustedRegexp.new("\\A#{name_regex_delete || name_regex}\\z") + regex_retain = ::Gitlab::UntrustedRegexp.new("\\A#{name_regex_keep}\\z") - tags.select do |tag| + @tags.select! do |tag| # regex_retain will override any overlapping matches by regex_delete regex_delete.match?(tag.name) && !regex_retain.match?(tag.name) end end - def filter_keep_n(container_repository, tags) - return tags unless params['keep_n'] + def filter_keep_n + return unless keep_n - tags = order_by_date(tags) - cache_tags(container_repository, tags.first(keep_n)) - tags.drop(keep_n) + order_by_date + cache_tags(@tags.first(keep_n_as_integer)) + @tags = @tags.drop(keep_n_as_integer) end - def filter_by_older_than(container_repository, tags) - return tags unless older_than + def filter_by_older_than + return unless older_than older_than_timestamp = older_than_in_seconds.ago - tags, tags_to_keep = tags.partition do |tag| + @tags, tags_to_keep = @tags.partition do |tag| tag.created_at && tag.created_at < older_than_timestamp end - cache_tags(container_repository, tags_to_keep) - - tags + cache_tags(tags_to_keep) end def can_destroy? - return true if params['container_expiration_policy'] + return true if container_expiration_policy - can?(current_user, :destroy_container_image, project) + can?(@current_user, :destroy_container_image, @project) end def valid_regex? %w(name_regex_delete name_regex name_regex_keep).each do |param_name| - regex = params[param_name] + regex = @params[param_name] ::Gitlab::UntrustedRegexp.new(regex) unless regex.blank? end true rescue RegexpError => e - ::Gitlab::ErrorTracking.log_exception(e, project_id: project.id) + ::Gitlab::ErrorTracking.log_exception(e, project_id: @project.id) false end - def truncate(tags) - return tags unless throttling_enabled? - return tags if max_list_size == 0 + def truncate + @counts[:before_truncate_size] = @tags.size + @counts[:after_truncate_size] = @tags.size + + return unless throttling_enabled? + return if max_list_size == 0 # truncate the list to make sure that after the #filter_keep_n # execution, the resulting list will be max_list_size - truncated_size = max_list_size + keep_n + truncated_size = max_list_size + keep_n_as_integer - return tags if tags.size <= truncated_size + return if @tags.size <= truncated_size - tags.sample(truncated_size) + @tags = @tags.sample(truncated_size) + @counts[:after_truncate_size] = @tags.size end - def populate_tags_from_cache(container_repository, tags) - cache(container_repository).populate(tags) if caching_enabled?(container_repository) + def populate_from_cache + @counts[:cached_tags_count] = cache.populate(@tags) if caching_enabled? end - def cache_tags(container_repository, tags) - cache(container_repository).insert(tags, older_than_in_seconds) if caching_enabled?(container_repository) + def cache_tags(tags) + cache.insert(tags, older_than_in_seconds) if caching_enabled? end - def cache(container_repository) - # TODO Implement https://gitlab.com/gitlab-org/gitlab/-/issues/340277 to avoid passing - # the container repository parameter which is bad for a memoized function + def cache strong_memoize(:cache) do - ::Projects::ContainerRepository::CacheTagsCreatedAtService.new(container_repository) + ::Projects::ContainerRepository::CacheTagsCreatedAtService.new(@container_repository) end end - def caching_enabled?(container_repository) - params['container_expiration_policy'] && + def caching_enabled? + container_expiration_policy && older_than.present? && - Feature.enabled?(:container_registry_expiration_policies_caching, container_repository.project) + Feature.enabled?(:container_registry_expiration_policies_caching, @project) end def throttling_enabled? @@ -155,7 +159,11 @@ module Projects end def keep_n - params['keep_n'].to_i + @params['keep_n'] + end + + def keep_n_as_integer + keep_n.to_i end def older_than_in_seconds @@ -165,7 +173,23 @@ module Projects end def older_than - params['older_than'] + @params['older_than'] + end + + def name_regex_delete + @params['name_regex_delete'] + end + + def name_regex + @params['name_regex'] + end + + def name_regex_keep + @params['name_regex_keep'] + end + + def container_expiration_policy + @params['container_expiration_policy'] end end end diff --git a/app/workers/ci/stuck_builds/drop_running_worker.rb b/app/workers/ci/stuck_builds/drop_running_worker.rb index 35326bf74c9..db571fdc38d 100644 --- a/app/workers/ci/stuck_builds/drop_running_worker.rb +++ b/app/workers/ci/stuck_builds/drop_running_worker.rb @@ -4,6 +4,7 @@ module Ci module StuckBuilds class DropRunningWorker include ApplicationWorker + include ExclusiveLeaseGuard idempotent! @@ -17,26 +18,16 @@ module Ci feature_category :continuous_integration - EXCLUSIVE_LEASE_KEY = 'ci_stuck_builds_drop_running_worker_lease' - def perform - return unless try_obtain_lease - - begin + try_obtain_lease do Ci::StuckBuilds::DropRunningService.new.execute - ensure - remove_lease end end private - def try_obtain_lease - @uuid = Gitlab::ExclusiveLease.new(EXCLUSIVE_LEASE_KEY, timeout: 30.minutes).try_obtain - end - - def remove_lease - Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid) + def lease_timeout + 30.minutes end end end diff --git a/app/workers/cleanup_container_repository_worker.rb b/app/workers/cleanup_container_repository_worker.rb index 9adc026ced2..7274ecf62f9 100644 --- a/app/workers/cleanup_container_repository_worker.rb +++ b/app/workers/cleanup_container_repository_worker.rb @@ -28,8 +28,8 @@ class CleanupContainerRepositoryWorker end result = Projects::ContainerRepository::CleanupTagsService - .new(project, current_user, params) - .execute(container_repository) + .new(container_repository, current_user, params) + .execute if run_by_container_expiration_policy? && result[:status] == :success container_repository.reset_expiration_policy_started_at! diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index 6282c9200b1..72fcf06dce1 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -2,6 +2,7 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + include ExclusiveLeaseGuard # rubocop:disable Scalability/CronWorkerContext # This is an instance-wide cleanup query, so there's no meaningful @@ -14,28 +15,18 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker feature_category :continuous_integration worker_resource_boundary :cpu - EXCLUSIVE_LEASE_KEY = 'stuck_ci_builds_worker_lease' - def perform Ci::StuckBuilds::DropRunningWorker.perform_in(20.minutes) Ci::StuckBuilds::DropScheduledWorker.perform_in(40.minutes) - return unless try_obtain_lease - - begin + try_obtain_lease do Ci::StuckBuilds::DropService.new.execute - ensure - remove_lease end end private - def try_obtain_lease - @uuid = Gitlab::ExclusiveLease.new(EXCLUSIVE_LEASE_KEY, timeout: 30.minutes).try_obtain - end - - def remove_lease - Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid) + def lease_timeout + 30.minutes end end diff --git a/db/post_migrate/20211004110500_add_temporary_index_to_issue_metrics.rb b/db/post_migrate/20211004110500_add_temporary_index_to_issue_metrics.rb new file mode 100644 index 00000000000..cfc37c55121 --- /dev/null +++ b/db/post_migrate/20211004110500_add_temporary_index_to_issue_metrics.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddTemporaryIndexToIssueMetrics < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + INDEX_NAME = 'index_issue_metrics_first_mentioned_in_commit' + + def up + add_concurrent_index :issue_metrics, :issue_id, where: 'EXTRACT(YEAR FROM first_mentioned_in_commit_at) > 2019', name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :issue_metrics, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20211004110927_schedule_fix_first_mentioned_in_commit_at_job.rb b/db/post_migrate/20211004110927_schedule_fix_first_mentioned_in_commit_at_job.rb new file mode 100644 index 00000000000..c7612db3aaf --- /dev/null +++ b/db/post_migrate/20211004110927_schedule_fix_first_mentioned_in_commit_at_job.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class ScheduleFixFirstMentionedInCommitAtJob < Gitlab::Database::Migration[1.0] + MIGRATION = 'FixFirstMentionedInCommitAt' + BATCH_SIZE = 10_000 + INTERVAL = 2.minutes.to_i + + disable_ddl_transaction! + + def up + scope = define_batchable_model('issue_metrics') + .where('EXTRACT(YEAR FROM first_mentioned_in_commit_at) > 2019') + + queue_background_migration_jobs_by_range_at_intervals( + scope, + MIGRATION, + INTERVAL, + batch_size: BATCH_SIZE, + track_jobs: true, + primary_column_name: :issue_id + ) + end + + def down + # noop + end +end diff --git a/db/schema_migrations/20211004110500 b/db/schema_migrations/20211004110500 new file mode 100644 index 00000000000..e22ed05de83 --- /dev/null +++ b/db/schema_migrations/20211004110500 @@ -0,0 +1 @@ +1b0b562aefb724afe24b8640a22013cea6fddd0e594d6723f6819f69804ba9f7
\ No newline at end of file diff --git a/db/schema_migrations/20211004110927 b/db/schema_migrations/20211004110927 new file mode 100644 index 00000000000..aa70a4aa0d8 --- /dev/null +++ b/db/schema_migrations/20211004110927 @@ -0,0 +1 @@ +50c937f979c83f6937364d92bf65ed42ef963f2d241eadcee6355c1b256c3ec9
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 5fd10356a24..189e4e8efaf 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -25448,6 +25448,8 @@ CREATE UNIQUE INDEX index_issue_links_on_source_id_and_target_id ON issue_links CREATE INDEX index_issue_links_on_target_id ON issue_links USING btree (target_id); +CREATE INDEX index_issue_metrics_first_mentioned_in_commit ON issue_metrics USING btree (issue_id) WHERE (date_part('year'::text, first_mentioned_in_commit_at) > (2019)::double precision); + CREATE INDEX index_issue_metrics_on_issue_id_and_timestamps ON issue_metrics USING btree (issue_id, first_mentioned_in_commit_at, first_associated_with_milestone_at, first_added_to_board_at); CREATE INDEX index_issue_on_project_id_state_id_and_blocking_issues_count ON issues USING btree (project_id, state_id, blocking_issues_count); diff --git a/doc/user/compliance/compliance_report/index.md b/doc/user/compliance/compliance_report/index.md index d30cedfb3cd..63dd06bcfdb 100644 --- a/doc/user/compliance/compliance_report/index.md +++ b/doc/user/compliance/compliance_report/index.md @@ -30,7 +30,7 @@ Prerequisites: To view the compliance report: 1. On the top bar, select **Menu > Groups** and find your group. -1. On the left sidebar, select **Security & Compliance > Compliance**. +1. On the left sidebar, select **Security & Compliance > Compliance report**. NOTE: The compliance report shows only the latest merge request on each project. @@ -87,7 +87,7 @@ Depending on the merge strategy, the merge commit SHA can be a merge commit, squ To download the Chain of Custody report: 1. On the top bar, select **Menu > Groups** and find your group. -1. On the left sidebar, select **Security & Compliance > Compliance**. +1. On the left sidebar, select **Security & Compliance > Compliance report**. 1. Select **List of all merge commits**. ### Commit-specific Chain of Custody Report **(ULTIMATE)** @@ -97,7 +97,7 @@ To download the Chain of Custody report: You can generate a commit-specific Chain of Custody report for a given commit SHA. 1. On the top bar, select **Menu > Groups** and find your group. -1. On the left sidebar, select **Security & Compliance > Compliance**. +1. On the left sidebar, select **Security & Compliance > Compliance report**. 1. At the top of the compliance report, to the right of **List of all merge commits**, select the down arrow (**{angle-down}**). 1. Enter the merge commit SHA, and then select **Export commit custody report**. SHA and then select **Export commit custody report**. diff --git a/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at.rb b/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at.rb new file mode 100644 index 00000000000..0dc1e36ff19 --- /dev/null +++ b/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class that fixes the incorrectly set authored_date within + # issue_metrics table + class FixFirstMentionedInCommitAt + SUB_BATCH_SIZE = 500 + + # rubocop: disable Style/Documentation + class TmpIssueMetrics < ActiveRecord::Base + include EachBatch + + self.table_name = 'issue_metrics' + + def self.from_2020 + where('EXTRACT(YEAR FROM first_mentioned_in_commit_at) > 2019') + end + end + # rubocop: enable Style/Documentation + + def perform(start_id, end_id) + scope(start_id, end_id).each_batch(of: SUB_BATCH_SIZE, column: :issue_id) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(issue_id), max(issue_id)')).first + + # The query need to be reconstructed because .each_batch modifies the default scope + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510 + inner_query = TmpIssueMetrics + .unscoped + .merge(scope(first, last)) + .from("issue_metrics, #{lateral_query}") + .select('issue_metrics.issue_id', 'first_authored_date.authored_date') + .where('issue_metrics.first_mentioned_in_commit_at > first_authored_date.authored_date') + + TmpIssueMetrics.connection.execute <<~UPDATE_METRICS + WITH cte AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + #{inner_query.to_sql} + ) + UPDATE issue_metrics + SET + first_mentioned_in_commit_at = cte.authored_date + FROM + cte + WHERE + cte.issue_id = issue_metrics.issue_id + UPDATE_METRICS + end + end + + private + + def scope(start_id, end_id) + TmpIssueMetrics.from_2020.where(issue_id: start_id..end_id) + end + + def lateral_query + <<~SQL + LATERAL ( + SELECT MIN(first_authored_date.authored_date) as authored_date + FROM merge_requests_closing_issues, + LATERAL ( + SELECT id + FROM merge_request_diffs + WHERE merge_request_id = merge_requests_closing_issues.merge_request_id + ORDER BY id DESC + LIMIT 1 + ) last_diff_id, + LATERAL ( + SELECT authored_date + FROM merge_request_diff_commits + WHERE + merge_request_diff_id = last_diff_id.id + ORDER BY relative_order DESC + LIMIT 1 + ) first_authored_date + WHERE merge_requests_closing_issues.issue_id = issue_metrics.issue_id + ) first_authored_date + SQL + end + end + end +end diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index 19d80ba1d64..bdaf0d35a83 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -106,7 +106,7 @@ module Gitlab final_delay = 0 batch_counter = 0 - model_class.each_batch(of: batch_size) do |relation, index| + model_class.each_batch(of: batch_size, column: primary_column_name) do |relation, index| max = relation.arel_table[primary_column_name].maximum min = relation.arel_table[primary_column_name].minimum diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 1b19945b7ea..8087f8563c8 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -8460,13 +8460,10 @@ msgstr "" msgid "Completed" msgstr "" -msgid "Compliance" -msgstr "" - -msgid "Compliance Dashboard" +msgid "Compliance framework" msgstr "" -msgid "Compliance framework" +msgid "Compliance report" msgstr "" msgid "ComplianceDashboard|created by:" @@ -21350,6 +21347,9 @@ msgstr "" msgid "Merge request events" msgstr "" +msgid "Merge request reports" +msgstr "" + msgid "Merge request was scheduled to merge after pipeline succeeds" msgstr "" diff --git a/qa/qa/specs/features/api/1_manage/bulk_import_project_spec.rb b/qa/qa/specs/features/api/1_manage/bulk_import_project_spec.rb index b88fdfdc757..9935908d55e 100644 --- a/qa/qa/specs/features/api/1_manage/bulk_import_project_spec.rb +++ b/qa/qa/specs/features/api/1_manage/bulk_import_project_spec.rb @@ -61,7 +61,10 @@ module QA end context 'with project' do - it 'successfully imports project' do + it( + 'successfully imports project', + testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/2297' + ) do expect { imported_group.import_status }.to eventually_eq('finished').within(import_wait_duration) imported_projects = imported_group.reload!.projects diff --git a/spec/controllers/search_controller_spec.rb b/spec/controllers/search_controller_spec.rb index 6bcb88278a0..73e8e0c7dd4 100644 --- a/spec/controllers/search_controller_spec.rb +++ b/spec/controllers/search_controller_spec.rb @@ -215,6 +215,16 @@ RSpec.describe SearchController do end end + it 'strips surrounding whitespace from search query' do + get :show, params: { scope: 'notes', search: ' foobar ' } + expect(assigns[:search_term]).to eq 'foobar' + end + + it 'strips surrounding whitespace from autocomplete term' do + expect(controller).to receive(:search_autocomplete_opts).with('youcompleteme') + get :autocomplete, params: { term: ' youcompleteme ' } + end + it 'finds issue comments' do project = create(:project, :public) note = create(:note_on_issue, project: project) diff --git a/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb b/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb new file mode 100644 index 00000000000..3e6de5390a7 --- /dev/null +++ b/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::FixFirstMentionedInCommitAt, :migration, schema: 20211004110500 do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:users) { table(:users) } + let(:merge_requests) { table(:merge_requests) } + let(:issues) { table(:issues) } + let(:issue_metrics) { table(:issue_metrics) } + let(:merge_requests_closing_issues) { table(:merge_requests_closing_issues) } + let(:diffs) { table(:merge_request_diffs) } + let(:ten_days_ago) { 10.days.ago } + let(:commits) do + table(:merge_request_diff_commits).tap do |t| + t.extend(SuppressCompositePrimaryKeyWarning) + end + end + + let(:namespace) { namespaces.create!(name: 'ns', path: 'ns') } + let(:project) { projects.create!(namespace_id: namespace.id) } + + let!(:issue1) do + issues.create!( + title: 'issue', + description: 'description', + project_id: project.id + ) + end + + let!(:issue2) do + issues.create!( + title: 'issue', + description: 'description', + project_id: project.id + ) + end + + let!(:merge_request1) do + merge_requests.create!( + source_branch: 'a', + target_branch: 'master', + target_project_id: project.id + ) + end + + let!(:merge_request2) do + merge_requests.create!( + source_branch: 'b', + target_branch: 'master', + target_project_id: project.id + ) + end + + let!(:merge_request_closing_issue1) do + merge_requests_closing_issues.create!(issue_id: issue1.id, merge_request_id: merge_request1.id) + end + + let!(:merge_request_closing_issue2) do + merge_requests_closing_issues.create!(issue_id: issue2.id, merge_request_id: merge_request2.id) + end + + let!(:diff1) { diffs.create!(merge_request_id: merge_request1.id) } + let!(:diff2) { diffs.create!(merge_request_id: merge_request1.id) } + + let!(:other_diff) { diffs.create!(merge_request_id: merge_request2.id) } + + let!(:commit1) do + commits.create!( + merge_request_diff_id: diff2.id, + relative_order: 0, + sha: Gitlab::Database::ShaAttribute.serialize('aaa'), + authored_date: 5.days.ago + ) + end + + let!(:commit2) do + commits.create!( + merge_request_diff_id: diff2.id, + relative_order: 1, + sha: Gitlab::Database::ShaAttribute.serialize('aaa'), + authored_date: 10.days.ago + ) + end + + let!(:commit3) do + commits.create!( + merge_request_diff_id: other_diff.id, + relative_order: 1, + sha: Gitlab::Database::ShaAttribute.serialize('aaa'), + authored_date: 5.days.ago + ) + end + + def run_migration + described_class + .new + .perform(issue_metrics.minimum(:issue_id), issue_metrics.maximum(:issue_id)) + end + + context 'when the persisted first_mentioned_in_commit_at is later than the first commit authored_date' do + it 'updates the issue_metrics record' do + record1 = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: Time.current) + record2 = issue_metrics.create!(issue_id: issue2.id, first_mentioned_in_commit_at: Time.current) + + run_migration + record1.reload + record2.reload + + expect(record1.first_mentioned_in_commit_at).to be_within(2.seconds).of(commit2.authored_date) + expect(record2.first_mentioned_in_commit_at).to be_within(2.seconds).of(commit3.authored_date) + end + end + + context 'when the persisted first_mentioned_in_commit_at is earlier than the first commit authored_date' do + it 'does not update the issue_metrics record' do + record = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: 20.days.ago) + + expect { run_migration }.not_to change { record.reload.first_mentioned_in_commit_at } + end + end + + context 'when the first_mentioned_in_commit_at is null' do + it 'does nothing' do + record = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: nil) + + expect { run_migration }.not_to change { record.reload.first_mentioned_in_commit_at } + end + end +end diff --git a/spec/models/protected_branch_spec.rb b/spec/models/protected_branch_spec.rb index 019c01af672..587a9683a8e 100644 --- a/spec/models/protected_branch_spec.rb +++ b/spec/models/protected_branch_spec.rb @@ -308,4 +308,15 @@ RSpec.describe ProtectedBranch do expect(described_class.by_name('')).to be_empty end end + + describe '.get_ids_by_name' do + let(:branch_name) { 'branch_name' } + let!(:protected_branch) { create(:protected_branch, name: branch_name) } + let(:branch_id) { protected_branch.id } + + it 'returns the id for each protected branch matching name' do + expect(described_class.get_ids_by_name([branch_name])) + .to match_array([branch_id]) + end + end end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index b763ea066be..db805a804c8 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -6180,4 +6180,14 @@ RSpec.describe User do it_behaves_like 'groups_with_developer_maintainer_project_access examples' end end + + describe '.get_ids_by_username' do + let(:user_name) { 'user_name' } + let!(:user) { create(:user, username: user_name) } + let(:user_id) { user.id } + + it 'returns the id of each record matching username' do + expect(described_class.get_ids_by_username([user_name])).to match_array([user_id]) + end + end end diff --git a/spec/services/container_expiration_policies/cleanup_service_spec.rb b/spec/services/container_expiration_policies/cleanup_service_spec.rb index c8bb8bfe06d..a1f76e5e5dd 100644 --- a/spec/services/container_expiration_policies/cleanup_service_spec.rb +++ b/spec/services/container_expiration_policies/cleanup_service_spec.rb @@ -24,8 +24,8 @@ RSpec.describe ContainerExpirationPolicies::CleanupService do it 'completely clean up the repository' do expect(Projects::ContainerRepository::CleanupTagsService) - .to receive(:new).with(project, nil, cleanup_tags_service_params).and_return(cleanup_tags_service) - expect(cleanup_tags_service).to receive(:execute).with(repository).and_return(status: :success) + .to receive(:new).with(repository, nil, cleanup_tags_service_params).and_return(cleanup_tags_service) + expect(cleanup_tags_service).to receive(:execute).and_return(status: :success) response = subject diff --git a/spec/services/issues/close_service_spec.rb b/spec/services/issues/close_service_spec.rb index 7a54647fb07..93ef046a632 100644 --- a/spec/services/issues/close_service_spec.rb +++ b/spec/services/issues/close_service_spec.rb @@ -168,7 +168,7 @@ RSpec.describe Issues::CloseService do context 'updating `metrics.first_mentioned_in_commit_at`' do context 'when `metrics.first_mentioned_in_commit_at` is not set' do it 'uses the first commit authored timestamp' do - expected = closing_merge_request.commits.first.authored_date + expected = closing_merge_request.commits.take(100).last.authored_date close_issue diff --git a/spec/services/projects/container_repository/cleanup_tags_service_spec.rb b/spec/services/projects/container_repository/cleanup_tags_service_spec.rb index 63a4d717df2..289bbf4540e 100644 --- a/spec/services/projects/container_repository/cleanup_tags_service_spec.rb +++ b/spec/services/projects/container_repository/cleanup_tags_service_spec.rb @@ -9,7 +9,7 @@ RSpec.describe Projects::ContainerRepository::CleanupTagsService, :clean_gitlab_ let_it_be(:project, reload: true) { create(:project, :private) } let(:repository) { create(:container_repository, :root, project: project) } - let(:service) { described_class.new(project, user, params) } + let(:service) { described_class.new(repository, user, params) } let(:tags) { %w[latest A Ba Bb C D E] } before do @@ -39,7 +39,7 @@ RSpec.describe Projects::ContainerRepository::CleanupTagsService, :clean_gitlab_ end describe '#execute' do - subject { service.execute(repository) } + subject { service.execute } shared_examples 'reading and removing tags' do |caching_enabled: true| context 'when no params are specified' do diff --git a/spec/workers/ci/stuck_builds/drop_running_worker_spec.rb b/spec/workers/ci/stuck_builds/drop_running_worker_spec.rb index 9837902c449..6d3aa71fe81 100644 --- a/spec/workers/ci/stuck_builds/drop_running_worker_spec.rb +++ b/spec/workers/ci/stuck_builds/drop_running_worker_spec.rb @@ -5,66 +5,24 @@ require 'spec_helper' RSpec.describe Ci::StuckBuilds::DropRunningWorker do include ExclusiveLeaseHelpers - let(:worker_lease_key) { Ci::StuckBuilds::DropRunningWorker::EXCLUSIVE_LEASE_KEY } - let(:worker_lease_uuid) { SecureRandom.uuid } - let(:worker2) { described_class.new } - - subject(:worker) { described_class.new } - - before do - stub_exclusive_lease(worker_lease_key, worker_lease_uuid) - end + let(:worker) { described_class.new } + let(:lease_uuid) { SecureRandom.uuid } describe '#perform' do + subject { worker.perform } + it_behaves_like 'an idempotent worker' it 'executes an instance of Ci::StuckBuilds::DropRunningService' do + expect_to_obtain_exclusive_lease(worker.lease_key, lease_uuid) + expect_next_instance_of(Ci::StuckBuilds::DropRunningService) do |service| expect(service).to receive(:execute).exactly(:once) end - worker.perform - end - - context 'with an exclusive lease' do - it 'does not execute concurrently' do - expect(worker).to receive(:remove_lease).exactly(:once) - expect(worker2).not_to receive(:remove_lease) - - worker.perform - - stub_exclusive_lease_taken(worker_lease_key) - - worker2.perform - end - - it 'can execute in sequence' do - expect(worker).to receive(:remove_lease).at_least(:once) - expect(worker2).to receive(:remove_lease).at_least(:once) - - worker.perform - worker2.perform - end - - it 'cancels exclusive leases after worker perform' do - expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid) + expect_to_cancel_exclusive_lease(worker.lease_key, lease_uuid) - worker.perform - end - - context 'when the DropRunningService fails' do - it 'ensures cancellation of the exclusive lease' do - expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid) - - allow_next_instance_of(Ci::StuckBuilds::DropRunningService) do |service| - expect(service).to receive(:execute) do - raise 'The query timed out' - end - end - - expect { worker.perform }.to raise_error(/The query timed out/) - end - end + subject end end end diff --git a/spec/workers/cleanup_container_repository_worker_spec.rb b/spec/workers/cleanup_container_repository_worker_spec.rb index 9cf8974a2a1..6ae4308bd46 100644 --- a/spec/workers/cleanup_container_repository_worker_spec.rb +++ b/spec/workers/cleanup_container_repository_worker_spec.rb @@ -17,7 +17,7 @@ RSpec.describe CleanupContainerRepositoryWorker, :clean_gitlab_redis_shared_stat it 'executes the destroy service' do expect(Projects::ContainerRepository::CleanupTagsService).to receive(:new) - .with(project, user, params.merge('container_expiration_policy' => false)) + .with(repository, user, params.merge('container_expiration_policy' => false)) .and_return(service) expect(service).to receive(:execute) @@ -49,7 +49,7 @@ RSpec.describe CleanupContainerRepositoryWorker, :clean_gitlab_redis_shared_stat expect(repository).to receive(:start_expiration_policy!).and_call_original expect(repository).to receive(:reset_expiration_policy_started_at!).and_call_original expect(Projects::ContainerRepository::CleanupTagsService).to receive(:new) - .with(project, nil, params.merge('container_expiration_policy' => true)) + .with(repository, nil, params.merge('container_expiration_policy' => true)) .and_return(service) expect(service).to receive(:execute).and_return(status: :success) @@ -62,7 +62,7 @@ RSpec.describe CleanupContainerRepositoryWorker, :clean_gitlab_redis_shared_stat expect(repository).to receive(:start_expiration_policy!).and_call_original expect(repository).not_to receive(:reset_expiration_policy_started_at!) expect(Projects::ContainerRepository::CleanupTagsService).to receive(:new) - .with(project, nil, params.merge('container_expiration_policy' => true)) + .with(repository, nil, params.merge('container_expiration_policy' => true)) .and_return(service) expect(service).to receive(:execute).and_return(status: :error, message: 'timeout while deleting tags') diff --git a/spec/workers/stuck_ci_jobs_worker_spec.rb b/spec/workers/stuck_ci_jobs_worker_spec.rb index 1eee3535f71..aedf7f1c0b5 100644 --- a/spec/workers/stuck_ci_jobs_worker_spec.rb +++ b/spec/workers/stuck_ci_jobs_worker_spec.rb @@ -5,76 +5,34 @@ require 'spec_helper' RSpec.describe StuckCiJobsWorker do include ExclusiveLeaseHelpers - let(:worker_lease_key) { StuckCiJobsWorker::EXCLUSIVE_LEASE_KEY } - let(:worker_lease_uuid) { SecureRandom.uuid } - let(:worker2) { described_class.new } - - subject(:worker) { described_class.new } - - before do - stub_exclusive_lease(worker_lease_key, worker_lease_uuid) - end + let(:worker) { described_class.new } + let(:lease_uuid) { SecureRandom.uuid } describe '#perform' do + subject { worker.perform } + it 'enqueues a Ci::StuckBuilds::DropRunningWorker job' do expect(Ci::StuckBuilds::DropRunningWorker).to receive(:perform_in).with(20.minutes).exactly(:once) - worker.perform + subject end it 'enqueues a Ci::StuckBuilds::DropScheduledWorker job' do expect(Ci::StuckBuilds::DropScheduledWorker).to receive(:perform_in).with(40.minutes).exactly(:once) - worker.perform + subject end it 'executes an instance of Ci::StuckBuilds::DropService' do + expect_to_obtain_exclusive_lease(worker.lease_key, lease_uuid) + expect_next_instance_of(Ci::StuckBuilds::DropService) do |service| expect(service).to receive(:execute).exactly(:once) end - worker.perform - end - - context 'with an exclusive lease' do - it 'does not execute concurrently' do - expect(worker).to receive(:remove_lease).exactly(:once) - expect(worker2).not_to receive(:remove_lease) - - worker.perform - - stub_exclusive_lease_taken(worker_lease_key) - - worker2.perform - end - - it 'can execute in sequence' do - expect(worker).to receive(:remove_lease).at_least(:once) - expect(worker2).to receive(:remove_lease).at_least(:once) - - worker.perform - worker2.perform - end - - it 'cancels exclusive leases after worker perform' do - expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid) + expect_to_cancel_exclusive_lease(worker.lease_key, lease_uuid) - worker.perform - end - - context 'when the DropService fails' do - it 'ensures cancellation of the exclusive lease' do - expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid) - - allow_next_instance_of(Ci::StuckBuilds::DropService) do |service| - expect(service).to receive(:execute) do - raise 'The query timed out' - end - end - - expect { worker.perform }.to raise_error(/The query timed out/) - end - end + subject end end end |