diff options
55 files changed, 611 insertions, 200 deletions
diff --git a/app/assets/javascripts/packages_and_registries/settings/group/components/dependency_proxy_settings.vue b/app/assets/javascripts/packages_and_registries/settings/group/components/dependency_proxy_settings.vue index 2dbe36def0e..5815c6393a7 100644 --- a/app/assets/javascripts/packages_and_registries/settings/group/components/dependency_proxy_settings.vue +++ b/app/assets/javascripts/packages_and_registries/settings/group/components/dependency_proxy_settings.vue @@ -103,6 +103,7 @@ export default { :disabled="isLoading" :label="$options.i18n.label" data-qa-selector="dependency_proxy_setting_toggle" + data-testid="dependency-proxy-setting-toggle" /> </div> </template> diff --git a/app/assets/javascripts/vue_shared/components/markdown/suggestion_diff_header.vue b/app/assets/javascripts/vue_shared/components/markdown/suggestion_diff_header.vue index 70ab1767f99..648e9c9462f 100644 --- a/app/assets/javascripts/vue_shared/components/markdown/suggestion_diff_header.vue +++ b/app/assets/javascripts/vue_shared/components/markdown/suggestion_diff_header.vue @@ -1,11 +1,11 @@ <script> -import { GlButton, GlLoadingIcon, GlTooltipDirective, GlIcon } from '@gitlab/ui'; +import { GlBadge, GlButton, GlLoadingIcon, GlTooltipDirective, GlIcon } from '@gitlab/ui'; import { isLoggedIn } from '~/lib/utils/common_utils'; import { __ } from '~/locale'; import ApplySuggestion from './apply_suggestion.vue'; export default { - components: { GlIcon, GlButton, GlLoadingIcon, ApplySuggestion }, + components: { GlBadge, GlIcon, GlButton, GlLoadingIcon, ApplySuggestion }, directives: { 'gl-tooltip': GlTooltipDirective }, props: { batchSuggestionsCount: { @@ -134,9 +134,9 @@ export default { <gl-icon name="question-o" css-classes="link-highlight" /> </a> </div> - <div v-if="isApplied" class="badge badge-success" data-qa-selector="applied_badge"> + <gl-badge v-if="isApplied" variant="success" data-qa-selector="applied_badge"> {{ __('Applied') }} - </div> + </gl-badge> <div v-else-if="isApplying" class="d-flex align-items-center text-secondary" diff --git a/app/models/key.rb b/app/models/key.rb index 64385953865..a478434538c 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -46,7 +46,7 @@ class Key < ApplicationRecord scope :order_last_used_at_desc, -> { reorder(::Gitlab::Database.nulls_last_order('last_used_at', 'DESC')) } # Date is set specifically in this scope to improve query time. - scope :expired_and_not_notified, -> { where(["date(expires_at AT TIME ZONE 'UTC') BETWEEN '2000-01-01' AND CURRENT_DATE AND expiry_notification_delivered_at IS NULL"]) } + scope :expired_today_and_not_notified, -> { where(["date(expires_at AT TIME ZONE 'UTC') = CURRENT_DATE AND expiry_notification_delivered_at IS NULL"]) } scope :expiring_soon_and_not_notified, -> { where(["date(expires_at AT TIME ZONE 'UTC') > CURRENT_DATE AND date(expires_at AT TIME ZONE 'UTC') < ? AND before_expiry_notification_delivered_at IS NULL", DAYS_TO_EXPIRE.days.from_now.to_date]) } def self.regular_keys diff --git a/app/models/user.rb b/app/models/user.rb index 54e25a0d70d..b42a47cec0a 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -123,7 +123,7 @@ class User < ApplicationRecord # Profile has_many :keys, -> { regular_keys }, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :expired_and_unnotified_keys, -> { expired_and_not_notified }, class_name: 'Key' + has_many :expired_today_and_unnotified_keys, -> { expired_today_and_not_notified }, class_name: 'Key' has_many :expiring_soon_and_unnotified_keys, -> { expiring_soon_and_not_notified }, class_name: 'Key' has_many :deploy_keys, -> { where(type: 'DeployKey') }, dependent: :nullify # rubocop:disable Cop/ActiveRecordDependent has_many :group_deploy_keys diff --git a/app/presenters/blob_presenter.rb b/app/presenters/blob_presenter.rb index 10e81c1ac34..e703b9d79bb 100644 --- a/app/presenters/blob_presenter.rb +++ b/app/presenters/blob_presenter.rb @@ -121,6 +121,19 @@ class BlobPresenter < Gitlab::View::Presenter::Delegated end def transformed_blob_data - @transformed_blob ||= ( blob.path.ends_with?('.ipynb') && IpynbDiff.transform(blob.data, options: { include_metadata: false, cell_decorator: :percent }) ) || blob.data + @transformed_blob ||= if blob.path.ends_with?('.ipynb') + new_blob = IpynbDiff.transform(blob.data, + raise_errors: true, + options: { include_metadata: false, cell_decorator: :percent }) + + Gitlab::AppLogger.info(new_blob ? 'IPYNBDIFF_BLOB_GENERATED' : 'IPYNBDIFF_BLOB_NIL') + + new_blob + end + + @transformed_blob ||= blob.data + rescue IpynbDiff::InvalidNotebookError => e + Gitlab::ErrorTracking.track_exception(e, issue_url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/344676') + blob.data end end diff --git a/app/services/loose_foreign_keys/batch_cleaner_service.rb b/app/services/loose_foreign_keys/batch_cleaner_service.rb index a623524b2b1..caee2dcf753 100644 --- a/app/services/loose_foreign_keys/batch_cleaner_service.rb +++ b/app/services/loose_foreign_keys/batch_cleaner_service.rb @@ -7,6 +7,10 @@ module LooseForeignKeys @deleted_parent_records = deleted_parent_records @modification_tracker = modification_tracker @models_by_table_name = models_by_table_name + @deleted_records_counter = Gitlab::Metrics.counter( + :loose_foreign_key_processed_deleted_records, + 'The number of processed loose foreign key deleted records' + ) end def execute @@ -21,13 +25,15 @@ module LooseForeignKeys return if modification_tracker.over_limit? # At this point, all associations are cleaned up, we can update the status of the parent records - LooseForeignKeys::DeletedRecord + update_count = LooseForeignKeys::DeletedRecord .mark_records_processed_for_table_between(deleted_parent_records.first.fully_qualified_table_name, deleted_parent_records.first, deleted_parent_records.last) + + deleted_records_counter.increment({ table: parent_klass.table_name, db_config_name: LooseForeignKeys::DeletedRecord.connection.pool.db_config.name }, update_count) end private - attr_reader :parent_klass, :deleted_parent_records, :modification_tracker, :models_by_table_name + attr_reader :parent_klass, :deleted_parent_records, :modification_tracker, :models_by_table_name, :deleted_records_counter def record_result(cleaner, result) if cleaner.async_delete? diff --git a/app/workers/ssh_keys/expired_notification_worker.rb b/app/workers/ssh_keys/expired_notification_worker.rb index d8553b5a9a2..dc1efce51ce 100644 --- a/app/workers/ssh_keys/expired_notification_worker.rb +++ b/app/workers/ssh_keys/expired_notification_worker.rb @@ -29,7 +29,7 @@ module SshKeys ) ]) - scope = Key.expired_and_not_notified.order(order) + scope = Key.expired_today_and_not_notified.order(order) iterator = Gitlab::Pagination::Keyset::Iterator.new(scope: scope, use_union_optimization: true) iterator.each_batch(of: BATCH_SIZE) do |relation| @@ -37,7 +37,7 @@ module SshKeys users.each do |user| with_context(user: user) do - Keys::ExpiryNotificationService.new(user, { keys: user.expired_and_unnotified_keys, expiring_soon: false }).execute + Keys::ExpiryNotificationService.new(user, { keys: user.expired_today_and_unnotified_keys, expiring_soon: false }).execute end end end diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index fa634da6fef..debaa1e6a82 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -571,7 +571,7 @@ Settings.cron_jobs['user_status_cleanup_batch_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['user_status_cleanup_batch_worker']['cron'] ||= '* * * * *' Settings.cron_jobs['user_status_cleanup_batch_worker']['job_class'] = 'UserStatusCleanup::BatchWorker' Settings.cron_jobs['ssh_keys_expired_notification_worker'] ||= Settingslogic.new({}) -Settings.cron_jobs['ssh_keys_expired_notification_worker']['cron'] ||= '0 2 * * *' +Settings.cron_jobs['ssh_keys_expired_notification_worker']['cron'] ||= '0 2,14 * * *' Settings.cron_jobs['ssh_keys_expired_notification_worker']['job_class'] = 'SshKeys::ExpiredNotificationWorker' Settings.cron_jobs['namespaces_in_product_marketing_emails_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['namespaces_in_product_marketing_emails_worker']['cron'] ||= '0 16 * * *' diff --git a/db/migrate/20211101132310_add_reindexing_queue.rb b/db/migrate/20211101132310_add_reindexing_queue.rb new file mode 100644 index 00000000000..d9d1f9dce89 --- /dev/null +++ b/db/migrate/20211101132310_add_reindexing_queue.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddReindexingQueue < Gitlab::Database::Migration[1.0] + def change + create_table :postgres_reindex_queued_actions do |t| + t.text :index_identifier, null: false, limit: 255 + t.integer :state, limit: 2, null: false, default: 0 + t.timestamps_with_timezone null: false + + t.index :state + end + + change_column_default :postgres_reindex_queued_actions, :created_at, from: nil, to: -> { 'NOW()' } + change_column_default :postgres_reindex_queued_actions, :updated_at, from: nil, to: -> { 'NOW()' } + end +end diff --git a/db/post_migrate/20211027112901_drop_index_keys_on_expires_at_and_before_expiry_notification_undelivered.rb b/db/post_migrate/20211027112901_drop_index_keys_on_expires_at_and_before_expiry_notification_undelivered.rb new file mode 100644 index 00000000000..3447c4da81a --- /dev/null +++ b/db/post_migrate/20211027112901_drop_index_keys_on_expires_at_and_before_expiry_notification_undelivered.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class DropIndexKeysOnExpiresAtAndBeforeExpiryNotificationUndelivered < Gitlab::Database::Migration[1.0] + DOWNTIME = false + INDEX_NAME = 'index_keys_on_expires_at_and_expiry_notification_undelivered' + disable_ddl_transaction! + + def up + remove_concurrent_index_by_name(:keys, INDEX_NAME) + end + + def down + add_concurrent_index :keys, + "date(timezone('UTC', expires_at)), expiry_notification_delivered_at", + where: 'expiry_notification_delivered_at IS NULL', name: INDEX_NAME + end +end diff --git a/db/schema_migrations/20211027112901 b/db/schema_migrations/20211027112901 new file mode 100644 index 00000000000..0e4cb808f00 --- /dev/null +++ b/db/schema_migrations/20211027112901 @@ -0,0 +1 @@ +83c1d699e5de98007ef815b5f9dfbc9a644c6289bf86832af1d5b8a6d3d83659
\ No newline at end of file diff --git a/db/schema_migrations/20211101132310 b/db/schema_migrations/20211101132310 new file mode 100644 index 00000000000..08a95d2747e --- /dev/null +++ b/db/schema_migrations/20211101132310 @@ -0,0 +1 @@ +3e02605ce307d0ce37c3830e6909e7cfe5632408a757adf59209a70da92c0bc6
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 8e0442d40f0..e8963c58955 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -17856,6 +17856,24 @@ CREATE SEQUENCE postgres_reindex_actions_id_seq ALTER SEQUENCE postgres_reindex_actions_id_seq OWNED BY postgres_reindex_actions.id; +CREATE TABLE postgres_reindex_queued_actions ( + id bigint NOT NULL, + index_identifier text NOT NULL, + state smallint DEFAULT 0 NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT check_e4b01395c0 CHECK ((char_length(index_identifier) <= 255)) +); + +CREATE SEQUENCE postgres_reindex_queued_actions_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE postgres_reindex_queued_actions_id_seq OWNED BY postgres_reindex_queued_actions.id; + CREATE TABLE programming_languages ( id integer NOT NULL, name character varying NOT NULL, @@ -21756,6 +21774,8 @@ ALTER TABLE ONLY postgres_async_indexes ALTER COLUMN id SET DEFAULT nextval('pos ALTER TABLE ONLY postgres_reindex_actions ALTER COLUMN id SET DEFAULT nextval('postgres_reindex_actions_id_seq'::regclass); +ALTER TABLE ONLY postgres_reindex_queued_actions ALTER COLUMN id SET DEFAULT nextval('postgres_reindex_queued_actions_id_seq'::regclass); + ALTER TABLE ONLY product_analytics_events_experimental ALTER COLUMN id SET DEFAULT nextval('product_analytics_events_experimental_id_seq'::regclass); ALTER TABLE ONLY programming_languages ALTER COLUMN id SET DEFAULT nextval('programming_languages_id_seq'::regclass); @@ -23583,6 +23603,9 @@ ALTER TABLE ONLY postgres_async_indexes ALTER TABLE ONLY postgres_reindex_actions ADD CONSTRAINT postgres_reindex_actions_pkey PRIMARY KEY (id); +ALTER TABLE ONLY postgres_reindex_queued_actions + ADD CONSTRAINT postgres_reindex_queued_actions_pkey PRIMARY KEY (id); + ALTER TABLE ONLY programming_languages ADD CONSTRAINT programming_languages_pkey PRIMARY KEY (id); @@ -25704,8 +25727,6 @@ CREATE INDEX index_jira_imports_on_user_id ON jira_imports USING btree (user_id) CREATE INDEX index_jira_tracker_data_on_service_id ON jira_tracker_data USING btree (service_id); -CREATE INDEX index_keys_on_expires_at_and_expiry_notification_undelivered ON keys USING btree (date(timezone('UTC'::text, expires_at)), expiry_notification_delivered_at) WHERE (expiry_notification_delivered_at IS NULL); - CREATE INDEX index_keys_on_expires_at_and_id ON keys USING btree (date(timezone('UTC'::text, expires_at)), id) WHERE (expiry_notification_delivered_at IS NULL); CREATE UNIQUE INDEX index_keys_on_fingerprint ON keys USING btree (fingerprint); @@ -26264,6 +26285,8 @@ CREATE UNIQUE INDEX index_postgres_async_indexes_on_name ON postgres_async_index CREATE INDEX index_postgres_reindex_actions_on_index_identifier ON postgres_reindex_actions USING btree (index_identifier); +CREATE INDEX index_postgres_reindex_queued_actions_on_state ON postgres_reindex_queued_actions USING btree (state); + CREATE UNIQUE INDEX index_programming_languages_on_name ON programming_languages USING btree (name); CREATE INDEX index_project_access_tokens_on_project_id ON project_access_tokens USING btree (project_id); diff --git a/doc/user/admin_area/settings/deprecated_api_rate_limits.md b/doc/user/admin_area/settings/deprecated_api_rate_limits.md index a2edb6da86e..9be703f3b82 100644 --- a/doc/user/admin_area/settings/deprecated_api_rate_limits.md +++ b/doc/user/admin_area/settings/deprecated_api_rate_limits.md @@ -47,7 +47,7 @@ To override the general user and IP rate limits for requests to deprecated API e 1. Select the **Maximum authenticated API requests per period per user**. 1. Select the **Authenticated API rate limit period in seconds**. -## Resources +## Related topics - [Rate limits](../../../security/rate_limits.md) - [User and IP rate limits](user_and_ip_rate_limits.md) diff --git a/doc/user/admin_area/settings/files_api_rate_limits.md b/doc/user/admin_area/settings/files_api_rate_limits.md index f91d4b2fb0c..4f0f50dbcd2 100644 --- a/doc/user/admin_area/settings/files_api_rate_limits.md +++ b/doc/user/admin_area/settings/files_api_rate_limits.md @@ -49,7 +49,7 @@ To override the general user and IP rate limits for requests to the Repository f 1. Select the **Max authenticated API requests per period per user**. 1. Select the **Authenticated API rate limit period in seconds**. -## Resources +## Related topics - [Rate limits](../../../security/rate_limits.md) - [Repository files API](../../../api/repository_files.md) diff --git a/doc/user/admin_area/settings/git_lfs_rate_limits.md b/doc/user/admin_area/settings/git_lfs_rate_limits.md index 8a0754374e2..adc6cc2b11b 100644 --- a/doc/user/admin_area/settings/git_lfs_rate_limits.md +++ b/doc/user/admin_area/settings/git_lfs_rate_limits.md @@ -29,7 +29,7 @@ supersede the [general user and IP rate limits](user_and_ip_rate_limits.md): 1. Enter a value for **Authenticated Git LFS rate limit period in seconds**. 1. Select **Save changes**. -## Resources +## Related topics - [Rate limiting](../../../security/rate_limits.md) - [User and IP rate limits](user_and_ip_rate_limits.md) diff --git a/doc/user/project/index.md b/doc/user/project/index.md index f3173736e9b..78cd2f8fb79 100644 --- a/doc/user/project/index.md +++ b/doc/user/project/index.md @@ -47,7 +47,7 @@ Projects include the following [features](https://about.gitlab.com/features/): strategy and get reviewed by your team. - [Merge Request Approvals](merge_requests/approvals/index.md): Ask for approval before implementing a change. - - [Fix merge conflicts from the UI](merge_requests/resolve_conflicts.md): View Git diffs from the GitLab UI. + - [Fix merge conflicts from the UI](merge_requests/conflicts.md): View Git diffs from the GitLab UI. - [Review Apps](../../ci/review_apps/index.md): By branch, preview the results of the changes proposed in a merge request. - [Labels](labels.md): Organize issues and merge requests by labels. diff --git a/doc/user/project/merge_requests/img/merge_request_tab_position_v13_11.png b/doc/user/project/merge_requests/img/merge_request_tab_position_v13_11.png Binary files differdeleted file mode 100644 index 52c715840f1..00000000000 --- a/doc/user/project/merge_requests/img/merge_request_tab_position_v13_11.png +++ /dev/null diff --git a/doc/user/project/merge_requests/img/project_merge_requests_list_view_v13_5.png b/doc/user/project/merge_requests/img/project_merge_requests_list_view_v13_5.png Binary files differdeleted file mode 100644 index 625d47b1142..00000000000 --- a/doc/user/project/merge_requests/img/project_merge_requests_list_view_v13_5.png +++ /dev/null diff --git a/doc/user/project/merge_requests/index.md b/doc/user/project/merge_requests/index.md index 4c440b486b0..54b97eb5732 100644 --- a/doc/user/project/merge_requests/index.md +++ b/doc/user/project/merge_requests/index.md @@ -17,69 +17,42 @@ Merge requests include: - A comment section for discussion threads. - The list of commits. -To get started, read the [introduction to merge requests](getting_started.md). +Read more about [how to get started](getting_started.md). -## Merge request tabs +## View merge requests -Merge requests contain tabs at the top of the page to help you navigate to -important parts of the merge request: +You can view merge requests for your project, group, or yourself. - +### View merge requests for a project -- **Overview**: Contains the description, notifications from pipelines, and a - discussion area for [comment threads](../../discussions/index.md#resolve-a-thread) - and [code suggestions](reviews/suggestions.md). The right sidebar provides fields - to add assignees, reviewers, labels, and a milestone to your work, and the - [merge request widgets area](widgets.md) reports results from pipelines and tests. -- **Commits**: Contains a list of commits added to this merge request. For more - information, read [Commits tab in merge requests](commits.md). -- **Pipelines**: If configured, contains a list of recent [GitLab CI/CD](../../../ci/index.md) - pipelines and their status. -- **Changes**: Contains the diffs of files changed by this merge request. You can - [configure the display](changes.md). +To view all merge requests for a project: -## View merge requests +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Merge requests**. + +Or, to use a [keyboard shortcut](../../shortcuts.md), press <kbd>g</kbd> + <kbd>m</kbd>. + +### View merge requests for all projects in a group -To view a list of merge requests: +To view merge requests for all projects in a group: -- **Merge requests for a project**: Go to your project and select **Merge requests**, or use - the <kbd>g</kbd> + <kbd>m</kbd> [keyboard shortcut](../../shortcuts.md) from a page in your project. -- **All projects in a group**: Go to your group and select **Merge requests**. - If your group contains subgroups, this view also displays merge requests from the subgroup projects. - GitLab displays a count of open merge requests in the left sidebar, but - [caches the value](reviews/index.md#cached-merge-request-count) for groups with a large number of - open merge requests. -- **Merge requests assigned to you**: On any GitLab page, select **Merge requests** - in the top bar, or use the <kbd>Shift</kbd> + <kbd>m</kbd> - [global keyboard shortcut](../../shortcuts.md). +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Merge requests**. -GitLab displays open merge requests, with tabs to filter the list by open and closed status: +If your group contains subgroups, this view also displays merge requests from the subgroup projects. - +## View all merge requests assigned to you + +To view all merge requests assigned to you: + +1. On the top bar, put your cursor in the **Search** box. +1. From the dropdown list, select **Merge requests assigned to me**. + +Or, to use a [keyboard shortcut](../../shortcuts.md), press <kbd>Shift</kbd> + <kbd>m</kbd>. You can [search and filter](../../search/index.md#filter-issue-and-merge-request-lists), the results, or select a merge request to begin a review. -## Merge request sidebar - -The **Overview** tab of a merge request displays a sidebar. In this sidebar, you -can assign, categorize, and track progress on a merge request: - -- [**Assignee**](getting_started.md#assignee): Designate the directly responsible - individual (DRI) for a merge request. With - [multiple assignees](getting_started.md#multiple-assignees), you can assign a - merge request to more than one person at a time. -- [**Reviewer**](reviews/index.md): Designate a team member to review a merge request. - Higher tiers can assign multiple reviewers, and [require approvals](approvals/index.md) - from these reviewers. -- [**Milestone**](../milestones/index.md): Track time-sensitive changes. -- [**Time tracking**](../time_tracking.md): Time spent on a merge request. -- [**Labels**](../labels.md): Categorize a merge request and display it on - appropriate [issue boards](../issue_board.md). -- **Participants**: A list of users participating or watching a merge request. -- [**Notifications**](../../profile/notifications.md): A toggle to select whether - or not to receive notifications for updates to a merge request. - ## Add changes to a merge request If you have permission to add changes to a merge request, you can add your changes @@ -160,3 +133,7 @@ For a web developer writing a webpage for your company's website: - [Authorization for merge requests](authorization_for_merge_requests.md) - [Testing and reports](testing_and_reports_in_merge_requests.md) - [GitLab keyboard shortcuts](../../shortcuts.md) +- [Comments and threads](../../discussions/index.md) +- [Suggest code changes](reviews/suggestions.md) +- [Commits](commits.md) +- [CI/CD pipelines](../../../ci/index.md) diff --git a/doc/user/project/merge_requests/resolve_conflicts.md b/doc/user/project/merge_requests/resolve_conflicts.md index 169dcbf5c9e..611f37a949b 100644 --- a/doc/user/project/merge_requests/resolve_conflicts.md +++ b/doc/user/project/merge_requests/resolve_conflicts.md @@ -3,7 +3,7 @@ redirect_to: 'conflicts.md' remove_date: '2022-01-26' --- -This document was moved to [another location](../path/to/file/index.md). +This document was moved to [another location](conflicts.md). <!-- This redirect file can be deleted after <2022-01-26>. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page -->
\ No newline at end of file +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/user/project/merge_requests/reviews/index.md b/doc/user/project/merge_requests/reviews/index.md index 9b1ea833958..597dcb3dfb9 100644 --- a/doc/user/project/merge_requests/reviews/index.md +++ b/doc/user/project/merge_requests/reviews/index.md @@ -211,7 +211,7 @@ These features are associated with merge requests: When viewing the commit details page, GitLab links to the merge request(s) containing that commit. - [Merge requests versions](../versions.md): Select and compare the different versions of merge request diffs -- [Resolve conflicts](../resolve_conflicts.md): +- [Resolve conflicts](../conflicts.md): GitLab can provide the option to resolve certain merge request conflicts in the GitLab UI. - [Revert changes](../revert_changes.md): Revert changes from any commit from a merge request. diff --git a/doc/user/project/repository/branches/default.md b/doc/user/project/repository/branches/default.md index 5cd025f017d..bc6bc799670 100644 --- a/doc/user/project/repository/branches/default.md +++ b/doc/user/project/repository/branches/default.md @@ -167,7 +167,7 @@ changed, GitLab records the name of the old default branch. If that branch is deleted, attempts to view a file or directory on it are redirected to the current default branch, instead of displaying the "not found" page. -## Resources +## Related topics - [Configure a default branch for your wiki](../../wiki/index.md) - [Discussion of default branch renaming](https://lore.kernel.org/git/pull.656.v4.git.1593009996.gitgitgadget@gmail.com/) diff --git a/doc/user/project/repository/vscode.md b/doc/user/project/repository/vscode.md index 11dee1c78f4..c021d999588 100644 --- a/doc/user/project/repository/vscode.md +++ b/doc/user/project/repository/vscode.md @@ -39,7 +39,7 @@ you can [configure](https://marketplace.visualstudio.com/items?itemName=GitLab.g Report any issues, bugs, or feature requests in the [`gitlab-vscode-extension` issue queue](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/issues). -## Resources +## Related topics - [Download the extension](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow) - [Extension documentation](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/blob/main/README.md) diff --git a/doc/user/project/repository/x509_signed_commits/index.md b/doc/user/project/repository/x509_signed_commits/index.md index 17031dd29af..2db7ae308bd 100644 --- a/doc/user/project/repository/x509_signed_commits/index.md +++ b/doc/user/project/repository/x509_signed_commits/index.md @@ -158,7 +158,7 @@ can start signing your tags: git config --global tag.gpgsign true ``` -## Resources +## Related topics - [Rake task for X.509 signatures](../../../../raketasks/x509_signatures.md) diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index e5ce3984bee..357ef48f3e0 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -263,14 +263,28 @@ module Gitlab # A patch over ActiveRecord::Base.transaction that provides # observability into transactional methods. def transaction(**options, &block) - if options[:requires_new] && connection.transaction_open? - ::Gitlab::Database::Metrics.subtransactions_increment(self.name) - end + transaction_type = get_transaction_type(connection.transaction_open?, options[:requires_new]) + + ::Gitlab::Database::Metrics.subtransactions_increment(self.name) if transaction_type == :sub_transaction + + payload = { connection: connection, transaction_type: transaction_type } - ActiveSupport::Notifications.instrument('transaction.active_record', { connection: connection }) do + ActiveSupport::Notifications.instrument('transaction.active_record', payload) do super(**options, &block) end end + + private + + def get_transaction_type(transaction_open, requires_new_flag) + if transaction_open + return :sub_transaction if requires_new_flag + + return :fake_transaction + end + + :real_transaction + end end end diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 9490fee1ae4..7969f6cda40 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -2,6 +2,13 @@ # This module gathers information about table to schema mapping # to understand table affinity +# +# Each table / view needs to have assigned gitlab_schema. Names supported today: +# +# - gitlab_shared - defines a set of tables that are found on all databases (data accessed is dependent on connection) +# - gitlab_main / gitlab_ci - defines a set of tables that can only exist on a given database +# + module Gitlab module Database module GitlabSchema diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 4fb48e932b4..544b6ce4ed0 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -32,6 +32,7 @@ approval_project_rules_users: :gitlab_main approvals: :gitlab_main approver_groups: :gitlab_main approvers: :gitlab_main +ar_internal_metadata: :gitlab_shared atlassian_identities: :gitlab_main audit_events_external_audit_event_destinations: :gitlab_main audit_events: :gitlab_main @@ -171,7 +172,7 @@ design_management_designs: :gitlab_main design_management_designs_versions: :gitlab_main design_management_versions: :gitlab_main design_user_mentions: :gitlab_main -detached_partitions: :gitlab_main +detached_partitions: :gitlab_shared diff_note_positions: :gitlab_main dora_daily_metrics: :gitlab_main draft_notes: :gitlab_main @@ -373,8 +374,14 @@ personal_access_tokens: :gitlab_main plan_limits: :gitlab_main plans: :gitlab_main pool_repositories: :gitlab_main -postgres_async_indexes: :gitlab_main -postgres_reindex_actions: :gitlab_main +postgres_async_indexes: :gitlab_shared +postgres_foreign_keys: :gitlab_shared +postgres_index_bloat_estimates: :gitlab_shared +postgres_indexes: :gitlab_shared +postgres_partitioned_tables: :gitlab_shared +postgres_partitions: :gitlab_shared +postgres_reindex_actions: :gitlab_shared +postgres_reindex_queued_actions: :gitlab_main product_analytics_events_experimental: :gitlab_main programming_languages: :gitlab_main project_access_tokens: :gitlab_main @@ -438,6 +445,7 @@ reviews: :gitlab_main routes: :gitlab_main saml_group_links: :gitlab_main saml_providers: :gitlab_main +schema_migrations: :gitlab_shared scim_identities: :gitlab_main scim_oauth_access_tokens: :gitlab_main security_findings: :gitlab_main diff --git a/lib/gitlab/database/migrations/observers.rb b/lib/gitlab/database/migrations/observers.rb index 140b3feed64..b890e62c2d0 100644 --- a/lib/gitlab/database/migrations/observers.rb +++ b/lib/gitlab/database/migrations/observers.rb @@ -9,7 +9,8 @@ module Gitlab TotalDatabaseSizeChange, QueryStatistics, QueryLog, - QueryDetails + QueryDetails, + TransactionDuration ] end end diff --git a/lib/gitlab/database/migrations/observers/transaction_duration.rb b/lib/gitlab/database/migrations/observers/transaction_duration.rb new file mode 100644 index 00000000000..a96b94334cf --- /dev/null +++ b/lib/gitlab/database/migrations/observers/transaction_duration.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module Observers + class TransactionDuration < MigrationObserver + def before + file_path = File.join(output_dir, "#{observation.version}_#{observation.name}-transaction-duration.json") + @file = File.open(file_path, 'wb') + @writer = Oj::StreamWriter.new(@file, {}) + @writer.push_array + @subscriber = ActiveSupport::Notifications.subscribe('transaction.active_record') do |*args| + record_sql_event(*args) + end + end + + def after + ActiveSupport::Notifications.unsubscribe(@subscriber) + @writer.pop_all + @writer.flush + @file.close + end + + def record + # no-op + end + + def record_sql_event(_name, started, finished, _unique_id, payload) + return if payload[:transaction_type] == :fake_transaction + + @writer.push_value({ + start_time: started.iso8601(6), + end_time: finished.iso8601(6), + transaction_type: payload[:transaction_type] + }) + end + end + end + end + end +end diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb index 4d938686ccf..4a9d8728c83 100644 --- a/lib/gitlab/database/postgres_index.rb +++ b/lib/gitlab/database/postgres_index.rb @@ -11,6 +11,7 @@ module Gitlab has_one :bloat_estimate, class_name: 'Gitlab::Database::PostgresIndexBloatEstimate', foreign_key: :identifier has_many :reindexing_actions, class_name: 'Gitlab::Database::Reindexing::ReindexAction', foreign_key: :index_identifier + has_many :queued_reindexing_actions, class_name: 'Gitlab::Database::Reindexing::QueuedAction', foreign_key: :index_identifier scope :by_identifier, ->(identifier) do raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 6e2bad05b71..7a22e324bdb 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -15,13 +15,45 @@ module Gitlab # on e.g. vacuum. REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30 - # candidate_indexes: Array of Gitlab::Database::PostgresIndex - def self.perform(candidate_indexes, how_many: DEFAULT_INDEXES_PER_INVOCATION) - IndexSelection.new(candidate_indexes).take(how_many).each do |index| + # Performs automatic reindexing for a limited number of indexes per call + # 1. Consume from the explicit reindexing queue + # 2. Apply bloat heuristic to find most bloated indexes and reindex those + def self.automatic_reindexing(maximum_records: DEFAULT_INDEXES_PER_INVOCATION) + # Cleanup leftover temporary indexes from previous, possibly aborted runs (if any) + cleanup_leftovers! + + # Consume from the explicit reindexing queue first + done_counter = perform_from_queue(maximum_records: maximum_records) + + return if done_counter >= maximum_records + + # Execute reindexing based on bloat heuristic + perform_with_heuristic(maximum_records: maximum_records - done_counter) + end + + # Reindex based on bloat heuristic for a limited number of indexes per call + # + # We use a bloat heuristic to estimate the index bloat and pick the + # most bloated indexes for reindexing. + def self.perform_with_heuristic(candidate_indexes = Gitlab::Database::PostgresIndex.reindexing_support, maximum_records: DEFAULT_INDEXES_PER_INVOCATION) + IndexSelection.new(candidate_indexes).take(maximum_records).each do |index| Coordinator.new(index).perform end end + # Reindex indexes that have been explicitly enqueued (for a limited number of indexes per call) + def self.perform_from_queue(maximum_records: DEFAULT_INDEXES_PER_INVOCATION) + QueuedAction.in_queue_order.limit(maximum_records).each do |queued_entry| + Coordinator.new(queued_entry.index).perform + + queued_entry.done! + rescue StandardError => e + queued_entry.failed! + + Gitlab::AppLogger.error("Failed to perform reindexing action on queued entry #{queued_entry}: #{e}") + end.size + end + def self.cleanup_leftovers! PostgresIndex.reindexing_leftovers.each do |index| Gitlab::AppLogger.info("Removing index #{index.identifier} which is a leftover, temporary index from previous reindexing activity") diff --git a/lib/gitlab/database/reindexing/queued_action.rb b/lib/gitlab/database/reindexing/queued_action.rb new file mode 100644 index 00000000000..c2039a289da --- /dev/null +++ b/lib/gitlab/database/reindexing/queued_action.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Reindexing + class QueuedAction < SharedModel + self.table_name = 'postgres_reindex_queued_actions' + + enum state: { queued: 0, done: 1, failed: 2 } + + belongs_to :index, foreign_key: :index_identifier, class_name: 'Gitlab::Database::PostgresIndex' + + scope :in_queue_order, -> { queued.order(:created_at) } + + def to_s + "queued action [ id = #{id}, index: #{index_identifier} ]" + end + end + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index cd08e9e2526..4cd5e999027 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -458,9 +458,17 @@ module Gitlab new_diff = IpynbDiff.diff(from, to, diff_opts: { context: 5, include_diff_info: true }, - transform_options: { cell_decorator: :percent } ) + transform_options: { cell_decorator: :percent }, + raise_if_invalid_notebook: true) diff.diff = new_diff.scan(/.*\n/)[2..-1].join('') if new_diff + + Gitlab::AppLogger.info({ message: new_diff ? 'IPYNB_DIFF_GENERATED' : 'IPYNB_DIFF_NIL', + from: from&.to_s, to: to&.to_s, + lib_version: Gem.loaded_specs["ipynbdiff"].version.version }) + + rescue IpynbDiff::InvalidNotebookError => e + Gitlab::ErrorTracking.track_exception(e, issue_url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/344676') end def alternate_viewer_class diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 4d8394a6a4d..730c140ee4e 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -160,39 +160,46 @@ namespace :gitlab do Rake::Task['gitlab:db:create_dynamic_partitions'].invoke end - desc 'reindex a regular index without downtime to eliminate bloat' - task :reindex, [:index_name, :database] => :environment do |_, args| - unless Feature.enabled?(:database_reindexing, type: :ops) + desc 'execute reindexing without downtime to eliminate bloat' + task reindex: :environment do + unless Feature.enabled?(:database_reindexing, type: :ops, default_enabled: :yaml) puts "This feature (database_reindexing) is currently disabled.".color(:yellow) exit end Gitlab::Database::EachDatabase.each_database_connection do |connection, connection_name| - indexes = Gitlab::Database::PostgresIndex.reindexing_support - - if (identifier = args[:index_name]) && (args.fetch(:database, 'main') == connection_name) - raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ - - indexes = indexes.where(identifier: identifier) - - raise "Index #{args[:index_name]} for #{connection_name} database not found or not supported" if indexes.empty? - end - Gitlab::Database::SharedModel.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false) - # Cleanup leftover temporary indexes from previous, possibly aborted runs (if any) - Gitlab::Database::Reindexing.cleanup_leftovers! - # Hack: Before we do actual reindexing work, create async indexes Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops) - Gitlab::Database::Reindexing.perform(indexes) + Gitlab::Database::Reindexing.automatic_reindexing end rescue StandardError => e Gitlab::AppLogger.error(e) raise end + desc 'Enqueue an index for reindexing' + task :enqueue_reindexing_action, [:index_name, :database] => :environment do |_, args| + connection = Gitlab::Database.databases[args.fetch(:database, Gitlab::Database::PRIMARY_DATABASE_NAME)] + + Gitlab::Database::SharedModel.using_connection(connection.scope.connection) do + queued_action = Gitlab::Database::PostgresIndex.find(args[:index_name]).queued_reindexing_actions.create! + + puts "Queued reindexing action: #{queued_action}" + puts "There are #{Gitlab::Database::Reindexing::QueuedAction.queued.size} queued actions in total." + end + + unless Feature.enabled?(:database_reindexing, type: :ops, default_enabled: :yaml) + puts <<~NOTE.color(:yellow) + Note: database_reindexing feature is currently disabled. + + Enable with: Feature.enable(:database_reindexing) + NOTE + end + end + desc 'Check if there have been user additions to the database' task active: :environment do if ActiveRecord::Base.connection.migration_context.needs_migration? diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 5a1248ee28a..bcb066ea996 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -22656,9 +22656,6 @@ msgstr "" msgid "NamespaceStorageSize|push to your repository, create pipelines, create issues or add comments. To reduce storage capacity, delete unused repositories, artifacts, wikis, issues, and pipelines." msgstr "" -msgid "NamespaceStorageSize|push to your repository, create pipelines, create issues or add comments. To reduce storage capacity, delete unused repositories, artifacts, wikis, issues, and pipelines. To learn more about reducing storage capacity please visit our docs." -msgstr "" - msgid "NamespaceUserCap|Pending users must be reviewed and approved by a group owner. Learn more about %{user_caps_link_start}User Caps%{link_end} and %{users_pending_approval_link_start}Users Pending Approval%{link_end}." msgstr "" diff --git a/package.json b/package.json index 79a3d16116a..791e475341b 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "@gitlab/favicon-overlay": "2.0.0", "@gitlab/svgs": "1.219.0", "@gitlab/tributejs": "1.0.0", - "@gitlab/ui": "32.28.0", + "@gitlab/ui": "32.31.0", "@gitlab/visual-review-tools": "1.6.1", "@rails/actioncable": "6.1.4-1", "@rails/ujs": "6.1.4-1", diff --git a/spec/factories/gitlab/database/reindexing/queued_action.rb b/spec/factories/gitlab/database/reindexing/queued_action.rb new file mode 100644 index 00000000000..30e12a81272 --- /dev/null +++ b/spec/factories/gitlab/database/reindexing/queued_action.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :reindexing_queued_action, class: 'Gitlab::Database::Reindexing::QueuedAction' do + association :index, factory: :postgres_index + + state { Gitlab::Database::Reindexing::QueuedAction.states[:queued] } + index_identifier { index.identifier } + end +end diff --git a/spec/features/groups/dependency_proxy_spec.rb b/spec/features/groups/dependency_proxy_spec.rb index d6b0bdc8ea4..623fb065bfc 100644 --- a/spec/features/groups/dependency_proxy_spec.rb +++ b/spec/features/groups/dependency_proxy_spec.rb @@ -56,9 +56,14 @@ RSpec.describe 'Group Dependency Proxy' do visit settings_path wait_for_requests - click_button 'Enable Proxy' + proxy_toggle = find('[data-testid="dependency-proxy-setting-toggle"]') + proxy_toggle_button = proxy_toggle.find('button') - expect(page).to have_button 'Enable Proxy', class: '!is-checked' + expect(proxy_toggle).to have_css("button.is-checked") + + proxy_toggle_button.click + + expect(proxy_toggle).not_to have_css("button.is-checked") visit path diff --git a/spec/features/issues/user_toggles_subscription_spec.rb b/spec/features/issues/user_toggles_subscription_spec.rb index 9809bb34d26..541bbc8a8e7 100644 --- a/spec/features/issues/user_toggles_subscription_spec.rb +++ b/spec/features/issues/user_toggles_subscription_spec.rb @@ -45,7 +45,7 @@ RSpec.describe "User toggles subscription", :js do it 'is disabled' do expect(page).to have_content('Disabled by project owner') - expect(page).to have_button('Notifications', class: 'is-disabled') + expect(page).to have_selector('[data-testid="subscription-toggle"]', class: 'is-disabled') end end end diff --git a/spec/features/projects/settings/packages_settings_spec.rb b/spec/features/projects/settings/packages_settings_spec.rb index 62f31fd027b..e70839e9720 100644 --- a/spec/features/projects/settings/packages_settings_spec.rb +++ b/spec/features/projects/settings/packages_settings_spec.rb @@ -19,7 +19,7 @@ RSpec.describe 'Projects > Settings > Packages', :js do let(:packages_enabled) { true } it 'displays the packages toggle button' do - expect(page).to have_button('Packages', class: 'gl-toggle') + expect(page).to have_selector('[data-testid="toggle-label"]', text: 'Packages') expect(page).to have_selector('input[name="project[packages_enabled]"] + button', visible: true) end end @@ -28,7 +28,7 @@ RSpec.describe 'Projects > Settings > Packages', :js do let(:packages_enabled) { false } it 'does not show up in UI' do - expect(page).not_to have_button('Packages', class: 'gl-toggle') + expect(page).not_to have_selector('[data-testid="toggle-label"]', text: 'Packages') end end end diff --git a/spec/frontend/alerts_settings/components/alerts_settings_form_spec.js b/spec/frontend/alerts_settings/components/alerts_settings_form_spec.js index 298596085ef..7e0d7ff93a1 100644 --- a/spec/frontend/alerts_settings/components/alerts_settings_form_spec.js +++ b/spec/frontend/alerts_settings/components/alerts_settings_form_spec.js @@ -58,7 +58,6 @@ describe('AlertsSettingsForm', () => { afterEach(() => { if (wrapper) { wrapper.destroy(); - wrapper = null; } }); @@ -69,7 +68,7 @@ describe('AlertsSettingsForm', () => { const enableIntegration = (index, value) => { findFormFields().at(index).setValue(value); - findFormToggle().trigger('click'); + findFormToggle().vm.$emit('change', true); }; describe('with default values', () => { diff --git a/spec/frontend/boards/components/sidebar/board_sidebar_subscription_spec.js b/spec/frontend/boards/components/sidebar/board_sidebar_subscription_spec.js index 8847f626c1f..6e1b528babc 100644 --- a/spec/frontend/boards/components/sidebar/board_sidebar_subscription_spec.js +++ b/spec/frontend/boards/components/sidebar/board_sidebar_subscription_spec.js @@ -14,8 +14,8 @@ describe('~/boards/components/sidebar/board_sidebar_subscription_spec.vue', () = let store; const findNotificationHeader = () => wrapper.find("[data-testid='notification-header-text']"); - const findToggle = () => wrapper.find(GlToggle); - const findGlLoadingIcon = () => wrapper.find(GlLoadingIcon); + const findToggle = () => wrapper.findComponent(GlToggle); + const findGlLoadingIcon = () => wrapper.findComponent(GlLoadingIcon); const createComponent = (activeBoardItem = { ...mockActiveIssue }) => { store = createStore(); @@ -32,7 +32,6 @@ describe('~/boards/components/sidebar/board_sidebar_subscription_spec.vue', () = afterEach(() => { wrapper.destroy(); - wrapper = null; store = null; jest.clearAllMocks(); }); @@ -104,7 +103,7 @@ describe('~/boards/components/sidebar/board_sidebar_subscription_spec.vue', () = expect(findGlLoadingIcon().exists()).toBe(false); - findToggle().trigger('click'); + findToggle().vm.$emit('change'); await wrapper.vm.$nextTick(); @@ -129,7 +128,7 @@ describe('~/boards/components/sidebar/board_sidebar_subscription_spec.vue', () = expect(findGlLoadingIcon().exists()).toBe(false); - findToggle().trigger('click'); + findToggle().vm.$emit('change'); await wrapper.vm.$nextTick(); @@ -152,7 +151,7 @@ describe('~/boards/components/sidebar/board_sidebar_subscription_spec.vue', () = }); jest.spyOn(wrapper.vm, 'setError').mockImplementation(() => {}); - findToggle().trigger('click'); + findToggle().vm.$emit('change'); await wrapper.vm.$nextTick(); expect(wrapper.vm.setError).toHaveBeenCalled(); diff --git a/spec/frontend/pipelines/graph/graph_component_wrapper_spec.js b/spec/frontend/pipelines/graph/graph_component_wrapper_spec.js index 2e8979f2b9d..db4de6deeb7 100644 --- a/spec/frontend/pipelines/graph/graph_component_wrapper_spec.js +++ b/spec/frontend/pipelines/graph/graph_component_wrapper_spec.js @@ -327,7 +327,7 @@ describe('Pipeline graph wrapper', () => { expect(getLinksLayer().exists()).toBe(true); expect(getLinksLayer().props('showLinks')).toBe(false); expect(getViewSelector().props('type')).toBe(LAYER_VIEW); - await getDependenciesToggle().trigger('click'); + await getDependenciesToggle().vm.$emit('change', true); jest.runOnlyPendingTimers(); await wrapper.vm.$nextTick(); expect(wrapper.findComponent(LinksLayer).props('showLinks')).toBe(true); diff --git a/spec/frontend/pipelines/graph/graph_view_selector_spec.js b/spec/frontend/pipelines/graph/graph_view_selector_spec.js index 5b2a29de443..f4faa25545b 100644 --- a/spec/frontend/pipelines/graph/graph_view_selector_spec.js +++ b/spec/frontend/pipelines/graph/graph_view_selector_spec.js @@ -111,7 +111,7 @@ describe('the graph view selector component', () => { expect(wrapper.emitted().updateShowLinksState).toBeUndefined(); expect(findToggleLoader().exists()).toBe(false); - await findDependenciesToggle().trigger('click'); + await findDependenciesToggle().vm.$emit('change', true); /* Loading happens before the event is emitted or timers are run. Then we run the timer because the event is emitted in setInterval diff --git a/spec/lib/gitlab/database/gitlab_schema_spec.rb b/spec/lib/gitlab/database/gitlab_schema_spec.rb index 02a9f6eef7b..4f94ce2e541 100644 --- a/spec/lib/gitlab/database/gitlab_schema_spec.rb +++ b/spec/lib/gitlab/database/gitlab_schema_spec.rb @@ -2,28 +2,37 @@ require 'spec_helper' RSpec.describe Gitlab::Database::GitlabSchema do - it 'matches all the tables in the database', :aggregate_failures do - # These tables do not need a gitlab_schema - excluded_tables = %w(ar_internal_metadata schema_migrations) + describe '.tables_to_schema' do + subject { described_class.tables_to_schema } - all_tables_in_database = ApplicationRecord.connection.tables + it 'all tables have assigned a known gitlab_schema' do + is_expected.to all( + match([be_a(String), be_in([:gitlab_shared, :gitlab_main, :gitlab_ci])]) + ) + end - all_tables_with_gitlab_schema = described_class.tables_to_schema.keys + # This being run across different databases indirectly also tests + # a general consistency of structure across databases + Gitlab::Database.database_base_models.each do |db_config_name, db_class| + let(:db_data_sources) { db_class.connection.data_sources } - missing = [] - all_tables_in_database.each do |table_in_database| - next if table_in_database.in?(excluded_tables) + context "for #{db_config_name} using #{db_class}" do + it 'new data sources are added' do + missing_tables = db_data_sources.to_set - subject.keys - missing << table_in_database unless all_tables_with_gitlab_schema.include?(table_in_database) - end + expect(missing_tables).to be_empty, \ + "Missing table(s) #{missing_tables.to_a} not found in #{described_class}.tables_to_schema. " \ + "Any new tables must be added to lib/gitlab/database/gitlab_schemas.yml." + end - extras = [] - all_tables_with_gitlab_schema.each do |table_with_gitlab_schema| - extras << table_with_gitlab_schema unless all_tables_in_database.include?(table_with_gitlab_schema) - end + it 'non-existing data sources are removed' do + extra_tables = subject.keys.to_set - db_data_sources - expect(missing).to be_empty, "Missing table(s) #{missing} not found in #{described_class}.tables_to_schema. Any new tables must be added to spec/support/database/gitlab_schemas.yml ." - - expect(extras).to be_empty, "Extra table(s) #{extras} found in #{described_class}.tables_to_schema. Any removed or renamed tables must be removed from spec/support/database/gitlab_schemas.yml ." + expect(extra_tables).to be_empty, \ + "Extra table(s) #{extra_tables.to_a} found in #{described_class}.tables_to_schema. " \ + "Any removed or renamed tables must be removed from lib/gitlab/database/gitlab_schemas.yml." + end + end + end end end diff --git a/spec/lib/gitlab/database/migrations/observers/transaction_duration_spec.rb b/spec/lib/gitlab/database/migrations/observers/transaction_duration_spec.rb new file mode 100644 index 00000000000..e65f89747c4 --- /dev/null +++ b/spec/lib/gitlab/database/migrations/observers/transaction_duration_spec.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe Gitlab::Database::Migrations::Observers::TransactionDuration do + subject(:transaction_duration_observer) { described_class.new(observation, directory_path) } + + let(:observation) { Gitlab::Database::Migrations::Observation.new(migration_version, migration_name) } + let(:directory_path) { Dir.mktmpdir } + let(:log_file) { "#{directory_path}/#{migration_version}_#{migration_name}-transaction-duration.json" } + let(:transaction_duration) { Gitlab::Json.parse(File.read(log_file)) } + let(:migration_version) { 20210422152437 } + let(:migration_name) { 'test' } + + after do + FileUtils.remove_entry(directory_path) + end + + it 'records real and sub transactions duration', :delete do + observe + + entry = transaction_duration[0] + start_time, end_time, transaction_type = entry.values_at('start_time', 'end_time', 'transaction_type') + start_time = DateTime.parse(start_time) + end_time = DateTime.parse(end_time) + + aggregate_failures do + expect(transaction_duration.size).to eq(3) + expect(start_time).to be_before(end_time) + expect(transaction_type).not_to be_nil + end + end + + context 'when there are sub-transactions' do + it 'records transaction duration' do + observe_sub_transaction + + expect(transaction_duration.size).to eq(1) + + entry = transaction_duration[0]['transaction_type'] + + expect(entry).to eql 'sub_transaction' + end + end + + context 'when there are real-transactions' do + it 'records transaction duration', :delete do + observe_real_transaction + + expect(transaction_duration.size).to eq(1) + + entry = transaction_duration[0]['transaction_type'] + + expect(entry).to eql 'real_transaction' + end + end + + private + + def observe + transaction_duration_observer.before + run_transaction + transaction_duration_observer.after + transaction_duration_observer.record + end + + def observe_sub_transaction + transaction_duration_observer.before + run_sub_transactions + transaction_duration_observer.after + transaction_duration_observer.record + end + + def observe_real_transaction + transaction_duration_observer.before + run_real_transactions + transaction_duration_observer.after + transaction_duration_observer.record + end + + def run_real_transactions + ActiveRecord::Base.transaction do + end + end + + def run_sub_transactions + ActiveRecord::Base.transaction(requires_new: true) do + end + end + + def run_transaction + ActiveRecord::Base.connection_pool.with_connection do |connection| + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::SharedModel.transaction do + Gitlab::Database::SharedModel.transaction(requires_new: true) do + Gitlab::Database::SharedModel.transaction do + Gitlab::Database::SharedModel.transaction do + Gitlab::Database::SharedModel.transaction(requires_new: true) do + end + end + end + end + end + end + end + end +end diff --git a/spec/lib/gitlab/database/reindexing_spec.rb b/spec/lib/gitlab/database/reindexing_spec.rb index 550f9db2b5b..13aff343432 100644 --- a/spec/lib/gitlab/database/reindexing_spec.rb +++ b/spec/lib/gitlab/database/reindexing_spec.rb @@ -4,10 +4,63 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Reindexing do include ExclusiveLeaseHelpers + include Database::DatabaseHelpers - describe '.perform' do - subject { described_class.perform(candidate_indexes) } + describe '.automatic_reindexing' do + subject { described_class.automatic_reindexing(maximum_records: limit) } + let(:limit) { 5 } + + before_all do + swapout_view_for_table(:postgres_indexes) + end + + before do + allow(Gitlab::Database::Reindexing).to receive(:cleanup_leftovers!) + allow(Gitlab::Database::Reindexing).to receive(:perform_from_queue).and_return(0) + allow(Gitlab::Database::Reindexing).to receive(:perform_with_heuristic).and_return(0) + end + + it 'cleans up leftovers, before consuming the queue' do + expect(Gitlab::Database::Reindexing).to receive(:cleanup_leftovers!).ordered + expect(Gitlab::Database::Reindexing).to receive(:perform_from_queue).ordered + + subject + end + + context 'with records in the queue' do + before do + create(:reindexing_queued_action) + end + + context 'with enough records in the queue to reach limit' do + let(:limit) { 1 } + + it 'does not perform reindexing with heuristic' do + expect(Gitlab::Database::Reindexing).to receive(:perform_from_queue).and_return(limit) + expect(Gitlab::Database::Reindexing).not_to receive(:perform_with_heuristic) + + subject + end + end + + context 'without enough records in the queue to reach limit' do + let(:limit) { 2 } + + it 'continues if the queue did not have enough records' do + expect(Gitlab::Database::Reindexing).to receive(:perform_from_queue).ordered.and_return(1) + expect(Gitlab::Database::Reindexing).to receive(:perform_with_heuristic).with(maximum_records: 1).ordered + + subject + end + end + end + end + + describe '.perform_with_heuristic' do + subject { described_class.perform_with_heuristic(candidate_indexes, maximum_records: limit) } + + let(:limit) { 2 } let(:coordinator) { instance_double(Gitlab::Database::Reindexing::Coordinator) } let(:index_selection) { instance_double(Gitlab::Database::Reindexing::IndexSelection) } let(:candidate_indexes) { double } @@ -15,7 +68,7 @@ RSpec.describe Gitlab::Database::Reindexing do it 'delegates to Coordinator' do expect(Gitlab::Database::Reindexing::IndexSelection).to receive(:new).with(candidate_indexes).and_return(index_selection) - expect(index_selection).to receive(:take).with(2).and_return(indexes) + expect(index_selection).to receive(:take).with(limit).and_return(indexes) indexes.each do |index| expect(Gitlab::Database::Reindexing::Coordinator).to receive(:new).with(index).and_return(coordinator) @@ -26,6 +79,59 @@ RSpec.describe Gitlab::Database::Reindexing do end end + describe '.perform_from_queue' do + subject { described_class.perform_from_queue(maximum_records: limit) } + + before_all do + swapout_view_for_table(:postgres_indexes) + end + + let(:limit) { 2 } + let(:queued_actions) { create_list(:reindexing_queued_action, 3) } + let(:coordinator) { instance_double(Gitlab::Database::Reindexing::Coordinator) } + + before do + queued_actions.take(limit).each do |action| + allow(Gitlab::Database::Reindexing::Coordinator).to receive(:new).with(action.index).and_return(coordinator) + allow(coordinator).to receive(:perform) + end + end + + it 'consumes the queue in order of created_at and applies the limit' do + queued_actions.take(limit).each do |action| + expect(Gitlab::Database::Reindexing::Coordinator).to receive(:new).ordered.with(action.index).and_return(coordinator) + expect(coordinator).to receive(:perform) + end + + subject + end + + it 'updates queued action and sets state to done' do + subject + + queue = queued_actions + + queue.shift(limit).each do |action| + expect(action.reload.state).to eq('done') + end + + queue.each do |action| + expect(action.reload.state).to eq('queued') + end + end + + it 'updates queued action upon error and sets state to failed' do + expect(Gitlab::Database::Reindexing::Coordinator).to receive(:new).ordered.with(queued_actions.first.index).and_return(coordinator) + expect(coordinator).to receive(:perform).and_raise('something went wrong') + + subject + + states = queued_actions.map(&:reload).map(&:state) + + expect(states).to eq(%w(failed done queued)) + end + end + describe '.cleanup_leftovers!' do subject { described_class.cleanup_leftovers! } diff --git a/spec/models/key_spec.rb b/spec/models/key_spec.rb index 7468c1b9f0a..d41a1604211 100644 --- a/spec/models/key_spec.rb +++ b/spec/models/key_spec.rb @@ -85,9 +85,9 @@ RSpec.describe Key, :mailer do let_it_be(:expiring_soon_notified) { create(:key, expires_at: 4.days.from_now, user: user, before_expiry_notification_delivered_at: Time.current) } let_it_be(:future_expiry) { create(:key, expires_at: 1.month.from_now, user: user) } - describe '.expired_and_not_notified' do + describe '.expired_today_and_not_notified' do it 'returns keys that expire today and in the past' do - expect(described_class.expired_and_not_notified).to contain_exactly(expired_today_not_notified, expired_yesterday) + expect(described_class.expired_today_and_not_notified).to contain_exactly(expired_today_not_notified) end end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 21c5aea514a..cad9ad81a32 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -98,7 +98,7 @@ RSpec.describe User do it { is_expected.to have_many(:group_members) } it { is_expected.to have_many(:groups) } it { is_expected.to have_many(:keys).dependent(:destroy) } - it { is_expected.to have_many(:expired_and_unnotified_keys) } + it { is_expected.to have_many(:expired_today_and_unnotified_keys) } it { is_expected.to have_many(:deploy_keys).dependent(:nullify) } it { is_expected.to have_many(:group_deploy_keys) } it { is_expected.to have_many(:events).dependent(:delete_all) } diff --git a/spec/services/loose_foreign_keys/batch_cleaner_service_spec.rb b/spec/services/loose_foreign_keys/batch_cleaner_service_spec.rb index dcef419cc9a..409bbfb4098 100644 --- a/spec/services/loose_foreign_keys/batch_cleaner_service_spec.rb +++ b/spec/services/loose_foreign_keys/batch_cleaner_service_spec.rb @@ -79,6 +79,8 @@ RSpec.describe LooseForeignKeys::BatchCleanerService do end context 'when parent records are deleted' do + let(:deleted_records_counter) { Gitlab::Metrics.registry.get(:loose_foreign_key_processed_deleted_records) } + before do parent_record_1.delete @@ -98,8 +100,15 @@ RSpec.describe LooseForeignKeys::BatchCleanerService do expect(loose_fk_child_table_2.where(parent_id_with_different_column: nil).count).to eq(2) end - it 'cleans up the parent DeletedRecord' do + it 'cleans up the pending parent DeletedRecord' do expect(LooseForeignKeys::DeletedRecord.status_pending.count).to eq(0) + expect(LooseForeignKeys::DeletedRecord.status_processed.count).to eq(1) + end + + it 'records the DeletedRecord status updates', :prometheus do + counter = Gitlab::Metrics.registry.get(:loose_foreign_key_processed_deleted_records) + + expect(counter.get(table: parent_model.table_name, db_config_name: 'main')).to eq(1) end it 'does not delete unrelated records' do diff --git a/spec/support/shared_examples/features/sidebar_shared_examples.rb b/spec/support/shared_examples/features/sidebar_shared_examples.rb index 5bfe929e957..d509d124de0 100644 --- a/spec/support/shared_examples/features/sidebar_shared_examples.rb +++ b/spec/support/shared_examples/features/sidebar_shared_examples.rb @@ -52,16 +52,17 @@ RSpec.shared_examples 'issue boards sidebar' do it 'shows toggle as on then as off as user toggles to subscribe and unsubscribe', :aggregate_failures do wait_for_requests + subscription_button = find('[data-testid="subscription-toggle"]') - click_button 'Notifications' + subscription_button.click - expect(page).to have_button('Notifications', class: 'is-checked') + expect(subscription_button).to have_css("button.is-checked") - click_button 'Notifications' + subscription_button.click wait_for_requests - expect(page).not_to have_button('Notifications', class: 'is-checked') + expect(subscription_button).to have_css("button:not(.is-checked)") end context 'when notifications have been disabled' do @@ -73,7 +74,7 @@ RSpec.shared_examples 'issue boards sidebar' do it 'displays a message that notifications have been disabled' do page.within('[data-testid="sidebar-notifications"]') do - expect(page).to have_button('Notifications', class: 'is-disabled') + expect(page).to have_selector('[data-testid="subscription-toggle"]', class: 'is-disabled') expect(page).to have_content('Disabled by project owner') end end diff --git a/spec/tasks/gitlab/db_rake_spec.rb b/spec/tasks/gitlab/db_rake_spec.rb index 5e88b71733d..a273c8cfd7d 100644 --- a/spec/tasks/gitlab/db_rake_spec.rb +++ b/spec/tasks/gitlab/db_rake_spec.rb @@ -215,7 +215,7 @@ RSpec.describe 'gitlab:db namespace rake task', :silence_stdout do stub_feature_flags(database_async_index_creation: true) expect(Gitlab::Database::AsyncIndexes).to receive(:create_pending_indexes!).ordered.exactly(databases_count).times - expect(Gitlab::Database::Reindexing).to receive(:perform).ordered.exactly(databases_count).times + expect(Gitlab::Database::Reindexing).to receive(:automatic_reindexing).ordered.once run_rake_task('gitlab:db:reindex') end @@ -231,56 +231,30 @@ RSpec.describe 'gitlab:db namespace rake task', :silence_stdout do end end - context 'when no index_name is given' do + context 'calls automatic reindexing' do it 'uses all candidate indexes' do - expect(Gitlab::Database::PostgresIndex).to receive(:reindexing_support).exactly(databases_count).times.and_return(indexes) - expect(Gitlab::Database::Reindexing).to receive(:perform).with(indexes).exactly(databases_count).times + expect(Gitlab::Database::Reindexing).to receive(:automatic_reindexing).once run_rake_task('gitlab:db:reindex') end end + end - context 'with index name given' do - let(:index) { double('index') } - - before do - allow(Gitlab::Database::PostgresIndex).to receive(:reindexing_support).and_return(indexes) - end - - it 'calls the index rebuilder with the proper arguments' do - allow(indexes).to receive(:where).with(identifier: 'public.foo_idx').and_return([index]) - expect(Gitlab::Database::Reindexing).to receive(:perform).with([index]).ordered - expect(Gitlab::Database::Reindexing).to receive(:perform).with(indexes).ordered if databases.many? - - run_rake_task('gitlab:db:reindex', '[public.foo_idx]') - end - - context 'when database name is provided' do - it 'calls the index rebuilder with the proper arguments when the database name match' do - allow(indexes).to receive(:where).with(identifier: 'public.foo_idx').and_return([index]) - expect(Gitlab::Database::Reindexing).to receive(:perform).with([index]).ordered - expect(Gitlab::Database::Reindexing).to receive(:perform).with(indexes).ordered if databases.many? - - run_rake_task('gitlab:db:reindex', '[public.foo_idx,main]') - end - - it 'ignores the index and uses all candidate indexes if database name does not match' do - expect(Gitlab::Database::PostgresIndex).to receive(:reindexing_support).exactly(databases_count).times.and_return(indexes) - expect(Gitlab::Database::Reindexing).to receive(:perform).with(indexes).exactly(databases_count).times - - run_rake_task('gitlab:db:reindex', '[public.foo_idx,no_such_database]') - end - end + describe 'enqueue_reindexing_action' do + let(:index_name) { 'public.users_pkey' } - it 'raises an error if the index does not exist' do - allow(indexes).to receive(:where).with(identifier: 'public.absent_index').and_return([]) + it 'creates an entry in the queue' do + expect do + run_rake_task('gitlab:db:enqueue_reindexing_action', "[#{index_name}, main]") + end.to change { Gitlab::Database::PostgresIndex.find(index_name).queued_reindexing_actions.size }.from(0).to(1) + end - expect { run_rake_task('gitlab:db:reindex', '[public.absent_index]') }.to raise_error(/Index public.absent_index for main database not found or not supported/) - end + it 'defaults to main database' do + expect(Gitlab::Database::SharedModel).to receive(:using_connection).with(Gitlab::Database.main.scope.connection).and_call_original - it 'raises an error if the index is not fully qualified with a schema' do - expect { run_rake_task('gitlab:db:reindex', '[foo_idx]') }.to raise_error(/Index name is not fully qualified/) - end + expect do + run_rake_task('gitlab:db:enqueue_reindexing_action', "[#{index_name}]") + end.to change { Gitlab::Database::PostgresIndex.find(index_name).queued_reindexing_actions.size }.from(0).to(1) end end diff --git a/spec/workers/ssh_keys/expired_notification_worker_spec.rb b/spec/workers/ssh_keys/expired_notification_worker_spec.rb index 109d24f03ab..be38391ff8c 100644 --- a/spec/workers/ssh_keys/expired_notification_worker_spec.rb +++ b/spec/workers/ssh_keys/expired_notification_worker_spec.rb @@ -20,7 +20,7 @@ RSpec.describe SshKeys::ExpiredNotificationWorker, type: :worker do stub_const("SshKeys::ExpiredNotificationWorker::BATCH_SIZE", 5) end - let_it_be_with_reload(:keys) { create_list(:key, 20, expires_at: 3.days.ago, user: user) } + let_it_be_with_reload(:keys) { create_list(:key, 20, expires_at: Time.current, user: user) } it 'updates all keys regardless of batch size' do worker.perform @@ -54,8 +54,8 @@ RSpec.describe SshKeys::ExpiredNotificationWorker, type: :worker do context 'when key has expired in the past' do let_it_be(:expired_past) { create(:key, expires_at: 1.day.ago, user: user) } - it 'does update notified column' do - expect { worker.perform }.to change { expired_past.reload.expiry_notification_delivered_at } + it 'does not update notified column' do + expect { worker.perform }.not_to change { expired_past.reload.expiry_notification_delivered_at } end context 'when key has already been notified of expiration' do diff --git a/yarn.lock b/yarn.lock index 3ae03618aa8..2377c548458 100644 --- a/yarn.lock +++ b/yarn.lock @@ -914,10 +914,10 @@ resolved "https://registry.yarnpkg.com/@gitlab/tributejs/-/tributejs-1.0.0.tgz#672befa222aeffc83e7d799b0500a7a4418e59b8" integrity sha512-nmKw1+hB6MHvlmPz63yPwVs1qQkycHwsKgxpEbzmky16Y6mL4EJMk3w1b8QlOAF/AIAzjCERPhe/R4MJiohbZw== -"@gitlab/ui@32.28.0": - version "32.28.0" - resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-32.28.0.tgz#afa0e1dee66ec438126b2706e0860e2c622fc867" - integrity sha512-DudY+jPIxjGymyB2KgdexmxV+sbjCy3or7LzviiI0jb5jCIiqNoZ4F4G+P0T+Z1FbLBgEXjgUaEZw09edka/lg== +"@gitlab/ui@32.31.0": + version "32.31.0" + resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-32.31.0.tgz#e379f79f0797c98d67e121739add8dec8281a5d4" + integrity sha512-a/03Jgh3TJx0W1lJjsYZiAKbRQHGvomrGhzDvBpxKve2FXrYdo4G6gbwlIKJGiooB5YmZ5OIWhgnhQ8FSy15Aw== dependencies: "@babel/standalone" "^7.0.0" bootstrap-vue "2.20.1" |
