diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-03-29 09:09:07 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-03-29 09:09:07 +0300 |
commit | 5169d58a9dd60d54cd40f325431658b1436497c2 (patch) | |
tree | ca3882d6de5615ce8077ea166b0019059256a901 | |
parent | 4535b534a87365512074ddf18410415705be730b (diff) |
Add latest changes from gitlab-org/gitlab@master
19 files changed, 301 insertions, 324 deletions
diff --git a/app/assets/javascripts/diffs/components/app.vue b/app/assets/javascripts/diffs/components/app.vue index 98f1ee9242f..cdea55f0885 100644 --- a/app/assets/javascripts/diffs/components/app.vue +++ b/app/assets/javascripts/diffs/components/app.vue @@ -349,14 +349,6 @@ export default { refetchDiffData() { this.fetchData(false); }, - startDiffRendering() { - requestIdleCallback( - () => { - this.startRenderDiffsQueue(); - }, - { timeout: 1000 }, - ); - }, needsReload() { return this.diffFiles.length && isSingleViewStyle(this.diffFiles[0]); }, @@ -368,8 +360,6 @@ export default { .then(({ real_size }) => { this.diffFilesLength = parseInt(real_size, 10); if (toggleTree) this.setTreeDisplay(); - - this.startDiffRendering(); }) .catch(() => { createFlash(__('Something went wrong on our end. Please try again!')); @@ -384,7 +374,6 @@ export default { // change when loading the other half of the diff files. this.setDiscussions(); }) - .then(() => this.startDiffRendering()) .catch(() => { createFlash(__('Something went wrong on our end. Please try again!')); }); diff --git a/app/assets/javascripts/diffs/store/actions.js b/app/assets/javascripts/diffs/store/actions.js index 8796016def9..70f7b24cd08 100644 --- a/app/assets/javascripts/diffs/store/actions.js +++ b/app/assets/javascripts/diffs/store/actions.js @@ -49,7 +49,6 @@ import { convertExpandLines, idleCallback, allDiscussionWrappersExpanded, - prepareDiffData, prepareLineForRenamedFile, } from './utils'; @@ -163,7 +162,15 @@ export const fetchDiffFilesBatch = ({ commit, state, dispatch }) => { return pagination.next_page; }) - .then((nextPage) => nextPage && getBatch(nextPage)) + .then((nextPage) => { + dispatch('startRenderDiffsQueue'); + + if (nextPage) { + return getBatch(nextPage); + } + + return null; + }) .catch(() => commit(types.SET_RETRIEVING_BATCHES, false)); return getBatch() @@ -197,13 +204,7 @@ export const fetchDiffFilesMeta = ({ commit, state }) => { commit(types.SET_MERGE_REQUEST_DIFFS, data.merge_request_diffs || []); commit(types.SET_DIFF_METADATA, strippedData); - worker.postMessage( - prepareDiffData({ - diff: data, - priorFiles: state.diffFiles, - meta: true, - }), - ); + worker.postMessage(data.diff_files); return data; }) @@ -304,33 +305,38 @@ export const renderFileForDiscussionId = ({ commit, rootState, state }, discussi }; export const startRenderDiffsQueue = ({ state, commit }) => { - const checkItem = () => - new Promise((resolve) => { - const nextFile = state.diffFiles.find( - (file) => - !file.renderIt && - file.viewer && - (!isCollapsed(file) || file.viewer.name !== diffViewerModes.text), - ); - - if (nextFile) { - requestAnimationFrame(() => { - commit(types.RENDER_FILE, nextFile); + const diffFilesToRender = state.diffFiles.filter( + (file) => + !file.renderIt && + file.viewer && + (!isCollapsed(file) || file.viewer.name !== diffViewerModes.text), + ); + let currentDiffFileIndex = 0; + + const checkItem = () => { + const nextFile = diffFilesToRender[currentDiffFileIndex]; + + if (nextFile) { + currentDiffFileIndex += 1; + commit(types.RENDER_FILE, nextFile); + + const requestIdle = () => + requestIdleCallback((idleDeadline) => { + // Wait for at least 5ms before trying to render + if (idleDeadline.timeRemaining() >= 6) { + checkItem(); + } else { + requestIdle(); + } }); - requestIdleCallback( - () => { - checkItem() - .then(resolve) - .catch(() => {}); - }, - { timeout: 1000 }, - ); - } else { - resolve(); - } - }); - return checkItem(); + requestIdle(); + } + }; + + if (diffFilesToRender.length) { + checkItem(); + } }; export const setRenderIt = ({ commit }, file) => commit(types.RENDER_FILE, file); diff --git a/app/assets/javascripts/diffs/store/utils.js b/app/assets/javascripts/diffs/store/utils.js index b37a75eb2a3..7fa51b9ddea 100644 --- a/app/assets/javascripts/diffs/store/utils.js +++ b/app/assets/javascripts/diffs/store/utils.js @@ -381,22 +381,13 @@ function prepareDiffFileLines(file) { inlineLines.forEach((line) => prepareLine(line, file)); // WARNING: In-Place Mutations! - Object.assign(file, { - inlineLinesCount: inlineLines.length, - }); - return file; } -function getVisibleDiffLines(file) { - return file.inlineLinesCount; -} - -function finalizeDiffFile(file) { - const lines = getVisibleDiffLines(file); - +function finalizeDiffFile(file, index) { Object.assign(file, { - renderIt: lines < LINES_TO_BE_RENDERED_DIRECTLY, + renderIt: + index < 3 ? file[INLINE_DIFF_LINES_KEY].length < LINES_TO_BE_RENDERED_DIRECTLY : false, isShowingFullFile: false, isLoadingFullFile: false, discussions: [], @@ -424,7 +415,7 @@ export function prepareDiffData({ diff, priorFiles = [], meta = false }) { .map((file, index, allFiles) => prepareRawDiffFile({ file, allFiles, meta })) .map(ensureBasicDiffFileLines) .map(prepareDiffFileLines) - .map(finalizeDiffFile); + .map((file, index) => finalizeDiffFile(file, priorFiles.length + index)); return deduplicateFilesList([...priorFiles, ...cleanedFiles]); } diff --git a/app/assets/javascripts/sidebar/components/copy_email_to_clipboard.vue b/app/assets/javascripts/sidebar/components/copy_email_to_clipboard.vue index 8c8241cf6a4..0d8cb8cb2b6 100644 --- a/app/assets/javascripts/sidebar/components/copy_email_to_clipboard.vue +++ b/app/assets/javascripts/sidebar/components/copy_email_to_clipboard.vue @@ -1,43 +1,24 @@ <script> -import { s__, __, sprintf } from '~/locale'; -import ClipboardButton from '~/vue_shared/components/clipboard_button.vue'; +import CopyableField from '../../vue_shared/components/sidebar/copyable_field.vue'; export default { - i18n: { - copyEmail: __('Copy email address'), - }, components: { - ClipboardButton, + CopyableField, }, props: { - copyText: { + issueEmailAddress: { type: String, required: true, }, }, - computed: { - emailText() { - return sprintf(s__('RightSidebar|Issue email: %{copyText}'), { copyText: this.copyText }); - }, - }, }; </script> <template> - <div + <copyable-field data-qa-selector="copy-forward-email" - class="copy-email-address gl-display-flex gl-align-items-center gl-justify-content-space-between" - > - <span - class="gl-overflow-hidden gl-text-overflow-ellipsis gl-white-space-nowrap hide-collapsed gl-w-85p" - >{{ emailText }}</span - > - <clipboard-button - class="copy-email-button gl-bg-none!" - category="tertiary" - :title="$options.i18n.copyEmail" - :text="copyText" - tooltip-placement="left" - /> - </div> + :name="s__('RightSidebar|Issue email')" + :clipboard-tooltip-text="s__('RightSidebar|Copy email address')" + :value="issueEmailAddress" + /> </template> diff --git a/app/assets/javascripts/sidebar/mount_sidebar.js b/app/assets/javascripts/sidebar/mount_sidebar.js index 312c0c89f29..b98c8a8c5dc 100644 --- a/app/assets/javascripts/sidebar/mount_sidebar.js +++ b/app/assets/javascripts/sidebar/mount_sidebar.js @@ -337,7 +337,7 @@ function mountCopyEmailComponent() { new Vue({ el, render: (createElement) => - createElement(CopyEmailToClipboard, { props: { copyText: createNoteEmail } }), + createElement(CopyEmailToClipboard, { props: { issueEmailAddress: createNoteEmail } }), }); } diff --git a/app/assets/javascripts/vue_shared/components/sidebar/copyable_field.vue b/app/assets/javascripts/vue_shared/components/sidebar/copyable_field.vue index 9c0952d3d2e..833bfdae1ab 100644 --- a/app/assets/javascripts/vue_shared/components/sidebar/copyable_field.vue +++ b/app/assets/javascripts/vue_shared/components/sidebar/copyable_field.vue @@ -27,6 +27,11 @@ export default { required: false, default: false, }, + clipboardTooltipText: { + type: String, + required: false, + default: undefined, + }, }, computed: { clipboardProps() { @@ -35,7 +40,9 @@ export default { tooltipBoundary: 'viewport', tooltipPlacement: 'left', text: this.value, - title: sprintf(this.$options.i18n.clipboardTooltip, { name: this.name }), + title: + this.clipboardTooltipText || + sprintf(this.$options.i18n.clipboardTooltip, { name: this.name }), }; }, loadingIconLabel() { diff --git a/app/assets/stylesheets/framework/sidebar.scss b/app/assets/stylesheets/framework/sidebar.scss index 241aaad015e..cb8a0c40f7f 100644 --- a/app/assets/stylesheets/framework/sidebar.scss +++ b/app/assets/stylesheets/framework/sidebar.scss @@ -58,19 +58,6 @@ height: $gl-padding; } } - - .copy-email-button { // TODO: replace with utility - @include gl-w-full; - @include gl-h-full; - } - - .copy-email-address { - height: 60px; - - &:hover { - background: $gray-100; - } - } } .right-sidebar-expanded { diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 133c6217bc0..720ee6f23d8 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -359,7 +359,7 @@ class MergeRequest < ApplicationRecord scope :preload_metrics, -> (relation) { preload(metrics: relation) } scope :preload_project_and_latest_diff, -> { preload(:source_project, :latest_merge_request_diff) } scope :preload_latest_diff_commit, -> { preload(latest_merge_request_diff: :merge_request_diff_commits) } - scope :with_web_entity_associations, -> { preload(:author, :target_project) } + scope :with_web_entity_associations, -> { preload(:author, target_project: [:project_feature, group: [:route, :parent], namespace: :route]) } scope :with_auto_merge_enabled, -> do with_state(:opened).where(auto_merge_enabled: true) diff --git a/changelogs/unreleased/fix-n-1-for-merge-requests-scope.yml b/changelogs/unreleased/fix-n-1-for-merge-requests-scope.yml new file mode 100644 index 00000000000..0b473f72893 --- /dev/null +++ b/changelogs/unreleased/fix-n-1-for-merge-requests-scope.yml @@ -0,0 +1,5 @@ +--- +title: Preload additional data to fix N+1 queries for merge request search +merge_request: 57284 +author: +type: performance diff --git a/doc/administration/geo/index.md b/doc/administration/geo/index.md index 296500e35e2..d14ab70c138 100644 --- a/doc/administration/geo/index.md +++ b/doc/administration/geo/index.md @@ -54,7 +54,7 @@ Geo provides: ### Gitaly Cluster Geo should not be confused with [Gitaly Cluster](../gitaly/praefect.md). For more information about -the difference between Geo and Gitaly Cluster, see [Gitaly Cluster compared to Geo](../gitaly/praefect.md#gitaly-cluster-compared-to-geo). +the difference between Geo and Gitaly Cluster, see [Gitaly Cluster compared to Geo](../gitaly/index.md#gitaly-cluster-compared-to-geo). ## How it works diff --git a/doc/administration/gitaly/configure_gitaly.md b/doc/administration/gitaly/configure_gitaly.md index 7e3647d1e34..6c3d276b117 100644 --- a/doc/administration/gitaly/configure_gitaly.md +++ b/doc/administration/gitaly/configure_gitaly.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Configure Gitaly +# Configure Gitaly **(FREE SELF)** The Gitaly service itself is configured by using a [TOML configuration file](reference.md). diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index b98dda76389..cf7c5bda77e 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Gitaly +# Gitaly and Gitaly Cluster **(FREE SELF)** [Gitaly](https://gitlab.com/gitlab-org/gitaly) provides high-level RPC access to Git repositories. It is used by GitLab to read and write Git data. @@ -68,8 +68,202 @@ GitLab installations for more than 2000 users should use Gitaly Cluster. ## Gitaly Cluster -Gitaly can run in a clustered configuration to scale Gitaly and increase fault tolerance. For more -information, see [Gitaly Cluster](praefect.md). +Gitaly, the service that provides storage for Git repositories, can +be run in a clustered configuration to scale the Gitaly service and increase +fault tolerance. In this configuration, every Git repository is stored on every +Gitaly node in the cluster. + +Using a Gitaly Cluster increases fault tolerance by: + +- Replicating write operations to warm standby Gitaly nodes. +- Detecting Gitaly node failures. +- Automatically routing Git requests to an available Gitaly node. + +NOTE: +Technical support for Gitaly clusters is limited to GitLab Premium and Ultimate +customers. + +The availability objectives for Gitaly clusters are: + +- **Recovery Point Objective (RPO):** Less than 1 minute. + + Writes are replicated asynchronously. Any writes that have not been replicated + to the newly promoted primary are lost. + + [Strong consistency](praefect.md#strong-consistency) can be used to avoid loss in some + circumstances. + +- **Recovery Time Objective (RTO):** Less than 10 seconds. + Outages are detected by a health check run by each Praefect node every + second. Failover requires ten consecutive failed health checks on each + Praefect node. + + [Faster outage detection](https://gitlab.com/gitlab-org/gitaly/-/issues/2608) + is planned to improve this to less than 1 second. + +Gitaly Cluster supports: + +- [Strong consistency](praefect.md#strong-consistency) of the secondary replicas. +- [Automatic failover](praefect.md#automatic-failover-and-leader-election) from the primary to the secondary. +- Reporting of possible data loss if replication queue is non-empty. +- Marking repositories as [read only](praefect.md#read-only-mode) if data loss is detected to prevent data inconsistencies. + +Follow the [Gitaly Cluster epic](https://gitlab.com/groups/gitlab-org/-/epics/1489) +for improvements including +[horizontally distributing reads](https://gitlab.com/groups/gitlab-org/-/epics/2013). + +### Overview + +Git storage is provided through the Gitaly service in GitLab, and is essential +to the operation of the GitLab application. When the number of +users, repositories, and activity grows, it is important to scale Gitaly +appropriately by: + +- Increasing the available CPU and memory resources available to Git before + resource exhaustion degrades Git, Gitaly, and GitLab application performance. +- Increase available storage before storage limits are reached causing write + operations to fail. +- Improve fault tolerance by removing single points of failure. Git should be + considered mission critical if a service degradation would prevent you from + deploying changes to production. + +### Moving beyond NFS + +WARNING: +From GitLab 13.0, using NFS for Git repositories is deprecated. In GitLab 14.0, +support for NFS for Git repositories is scheduled to be removed. Upgrade to +Gitaly Cluster as soon as possible. + +[Network File System (NFS)](https://en.wikipedia.org/wiki/Network_File_System) +is not well suited to Git workloads which are CPU and IOPS sensitive. +Specifically: + +- Git is sensitive to file system latency. Even simple operations require many + read operations. Operations that are fast on block storage can become an order of + magnitude slower. This significantly impacts GitLab application performance. +- NFS performance optimizations that prevent the performance gap between + block storage and NFS being even wider are vulnerable to race conditions. We have observed + [data inconsistencies](https://gitlab.com/gitlab-org/gitaly/-/issues/2589) + in production environments caused by simultaneous writes to different NFS + clients. Data corruption is not an acceptable risk. + +Gitaly Cluster is purpose built to provide reliable, high performance, fault +tolerant Git storage. + +Further reading: + +- Blog post: [The road to Gitaly v1.0 (aka, why GitLab doesn't require NFS for storing Git data anymore)](https://about.gitlab.com/blog/2018/09/12/the-road-to-gitaly-1-0/) +- Blog post: [How we spent two weeks hunting an NFS bug in the Linux kernel](https://about.gitlab.com/blog/2018/11/14/how-we-spent-two-weeks-hunting-an-nfs-bug/) + +### Where Gitaly Cluster fits + +GitLab accesses [repositories](../../user/project/repository/index.md) through the configured +[repository storages](../repository_storage_paths.md). Each new repository is stored on one of the +repository storages based on their configured weights. Each repository storage is either: + +- A Gitaly storage served directly by Gitaly. These map to a directory on the file system of a + Gitaly node. +- A [virtual storage](#virtual-storage-or-direct-gitaly-storage) served by Praefect. A virtual + storage is a cluster of Gitaly storages that appear as a single repository storage. + +Virtual storages are a feature of Gitaly Cluster. They support replicating the repositories to +multiple storages for fault tolerance. Virtual storages can improve performance by distributing +requests across Gitaly nodes. Their distributed nature makes it viable to have a single repository +storage in GitLab to simplify repository management. + +### Components of Gitaly Cluster + +Gitaly Cluster consists of multiple components: + +- [Load balancer](praefect.md#load-balancer) for distributing requests and providing fault-tolerant access to + Praefect nodes. +- [Praefect](praefect.md#praefect) nodes for managing the cluster and routing requests to Gitaly nodes. +- [PostgreSQL database](praefect.md#postgresql) for persisting cluster metadata and [PgBouncer](praefect.md#pgbouncer), + recommended for pooling Praefect's database connections. +- Gitaly nodes to provide repository storage and Git access. + +![Cluster example](img/cluster_example_v13_3.png) + +In this example: + +- Repositories are stored on a virtual storage called `storage-1`. +- Three Gitaly nodes provide `storage-1` access: `gitaly-1`, `gitaly-2`, and `gitaly-3`. +- The three Gitaly nodes store data on their file systems. + +### Virtual storage or direct Gitaly storage + +Gitaly supports multiple models of scaling: + +- Clustering using Gitaly Cluster, where each repository is stored on multiple Gitaly nodes in the + cluster. Read requests are distributed between repository replicas and write requests are + broadcast to repository replicas. GitLab accesses virtual storage. +- Direct access to Gitaly storage using [repository storage paths](../repository_storage_paths.md), + where each repository is stored on the assigned Gitaly node. All requests are routed to this node. + +The following is Gitaly set up to use direct access to Gitaly instead of Gitaly Cluster: + +![Shard example](img/shard_example_v13_3.png) + +In this example: + +- Each repository is stored on one of three Gitaly storages: `storage-1`, `storage-2`, + or `storage-3`. +- Each storage is serviced by a Gitaly node. +- The three Gitaly nodes store data in three separate hashed storage locations. + +Generally, virtual storage with Gitaly Cluster can replace direct Gitaly storage configurations, at +the expense of additional storage needed to store each repository on multiple Gitaly nodes. The +benefit of using Gitaly Cluster over direct Gitaly storage is: + +- Improved fault tolerance, because each Gitaly node has a copy of every repository. +- Improved resource utilization, reducing the need for over-provisioning for shard-specific peak + loads, because read loads are distributed across replicas. +- Manual rebalancing for performance is not required, because read loads are distributed across + replicas. +- Simpler management, because all Gitaly nodes are identical. + +Under some workloads, CPU and memory requirements may require a large fleet of Gitaly nodes. It +can be uneconomical to have one to one replication factor. + +A hybrid approach can be used in these instances, where each shard is configured as a smaller +cluster. [Variable replication factor](https://gitlab.com/groups/gitlab-org/-/epics/3372) is planned +to provide greater flexibility for extremely large GitLab instances. + +### Gitaly Cluster compared to Geo + +Gitaly Cluster and [Geo](../geo/index.md) both provide redundancy. However the redundancy of: + +- Gitaly Cluster provides fault tolerance for data storage and is invisible to the user. Users are + not aware when Gitaly Cluster is used. +- Geo provides [replication](../geo/index.md) and [disaster recovery](../geo/disaster_recovery/index.md) for + an entire instance of GitLab. Users know when they are using Geo for + [replication](../geo/index.md). Geo [replicates multiple data types](../geo/replication/datatypes.md#limitations-on-replicationverification), + including Git data. + +The following table outlines the major differences between Gitaly Cluster and Geo: + +| Tool | Nodes | Locations | Latency tolerance | Failover | Consistency | Provides redundancy for | +|:---------------|:---------|:----------|:-------------------|:----------------------------------------------------------------|:-----------------------------------------|:------------------------| +| Gitaly Cluster | Multiple | Single | Approximately 1 ms | [Automatic](praefect.md#automatic-failover-and-leader-election) | [Strong](praefect.md#strong-consistency) | Data storage in Git | +| Geo | Multiple | Multiple | Up to one minute | [Manual](../geo/disaster_recovery/index.md) | Eventual | Entire GitLab instance | + +For more information, see: + +- Geo [use cases](../geo/index.md#use-cases). +- Geo [architecture](../geo/index.md#architecture). + +### Architecture + +Praefect is a router and transaction manager for Gitaly, and a required +component for running a Gitaly Cluster. + +![Architecture diagram](img/praefect_architecture_v12_10.png) + +For more information, see [Gitaly High Availability (HA) Design](https://gitlab.com/gitlab-org/gitaly/-/blob/master/doc/design_ha.md). + +### Configure Gitaly Cluster + +For more information on configuring Gitaly Cluster, see [Configure Gitaly Cluster](praefect.md). ## Do not bypass Gitaly @@ -180,7 +374,7 @@ There are two facets to our efforts to remove direct Git access in GitLab: NFS. The second facet presents the only real solution. For this, we developed -[Gitaly Cluster](praefect.md). +[Gitaly Cluster](#gitaly-cluster). ## NFS deprecation notice @@ -198,7 +392,7 @@ Additional information: GitLab recommends: -- Creating a [Gitaly Cluster](praefect.md) as soon as possible. +- Creating a [Gitaly Cluster](#gitaly-cluster) as soon as possible. - [Moving your projects](praefect.md#migrate-existing-repositories-to-gitaly-cluster) from NFS-based storage to the Gitaly Cluster. diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index 72e27a1c631..f4df5a900d6 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -5,201 +5,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Gitaly Cluster **(FREE SELF)** +# Configure Gitaly Cluster **(FREE SELF)** -[Gitaly](index.md), the service that provides storage for Git repositories, can -be run in a clustered configuration to scale the Gitaly service and increase -fault tolerance. In this configuration, every Git repository is stored on every -Gitaly node in the cluster. - -Using a Gitaly Cluster increases fault tolerance by: - -- Replicating write operations to warm standby Gitaly nodes. -- Detecting Gitaly node failures. -- Automatically routing Git requests to an available Gitaly node. - -NOTE: -Technical support for Gitaly clusters is limited to GitLab Premium and Ultimate -customers. - -The availability objectives for Gitaly clusters are: - -- **Recovery Point Objective (RPO):** Less than 1 minute. - - Writes are replicated asynchronously. Any writes that have not been replicated - to the newly promoted primary are lost. - - [Strong consistency](#strong-consistency) can be used to avoid loss in some - circumstances. - -- **Recovery Time Objective (RTO):** Less than 10 seconds. - - Outages are detected by a health checks run by each Praefect node every - second. Failover requires ten consecutive failed health checks on each - Praefect node. - - [Faster outage detection](https://gitlab.com/gitlab-org/gitaly/-/issues/2608) - is planned to improve this to less than 1 second. - -Gitaly Cluster supports: - -- [Strong consistency](#strong-consistency) of the secondary replicas. -- [Automatic failover](#automatic-failover-and-leader-election) from the primary to the secondary. -- Reporting of possible data loss if replication queue is non-empty. -- Marking repositories as [read only](#read-only-mode) if data loss is detected to prevent data inconsistencies. - -Follow the [Gitaly Cluster epic](https://gitlab.com/groups/gitlab-org/-/epics/1489) -for improvements including -[horizontally distributing reads](https://gitlab.com/groups/gitlab-org/-/epics/2013). - -## Overview - -Git storage is provided through the Gitaly service in GitLab, and is essential -to correct proper operation of the GitLab application. When the number of -users, repositories, and activity grows, it is important to scale Gitaly -appropriately by: - -- Increasing the available CPU and memory resources available to Git before - resource exhaustion degrades Git, Gitaly, and GitLab application performance. -- Increase available storage before storage limits are reached causing write - operations to fail. -- Improve fault tolerance by removing single points of failure. Git should be - considered mission critical if a service degradation would prevent you from - deploying changes to production. - -### Moving beyond NFS - -WARNING: -From GitLab 13.0, using NFS for Git repositories is deprecated. In GitLab 14.0, -support for NFS for Git repositories is scheduled to be removed. Upgrade to -Gitaly Cluster as soon as possible. - -[Network File System (NFS)](https://en.wikipedia.org/wiki/Network_File_System) -is not well suited to Git workloads which are CPU and IOPS sensitive. -Specifically: - -- Git is sensitive to file system latency. Even simple operations require many - read operations. Operations that are fast on block storage can become an order of - magnitude slower. This significantly impacts GitLab application performance. -- NFS performance optimizations that prevent the performance gap between - block storage and NFS being even wider are vulnerable to race conditions. We have observed - [data inconsistencies](https://gitlab.com/gitlab-org/gitaly/-/issues/2589) - in production environments caused by simultaneous writes to different NFS - clients. Data corruption is not an acceptable risk. - -Gitaly Cluster is purpose built to provide reliable, high performance, fault -tolerant Git storage. - -Further reading: - -- Blog post: [The road to Gitaly v1.0 (aka, why GitLab doesn't require NFS for storing Git data anymore)](https://about.gitlab.com/blog/2018/09/12/the-road-to-gitaly-1-0/) -- Blog post: [How we spent two weeks hunting an NFS bug in the Linux kernel](https://about.gitlab.com/blog/2018/11/14/how-we-spent-two-weeks-hunting-an-nfs-bug/) - -## Where Gitaly Cluster fits - -GitLab accesses [repositories](../../user/project/repository/index.md) through the configured -[repository storages](../repository_storage_paths.md). Each new repository is stored on one of the -repository storages based on their configured weights. Each repository storage is either: - -- A Gitaly storage served directly by Gitaly. These map to a directory on the file system of a - Gitaly node. -- A [virtual storage](#virtual-storage-or-direct-gitaly-storage) served by Praefect. A virtual - storage is a cluster of Gitaly storages that appear as a single repository storage. - -Virtual storages are a feature of Gitaly Cluster. They support replicating the repositories to -multiple storages for fault tolerance. Virtual storages can improve performance by distributing -requests across Gitaly nodes. Their distributed nature makes it viable to have a single repository -storage in GitLab to simplify repository management. - -## Components of Gitaly Cluster - -Gitaly Cluster consists of multiple components: - -- [Load balancer](#load-balancer) for distributing requests and providing fault-tolerant access to - Praefect nodes. -- [Praefect](#praefect) nodes for managing the cluster and routing requests to Gitaly nodes. -- [PostgreSQL database](#postgresql) for persisting cluster metadata and [PgBouncer](#pgbouncer), - recommended for pooling Praefect's database connections. -- [Gitaly](index.md) nodes to provide repository storage and Git access. - -![Cluster example](img/cluster_example_v13_3.png) - -In this example: - -- Repositories are stored on a virtual storage called `storage-1`. -- Three Gitaly nodes provide `storage-1` access: `gitaly-1`, `gitaly-2`, and `gitaly-3`. -- The three Gitaly nodes store data on their file systems. - -### Virtual storage or direct Gitaly storage - -Gitaly supports multiple models of scaling: - -- Clustering using Gitaly Cluster, where each repository is stored on multiple Gitaly nodes in the - cluster. Read requests are distributed between repository replicas and write requests are - broadcast to repository replicas. GitLab accesses virtual storage. -- Direct access to Gitaly storage using [repository storage paths](../repository_storage_paths.md), - where each repository is stored on the assigned Gitaly node. All requests are routed to this node. - -The following is Gitaly set up to use direct access to Gitaly instead of Gitaly Cluster: - -![Shard example](img/shard_example_v13_3.png) - -In this example: - -- Each repository is stored on one of three Gitaly storages: `storage-1`, `storage-2`, - or `storage-3`. -- Each storage is serviced by a Gitaly node. -- The three Gitaly nodes store data in three separate hashed storage locations. - -Generally, virtual storage with Gitaly Cluster can replace direct Gitaly storage configurations, at -the expense of additional storage needed to store each repository on multiple Gitaly nodes. The -benefit of using Gitaly Cluster over direct Gitaly storage is: - -- Improved fault tolerance, because each Gitaly node has a copy of every repository. -- Improved resource utilization, reducing the need for over-provisioning for shard-specific peak - loads, because read loads are distributed across replicas. -- Manual rebalancing for performance is not required, because read loads are distributed across - replicas. -- Simpler management, because all Gitaly nodes are identical. - -Under some workloads, CPU and memory requirements may require a large fleet of Gitaly nodes. It -can be uneconomical to have one to one replication factor. - -A hybrid approach can be used in these instances, where each shard is configured as a smaller -cluster. [Variable replication factor](https://gitlab.com/groups/gitlab-org/-/epics/3372) is planned -to provide greater flexibility for extremely large GitLab instances. - -### Gitaly Cluster compared to Geo - -Gitaly Cluster and [Geo](../geo/index.md) both provide redundancy. However the redundancy of: - -- Gitaly Cluster provides fault tolerance for data storage and is invisible to the user. Users are - not aware when Gitaly Cluster is used. -- Geo provides [replication](../geo/index.md) and [disaster recovery](../geo/disaster_recovery/index.md) for - an entire instance of GitLab. Users know when they are using Geo for - [replication](../geo/index.md). Geo [replicates multiple datatypes](../geo/replication/datatypes.md#limitations-on-replicationverification), - including Git data. - -The following table outlines the major differences between Gitaly Cluster and Geo: - -| Tool | Nodes | Locations | Latency tolerance | Failover | Consistency | Provides redundancy for | -|:---------------|:---------|:----------|:-------------------|:-----------------------------------------------------|:------------------------------|:------------------------| -| Gitaly Cluster | Multiple | Single | Approximately 1 ms | [Automatic](#automatic-failover-and-leader-election) | [Strong](#strong-consistency) | Data storage in Git | -| Geo | Multiple | Multiple | Up to one minute | [Manual](../geo/disaster_recovery/index.md) | Eventual | Entire GitLab instance | - -For more information, see: - -- [Gitaly](index.md). -- Geo [use cases](../geo/index.md#use-cases) and [architecture](../geo/index.md#architecture). - -## Architecture - -Praefect is a router and transaction manager for Gitaly, and a required -component for running a Gitaly Cluster. - -![Architecture diagram](img/praefect_architecture_v12_10.png) - -For more information, see [Gitaly HA Design](https://gitlab.com/gitlab-org/gitaly/-/blob/master/doc/design_ha.md) +In addition to Gitaly Cluster configuration instructions available as part of +[reference architectures](../reference_architectures/index.md) for installations for more than +2000 users, advanced configuration instructions are available below. ## Requirements for configuring a Gitaly Cluster diff --git a/doc/administration/gitaly/reference.md b/doc/administration/gitaly/reference.md index f08b03017e4..ec5a8d47ae2 100644 --- a/doc/administration/gitaly/reference.md +++ b/doc/administration/gitaly/reference.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Gitaly reference +# Gitaly reference **(FREE SELF)** Gitaly is configured via a [TOML](https://github.com/toml-lang/toml) configuration file. Unlike installations from source, in Omnibus GitLab, you diff --git a/locale/gitlab.pot b/locale/gitlab.pot index f58c1811735..7358ffdb6c7 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -8582,9 +8582,6 @@ msgstr "" msgid "Copy commit SHA" msgstr "" -msgid "Copy email address" -msgstr "" - msgid "Copy environment" msgstr "" @@ -20749,6 +20746,9 @@ msgstr "" msgid "No authentication methods configured." msgstr "" +msgid "No available branches" +msgstr "" + msgid "No available groups to fork the project." msgstr "" @@ -21540,6 +21540,9 @@ msgstr "" msgid "OnDemandScans|You cannot run an active scan against an unvalidated site." msgstr "" +msgid "OnDemandScans|You must create a repository within your project to run an on-demand scan." +msgstr "" + msgid "Once a project is permanently deleted, it %{strongStart}cannot be recovered%{strongEnd}. Permanently deleting this project will %{strongStart}immediately delete%{strongEnd} its repositories and %{strongStart}all related resources%{strongEnd}, including issues, merge requests etc." msgstr "" @@ -26365,7 +26368,10 @@ msgstr "" msgid "Revoked project access token %{project_access_token_name}!" msgstr "" -msgid "RightSidebar|Issue email: %{copyText}" +msgid "RightSidebar|Copy email address" +msgstr "" + +msgid "RightSidebar|Issue email" msgstr "" msgid "RightSidebar|adding a" diff --git a/spec/frontend/diffs/components/app_spec.js b/spec/frontend/diffs/components/app_spec.js index 34547238c23..f2572d203b4 100644 --- a/spec/frontend/diffs/components/app_spec.js +++ b/spec/frontend/diffs/components/app_spec.js @@ -105,7 +105,6 @@ describe('diffs/components/app', () => { jest.spyOn(wrapper.vm, 'fetchDiffFilesBatch').mockImplementation(fetchResolver); jest.spyOn(wrapper.vm, 'fetchCoverageFiles').mockImplementation(fetchResolver); jest.spyOn(wrapper.vm, 'setDiscussions').mockImplementation(() => {}); - jest.spyOn(wrapper.vm, 'startRenderDiffsQueue').mockImplementation(() => {}); jest.spyOn(wrapper.vm, 'unwatchDiscussions').mockImplementation(() => {}); jest.spyOn(wrapper.vm, 'unwatchRetrievingBatches').mockImplementation(() => {}); store.state.diffs.retrievingBatches = true; @@ -119,7 +118,6 @@ describe('diffs/components/app', () => { await nextTick(); - expect(wrapper.vm.startRenderDiffsQueue).toHaveBeenCalled(); expect(wrapper.vm.fetchDiffFilesMeta).toHaveBeenCalled(); expect(wrapper.vm.fetchDiffFilesBatch).toHaveBeenCalled(); expect(wrapper.vm.fetchCoverageFiles).toHaveBeenCalled(); @@ -134,7 +132,6 @@ describe('diffs/components/app', () => { await nextTick(); - expect(wrapper.vm.startRenderDiffsQueue).toHaveBeenCalled(); expect(wrapper.vm.fetchDiffFilesMeta).toHaveBeenCalled(); expect(wrapper.vm.fetchDiffFilesBatch).toHaveBeenCalled(); expect(wrapper.vm.fetchCoverageFiles).toHaveBeenCalled(); diff --git a/spec/frontend/diffs/store/actions_spec.js b/spec/frontend/diffs/store/actions_spec.js index ed3210ecfaf..de41e1724a3 100644 --- a/spec/frontend/diffs/store/actions_spec.js +++ b/spec/frontend/diffs/store/actions_spec.js @@ -80,7 +80,7 @@ describe('DiffsStoreActions', () => { jest.spyOn(utils, 'idleCallback').mockImplementation(() => null); ['requestAnimationFrame', 'requestIdleCallback'].forEach((method) => { global[method] = (cb) => { - cb(); + cb({ timeRemaining: () => 10 }); }; }); }); @@ -198,7 +198,7 @@ describe('DiffsStoreActions', () => { { type: types.VIEW_DIFF_FILE, payload: 'test2' }, { type: types.SET_RETRIEVING_BATCHES, payload: false }, ], - [], + [{ type: 'startRenderDiffsQueue' }, { type: 'startRenderDiffsQueue' }], done, ); }); diff --git a/spec/frontend/sidebar/components/copy_email_to_clipboard_spec.js b/spec/frontend/sidebar/components/copy_email_to_clipboard_spec.js index 704847f65bf..699b2bbd0b1 100644 --- a/spec/frontend/sidebar/components/copy_email_to_clipboard_spec.js +++ b/spec/frontend/sidebar/components/copy_email_to_clipboard_spec.js @@ -1,22 +1,17 @@ -import { getByText } from '@testing-library/dom'; -import { mount } from '@vue/test-utils'; +import { shallowMount } from '@vue/test-utils'; import CopyEmailToClipboard from '~/sidebar/components/copy_email_to_clipboard.vue'; -import ClipboardButton from '~/vue_shared/components/clipboard_button.vue'; +import CopyableField from '~/vue_shared/components/sidebar/copyable_field.vue'; describe('CopyEmailToClipboard component', () => { - const sampleEmail = 'sample+email@test.com'; + const mockIssueEmailAddress = 'sample+email@test.com'; - const wrapper = mount(CopyEmailToClipboard, { + const wrapper = shallowMount(CopyEmailToClipboard, { propsData: { - copyText: sampleEmail, + issueEmailAddress: mockIssueEmailAddress, }, }); - it('renders the Issue email text with the forwardable email', () => { - expect(getByText(wrapper.element, `Issue email: ${sampleEmail}`)).not.toBeNull(); - }); - - it('finds ClipboardButton with the correct props', () => { - expect(wrapper.find(ClipboardButton).props('text')).toBe(sampleEmail); + it('sets CopyableField `value` prop to issueEmailAddress', () => { + expect(wrapper.find(CopyableField).props('value')).toBe(mockIssueEmailAddress); }); }); diff --git a/spec/frontend/vue_shared/components/sidebar/copyable_field_spec.js b/spec/frontend/vue_shared/components/sidebar/copyable_field_spec.js index 13bc22d0fde..b99b1a66b79 100644 --- a/spec/frontend/vue_shared/components/sidebar/copyable_field_spec.js +++ b/spec/frontend/vue_shared/components/sidebar/copyable_field_spec.js @@ -61,5 +61,14 @@ describe('SidebarCopyableField', () => { expect(findClipboardButton().exists()).toBe(false); }); }); + + describe('with `clipboardTooltipText` prop', () => { + it('sets ClipboardButton `title` prop to `clipboardTooltipText` value', () => { + const mockClipboardTooltipText = 'Copy my custom value'; + createComponent({ ...defaultProps, clipboardTooltipText: mockClipboardTooltipText }); + + expect(findClipboardButton().props('title')).toBe(mockClipboardTooltipText); + }); + }); }); }); |