diff options
103 files changed, 3485 insertions, 2708 deletions
@@ -313,7 +313,7 @@ gem 'pg_query', '~> 1.3.0' gem 'premailer-rails', '~> 1.10.3' # LabKit: Tracing and Correlation -gem 'gitlab-labkit', '0.14.0' +gem 'gitlab-labkit', '~> 0.16.0' # I18n gem 'ruby_parser', '~> 3.15', require: false diff --git a/Gemfile.lock b/Gemfile.lock index d47cec9f06d..d5d700e4721 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -446,19 +446,18 @@ GEM fog-json (~> 1.2.0) mime-types ms_rest_azure (~> 0.12.0) - gitlab-labkit (0.14.0) + gitlab-labkit (0.16.0) actionpack (>= 5.0.0, < 7.0.0) activesupport (>= 5.0.0, < 7.0.0) - gitlab-pg_query (~> 1.3) grpc (~> 1.19) jaeger-client (~> 1.1) opentracing (~> 0.4) + pg_query (~> 1.3) redis (> 3.0.0, < 5.0.0) gitlab-license (1.3.1) gitlab-mail_room (0.0.8) gitlab-markup (1.7.1) gitlab-net-dns (0.9.1) - gitlab-pg_query (1.3.1) gitlab-pry-byebug (3.9.0) byebug (~> 11.0) pry (~> 0.13.0) @@ -1220,7 +1219,7 @@ GEM rack (>= 1, < 3) thor (1.1.0) thread_safe (0.3.6) - thrift (0.13.0) + thrift (0.14.0) tilt (2.0.10) timecop (0.9.1) timeliness (0.3.10) @@ -1416,7 +1415,7 @@ DEPENDENCIES gitlab-chronic (~> 0.10.5) gitlab-experiment (~> 0.4.12) gitlab-fog-azure-rm (~> 1.0.1) - gitlab-labkit (= 0.14.0) + gitlab-labkit (~> 0.16.0) gitlab-license (~> 1.3) gitlab-mail_room (~> 0.0.8) gitlab-markup (~> 1.7.1) diff --git a/app/assets/javascripts/boards/components/board_sidebar.js b/app/assets/javascripts/boards/components/board_sidebar.js index 6d5a13be3ac..9198a9019f8 100644 --- a/app/assets/javascripts/boards/components/board_sidebar.js +++ b/app/assets/javascripts/boards/components/board_sidebar.js @@ -107,8 +107,8 @@ export default Vue.extend({ closeSidebar() { this.detail.issue = {}; }, - setAssignees(data) { - boardsStore.detail.issue.setAssignees(data.issueSetAssignees.issue.assignees.nodes); + setAssignees(assignees) { + boardsStore.detail.issue.setAssignees(assignees); }, showScopedLabels(label) { return boardsStore.scopedLabels.enabled && isScopedLabel(label); diff --git a/app/assets/javascripts/graphql_shared/queries/users_search.query.graphql b/app/assets/javascripts/graphql_shared/queries/users_search.query.graphql index 88c33398b40..aaaaf3485ad 100644 --- a/app/assets/javascripts/graphql_shared/queries/users_search.query.graphql +++ b/app/assets/javascripts/graphql_shared/queries/users_search.query.graphql @@ -1,7 +1,7 @@ #import "../fragments/user.fragment.graphql" query usersSearch($search: String!, $fullPath: ID!) { - issuable: project(fullPath: $fullPath) { + workspace: project(fullPath: $fullPath) { users: projectMembers(search: $search) { nodes { user { diff --git a/app/assets/javascripts/notes/components/comment_form.vue b/app/assets/javascripts/notes/components/comment_form.vue index b865826051e..dd6f05a6ae7 100644 --- a/app/assets/javascripts/notes/components/comment_form.vue +++ b/app/assets/javascripts/notes/components/comment_form.vue @@ -132,9 +132,6 @@ export default { noteable: this.noteableDisplayName, }); }, - buttonVariant() { - return this.isOpen ? 'warning' : 'default'; - }, actionButtonClassNames() { return { 'btn-reopen': !this.isOpen, @@ -422,8 +419,6 @@ export default { <gl-button v-if="canToggleIssueState" :loading="isToggleStateButtonLoading" - category="secondary" - :variant="buttonVariant" :class="[actionButtonClassNames, 'btn-comment btn-comment-and-close']" :disabled="isSubmitting" data-testid="close-reopen-button" diff --git a/app/assets/javascripts/notes/components/sidebar_subscription.vue b/app/assets/javascripts/notes/components/sidebar_subscription.vue index be0efaa7b1c..047c04c8482 100644 --- a/app/assets/javascripts/notes/components/sidebar_subscription.vue +++ b/app/assets/javascripts/notes/components/sidebar_subscription.vue @@ -19,7 +19,7 @@ export default { computed: { fullPath() { if (this.noteableData.web_url) { - return this.noteableData.web_url.split('/-/')[0].substring(1); + return this.noteableData.web_url.split('/-/')[0].substring(1).replace('groups/', ''); } return null; }, @@ -28,7 +28,7 @@ export default { }, }, created() { - if (this.issuableType !== IssuableType.Issue) { + if (this.issuableType !== IssuableType.Issue && this.issuableType !== IssuableType.Epic) { return; } diff --git a/app/assets/javascripts/pages/groups/shared/group_details.js b/app/assets/javascripts/pages/groups/shared/group_details.js index b076ce68250..9e75985c130 100644 --- a/app/assets/javascripts/pages/groups/shared/group_details.js +++ b/app/assets/javascripts/pages/groups/shared/group_details.js @@ -3,8 +3,6 @@ import ShortcutsNavigation from '~/behaviors/shortcuts/shortcuts_navigation'; import { ACTIVE_TAB_SHARED, ACTIVE_TAB_ARCHIVED } from '~/groups/constants'; import initInviteMembersBanner from '~/groups/init_invite_members_banner'; -import initInviteMembersModal from '~/invite_members/init_invite_members_modal'; -import initInviteMembersTrigger from '~/invite_members/init_invite_members_trigger'; import { getPagePath, getDashPath } from '~/lib/utils/common_utils'; import initNotificationsDropdown from '~/notifications'; import ProjectsList from '~/projects_list'; @@ -26,6 +24,4 @@ export default function initGroupDetails(actionName = 'show') { new ProjectsList(); initInviteMembersBanner(); - initInviteMembersModal(); - initInviteMembersTrigger(); } diff --git a/app/assets/javascripts/pages/projects/show/index.js b/app/assets/javascripts/pages/projects/show/index.js index 7bb7be45f59..a0831c7df41 100644 --- a/app/assets/javascripts/pages/projects/show/index.js +++ b/app/assets/javascripts/pages/projects/show/index.js @@ -3,8 +3,6 @@ import Activities from '~/activities'; import ShortcutsNavigation from '~/behaviors/shortcuts/shortcuts_navigation'; import BlobViewer from '~/blob/viewer/index'; import { initUploadForm } from '~/blob_edit/blob_bundle'; -import initInviteMembersModal from '~/invite_members/init_invite_members_modal'; -import initInviteMembersTrigger from '~/invite_members/init_invite_members_trigger'; import leaveByUrl from '~/namespaces/leave_by_url'; import initVueNotificationsDropdown from '~/notifications'; import initReadMore from '~/read_more'; @@ -43,6 +41,3 @@ leaveByUrl('project'); initVueNotificationsDropdown(); new ShortcutsNavigation(); // eslint-disable-line no-new - -initInviteMembersTrigger(); -initInviteMembersModal(); diff --git a/app/assets/javascripts/pipeline_new/components/pipeline_new_form.vue b/app/assets/javascripts/pipeline_new/components/pipeline_new_form.vue index bd112697b49..ff6a354f673 100644 --- a/app/assets/javascripts/pipeline_new/components/pipeline_new_form.vue +++ b/app/assets/javascripts/pipeline_new/components/pipeline_new_form.vue @@ -9,10 +9,6 @@ import { GlFormSelect, GlFormTextarea, GlLink, - GlDropdown, - GlDropdownItem, - GlDropdownSectionHeader, - GlSearchBoxByType, GlSprintf, GlLoadingIcon, GlSafeHtmlDirective as SafeHtml, @@ -26,19 +22,26 @@ import httpStatusCodes from '~/lib/utils/http_status'; import { redirectTo } from '~/lib/utils/url_utility'; import { s__, __, n__ } from '~/locale'; import { VARIABLE_TYPE, FILE_TYPE, CONFIG_VARIABLES_TIMEOUT } from '../constants'; +import RefsDropdown from './refs_dropdown.vue'; + +const i18n = { + variablesDescription: s__( + 'Pipeline|Specify variable values to be used in this run. The values specified in %{linkStart}CI/CD settings%{linkEnd} will be used by default.', + ), + defaultError: __('Something went wrong on our end. Please try again.'), + refsLoadingErrorTitle: s__('Pipeline|Branches or tags could not be loaded.'), + submitErrorTitle: s__('Pipeline|Pipeline cannot be run.'), + warningTitle: __('The form contains the following warning:'), + maxWarningsSummary: __('%{total} warnings found: showing first %{warningsDisplayed}'), +}; export default { typeOptions: [ { value: VARIABLE_TYPE, text: __('Variable') }, { value: FILE_TYPE, text: __('File') }, ], - variablesDescription: s__( - 'Pipeline|Specify variable values to be used in this run. The values specified in %{linkStart}CI/CD settings%{linkEnd} will be used by default.', - ), + i18n, formElementClasses: 'gl-mr-3 gl-mb-3 gl-flex-basis-quarter gl-flex-shrink-0 gl-flex-grow-0', - errorTitle: __('Pipeline cannot be run.'), - warningTitle: __('The form contains the following warning:'), - maxWarningsSummary: __('%{total} warnings found: showing first %{warningsDisplayed}'), // this height value is used inline on the textarea to match the input field height // it's used to prevent the overwrite if 'gl-h-7' or 'gl-h-7!' were used textAreaStyle: { height: '32px' }, @@ -52,12 +55,9 @@ export default { GlFormSelect, GlFormTextarea, GlLink, - GlDropdown, - GlDropdownItem, - GlDropdownSectionHeader, - GlSearchBoxByType, GlSprintf, GlLoadingIcon, + RefsDropdown, }, directives: { SafeHtml }, props: { @@ -77,14 +77,6 @@ export default { type: String, required: true, }, - branches: { - type: Array, - required: true, - }, - tags: { - type: Array, - required: true, - }, settingsLink: { type: String, required: true, @@ -111,11 +103,11 @@ export default { }, data() { return { - searchTerm: '', refValue: { shortName: this.refParam, }, form: {}, + errorTitle: null, error: null, warnings: [], totalWarnings: 0, @@ -125,22 +117,6 @@ export default { }; }, computed: { - lowerCasedSearchTerm() { - return this.searchTerm.toLowerCase(); - }, - filteredBranches() { - return this.branches.filter((branch) => - branch.shortName.toLowerCase().includes(this.lowerCasedSearchTerm), - ); - }, - filteredTags() { - return this.tags.filter((tag) => - tag.shortName.toLowerCase().includes(this.lowerCasedSearchTerm), - ); - }, - hasTags() { - return this.tags.length > 0; - }, overMaxWarningsLimit() { return this.totalWarnings > this.maxWarnings; }, @@ -148,7 +124,7 @@ export default { return n__('%d warning found:', '%d warnings found:', this.warnings.length); }, summaryMessage() { - return this.overMaxWarningsLimit ? this.$options.maxWarningsSummary : this.warningsSummary; + return this.overMaxWarningsLimit ? i18n.maxWarningsSummary : this.warningsSummary; }, shouldShowWarning() { return this.warnings.length > 0 && !this.isWarningDismissed; @@ -166,6 +142,11 @@ export default { return this.form[this.refFullName]?.descriptions ?? {}; }, }, + watch: { + refValue() { + this.loadConfigVariablesForm(); + }, + }, created() { // this is needed until we add support for ref type in url query strings // ensure default branch is called with full ref on load @@ -174,7 +155,7 @@ export default { this.refValue.fullName = `refs/heads/${this.refValue.shortName}`; } - this.setRefSelected(this.refValue); + this.loadConfigVariablesForm(); }, methods: { addEmptyVariable(refValue) { @@ -213,49 +194,47 @@ export default { this.setVariable(refValue, type, key, value); }); }, - setRefSelected(refValue) { - this.refValue = refValue; - - if (!this.form[this.refFullName]) { - this.fetchConfigVariables(this.refFullName || this.refShortName) - .then(({ descriptions, params }) => { - Vue.set(this.form, this.refFullName, { - variables: [], - descriptions, - }); - - // Add default variables from yml - this.setVariableParams(this.refFullName, VARIABLE_TYPE, params); - }) - .catch(() => { - Vue.set(this.form, this.refFullName, { - variables: [], - descriptions: {}, - }); - }) - .finally(() => { - // Add/update variables, e.g. from query string - if (this.variableParams) { - this.setVariableParams(this.refFullName, VARIABLE_TYPE, this.variableParams); - } - if (this.fileParams) { - this.setVariableParams(this.refFullName, FILE_TYPE, this.fileParams); - } - - // Adds empty var at the end of the form - this.addEmptyVariable(this.refFullName); - }); - } - }, - isSelected(ref) { - return ref.fullName === this.refValue.fullName; - }, removeVariable(index) { this.variables.splice(index, 1); }, canRemove(index) { return index < this.variables.length - 1; }, + loadConfigVariablesForm() { + // Skip when variables already cached in `form` + if (this.form[this.refFullName]) { + return; + } + + this.fetchConfigVariables(this.refFullName || this.refShortName) + .then(({ descriptions, params }) => { + Vue.set(this.form, this.refFullName, { + variables: [], + descriptions, + }); + + // Add default variables from yml + this.setVariableParams(this.refFullName, VARIABLE_TYPE, params); + }) + .catch(() => { + Vue.set(this.form, this.refFullName, { + variables: [], + descriptions: {}, + }); + }) + .finally(() => { + // Add/update variables, e.g. from query string + if (this.variableParams) { + this.setVariableParams(this.refFullName, VARIABLE_TYPE, this.variableParams); + } + if (this.fileParams) { + this.setVariableParams(this.refFullName, FILE_TYPE, this.fileParams); + } + + // Adds empty var at the end of the form + this.addEmptyVariable(this.refFullName); + }); + }, fetchConfigVariables(refValue) { this.isLoading = true; @@ -330,11 +309,25 @@ export default { } = err?.response?.data; const [error] = errors; - this.error = error; - this.warnings = warnings; - this.totalWarnings = totalWarnings; + this.reportError({ + title: i18n.submitErrorTitle, + error, + warnings, + totalWarnings, + }); }); }, + onRefsLoadingError(error) { + this.reportError({ title: i18n.refsLoadingErrorTitle }); + + Sentry.captureException(error); + }, + reportError({ title = null, error = i18n.defaultError, warnings = [], totalWarnings = 0 }) { + this.errorTitle = title; + this.error = error; + this.warnings = warnings; + this.totalWarnings = totalWarnings; + }, }, }; </script> @@ -343,7 +336,7 @@ export default { <gl-form @submit.prevent="createPipeline"> <gl-alert v-if="error" - :title="$options.errorTitle" + :title="errorTitle" :dismissible="false" variant="danger" class="gl-mb-4" @@ -353,7 +346,7 @@ export default { </gl-alert> <gl-alert v-if="shouldShowWarning" - :title="$options.warningTitle" + :title="$options.i18n.warningTitle" variant="warning" class="gl-mb-4" data-testid="run-pipeline-warning-alert" @@ -380,31 +373,7 @@ export default { </details> </gl-alert> <gl-form-group :label="s__('Pipeline|Run for branch name or tag')"> - <gl-dropdown :text="refShortName" block> - <gl-search-box-by-type v-model.trim="searchTerm" :placeholder="__('Search refs')" /> - <gl-dropdown-section-header>{{ __('Branches') }}</gl-dropdown-section-header> - <gl-dropdown-item - v-for="branch in filteredBranches" - :key="branch.fullName" - class="gl-font-monospace" - is-check-item - :is-checked="isSelected(branch)" - @click="setRefSelected(branch)" - > - {{ branch.shortName }} - </gl-dropdown-item> - <gl-dropdown-section-header v-if="hasTags">{{ __('Tags') }}</gl-dropdown-section-header> - <gl-dropdown-item - v-for="tag in filteredTags" - :key="tag.fullName" - class="gl-font-monospace" - is-check-item - :is-checked="isSelected(tag)" - @click="setRefSelected(tag)" - > - {{ tag.shortName }} - </gl-dropdown-item> - </gl-dropdown> + <refs-dropdown v-model="refValue" @loadingError="onRefsLoadingError" /> </gl-form-group> <gl-loading-icon v-if="isLoading" class="gl-mb-5" size="lg" /> @@ -465,7 +434,7 @@ export default { </div> <template #description - ><gl-sprintf :message="$options.variablesDescription"> + ><gl-sprintf :message="$options.i18n.variablesDescription"> <template #link="{ content }"> <gl-link :href="settingsLink">{{ content }}</gl-link> </template> diff --git a/app/assets/javascripts/pipeline_new/components/refs_dropdown.vue b/app/assets/javascripts/pipeline_new/components/refs_dropdown.vue new file mode 100644 index 00000000000..ed5c659d1df --- /dev/null +++ b/app/assets/javascripts/pipeline_new/components/refs_dropdown.vue @@ -0,0 +1,113 @@ +<script> +import { GlDropdown, GlDropdownItem, GlDropdownSectionHeader, GlSearchBoxByType } from '@gitlab/ui'; +import { debounce } from 'lodash'; +import axios from '~/lib/utils/axios_utils'; +import { BRANCH_REF_TYPE, TAG_REF_TYPE, DEBOUNCE_REFS_SEARCH_MS } from '../constants'; +import formatRefs from '../utils/format_refs'; + +export default { + components: { + GlDropdown, + GlDropdownItem, + GlDropdownSectionHeader, + GlSearchBoxByType, + }, + inject: ['projectRefsEndpoint'], + props: { + value: { + type: Object, + required: false, + default: () => ({}), + }, + }, + data() { + return { + isLoading: false, + searchTerm: '', + branches: [], + tags: [], + }; + }, + computed: { + lowerCasedSearchTerm() { + return this.searchTerm.toLowerCase(); + }, + refShortName() { + return this.value.shortName; + }, + hasTags() { + return this.tags.length > 0; + }, + }, + watch: { + searchTerm() { + this.debouncedLoadRefs(); + }, + }, + methods: { + loadRefs() { + this.isLoading = true; + + axios + .get(this.projectRefsEndpoint, { + params: { + search: this.lowerCasedSearchTerm, + }, + }) + .then(({ data }) => { + // Note: These keys are uppercase in API + const { Branches = [], Tags = [] } = data; + + this.branches = formatRefs(Branches, BRANCH_REF_TYPE); + this.tags = formatRefs(Tags, TAG_REF_TYPE); + }) + .catch((e) => { + this.$emit('loadingError', e); + }) + .finally(() => { + this.isLoading = false; + }); + }, + debouncedLoadRefs: debounce(function debouncedLoadRefs() { + this.loadRefs(); + }, DEBOUNCE_REFS_SEARCH_MS), + setRefSelected(ref) { + this.$emit('input', ref); + }, + isSelected(ref) { + return ref.fullName === this.value.fullName; + }, + }, +}; +</script> +<template> + <gl-dropdown :text="refShortName" block @show.once="loadRefs"> + <gl-search-box-by-type + v-model.trim="searchTerm" + :is-loading="isLoading" + :placeholder="__('Search refs')" + /> + <gl-dropdown-section-header>{{ __('Branches') }}</gl-dropdown-section-header> + <gl-dropdown-item + v-for="branch in branches" + :key="branch.fullName" + class="gl-font-monospace" + is-check-item + :is-checked="isSelected(branch)" + @click="setRefSelected(branch)" + > + {{ branch.shortName }} + </gl-dropdown-item> + <gl-dropdown-section-header v-if="hasTags">{{ __('Tags') }}</gl-dropdown-section-header> + <gl-dropdown-item + v-for="tag in tags" + :key="tag.fullName" + class="gl-font-monospace" + is-check-item + :is-checked="isSelected(tag)" + @click="setRefSelected(tag)" + > + {{ tag.shortName }} + </gl-dropdown-item> + </gl-dropdown> +</template> diff --git a/app/assets/javascripts/pipeline_new/constants.js b/app/assets/javascripts/pipeline_new/constants.js index 004bbe7daf4..681755dc6ab 100644 --- a/app/assets/javascripts/pipeline_new/constants.js +++ b/app/assets/javascripts/pipeline_new/constants.js @@ -1,5 +1,6 @@ export const VARIABLE_TYPE = 'env_var'; export const FILE_TYPE = 'file'; +export const DEBOUNCE_REFS_SEARCH_MS = 250; export const CONFIG_VARIABLES_TIMEOUT = 5000; export const BRANCH_REF_TYPE = 'branch'; export const TAG_REF_TYPE = 'tag'; diff --git a/app/assets/javascripts/pipeline_new/index.js b/app/assets/javascripts/pipeline_new/index.js index 0b85184ec90..a645ea8603b 100644 --- a/app/assets/javascripts/pipeline_new/index.js +++ b/app/assets/javascripts/pipeline_new/index.js @@ -1,10 +1,13 @@ import Vue from 'vue'; import PipelineNewForm from './components/pipeline_new_form.vue'; -import formatRefs from './utils/format_refs'; export default () => { const el = document.getElementById('js-new-pipeline'); const { + // provide/inject + projectRefsEndpoint, + + // props projectId, pipelinesPath, configVariablesPath, @@ -12,19 +15,18 @@ export default () => { refParam, varParam, fileParam, - branchRefs, - tagRefs, settingsLink, maxWarnings, } = el?.dataset; const variableParams = JSON.parse(varParam); const fileParams = JSON.parse(fileParam); - const branches = formatRefs(JSON.parse(branchRefs), 'branch'); - const tags = formatRefs(JSON.parse(tagRefs), 'tag'); return new Vue({ el, + provide: { + projectRefsEndpoint, + }, render(createElement) { return createElement(PipelineNewForm, { props: { @@ -35,8 +37,6 @@ export default () => { refParam, variableParams, fileParams, - branches, - tags, settingsLink, maxWarnings: Number(maxWarnings), }, diff --git a/app/assets/javascripts/pipelines/components/pipelines_list/stage.vue b/app/assets/javascripts/pipelines/components/pipelines_list/stage.vue index 9b82693ecb4..dfa2c198158 100644 --- a/app/assets/javascripts/pipelines/components/pipelines_list/stage.vue +++ b/app/assets/javascripts/pipelines/components/pipelines_list/stage.vue @@ -38,6 +38,11 @@ export default { required: false, default: false, }, + isMergeTrain: { + type: Boolean, + required: false, + default: true, + }, }, data() { return { @@ -126,6 +131,21 @@ export default { @pipelineActionRequestComplete="pipelineActionRequestComplete" /> </li> + <template v-if="isMergeTrain"> + <li class="gl-new-dropdown-divider" role="presentation"> + <hr role="separator" aria-orientation="horizontal" class="dropdown-divider" /> + </li> + <li> + <div + class="gl-display-flex gl-align-items-center" + data-testid="warning-message-merge-trains" + > + <div class="menu-item gl-font-sm gl-text-gray-300!"> + {{ s__('Pipeline|Merge train pipeline jobs can not be retried') }} + </div> + </div> + </li> + </template> </ul> </gl-dropdown> </template> diff --git a/app/assets/javascripts/sidebar/components/assignees/sidebar_assignees_widget.vue b/app/assets/javascripts/sidebar/components/assignees/sidebar_assignees_widget.vue index 6d1e6495ac8..cc2201ad359 100644 --- a/app/assets/javascripts/sidebar/components/assignees/sidebar_assignees_widget.vue +++ b/app/assets/javascripts/sidebar/components/assignees/sidebar_assignees_widget.vue @@ -15,13 +15,12 @@ import { IssuableType } from '~/issue_show/constants'; import { __, n__ } from '~/locale'; import IssuableAssignees from '~/sidebar/components/assignees/issuable_assignees.vue'; import SidebarEditableItem from '~/sidebar/components/sidebar_editable_item.vue'; -import { assigneesQueries } from '~/sidebar/constants'; +import { assigneesQueries, ASSIGNEES_DEBOUNCE_DELAY } from '~/sidebar/constants'; import MultiSelectDropdown from '~/vue_shared/components/sidebar/multiselect_dropdown.vue'; export const assigneesWidget = Vue.observable({ updateAssignees: null, }); - export default { i18n: { unassigned: __('Unassigned'), @@ -88,10 +87,10 @@ export default { return this.queryVariables; }, update(data) { - return data.issuable || data.project?.issuable; + return data.workspace?.issuable; }, result({ data }) { - const issuable = data.issuable || data.project?.issuable; + const issuable = data.workspace?.issuable; if (issuable) { this.selected = this.moveCurrentUserToStart(cloneDeep(issuable.assignees.nodes)); } @@ -109,7 +108,7 @@ export default { }; }, update(data) { - const searchResults = data.issuable?.users?.nodes.map(({ user }) => user) || []; + const searchResults = data.workspace?.users?.nodes.map(({ user }) => user) || []; const mergedSearchResults = this.participants.reduce((acc, current) => { if ( !acc.some((user) => current.username === user.username) && @@ -121,7 +120,7 @@ export default { }, searchResults); return mergedSearchResults; }, - debounce: 250, + debounce: ASSIGNEES_DEBOUNCE_DELAY, skip() { return this.isSearchEmpty; }, @@ -229,7 +228,7 @@ export default { }, }) .then(({ data }) => { - this.$emit('assignees-updated', data); + this.$emit('assignees-updated', data.issuableSetAssignees.issuable.assignees.nodes); return data; }) .catch(() => { @@ -378,7 +377,7 @@ export default { <template v-if="showCurrentUser"> <gl-dropdown-divider /> <gl-dropdown-item - data-testid="unselected-participant" + data-testid="current-user" @click.stop="selectAssignee(currentUser)" > <gl-avatar-link> @@ -409,7 +408,7 @@ export default { /> </gl-avatar-link> </gl-dropdown-item> - <gl-dropdown-item v-if="noUsersFound && !isSearching"> + <gl-dropdown-item v-if="noUsersFound && !isSearching" data-testid="empty-results"> {{ __('No matching results') }} </gl-dropdown-item> </template> diff --git a/app/assets/javascripts/sidebar/components/confidential/confidential_issue_sidebar.vue b/app/assets/javascripts/sidebar/components/confidential/confidential_issue_sidebar.vue deleted file mode 100644 index 57b3705e803..00000000000 --- a/app/assets/javascripts/sidebar/components/confidential/confidential_issue_sidebar.vue +++ /dev/null @@ -1,113 +0,0 @@ -<script> -import { GlIcon, GlTooltipDirective } from '@gitlab/ui'; -import { mapState } from 'vuex'; -import { __, sprintf } from '~/locale'; -import eventHub from '~/sidebar/event_hub'; -import EditForm from './edit_form.vue'; - -export default { - components: { - EditForm, - GlIcon, - }, - directives: { - GlTooltip: GlTooltipDirective, - }, - props: { - fullPath: { - required: true, - type: String, - }, - isEditable: { - required: true, - type: Boolean, - }, - issuableType: { - required: false, - type: String, - default: 'issue', - }, - }, - data() { - return { - edit: false, - }; - }, - computed: { - ...mapState({ - confidential: ({ noteableData, confidential }) => { - if (noteableData) { - return noteableData.confidential; - } - return Boolean(confidential); - }, - }), - confidentialityIcon() { - return this.confidential ? 'eye-slash' : 'eye'; - }, - tooltipLabel() { - return this.confidential ? __('Confidential') : __('Not confidential'); - }, - confidentialText() { - return sprintf(__('This %{issuableType} is confidential'), { - issuableType: this.issuableType, - }); - }, - }, - created() { - eventHub.$on('closeConfidentialityForm', this.toggleForm); - }, - beforeDestroy() { - eventHub.$off('closeConfidentialityForm', this.toggleForm); - }, - methods: { - toggleForm() { - this.edit = !this.edit; - }, - }, -}; -</script> - -<template> - <div class="block issuable-sidebar-item confidentiality"> - <div - ref="collapseIcon" - v-gl-tooltip.viewport.left - :title="tooltipLabel" - class="sidebar-collapsed-icon" - @click="toggleForm" - > - <gl-icon :name="confidentialityIcon" /> - </div> - <div class="title hide-collapsed"> - {{ __('Confidentiality') }} - <a - v-if="isEditable" - ref="editLink" - class="float-right confidential-edit" - href="#" - data-track-event="click_edit_button" - data-track-label="right_sidebar" - data-track-property="confidentiality" - @click.prevent="toggleForm" - >{{ __('Edit') }}</a - > - </div> - <div class="value sidebar-item-value hide-collapsed"> - <edit-form - v-if="edit" - :confidential="confidential" - :full-path="fullPath" - :issuable-type="issuableType" - /> - <div v-if="!confidential" class="no-value sidebar-item-value" data-testid="not-confidential"> - <gl-icon :size="16" name="eye" class="sidebar-item-icon inline" /> - {{ __('Not confidential') }} - </div> - <div v-else class="value sidebar-item-value hide-collapsed"> - <gl-icon :size="16" name="eye-slash" class="sidebar-item-icon inline is-active" /> - {{ confidentialText }} - </div> - </div> - </div> -</template> diff --git a/app/assets/javascripts/sidebar/components/confidential/edit_form.vue b/app/assets/javascripts/sidebar/components/confidential/edit_form.vue deleted file mode 100644 index 057224d5918..00000000000 --- a/app/assets/javascripts/sidebar/components/confidential/edit_form.vue +++ /dev/null @@ -1,64 +0,0 @@ -<script> -import { GlSprintf } from '@gitlab/ui'; -import { __ } from '../../../locale'; -import editFormButtons from './edit_form_buttons.vue'; - -export default { - components: { - editFormButtons, - GlSprintf, - }, - props: { - confidential: { - required: true, - type: Boolean, - }, - fullPath: { - required: true, - type: String, - }, - issuableType: { - required: true, - type: String, - }, - }, - computed: { - confidentialityOnWarning() { - return __( - 'You are going to turn on the confidentiality. This means that only team members with %{strongStart}at least Reporter access%{strongEnd} are able to see and leave comments on the %{issuableType}.', - ); - }, - confidentialityOffWarning() { - return __( - 'You are going to turn off the confidentiality. This means %{strongStart}everyone%{strongEnd} will be able to see and leave a comment on this %{issuableType}.', - ); - }, - }, -}; -</script> - -<template> - <div class="dropdown show"> - <div class="dropdown-menu sidebar-item-warning-message"> - <div> - <p v-if="!confidential"> - <gl-sprintf :message="confidentialityOnWarning"> - <template #strong="{ content }"> - <strong>{{ content }}</strong> - </template> - <template #issuableType>{{ issuableType }}</template> - </gl-sprintf> - </p> - <p v-else> - <gl-sprintf :message="confidentialityOffWarning"> - <template #strong="{ content }"> - <strong>{{ content }}</strong> - </template> - <template #issuableType>{{ issuableType }}</template> - </gl-sprintf> - </p> - <edit-form-buttons :full-path="fullPath" :confidential="confidential" /> - </div> - </div> - </div> -</template> diff --git a/app/assets/javascripts/sidebar/components/confidential/edit_form_buttons.vue b/app/assets/javascripts/sidebar/components/confidential/edit_form_buttons.vue deleted file mode 100644 index 154a228c978..00000000000 --- a/app/assets/javascripts/sidebar/components/confidential/edit_form_buttons.vue +++ /dev/null @@ -1,81 +0,0 @@ -<script> -import { GlButton } from '@gitlab/ui'; -import $ from 'jquery'; -import { mapActions } from 'vuex'; -import { deprecatedCreateFlash as Flash } from '~/flash'; -import { __ } from '~/locale'; -import eventHub from '../../event_hub'; - -export default { - components: { - GlButton, - }, - props: { - fullPath: { - required: true, - type: String, - }, - confidential: { - required: true, - type: Boolean, - }, - }, - data() { - return { - isLoading: false, - }; - }, - computed: { - toggleButtonText() { - if (this.isLoading) { - return __('Applying'); - } - - return this.confidential ? __('Turn Off') : __('Turn On'); - }, - }, - methods: { - ...mapActions(['updateConfidentialityOnIssuable']), - closeForm() { - eventHub.$emit('closeConfidentialityForm'); - $(this.$el).trigger('hidden.gl.dropdown'); - }, - submitForm() { - this.isLoading = true; - const confidential = !this.confidential; - - this.updateConfidentialityOnIssuable({ confidential, fullPath: this.fullPath }) - .then(() => { - eventHub.$emit('updateIssuableConfidentiality', confidential); - }) - .catch((err) => { - Flash( - err || __('Something went wrong trying to change the confidentiality of this issue'), - ); - }) - .finally(() => { - this.closeForm(); - this.isLoading = false; - }); - }, - }, -}; -</script> - -<template> - <div class="sidebar-item-warning-message-actions"> - <gl-button class="gl-mr-3" @click="closeForm"> - {{ __('Cancel') }} - </gl-button> - <gl-button - category="secondary" - variant="warning" - :disabled="isLoading" - :loading="isLoading" - data-testid="confidential-toggle" - @click.prevent="submitForm" - > - {{ toggleButtonText }} - </gl-button> - </div> -</template> diff --git a/app/assets/javascripts/sidebar/components/confidential/mutations/update_issue_confidential.mutation.graphql b/app/assets/javascripts/sidebar/components/confidential/mutations/update_issue_confidential.mutation.graphql deleted file mode 100644 index 5caf5f6b555..00000000000 --- a/app/assets/javascripts/sidebar/components/confidential/mutations/update_issue_confidential.mutation.graphql +++ /dev/null @@ -1,8 +0,0 @@ -mutation updateIssueConfidential($input: IssueSetConfidentialInput!) { - issueSetConfidential(input: $input) { - issue { - confidential - } - errors - } -} diff --git a/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_content.vue b/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_content.vue index 7919021849c..37a44eb8f01 100644 --- a/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_content.vue +++ b/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_content.vue @@ -14,6 +14,10 @@ export default { type: Boolean, required: true, }, + issuableType: { + type: String, + required: true, + }, }, computed: { confidentialText() { @@ -35,7 +39,13 @@ export default { <template> <div> - <div v-gl-tooltip.viewport.left :title="tooltipLabel" class="sidebar-collapsed-icon"> + <div + v-gl-tooltip.viewport.left + :title="tooltipLabel" + class="sidebar-collapsed-icon" + data-testid="sidebar-collapsed-icon" + @click="$emit('expandSidebar')" + > <gl-icon :size="16" :name="confidentialIcon" diff --git a/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_form.vue b/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_form.vue index d0e1637607d..a21ac73f131 100644 --- a/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_form.vue +++ b/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_form.vue @@ -1,6 +1,7 @@ <script> import { GlSprintf, GlButton } from '@gitlab/ui'; import createFlash from '~/flash'; +import { IssuableType } from '~/issue_show/constants'; import { __, sprintf } from '~/locale'; import { confidentialityQueries } from '~/sidebar/constants'; @@ -45,6 +46,15 @@ export default { ? this.$options.i18n.confidentialityOffWarning : this.$options.i18n.confidentialityOnWarning; }, + workspacePath() { + return this.issuableType === IssuableType.Issue + ? { + projectPath: this.fullPath, + } + : { + groupPath: this.fullPath, + }; + }, }, methods: { submitForm() { @@ -54,7 +64,7 @@ export default { mutation: confidentialityQueries[this.issuableType].mutation, variables: { input: { - projectPath: this.fullPath, + ...this.workspacePath, iid: this.iid, confidential: !this.confidential, }, diff --git a/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_widget.vue b/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_widget.vue index 2af1bc92e4b..1db68d3d5b1 100644 --- a/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_widget.vue +++ b/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_widget.vue @@ -47,12 +47,15 @@ export default { variables() { return { fullPath: this.fullPath, - iid: this.iid, + iid: String(this.iid), }; }, update(data) { return data.workspace?.issuable?.confidential || false; }, + result({ data }) { + this.$emit('confidentialityUpdated', data.workspace?.issuable?.confidential); + }, error() { createFlash({ message: sprintf( @@ -80,6 +83,7 @@ export default { closeForm() { this.$refs.editable.collapse(); this.$el.dispatchEvent(hideDropdownEvent); + this.$emit('closeForm'); }, // synchronizing the quick action with the sidebar widget // this is a temporary solution until we have confidentiality real-time updates @@ -101,6 +105,10 @@ export default { data, }); }, + expandSidebar() { + this.$refs.editable.expand(); + this.$emit('expandSidebar'); + }, }, }; </script> @@ -115,11 +123,16 @@ export default { > <template #collapsed> <div> - <sidebar-confidentiality-content v-if="!isLoading" :confidential="confidential" /> + <sidebar-confidentiality-content + v-if="!isLoading" + :confidential="confidential" + :issuable-type="issuableType" + @expandSidebar="expandSidebar" + /> </div> </template> <template #default> - <sidebar-confidentiality-content :confidential="confidential" /> + <sidebar-confidentiality-content :confidential="confidential" :issuable-type="issuableType" /> <sidebar-confidentiality-form :confidential="confidential" :issuable-type="issuableType" diff --git a/app/assets/javascripts/sidebar/components/sidebar_editable_item.vue b/app/assets/javascripts/sidebar/components/sidebar_editable_item.vue index c9eca2bfc5c..28ae017078e 100644 --- a/app/assets/javascripts/sidebar/components/sidebar_editable_item.vue +++ b/app/assets/javascripts/sidebar/components/sidebar_editable_item.vue @@ -87,7 +87,7 @@ export default { <gl-button v-if="canUpdate" variant="link" - class="gl-text-gray-900! gl-hover-text-blue-800! gl-ml-auto js-sidebar-dropdown-toggle hide-collapsed" + class="gl-text-gray-900! gl-hover-text-blue-800! gl-ml-auto hide-collapsed" data-testid="edit-button" :data-track-event="tracking.event" :data-track-label="tracking.label" diff --git a/app/assets/javascripts/sidebar/constants.js b/app/assets/javascripts/sidebar/constants.js index 8221fdcb938..e3929499009 100644 --- a/app/assets/javascripts/sidebar/constants.js +++ b/app/assets/javascripts/sidebar/constants.js @@ -1,11 +1,15 @@ import { IssuableType } from '~/issue_show/constants'; +import epicConfidentialQuery from '~/sidebar/queries/epic_confidential.query.graphql'; import issueConfidentialQuery from '~/sidebar/queries/issue_confidential.query.graphql'; +import updateEpicMutation from '~/sidebar/queries/update_epic_confidential.mutation.graphql'; import updateIssueConfidentialMutation from '~/sidebar/queries/update_issue_confidential.mutation.graphql'; import getIssueParticipants from '~/vue_shared/components/sidebar/queries/get_issue_participants.query.graphql'; import getMergeRequestParticipants from '~/vue_shared/components/sidebar/queries/get_mr_participants.query.graphql'; import updateAssigneesMutation from '~/vue_shared/components/sidebar/queries/update_issue_assignees.mutation.graphql'; import updateMergeRequestParticipantsMutation from '~/vue_shared/components/sidebar/queries/update_mr_assignees.mutation.graphql'; +export const ASSIGNEES_DEBOUNCE_DELAY = 250; + export const assigneesQueries = { [IssuableType.Issue]: { query: getIssueParticipants, @@ -22,4 +26,8 @@ export const confidentialityQueries = { query: issueConfidentialQuery, mutation: updateIssueConfidentialMutation, }, + [IssuableType.Epic]: { + query: epicConfidentialQuery, + mutation: updateEpicMutation, + }, }; diff --git a/app/assets/javascripts/sidebar/queries/epic_confidential.query.graphql b/app/assets/javascripts/sidebar/queries/epic_confidential.query.graphql new file mode 100644 index 00000000000..7a1fdb40e93 --- /dev/null +++ b/app/assets/javascripts/sidebar/queries/epic_confidential.query.graphql @@ -0,0 +1,10 @@ +query epicConfidential($fullPath: ID!, $iid: ID) { + workspace: group(fullPath: $fullPath) { + __typename + issuable: epic(iid: $iid) { + __typename + id + confidential + } + } +} diff --git a/app/assets/javascripts/sidebar/queries/update_epic_confidential.mutation.graphql b/app/assets/javascripts/sidebar/queries/update_epic_confidential.mutation.graphql new file mode 100644 index 00000000000..69927ddd205 --- /dev/null +++ b/app/assets/javascripts/sidebar/queries/update_epic_confidential.mutation.graphql @@ -0,0 +1,9 @@ +mutation updateEpic($input: UpdateEpicInput!) { + issuableSetConfidential: updateEpic(input: $input) { + issuable: epic { + id + confidential + } + errors + } +} diff --git a/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline.vue b/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline.vue index adc4bdf745e..4051005b58b 100644 --- a/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline.vue +++ b/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline.vue @@ -15,6 +15,7 @@ import PipelineArtifacts from '~/pipelines/components/pipelines_list/pipelines_a import PipelineStage from '~/pipelines/components/pipelines_list/stage.vue'; import CiIcon from '~/vue_shared/components/ci_icon.vue'; import TooltipOnTruncate from '~/vue_shared/components/tooltip_on_truncate.vue'; +import { MT_MERGE_STRATEGY } from '../constants'; export default { name: 'MRWidgetPipeline', @@ -80,6 +81,11 @@ export default { type: String, required: true, }, + mergeStrategy: { + type: String, + required: false, + default: '', + }, }, computed: { hasPipeline() { @@ -130,6 +136,9 @@ export default { this.buildsWithCoverage.length, ); }, + isMergeTrain() { + return this.mergeStrategy === MT_MERGE_STRATEGY; + }, }, errorText: s__( 'Pipeline|Could not retrieve the pipeline status. For troubleshooting steps, read the %{linkStart}documentation%{linkEnd}.', @@ -249,7 +258,7 @@ export default { class="stage-container dropdown mr-widget-pipeline-stages" data-testid="widget-mini-pipeline-graph" > - <pipeline-stage :stage="stage" /> + <pipeline-stage :stage="stage" :is-merge-train="isMergeTrain" /> </div> </template> </span> diff --git a/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline_container.vue b/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline_container.vue index 6659041d6bf..9eb59b65e93 100644 --- a/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline_container.vue +++ b/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline_container.vue @@ -4,6 +4,7 @@ import { isNumber } from 'lodash'; import { sanitize } from '~/lib/dompurify'; import { n__ } from '~/locale'; import glFeatureFlagsMixin from '~/vue_shared/mixins/gl_feature_flags_mixin'; +import MergeRequestStore from '../stores/mr_widget_store'; import ArtifactsApp from './artifacts_list_app.vue'; import MrCollapsibleExtension from './mr_collapsible_extension.vue'; import MrWidgetContainer from './mr_widget_container.vue'; @@ -84,6 +85,15 @@ export default { this.deployments.length, ); }, + preferredAutoMergeStrategy() { + if (this.glFeatures.mergeRequestWidgetGraphql) { + return MergeRequestStore.getPreferredAutoMergeStrategy( + this.mr.availableAutoMergeStrategies, + ); + } + + return this.mr.preferredAutoMergeStrategy; + }, }, }; </script> @@ -100,6 +110,7 @@ export default { :source-branch-link="branchLink" :mr-troubleshooting-docs-path="mr.mrTroubleshootingDocsPath" :ci-troubleshooting-docs-path="mr.ciTroubleshootingDocsPath" + :merge-strategy="preferredAutoMergeStrategy" /> <template #footer> <div v-if="mr.exposedArtifactsPath" class="js-exposed-artifacts"> diff --git a/app/assets/javascripts/vue_shared/components/sidebar/queries/get_issue_participants.query.graphql b/app/assets/javascripts/vue_shared/components/sidebar/queries/get_issue_participants.query.graphql index 62c0b05426b..459ea27e9cd 100644 --- a/app/assets/javascripts/vue_shared/components/sidebar/queries/get_issue_participants.query.graphql +++ b/app/assets/javascripts/vue_shared/components/sidebar/queries/get_issue_participants.query.graphql @@ -1,8 +1,10 @@ #import "~/graphql_shared/fragments/user.fragment.graphql" query issueParticipants($fullPath: ID!, $iid: String!) { - project(fullPath: $fullPath) { + workspace: project(fullPath: $fullPath) { + __typename issuable: issue(iid: $iid) { + __typename id participants { nodes { diff --git a/app/assets/javascripts/vue_shared/components/sidebar/queries/get_mr_participants.query.graphql b/app/assets/javascripts/vue_shared/components/sidebar/queries/get_mr_participants.query.graphql index a75ce85a1dc..43bd9f17e9a 100644 --- a/app/assets/javascripts/vue_shared/components/sidebar/queries/get_mr_participants.query.graphql +++ b/app/assets/javascripts/vue_shared/components/sidebar/queries/get_mr_participants.query.graphql @@ -1,7 +1,7 @@ #import "~/graphql_shared/fragments/user.fragment.graphql" query getMrParticipants($fullPath: ID!, $iid: String!) { - project(fullPath: $fullPath) { + workspace: project(fullPath: $fullPath) { issuable: mergeRequest(iid: $iid) { id participants { diff --git a/app/assets/javascripts/vue_shared/components/sidebar/queries/update_issue_assignees.mutation.graphql b/app/assets/javascripts/vue_shared/components/sidebar/queries/update_issue_assignees.mutation.graphql index 2eb9bb4b07b..8ee8de2cb5c 100644 --- a/app/assets/javascripts/vue_shared/components/sidebar/queries/update_issue_assignees.mutation.graphql +++ b/app/assets/javascripts/vue_shared/components/sidebar/queries/update_issue_assignees.mutation.graphql @@ -1,10 +1,10 @@ #import "~/graphql_shared/fragments/user.fragment.graphql" mutation issueSetAssignees($iid: String!, $assigneeUsernames: [String!]!, $fullPath: ID!) { - issueSetAssignees( + issuableSetAssignees: issueSetAssignees( input: { iid: $iid, assigneeUsernames: $assigneeUsernames, projectPath: $fullPath } ) { - issue { + issuable: issue { id assignees { nodes { diff --git a/app/assets/stylesheets/framework/buttons.scss b/app/assets/stylesheets/framework/buttons.scss index 91af41c68dd..9dee1d77197 100644 --- a/app/assets/stylesheets/framework/buttons.scss +++ b/app/assets/stylesheets/framework/buttons.scss @@ -196,10 +196,6 @@ @include btn-orange; } - &.btn-close { - @include btn-outline($white, $orange-500, $orange-500, $orange-50, $orange-600, $orange-600, $orange-100, $orange-700, $orange-700); - } - &.btn-danger { @include btn-red; } diff --git a/app/graphql/resolvers/project_merge_requests_resolver.rb b/app/graphql/resolvers/project_merge_requests_resolver.rb index 9628a6dfd7a..66c020a0c14 100644 --- a/app/graphql/resolvers/project_merge_requests_resolver.rb +++ b/app/graphql/resolvers/project_merge_requests_resolver.rb @@ -10,7 +10,7 @@ module Resolvers def resolve(**args) scope = super - if only_count_is_selected_with_merged_at_filter?(args) && Feature.enabled?(:optimized_merge_request_count_with_merged_at_filter, default_enabled: :yaml) + if only_count_is_selected_with_merged_at_filter?(args) MergeRequest::MetricsFinder .new(current_user, args.merge(target_project: project)) .execute diff --git a/app/services/boards/update_service.rb b/app/services/boards/update_service.rb index 0340836fd78..48c6e44d55e 100644 --- a/app/services/boards/update_service.rb +++ b/app/services/boards/update_service.rb @@ -2,9 +2,20 @@ module Boards class UpdateService < Boards::BaseService + PERMITTED_PARAMS = %i(name hide_backlog_list hide_closed_list).freeze + def execute(board) + filter_params board.update(params) end + + def filter_params + params.slice!(*permitted_params) + end + + def permitted_params + PERMITTED_PARAMS + end end end diff --git a/app/services/issuable/clone/base_service.rb b/app/services/issuable/clone/base_service.rb index b2f9c083b5b..3c2bc527b12 100644 --- a/app/services/issuable/clone/base_service.rb +++ b/app/services/issuable/clone/base_service.rb @@ -3,12 +3,13 @@ module Issuable module Clone class BaseService < IssuableBaseService - attr_reader :original_entity, :new_entity + attr_reader :original_entity, :new_entity, :target_project alias_method :old_project, :project - def execute(original_entity, new_project = nil) + def execute(original_entity, target_project = nil) @original_entity = original_entity + @target_project = target_project # Using transaction because of a high resources footprint # on rewriting notes (unfolding references) @@ -77,6 +78,12 @@ module Issuable new_entity.project.group end end + + def relative_position + return if original_entity.project.root_ancestor.id != target_project.root_ancestor.id + + original_entity.relative_position + end end end end diff --git a/app/services/issues/clone_service.rb b/app/services/issues/clone_service.rb index 4c9c34f1247..b64e4687a87 100644 --- a/app/services/issues/clone_service.rb +++ b/app/services/issues/clone_service.rb @@ -47,6 +47,7 @@ module Issues new_params = { id: nil, iid: nil, + relative_position: relative_position, project: target_project, author: current_user, assignee_ids: original_entity.assignee_ids diff --git a/app/services/issues/move_service.rb b/app/services/issues/move_service.rb index 90ccbd8ed21..c1afb8f456d 100644 --- a/app/services/issues/move_service.rb +++ b/app/services/issues/move_service.rb @@ -48,13 +48,14 @@ module Issues def create_new_entity new_params = { - id: nil, - iid: nil, - project: target_project, - author: original_entity.author, - assignee_ids: original_entity.assignee_ids, - moved_issue: true - } + id: nil, + iid: nil, + relative_position: relative_position, + project: target_project, + author: original_entity.author, + assignee_ids: original_entity.assignee_ids, + moved_issue: true + } new_params = original_entity.serializable_hash.symbolize_keys.merge(new_params) diff --git a/app/views/groups/show.html.haml b/app/views/groups/show.html.haml index d1787d36cd2..a1557cda071 100644 --- a/app/views/groups/show.html.haml +++ b/app/views/groups/show.html.haml @@ -16,11 +16,6 @@ = content_for :meta_tags do = auto_discovery_link_tag(:atom, group_url(@group, rss_url_options), title: "#{@group.name} activity") -= content_for :invite_members_sidebar do - - if can_invite_members_for_group?(@group) - %li - .js-invite-members-trigger{ data: { icon: 'plus', classes: 'gl-text-decoration-none! gl-shadow-none!', display_text: _('Invite team members') } } - = render partial: 'flash_messages' = render_if_exists 'trials/banner', namespace: @group diff --git a/app/views/layouts/nav/sidebar/_group.html.haml b/app/views/layouts/nav/sidebar/_group.html.haml index e99b2f443be..9b5db5b6bb4 100644 --- a/app/views/layouts/nav/sidebar/_group.html.haml +++ b/app/views/layouts/nav/sidebar/_group.html.haml @@ -137,8 +137,6 @@ %strong.fly-out-top-item-name = _('Members') - = content_for :invite_members_sidebar - - if group_sidebar_link?(:settings) = nav_link(path: group_settings_nav_link_paths) do = link_to edit_group_path(@group) do diff --git a/app/views/layouts/nav/sidebar/_project.html.haml b/app/views/layouts/nav/sidebar/_project.html.haml index bbf2f242a63..b36f088959e 100644 --- a/app/views/layouts/nav/sidebar/_project.html.haml +++ b/app/views/layouts/nav/sidebar/_project.html.haml @@ -378,8 +378,6 @@ %strong.fly-out-top-item-name = _('Members') - = content_for :invite_members_sidebar - - if project_nav_tab? :settings = nav_link(path: sidebar_settings_paths) do = link_to edit_project_path(@project) do diff --git a/app/views/projects/_home_panel.html.haml b/app/views/projects/_home_panel.html.haml index 5ebe4a4b741..5085798238e 100644 --- a/app/views/projects/_home_panel.html.haml +++ b/app/views/projects/_home_panel.html.haml @@ -3,8 +3,6 @@ - max_project_topic_length = 15 - emails_disabled = @project.emails_disabled? -= render 'projects/invite_members_modal', project: @project - .project-home-panel.js-show-on-project-root.gl-my-5{ class: [("empty-project" if empty_repo)] } .row.gl-mb-3 .home-panel-title-row.col-md-12.col-lg-6.d-flex diff --git a/app/views/projects/_invite_members_link.html.haml b/app/views/projects/_invite_members_link.html.haml deleted file mode 100644 index 95cfc75d955..00000000000 --- a/app/views/projects/_invite_members_link.html.haml +++ /dev/null @@ -1,4 +0,0 @@ -- return unless can_invite_members_for_project?(@project) - -%li - .js-invite-members-trigger{ data: { icon: 'plus', classes: 'gl-text-decoration-none! gl-shadow-none!', display_text: _('Invite team members') } } diff --git a/app/views/projects/empty.html.haml b/app/views/projects/empty.html.haml index d16f271c8d8..0c682226df3 100644 --- a/app/views/projects/empty.html.haml +++ b/app/views/projects/empty.html.haml @@ -2,9 +2,6 @@ - default_branch_name = @project.default_branch_or_master - @skip_current_level_breadcrumb = true -= content_for :invite_members_sidebar do - = render partial: 'projects/invite_members_link' - = render partial: 'flash_messages', locals: { project: @project } = render "home_panel" diff --git a/app/views/projects/no_repo.html.haml b/app/views/projects/no_repo.html.haml index 3c7afff57f6..c88ea313287 100644 --- a/app/views/projects/no_repo.html.haml +++ b/app/views/projects/no_repo.html.haml @@ -12,8 +12,6 @@ #{ _('This means you can not push code until you create an empty repository or import existing one.') } %hr -= render 'projects/invite_members_modal', project: @project - .no-repo-actions = link_to project_repository_path(@project), method: :post, class: 'btn gl-button btn-confirm' do #{ _('Create empty repository') } diff --git a/app/views/projects/pipelines/new.html.haml b/app/views/projects/pipelines/new.html.haml index a760aaaf9b3..7a3817fe87b 100644 --- a/app/views/projects/pipelines/new.html.haml +++ b/app/views/projects/pipelines/new.html.haml @@ -14,8 +14,7 @@ ref_param: params[:ref] || @project.default_branch, var_param: params[:var].to_json, file_param: params[:file_var].to_json, - branch_refs: @project.repository.branch_names.to_json.html_safe, - tag_refs: @project.repository.tag_names.to_json.html_safe, + project_refs_endpoint: refs_project_path(@project, sort: 'updated_desc'), settings_link: project_settings_ci_cd_path(@project), max_warnings: ::Gitlab::Ci::Warnings::MAX_LIMIT } } diff --git a/app/views/projects/show.html.haml b/app/views/projects/show.html.haml index e1774c955bc..40faf91eadf 100644 --- a/app/views/projects/show.html.haml +++ b/app/views/projects/show.html.haml @@ -6,9 +6,6 @@ = content_for :meta_tags do = auto_discovery_link_tag(:atom, project_path(@project, rss_url_options), title: "#{@project.name} activity") -= content_for :invite_members_sidebar do - = render partial: 'projects/invite_members_link' - = render partial: 'flash_messages', locals: { project: @project } = render "projects/last_push" diff --git a/changelogs/unreleased/198562-merge-request-user-interface-encourages-accidentally-closing-the-r.yml b/changelogs/unreleased/198562-merge-request-user-interface-encourages-accidentally-closing-the-r.yml new file mode 100644 index 00000000000..ccc69cfcefe --- /dev/null +++ b/changelogs/unreleased/198562-merge-request-user-interface-encourages-accidentally-closing-the-r.yml @@ -0,0 +1,5 @@ +--- +title: Deemphasize comment and close button +merge_request: 55075 +author: +type: other diff --git a/changelogs/unreleased/321790-load-refs-on-demand.yml b/changelogs/unreleased/321790-load-refs-on-demand.yml new file mode 100644 index 00000000000..c20ad02500d --- /dev/null +++ b/changelogs/unreleased/321790-load-refs-on-demand.yml @@ -0,0 +1,5 @@ +--- +title: Improve performance of manual pipeline form by limiting the refs loaded on page load. +merge_request: 55394 +author: +type: performance diff --git a/changelogs/unreleased/322059-remove-api_v3_repos_events_optimization-flag.yml b/changelogs/unreleased/322059-remove-api_v3_repos_events_optimization-flag.yml new file mode 100644 index 00000000000..dafad2cb1eb --- /dev/null +++ b/changelogs/unreleased/322059-remove-api_v3_repos_events_optimization-flag.yml @@ -0,0 +1,5 @@ +--- +title: Fix N+1 queries in api/v3/repos/:namespace/:project/events endpoint +merge_request: 55442 +author: +type: performance diff --git a/changelogs/unreleased/add-rails-application-config-hosts.yml b/changelogs/unreleased/add-rails-application-config-hosts.yml new file mode 100644 index 00000000000..bfa339a5759 --- /dev/null +++ b/changelogs/unreleased/add-rails-application-config-hosts.yml @@ -0,0 +1,5 @@ +--- +title: Add setting to control Rails.application.config.hosts +merge_request: 55491 +author: +type: added diff --git a/changelogs/unreleased/fix-relative-position-on-move-and-copy-issue.yml b/changelogs/unreleased/fix-relative-position-on-move-and-copy-issue.yml new file mode 100644 index 00000000000..03eb4dcb767 --- /dev/null +++ b/changelogs/unreleased/fix-relative-position-on-move-and-copy-issue.yml @@ -0,0 +1,5 @@ +--- +title: Handle relative position on issue move or clone +merge_request: 55555 +author: +type: fixed diff --git a/changelogs/unreleased/remove-merge-request-count-with-merged-at-ff.yml b/changelogs/unreleased/remove-merge-request-count-with-merged-at-ff.yml new file mode 100644 index 00000000000..105ffae9841 --- /dev/null +++ b/changelogs/unreleased/remove-merge-request-count-with-merged-at-ff.yml @@ -0,0 +1,5 @@ +--- +title: Remove the optimized_merge_request_count_with_merged_at_filter feature flag +merge_request: 55600 +author: +type: other diff --git a/changelogs/unreleased/sh-improve-api-marginalia-comments.yml b/changelogs/unreleased/sh-improve-api-marginalia-comments.yml new file mode 100644 index 00000000000..ad4823edadf --- /dev/null +++ b/changelogs/unreleased/sh-improve-api-marginalia-comments.yml @@ -0,0 +1,5 @@ +--- +title: Improve Marginalia comments for API +merge_request: 55564 +author: +type: changed diff --git a/config/feature_flags/development/api_v3_repos_events_optimization.yml b/config/feature_flags/development/new_route_ci_minutes_purchase.yml index 114a4856a1d..c34fb14a9f0 100644 --- a/config/feature_flags/development/api_v3_repos_events_optimization.yml +++ b/config/feature_flags/development/new_route_ci_minutes_purchase.yml @@ -1,8 +1,8 @@ --- -name: api_v3_repos_events_optimization -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/54618 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/322059 +name: new_route_ci_minutes_purchase +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/54934 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/322582 milestone: '13.10' type: development -group: group::ecosystem +group: group::purchase default_enabled: false diff --git a/config/feature_flags/development/optimized_merge_request_count_with_merged_at_filter.yml b/config/feature_flags/development/optimized_merge_request_count_with_merged_at_filter.yml deleted file mode 100644 index a7a458e3d29..00000000000 --- a/config/feature_flags/development/optimized_merge_request_count_with_merged_at_filter.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: optimized_merge_request_count_with_merged_at_filter -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/52113 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/299347 -milestone: '13.9' -type: development -group: group::optimize -default_enabled: true diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 76eef1a81d3..57ece521301 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -73,6 +73,8 @@ production: &base worker_src: "'self' blob:" report_uri: + allowed_hosts: [] + # Trusted Proxies # Customize if you have GitLab behind a reverse proxy which is running on a different machine. # Add the IP address for your reverse proxy to the list, otherwise users will appear signed in from that address. diff --git a/config/initializers/0_marginalia.rb b/config/initializers/0_marginalia.rb index 33d677fdf7f..ab21f936cd8 100644 --- a/config/initializers/0_marginalia.rb +++ b/config/initializers/0_marginalia.rb @@ -13,7 +13,7 @@ require 'marginalia' # matching against the raw SQL, and prepending the comment prevents color # coding from working in the development log. Marginalia::Comment.prepend_comment = true if Rails.env.production? -Marginalia::Comment.components = [:application, :controller, :action, :correlation_id, :jid, :job_class] +Marginalia::Comment.components = [:application, :controller, :action, :correlation_id, :jid, :job_class, :endpoint_id] # As mentioned in https://github.com/basecamp/marginalia/pull/93/files, # adding :line has some overhead because a regexp on the backtrace has diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 151af995ff6..8de8584b748 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -210,6 +210,7 @@ Settings.gitlab['domain_allowlist'] ||= [] Settings.gitlab['import_sources'] ||= Gitlab::ImportSources.values Settings.gitlab['trusted_proxies'] ||= [] Settings.gitlab['content_security_policy'] ||= Gitlab::ContentSecurityPolicy::ConfigLoader.default_settings_hash +Settings.gitlab['allowed_hosts'] ||= [] Settings.gitlab['no_todos_messages'] ||= YAML.load_file(Rails.root.join('config', 'no_todos_messages.yml')) Settings.gitlab['impersonation_enabled'] ||= true if Settings.gitlab['impersonation_enabled'].nil? Settings.gitlab['usage_ping_enabled'] = true if Settings.gitlab['usage_ping_enabled'].nil? diff --git a/config/initializers/rails_host_authorization.rb b/config/initializers/rails_host_authorization.rb index 7d719dd519f..22bb6fb7061 100644 --- a/config/initializers/rails_host_authorization.rb +++ b/config/initializers/rails_host_authorization.rb @@ -2,6 +2,11 @@ # This file requires config/initializers/1_settings.rb +if Gitlab.config.gitlab.allowed_hosts.present? + Rails.application.config.hosts << Gitlab.config.gitlab.host << 'unix' + Rails.application.config.hosts += Gitlab.config.gitlab.allowed_hosts +end + if Rails.env.development? Rails.application.config.hosts += [Gitlab.config.gitlab.host, 'unix', 'host.docker.internal'] diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index d6a38e1b713..cdc9baf7573 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -17,21 +17,21 @@ full list of reference architectures, see | Service | Nodes | Configuration | GCP | AWS | Azure | |--------------------------------------------|-------------|-------------------------|-----------------|-------------|----------| -| External load balancing node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | -| Consul | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | -| PostgreSQL | 3 | 8 vCPU, 30 GB memory | n1-standard-8 | `m5.2xlarge` | D8s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | -| Internal load balancing node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | -| Redis - Cache | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | `m5.xlarge` | D4s v3 | -| Redis - Queues / Shared State | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | `m5.xlarge` | D4s v3 | -| Redis Sentinel - Cache | 3 | 1 vCPU, 1.7 GB memory | g1-small | `t3.small` | B1MS | -| Redis Sentinel - Queues / Shared State | 3 | 1 vCPU, 1.7 GB memory | g1-small | `t3.small` | B1MS | -| Gitaly Cluster | 3 | 16 vCPU, 60 GB memory | n1-standard-16 | `m5.4xlarge` | D16s v3 | -| Praefect | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | -| Praefect PostgreSQL | 1+* | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | -| Sidekiq | 4 | 4 vCPU, 15 GB memory | n1-standard-4 | `m5.xlarge` | D4s v3 | -| GitLab Rails | 3 | 32 vCPU, 28.8 GB memory | n1-highcpu-32 | `c5.9xlarge` | F32s v2 | -| Monitoring node | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | `c5.xlarge` | F4s v2 | +| External load balancing node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Consul | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| PostgreSQL | 3 | 8 vCPU, 30 GB memory | n1-standard-8 | m5.2xlarge | D8s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Redis - Cache | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis - Queues / Shared State | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis Sentinel - Cache | 3 | 1 vCPU, 1.7 GB memory | g1-small | t3.small | B1MS | +| Redis Sentinel - Queues / Shared State | 3 | 1 vCPU, 1.7 GB memory | g1-small | t3.small | B1MS | +| Gitaly | 3 | 16 vCPU, 60 GB memory | n1-standard-16 | m5.4xlarge | D16s v3 | +| Praefect | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Praefect PostgreSQL | 1+* | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Sidekiq | 4 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| GitLab Rails | 3 | 32 vCPU, 28.8 GB memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | +| Monitoring node | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | | Object storage | n/a | n/a | n/a | n/a | n/a | | NFS server | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | `c5.xlarge` | F4s v2 | @@ -206,7 +206,7 @@ The following list includes descriptions of each server and its assigned IP: - `10.6.0.111`: GitLab application 1 - `10.6.0.112`: GitLab application 2 - `10.6.0.113`: GitLab application 3 -- `10.6.0.121`: Prometheus +- `10.6.0.151`: Prometheus ## Configure the external load balancer @@ -1927,7 +1927,7 @@ To configure the Sidekiq nodes, on each one: node_exporter['listen_address'] = '0.0.0.0:9100' # Rails Status for prometheus - gitlab_rails['monitoring_whitelist'] = ['10.6.0.121/32', '127.0.0.0/8'] + gitlab_rails['monitoring_whitelist'] = ['10.6.0.151/32', '127.0.0.0/8'] ############################# ### Object storage ### @@ -2055,8 +2055,8 @@ On each node perform the following: # Add the monitoring node's IP address to the monitoring whitelist and allow it to # scrape the NGINX metrics - gitlab_rails['monitoring_whitelist'] = ['10.6.0.121/32', '127.0.0.0/8'] - nginx['status']['options']['allow'] = ['10.6.0.121/32', '127.0.0.0/8'] + gitlab_rails['monitoring_whitelist'] = ['10.6.0.151/32', '127.0.0.0/8'] + nginx['status']['options']['allow'] = ['10.6.0.151/32', '127.0.0.0/8'] ############################# ### Object storage ### @@ -2192,7 +2192,7 @@ running [Prometheus](../monitoring/prometheus/index.md) and The following IP will be used as an example: -- `10.6.0.121`: Prometheus +- `10.6.0.151`: Prometheus To configure the Monitoring node: diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index d02f7ea66ac..188d298c58b 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -12,100 +12,110 @@ full list of reference architectures, see [Available reference architectures](index.md#available-reference-architectures). > - **Supported users (approximate):** 25,000 -> - **High Availability:** Yes +> - **High Availability:** Yes ([Praefect](#configure-praefect-postgresql) needs a third-party PostgreSQL solution for HA) > - **Test requests per second (RPS) rates:** API: 500 RPS, Web: 50 RPS, Git (Pull): 50 RPS, Git (Push): 10 RPS | Service | Nodes | Configuration | GCP | AWS | Azure | |-----------------------------------------|-------------|-------------------------|-----------------|-------------|----------| -| External load balancing node | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | `c5.xlarge` | F4s v2 | -| Consul | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | -| PostgreSQL | 3 | 8 vCPU, 30 GB memory | n1-standard-8 | `m5.2xlarge` | D8s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | -| Internal load balancing node | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | `c5.large` | F2s v2 | -| Redis - Cache | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | `m5.xlarge` | D4s v3 | -| Redis - Queues / Shared State | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | `m5.xlarge` | D4s v3 | -| Redis Sentinel - Cache | 3 | 1 vCPU, 1.7 GB memory | g1-small | `t3.small` | B1MS | -| Redis Sentinel - Queues / Shared State | 3 | 1 vCPU, 1.7 GB memory | g1-small | `t3.small` | B1MS | -| Gitaly | 2 (minimum) | 32 vCPU, 120 GB memory | n1-standard-32 | `m5.8xlarge` | D32s v3 | -| Sidekiq | 4 | 4 vCPU, 15 GB memory | n1-standard-4 | `m5.xlarge` | D4s v3 | -| GitLab Rails | 5 | 32 vCPU, 28.8 GB memory | n1-highcpu-32 | `c5.9xlarge` | F32s v2 | -| Monitoring node | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | `c5.xlarge` | F4s v2 | +| External load balancing node | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Consul | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| PostgreSQL | 3 | 16 vCPU, 60 GB memory | n1-standard-16 | m5.4xlarge | D16s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.large | F2s v2 | +| Redis - Cache | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis - Queues / Shared State | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis Sentinel - Cache | 3 | 1 vCPU, 1.7 GB memory | g1-small | t3.small | B1MS | +| Redis Sentinel - Queues / Shared State | 3 | 1 vCPU, 1.7 GB memory | g1-small | t3.small | B1MS | +| Gitaly | 3 | 32 vCPU, 120 GB memory | n1-standard-32 | m5.8xlarge | D32s v3 | +| Praefect | 3 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Praefect PostgreSQL | 1+* | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Sidekiq | 4 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| GitLab Rails | 5 | 32 vCPU, 28.8 GB memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | +| Monitoring node | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | | Object storage | n/a | n/a | n/a | n/a | n/a | -| NFS server | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | `c5.xlarge` | F4s v2 | - -```mermaid -stateDiagram-v2 - [*] --> LoadBalancer - LoadBalancer --> ApplicationServer - - ApplicationServer --> BackgroundJobs - ApplicationServer --> Gitaly - ApplicationServer --> Redis_Cache - ApplicationServer --> Redis_Queues - ApplicationServer --> PgBouncer - PgBouncer --> Database - ApplicationServer --> ObjectStorage - BackgroundJobs --> ObjectStorage - - ApplicationMonitoring -->ApplicationServer - ApplicationMonitoring -->PgBouncer - ApplicationMonitoring -->Database - ApplicationMonitoring -->BackgroundJobs - - ApplicationServer --> Consul - - Consul --> Database - Consul --> PgBouncer - Redis_Cache --> Consul - Redis_Queues --> Consul - BackgroundJobs --> Consul - - state Consul { - "Consul_1..3" - } - - state Database { - "PG_Primary_Node" - "PG_Secondary_Node_1..2" - } - - state Redis_Cache { - "R_Cache_Primary_Node" - "R_Cache_Replica_Node_1..2" - "R_Cache_Sentinel_1..3" - } - - state Redis_Queues { - "R_Queues_Primary_Node" - "R_Queues_Replica_Node_1..2" - "R_Queues_Sentinel_1..3" - } - - state Gitaly { - "Gitaly_1..2" - } - - state BackgroundJobs { - "Sidekiq_1..4" - } - - state ApplicationServer { - "GitLab_Rails_1..5" - } - - state LoadBalancer { - "LoadBalancer_1" - } - - state ApplicationMonitoring { - "Prometheus" - "Grafana" - } - - state PgBouncer { - "Internal_Load_Balancer" - "PgBouncer_1..3" - } +| NFS server | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | + +```plantuml +@startuml 25k +card "**External Load Balancer**" as elb #6a9be7 +card "**Internal Load Balancer**" as ilb #9370DB + +together { + collections "**GitLab Rails** x5" as gitlab #32CD32 + collections "**Sidekiq** x4" as sidekiq #ff8dd1 +} + +together { + card "**Prometheus + Grafana**" as monitor #7FFFD4 + collections "**Consul** x3" as consul #e76a9b +} + +card "Gitaly Cluster" as gitaly_cluster { + collections "**Praefect** x3" as praefect #FF8C00 + collections "**Gitaly** x3" as gitaly #FF8C00 + card "**Praefect PostgreSQL***\n//Non fault-tolerant//" as praefect_postgres #FF8C00 + + praefect -[#FF8C00]-> gitaly + praefect -[#FF8C00]> praefect_postgres +} + +card "Database" as database { + collections "**PGBouncer** x3" as pgbouncer #4EA7FF + card "**PostgreSQL** (Primary)" as postgres_primary #4EA7FF + collections "**PostgreSQL** (Secondary) x2" as postgres_secondary #4EA7FF + + pgbouncer -[#4EA7FF]-> postgres_primary + postgres_primary .[#4EA7FF]> postgres_secondary +} + +card "redis" as redis { + collections "**Redis Persistent** x3" as redis_persistent #FF6347 + collections "**Redis Cache** x3" as redis_cache #FF6347 + collections "**Redis Persistent Sentinel** x3" as redis_persistent_sentinel #FF6347 + collections "**Redis Cache Sentinel** x3"as redis_cache_sentinel #FF6347 + + redis_persistent <.[#FF6347]- redis_persistent_sentinel + redis_cache <.[#FF6347]- redis_cache_sentinel +} + +cloud "**Object Storage**" as object_storage #white + +elb -[#6a9be7]-> gitlab +elb -[#6a9be7]--> monitor + +gitlab -[#32CD32]> sidekiq +gitlab -[#32CD32]--> ilb +gitlab -[#32CD32]-> object_storage +gitlab -[#32CD32]---> redis +gitlab -[hidden]-> monitor +gitlab -[hidden]-> consul + +sidekiq -[#ff8dd1]--> ilb +sidekiq -[#ff8dd1]-> object_storage +sidekiq -[#ff8dd1]---> redis +sidekiq -[hidden]-> monitor +sidekiq -[hidden]-> consul + +ilb -[#9370DB]-> gitaly_cluster +ilb -[#9370DB]-> database + +consul .[#e76a9b]u-> gitlab +consul .[#e76a9b]u-> sidekiq +consul .[#e76a9b]> monitor +consul .[#e76a9b]-> database +consul .[#e76a9b]-> gitaly_cluster +consul .[#e76a9b,norank]--> redis + +monitor .[#7FFFD4]u-> gitlab +monitor .[#7FFFD4]u-> sidekiq +monitor .[#7FFFD4]> consul +monitor .[#7FFFD4]-> database +monitor .[#7FFFD4]-> gitaly_cluster +monitor .[#7FFFD4,norank]--> redis +monitor .[#7FFFD4]> ilb +monitor .[#7FFFD4,norank]u--> elb + +@enduml ``` The Google Cloud Platform (GCP) architectures were built and tested using the @@ -120,19 +130,25 @@ uploads, or artifacts), using an [object storage service](#configure-the-object- is recommended instead of using NFS. Using an object storage service also doesn't require you to provision and maintain a node. +It's also worth noting that at this time [Praefect requires its own database server](../gitaly/praefect.md#postgresql) and +that to achieve full High Availability a third party PostgreSQL database solution will be required. +We hope to offer a built in solutions for these restrictions in the future but in the meantime a non HA PostgreSQL server +can be set up via Omnibus GitLab, which the above specs reflect. Refer to the following issues for more information: [`omnibus-gitlab#5919`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5919) & [`gitaly#3398`](https://gitlab.com/gitlab-org/gitaly/-/issues/3398) + ## Setup components To set up GitLab and its components to accommodate up to 25,000 users: -1. [Configure the external load balancing node](#configure-the-external-load-balancer) +1. [Configure the external load balancer](#configure-the-external-load-balancer) to handle the load balancing of the GitLab application services nodes. +1. [Configure the internal load balancer](#configure-the-internal-load-balancer). + to handle the load balancing of GitLab application internal connections. 1. [Configure Consul](#configure-consul). 1. [Configure PostgreSQL](#configure-postgresql), the database for GitLab. 1. [Configure PgBouncer](#configure-pgbouncer). -1. [Configure the internal load balancing node](#configure-the-internal-load-balancer). 1. [Configure Redis](#configure-redis). -1. [Configure Gitaly](#configure-gitaly), - which provides access to the Git repositories. +1. [Configure Gitaly Cluster](#configure-gitaly-cluster), + provides access to the Git repositories. 1. [Configure Sidekiq](#configure-sidekiq). 1. [Configure the main GitLab Rails application](#configure-gitlab-rails) to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend @@ -178,6 +194,11 @@ The following list includes descriptions of each server and its assigned IP: - `10.6.0.83`: Sentinel - Queues 3 - `10.6.0.91`: Gitaly 1 - `10.6.0.92`: Gitaly 2 +- `10.6.0.93`: Gitaly 3 +- `10.6.0.131`: Praefect 1 +- `10.6.0.132`: Praefect 2 +- `10.6.0.133`: Praefect 3 +- `10.6.0.141`: Praefect PostgreSQL 1 (non HA) - `10.6.0.101`: Sidekiq 1 - `10.6.0.102`: Sidekiq 2 - `10.6.0.103`: Sidekiq 3 @@ -185,7 +206,9 @@ The following list includes descriptions of each server and its assigned IP: - `10.6.0.111`: GitLab application 1 - `10.6.0.112`: GitLab application 2 - `10.6.0.113`: GitLab application 3 -- `10.6.0.121`: Prometheus +- `10.6.0.114`: GitLab application 4 +- `10.6.0.115`: GitLab application 5 +- `10.6.0.151`: Prometheus ## Configure the external load balancer @@ -308,6 +331,71 @@ Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. </a> </div> +## Configure the internal load balancer + +The Internal Load Balancer is used to balance any internal connections the GitLab environment requires +such as connections to [PgBouncer](#configure-pgbouncer) and [Praefect](#configure-praefect) (Gitaly Cluster). + +Note that it's a separate node from the External Load Balancer and shouldn't have any access externally. + +The following IP will be used as an example: + +- `10.6.0.40`: Internal Load Balancer + +Here's how you could do it with [HAProxy](https://www.haproxy.org/): + +```plaintext +global + log /dev/log local0 + log localhost local1 notice + log stdout format raw local0 + +defaults + log global + default-server inter 10s fall 3 rise 2 + balance leastconn + +frontend internal-pgbouncer-tcp-in + bind *:6432 + mode tcp + option tcplog + + default_backend pgbouncer + +frontend internal-praefect-tcp-in + bind *:2305 + mode tcp + option tcplog + option clitcpka + + default_backend praefect + +backend pgbouncer + mode tcp + option tcp-check + + server pgbouncer1 10.6.0.21:6432 check + server pgbouncer2 10.6.0.22:6432 check + server pgbouncer3 10.6.0.23:6432 check + +backend praefect + mode tcp + option tcp-check + option srvtcpka + + server praefect1 10.6.0.131:2305 check + server praefect2 10.6.0.132:2305 check + server praefect3 10.6.0.133:2305 check +``` + +Refer to your preferred Load Balancer's documentation for further guidance. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + ## Configure Consul The following IPs will be used as an example: @@ -662,52 +750,6 @@ The following IPs will be used as an example: </a> </div> -### Configure the internal load balancer - -If you're running more than one PgBouncer node as recommended, then at this time you'll need to set -up a TCP internal load balancer to serve each correctly. - -The following IP will be used as an example: - -- `10.6.0.40`: Internal Load Balancer - -Here's how you could do it with [HAProxy](https://www.haproxy.org/): - -```plaintext -global - log /dev/log local0 - log localhost local1 notice - log stdout format raw local0 - -defaults - log global - default-server inter 10s fall 3 rise 2 - balance leastconn - -frontend internal-pgbouncer-tcp-in - bind *:6432 - mode tcp - option tcplog - - default_backend pgbouncer - -backend pgbouncer - mode tcp - option tcp-check - - server pgbouncer1 10.6.0.21:6432 check - server pgbouncer2 10.6.0.22:6432 check - server pgbouncer3 10.6.0.23:6432 check -``` - -Refer to your preferred Load Balancer's documentation for further guidance. - -<div align="right"> - <a type="button" class="btn btn-default" href="#setup-components"> - Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> - </a> -</div> - ## Configure Redis Using [Redis](https://redis.io/) in scalable environment is possible using a **Primary** x **Replica** @@ -1302,19 +1344,283 @@ To configure the Sentinel Queues server: </a> </div> -## Configure Gitaly +## Configure Gitaly Cluster -NOTE: -[Gitaly Cluster](../gitaly/praefect.md) support -for the Reference Architectures is being -worked on as a [collaborative effort](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/1) between the Quality Engineering and Gitaly teams. When this component has been verified -some Architecture specs will likely change as a result to support the new -and improved designed. +[Gitaly Cluster](../gitaly/praefect.md) is a GitLab provided and recommended fault tolerant solution for storing Git repositories. +In this configuration, every Git repository is stored on every Gitaly node in the cluster, with one being designated the primary, and failover occurs automatically if the primary node goes down. + +The recommended cluster setup includes the following components: + +- 3 Gitaly nodes: Replicated storage of Git repositories. +- 3 Praefect nodes: Router and transaction manager for Gitaly Cluster. +- 1 Praefect PostgreSQL node: Database server for Praefect. A third-party solution + is required for Praefect database connections to be made highly available. +- 1 load balancer: A load balancer is required for Praefect. The + [internal load balancer](#configure-the-internal-load-balancer) will be used. + +This section will detail how to configure the recommended standard setup in order. +For more advanced setups refer to the [standalone Gitaly Cluster documentation](../gitaly/praefect.md). + +### Configure Praefect PostgreSQL + +Praefect, the routing and transaction manager for Gitaly Cluster, requires its own database server to store data on Gitaly Cluster status. + +If you want to have a highly available setup, Praefect requires a third-party PostgreSQL database. +A built-in solution is being [worked on](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5919). + +#### Praefect non-HA PostgreSQL standalone using Omnibus GitLab + +The following IPs will be used as an example: + +- `10.6.0.141`: Praefect PostgreSQL + +First, make sure to [install](https://about.gitlab.com/install/) +the Linux GitLab package in the Praefect PostgreSQL node. Following the steps, +install the necessary dependencies from step 1, and add the +GitLab package repository from step 2. When installing GitLab +in the second step, do not supply the `EXTERNAL_URL` value. + +1. SSH in to the Praefect PostgreSQL node. +1. Create a strong password to be used for the Praefect PostgreSQL user. Take note of this password as `<praefect_postgresql_password>`. +1. Generate the password hash for the Praefect PostgreSQL username/password pair. This assumes you will use the default + username of `praefect` (recommended). The command will request the password `<praefect_postgresql_password>` + and confirmation. Use the value that is output by this command in the next + step as the value of `<praefect_postgresql_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 praefect + ``` + +1. Edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: + + ```ruby + # Disable all components except PostgreSQL and Consul + roles ['postgres_role'] + repmgr['enable'] = false + patroni['enable'] = false + + # PostgreSQL configuration + postgresql['listen_address'] = '0.0.0.0' + postgresql['max_connections'] = 200 + + gitlab_rails['auto_migrate'] = false -[Gitaly](../gitaly/index.md) server node requirements are dependent on data, -specifically the number of projects and those projects' sizes. It's recommended -that a Gitaly server node stores no more than 5 TB of data. Depending on your -repository storage requirements, you may require additional Gitaly server nodes. + # Configure the Consul agent + consul['enable'] = true + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # START user configuration + # Please set the real values as explained in Required Information section + # + # Replace PRAEFECT_POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = "<praefect_postgresql_password_hash>" + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) + + # Set the network addresses that the exporters will listen on for monitoring + node_exporter['listen_address'] = '0.0.0.0:9100' + postgres_exporter['listen_address'] = '0.0.0.0:9187' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + # + # END user configuration + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Follow the [post configuration](#praefect-postgresql-post-configuration). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +#### Praefect HA PostgreSQL third-party solution + +[As noted](#configure-praefect-postgresql), a third-party PostgreSQL solution for +Praefect's database is recommended if aiming for full High Availability. + +There are many third-party solutions for PostgreSQL HA. The solution selected must have the following to work with Praefect: + +- A static IP for all connections that doesn't change on failover. +- [`LISTEN`](https://www.postgresql.org/docs/12/sql-listen.html) SQL functionality must be supported. + +Examples of the above could include [Google's Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) or [Amazon RDS](https://aws.amazon.com/rds/). + +Once the database is set up, follow the [post configuration](#praefect-postgresql-post-configuration). + +#### Praefect PostgreSQL post-configuration + +After the Praefect PostgreSQL server has been set up, you'll then need to configure the user and database for Praefect to use. + +We recommend the user be named `praefect` and the database `praefect_production`, and these can be configured as standard in PostgreSQL. +The password for the user is the same as the one you configured earlier as `<praefect_postgresql_password>`. + +This is how this would work with a Omnibus GitLab PostgreSQL setup: + +1. SSH in to the Praefect PostgreSQL node. +1. Connect to the PostgreSQL server with administrative access. + The `gitlab-psql` user should be used here for this as it's added by default in Omnibus. + The database `template1` is used because it is created by default on all PostgreSQL servers. + + ```shell + /opt/gitlab/embedded/bin/psql -U gitlab-psql -d template1 -h POSTGRESQL_SERVER_ADDRESS + ``` + +1. Create the new user `praefect`, replacing `<praefect_postgresql_password>`: + + ```shell + CREATE ROLE praefect WITH LOGIN CREATEDB PASSWORD <praefect_postgresql_password>; + ``` + +1. Reconnect to the PostgreSQL server, this time as the `praefect` user: + + ```shell + /opt/gitlab/embedded/bin/psql -U praefect -d template1 -h POSTGRESQL_SERVER_ADDRESS + ``` + +1. Create a new database `praefect_production`: + + ```shell + CREATE DATABASE praefect_production WITH ENCODING=UTF8; + ``` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +### Configure Praefect + +Praefect is the router and transaction manager for Gitaly Cluster and all connections to Gitaly go through +it. This section details how to configure it. + +Praefect requires several secret tokens to secure communications across the Cluster: + +- `<praefect_external_token>`: Used for repositories hosted on your Gitaly cluster and can only be accessed by Gitaly clients that carry this token. +- `<praefect_internal_token>`: Used for replication traffic inside your Gitaly cluster. This is distinct from `praefect_external_token` because Gitaly clients must not be able to access internal nodes of the Praefect cluster directly; that could lead to data loss. +- `<praefect_postgresql_password>`: The Praefect PostgreSQL password defined in the previous section is also required as part of this setup. + +Gitaly Cluster nodes are configured in Praefect via a `virtual storage`. Each storage contains +the details of each Gitaly node that makes up the cluster. Each storage is also given a name +and this name is used in several areas of the config. In this guide, the name of the storage will be +`default`. Also, this guide is geared towards new installs, if upgrading an existing environment +to use Gitaly Cluster, you may need to use a different name. +Refer to the [Praefect documentation](../gitaly/praefect.md#praefect) for more info. + +The following IPs will be used as an example: + +- `10.6.0.131`: Praefect 1 +- `10.6.0.132`: Praefect 2 +- `10.6.0.133`: Praefect 3 + +To configure the Praefect nodes, on each one: + +1. SSH in to the Praefect server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. +1. Edit the `/etc/gitlab/gitlab.rb` file to configure Praefect: + + ```ruby + # Avoid running unnecessary services on the Gitaly server + postgresql['enable'] = false + redis['enable'] = false + nginx['enable'] = false + puma['enable'] = false + unicorn['enable'] = false + sidekiq['enable'] = false + gitlab_workhorse['enable'] = false + grafana['enable'] = false + + # If you run a separate monitoring node you can disable these services + alertmanager['enable'] = false + prometheus['enable'] = false + + # Praefect Configuration + praefect['enable'] = true + praefect['listen_addr'] = '0.0.0.0:2305' + + gitlab_rails['rake_cache_clear'] = false + gitlab_rails['auto_migrate'] = false + + # Configure the Consul agent + consul['enable'] = true + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # START user configuration + # Please set the real values as explained in Required Information section + # + + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + praefect['auth_token'] = '<praefect_external_token>' + + # Praefect Database Settings + praefect['database_host'] = '10.6.0.141' + praefect['database_port'] = 5432 + # `no_proxy` settings must always be a direct connection for caching + praefect['database_host_no_proxy'] = '10.6.0.141' + praefect['database_port_no_proxy'] = 5432 + praefect['database_dbname'] = 'praefect_production' + praefect['database_user'] = 'praefect' + praefect['database_password'] = '<praefect_postgresql_password>' + + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + praefect['virtual_storages'] = { + 'default' => { + 'gitaly-1' => { + 'address' => 'tcp://10.6.0.91:8075', + 'token' => '<praefect_internal_token>', + 'primary' => true + }, + 'gitaly-2' => { + 'address' => 'tcp://10.6.0.92:8075', + 'token' => '<praefect_internal_token>' + }, + 'gitaly-3' => { + 'address' => 'tcp://10.6.0.93:8075', + 'token' => '<praefect_internal_token>' + }, + } + } + + # Set the network addresses that the exporters will listen on for monitoring + node_exporter['listen_address'] = '0.0.0.0:9100' + praefect['prometheus_listen_addr'] = '0.0.0.0:9652' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + # + # END user configuration + ``` + + 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and + then replace the file of the same name on this server. If that file isn't on + this server, add the file from your Consul server to this server. + + 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +### Configure Gitaly + +The [Gitaly](../gitaly/index.md) server nodes that make up the cluster have +requirements that are dependent on data, specifically the number of projects +and those projects' sizes. It's recommended that a Gitaly Cluster stores +no more than 5 TB of data on each node. Depending on your +repository storage requirements, you may require additional Gitaly Clusters. Due to Gitaly having notable input and output requirements, we strongly recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs @@ -1325,36 +1631,21 @@ adjusted to greater or lesser values depending on the scale of your environment's workload. If you're running the environment on a Cloud provider, refer to their documentation about how to configure IOPS correctly. -Be sure to note the following items: +Gitaly servers must not be exposed to the public internet, as Gitaly's network +traffic is unencrypted by default. The use of a firewall is highly recommended +to restrict access to the Gitaly server. Another option is to +[use TLS](#gitaly-cluster-tls-support). -- The GitLab Rails application shards repositories into - [repository storage paths](../repository_storage_paths.md). -- A Gitaly server can host one or more storage paths. -- A GitLab server can use one or more Gitaly server nodes. -- Gitaly addresses must be specified to be correctly resolvable for all Gitaly - clients. -- Gitaly servers must not be exposed to the public internet, as Gitaly's network - traffic is unencrypted by default. The use of a firewall is highly recommended - to restrict access to the Gitaly server. Another option is to - [use TLS](#gitaly-tls-support). +For configuring Gitaly you should note the following: -NOTE: -The token referred to throughout the Gitaly documentation is an arbitrary -password selected by the administrator. This token is unrelated to tokens -created for the GitLab API or other similar web API tokens. - -This section describes how to configure two Gitaly servers, with the following -IPs and domain names: - -- `10.6.0.91`: Gitaly 1 (`gitaly1.internal`) -- `10.6.0.92`: Gitaly 2 (`gitaly2.internal`) +- `git_data_dirs` should be configured to reflect the storage path for the specific Gitaly node +- `auth_token` should be the same as `praefect_internal_token` -Assumptions about your servers include having the secret token be `gitalysecret`, -and that your GitLab installation has three repository storages: +The following IPs will be used as an example: -- `default` on Gitaly 1 -- `storage1` on Gitaly 1 -- `storage2` on Gitaly 2 +- `10.6.0.91`: Gitaly 1 +- `10.6.0.92`: Gitaly 2 +- `10.6.0.93`: Gitaly 3 On each node: @@ -1364,21 +1655,9 @@ On each node: 1. Edit the Gitaly server node's `/etc/gitlab/gitlab.rb` file to configure storage paths, enable the network listener, and to configure the token: - <!-- - updates to following example must also be made at - https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab - --> - ```ruby # /etc/gitlab/gitlab.rb - # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests - # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. - # The following two values must be the same as their respective values - # of the GitLab Rails application setup - gitaly['auth_token'] = 'gitalysecret' - gitlab_shell['secret_token'] = 'shellsecret' - # Avoid running unnecessary services on the Gitaly server postgresql['enable'] = false redis['enable'] = false @@ -1407,36 +1686,42 @@ On each node: # firewalls to restrict access to this address/port. # Comment out following line if you only want to support TLS connections gitaly['listen_addr'] = "0.0.0.0:8075" + + # Gitaly Auth Token + # Should be the same as praefect_internal_token + gitaly['auth_token'] = '<praefect_internal_token>' ``` 1. Append the following to `/etc/gitlab/gitlab.rb` for each respective server: - - On `gitaly1.internal`: + - On Gitaly node 1: ```ruby git_data_dirs({ - 'default' => { - 'path' => '/var/opt/gitlab/git-data' - }, - 'storage1' => { - 'path' => '/mnt/gitlab/git-data' - }, + "gitaly-1" => { + "path" => "/var/opt/gitlab/git-data" + } }) ``` - - On `gitaly2.internal`: + - On Gitaly node 2: ```ruby git_data_dirs({ - 'storage2' => { - 'path' => '/mnt/gitlab/git-data' - }, + "gitaly-2" => { + "path" => "/var/opt/gitlab/git-data" + } }) ``` - <!-- - updates to following example must also be made at - https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab - --> + - On Gitaly node 3: + + ```ruby + git_data_dirs({ + "gitaly-3" => { + "path" => "/var/opt/gitlab/git-data" + } + }) + ``` 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and then replace the file of the same name on this server. If that file isn't on @@ -1444,34 +1729,44 @@ On each node: 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -### Gitaly TLS support +### Gitaly Cluster TLS support -Gitaly supports TLS encryption. To be able to communicate -with a Gitaly instance that listens for secure connections you will need to use `tls://` URL -scheme in the `gitaly_address` of the corresponding storage entry in the GitLab configuration. +Praefect supports TLS encryption. To communicate with a Praefect instance that listens +for secure connections, you must: -You will need to bring your own certificates as this isn't provided automatically. -The certificate, or its certificate authority, must be installed on all Gitaly -nodes (including the Gitaly node using the certificate) and on all client nodes -that communicate with it following the procedure described in -[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). +- Use a `tls://` URL scheme in the `gitaly_address` of the corresponding storage entry + in the GitLab configuration. +- Bring your own certificates because this isn't provided automatically. The certificate + corresponding to each Praefect server must be installed on that Praefect server. -NOTE: -The self-signed certificate must specify the address you use to access the -Gitaly server. If you are addressing the Gitaly server by a hostname, you can -either use the Common Name field for this, or add it as a Subject Alternative -Name. If you are addressing the Gitaly server by its IP address, you must add it -as a Subject Alternative Name to the certificate. -[gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). +Additionally the certificate, or its certificate authority, must be installed on all Gitaly servers +and on all Praefect clients that communicate with it following the procedure described in +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) (and repeated below). -It's possible to configure Gitaly servers with both an unencrypted listening -address (`listen_addr`) and an encrypted listening address (`tls_listen_addr`) -at the same time. This allows you to do a gradual transition from unencrypted to -encrypted traffic, if necessary. +Note the following: -To configure Gitaly with TLS: +- The certificate must specify the address you use to access the Praefect server. If + addressing the Praefect server by: -1. Create the `/etc/gitlab/ssl` directory and copy your key and certificate there: + - Hostname, you can either use the Common Name field for this, or add it as a Subject + Alternative Name. + - IP address, you must add it as a Subject Alternative Name to the certificate. + +- You can configure Praefect servers with both an unencrypted listening address + `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. + This allows you to do a gradual transition from unencrypted to encrypted traffic, if + necessary. + +- The Internal Load Balancer will also access to the certificates and need to be configured + to allow for TLS passthrough. + Refer to the load balancers documentation on how to configure this. + +To configure Praefect with TLS: + +1. Create certificates for Praefect servers. + +1. On the Praefect servers, create the `/etc/gitlab/ssl` directory and copy your key + and certificate there: ```shell sudo mkdir -p /etc/gitlab/ssl @@ -1480,27 +1775,34 @@ To configure Gitaly with TLS: sudo chmod 644 key.pem cert.pem ``` -1. Copy the cert to `/etc/gitlab/trusted-certs` so Gitaly will trust the cert when - calling into itself: +1. Edit `/etc/gitlab/gitlab.rb` and add: - ```shell - sudo cp /etc/gitlab/ssl/cert.pem /etc/gitlab/trusted-certs/ + ```ruby + praefect['tls_listen_addr'] = "0.0.0.0:3305" + praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" + praefect['key_path'] = "/etc/gitlab/ssl/key.pem" ``` -1. Edit `/etc/gitlab/gitlab.rb` and add: +1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). - <!-- - updates to following example must also be made at - https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab - --> +1. On the Praefect clients (including each Gitaly server), copy the certificates, + or their certificate authority, into `/etc/gitlab/trusted-certs`: - ```ruby - gitaly['tls_listen_addr'] = "0.0.0.0:9999" - gitaly['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - gitaly['key_path'] = "/etc/gitlab/ssl/key.pem" + ```shell + sudo cp cert.pem /etc/gitlab/trusted-certs/ ``` -1. Delete `gitaly['listen_addr']` to allow only encrypted connections. +1. On the Praefect clients (except Gitaly servers), edit `git_data_dirs` in + `/etc/gitlab/gitlab.rb` as follows: + + ```ruby + git_data_dirs({ + "default" => { + "gitaly_address" => 'tls://LOAD_BALANCER_SERVER_ADDRESS:2305', + "gitaly_token" => 'PRAEFECT_EXTERNAL_TOKEN' + } + }) + ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -1587,12 +1889,15 @@ To configure the Sidekiq nodes, on each one: ### Gitaly ### ####################################### + # git_data_dirs get configured for the Praefect virtual storage + # Address is Internal Load Balancer for Praefect + # Token is praefect_external_token git_data_dirs({ - 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, - 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, - 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + "default" => { + "gitaly_address" => "tcp://10.6.0.40:2305", # internal load balancer IP + "gitaly_token" => '<praefect_external_token>' + } }) - gitlab_rails['gitaly_token'] = 'YOUR_TOKEN' ####################################### ### Postgres ### @@ -1624,7 +1929,7 @@ To configure the Sidekiq nodes, on each one: node_exporter['listen_address'] = '0.0.0.0:9100' # Rails Status for prometheus - gitlab_rails['monitoring_whitelist'] = ['10.6.0.121/32', '127.0.0.0/8'] + gitlab_rails['monitoring_whitelist'] = ['10.6.0.151/32', '127.0.0.0/8'] ############################# ### Object storage ### @@ -1671,6 +1976,8 @@ The following IPs will be used as an example: - `10.6.0.111`: GitLab application 1 - `10.6.0.112`: GitLab application 2 - `10.6.0.113`: GitLab application 3 +- `10.6.0.114`: GitLab application 4 +- `10.6.0.115`: GitLab application 5 On each node perform the following: @@ -1690,17 +1997,14 @@ On each node perform the following: ```ruby external_url 'https://gitlab.example.com' - # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests - # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. - # The following two values must be the same as their respective values - # of the Gitaly setup - gitlab_rails['gitaly_token'] = 'gitalysecret' - gitlab_shell['secret_token'] = 'shellsecret' - + # git_data_dirs get configured for the Praefect virtual storage + # Address is Internal Load Balancer for Praefect + # Token is praefect_external_token git_data_dirs({ - 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, - 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, - 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + "default" => { + "gitaly_address" => "tcp://10.6.0.40:2305", # internal load balancer IP + "gitaly_token" => '<praefect_external_token>' + } }) ## Disable components that will not be on the GitLab application server @@ -1755,8 +2059,8 @@ On each node perform the following: # Add the monitoring node's IP address to the monitoring whitelist and allow it to # scrape the NGINX metrics - gitlab_rails['monitoring_whitelist'] = ['10.6.0.121/32', '127.0.0.0/8'] - nginx['status']['options']['allow'] = ['10.6.0.121/32', '127.0.0.0/8'] + gitlab_rails['monitoring_whitelist'] = ['10.6.0.151/32', '127.0.0.0/8'] + nginx['status']['options']['allow'] = ['10.6.0.151/32', '127.0.0.0/8'] ############################# ### Object storage ### @@ -1779,14 +2083,15 @@ On each node perform the following: ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -1. If you're using [Gitaly with TLS support](#gitaly-tls-support), make sure the +1. If you're using [Gitaly with TLS support](#gitaly-cluster-tls-support), make sure the `git_data_dirs` entry is configured with `tls` instead of `tcp`: ```ruby git_data_dirs({ - 'default' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, - 'storage1' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, - 'storage2' => { 'gitaly_address' => 'tls://gitaly2.internal:9999' }, + "default" => { + "gitaly_address" => "tls://10.6.0.40:2305", # internal load balancer IP + "gitaly_token" => '<praefect_external_token>' + } }) ``` @@ -1891,7 +2196,7 @@ running [Prometheus](../monitoring/prometheus/index.md) and The following IP will be used as an example: -- `10.6.0.121`: Prometheus +- `10.6.0.151`: Prometheus To configure the Monitoring node: diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 593bfaf7282..43c93000ed6 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -20,78 +20,107 @@ For a full list of reference architectures, see [Available reference architectures](index.md#available-reference-architectures). > - **Supported users (approximate):** 3,000 -> - **High Availability:** Yes +> - **High Availability:** Yes ([Praefect](#configure-praefect-postgresql) needs a third-party PostgreSQL solution for HA) > - **Test requests per second (RPS) rates:** API: 60 RPS, Web: 6 RPS, Git (Pull): 6 RPS, Git (Push): 1 RPS | Service | Nodes | Configuration | GCP | AWS | Azure | |--------------------------------------------|-------------|-----------------------|----------------|-------------|---------| -| External load balancing node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | -| Redis | 3 | 2 vCPU, 7.5 GB memory | n1-standard-2 | `m5.large` | D2s v3 | -| Consul + Sentinel | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | -| PostgreSQL | 3 | 2 vCPU, 7.5 GB memory | n1-standard-2 | `m5.large` | D2s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | -| Internal load balancing node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | -| Gitaly | 2 (minimum) | 4 vCPU, 15 GB memory | n1-standard-4 | `m5.xlarge` | D4s v3 | -| Sidekiq | 4 | 2 vCPU, 7.5 GB memory | n1-standard-2 | `m5.large` | D2s v3 | -| GitLab Rails | 3 | 8 vCPU, 7.2 GB memory | n1-highcpu-8 | `c5.2xlarge` | F8s v2 | -| Monitoring node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | +| External load balancing node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Redis | 3 | 2 vCPU, 7.5 GB memory | n1-standard-2 | m5.large | D2s v3 | +| Consul + Sentinel | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| PostgreSQL | 3 | 2 vCPU, 7.5 GB memory | n1-standard-2 | m5.large | D2s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Gitaly | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Praefect | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Praefect PostgreSQL | 1+* | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Sidekiq | 4 | 2 vCPU, 7.5 GB memory | n1-standard-2 | m5.large | D2s v3 | +| GitLab Rails | 3 | 8 vCPU, 7.2 GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | +| Monitoring node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | | Object storage | n/a | n/a | n/a | n/a | n/a | -| NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | `c5.xlarge` | F4s v2 | - -```mermaid -stateDiagram-v2 - [*] --> LoadBalancer - LoadBalancer --> ApplicationServer - - ApplicationServer --> BackgroundJobs - ApplicationServer --> Gitaly - ApplicationServer --> Redis - ApplicationServer --> PgBouncer - PgBouncer --> Database - ApplicationServer --> ObjectStorage - BackgroundJobs --> ObjectStorage - - ApplicationMonitoring -->ApplicationServer - ApplicationMonitoring -->Redis - ApplicationMonitoring -->PgBouncer - ApplicationMonitoring -->Database - ApplicationMonitoring -->BackgroundJobs - - state Database { - "PG_Primary_Node" - "PG_Secondary_Node_1..2" - } - state Redis { - "R_Primary_Node" - "R_Replica_Node_1..2" - "R_Consul/Sentinel_1..3" - } - - state Gitaly { - "Gitaly_1..2" - } - - state BackgroundJobs { - "Sidekiq_1..4" - } - - state ApplicationServer { - "GitLab_Rails_1..3" - } - - state LoadBalancer { - "LoadBalancer_1" - } - - state ApplicationMonitoring { - "Prometheus" - "Grafana" - } - - state PgBouncer { - "Internal_Load_Balancer" - "PgBouncer_1..3" - } +| NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | + +```plantuml +@startuml 3k +card "**External Load Balancer**" as elb #6a9be7 +card "**Internal Load Balancer**" as ilb #9370DB + +together { + collections "**GitLab Rails** x3" as gitlab #32CD32 + collections "**Sidekiq** x4" as sidekiq #ff8dd1 +} + +together { + card "**Prometheus + Grafana**" as monitor #7FFFD4 + collections "**Consul** x3" as consul #e76a9b +} + +card "Gitaly Cluster" as gitaly_cluster { + collections "**Praefect** x3" as praefect #FF8C00 + collections "**Gitaly** x3" as gitaly #FF8C00 + card "**Praefect PostgreSQL***\n//Non fault-tolerant//" as praefect_postgres #FF8C00 + + praefect -[#FF8C00]-> gitaly + praefect -[#FF8C00]> praefect_postgres +} + +card "Database" as database { + collections "**PGBouncer** x3" as pgbouncer #4EA7FF + card "**PostgreSQL** (Primary)" as postgres_primary #4EA7FF + collections "**PostgreSQL** (Secondary) x2" as postgres_secondary #4EA7FF + + pgbouncer -[#4EA7FF]-> postgres_primary + postgres_primary .[#4EA7FF]> postgres_secondary +} + +card "redis" as redis { + collections "**Redis Persistent** x3" as redis_persistent #FF6347 + collections "**Redis Cache** x3" as redis_cache #FF6347 + collections "**Redis Persistent Sentinel** x3" as redis_persistent_sentinel #FF6347 + collections "**Redis Cache Sentinel** x3"as redis_cache_sentinel #FF6347 + + redis_persistent <.[#FF6347]- redis_persistent_sentinel + redis_cache <.[#FF6347]- redis_cache_sentinel +} + +cloud "**Object Storage**" as object_storage #white + +elb -[#6a9be7]-> gitlab +elb -[#6a9be7]--> monitor + +gitlab -[#32CD32]> sidekiq +gitlab -[#32CD32]--> ilb +gitlab -[#32CD32]-> object_storage +gitlab -[#32CD32]---> redis +gitlab -[hidden]-> monitor +gitlab -[hidden]-> consul + +sidekiq -[#ff8dd1]--> ilb +sidekiq -[#ff8dd1]-> object_storage +sidekiq -[#ff8dd1]---> redis +sidekiq -[hidden]-> monitor +sidekiq -[hidden]-> consul + +ilb -[#9370DB]-> gitaly_cluster +ilb -[#9370DB]-> database + +consul .[#e76a9b]u-> gitlab +consul .[#e76a9b]u-> sidekiq +consul .[#e76a9b]> monitor +consul .[#e76a9b]-> database +consul .[#e76a9b]-> gitaly_cluster +consul .[#e76a9b,norank]--> redis + +monitor .[#7FFFD4]u-> gitlab +monitor .[#7FFFD4]u-> sidekiq +monitor .[#7FFFD4]> consul +monitor .[#7FFFD4]-> database +monitor .[#7FFFD4]-> gitaly_cluster +monitor .[#7FFFD4,norank]--> redis +monitor .[#7FFFD4]> ilb +monitor .[#7FFFD4,norank]u--> elb + +@enduml ``` The Google Cloud Platform (GCP) architectures were built and tested using the @@ -106,19 +135,25 @@ uploads, or artifacts), using an [object storage service](#configure-the-object- is recommended instead of using NFS. Using an object storage service also doesn't require you to provision and maintain a node. +It's also worth noting that at this time [Praefect requires its own database server](../gitaly/praefect.md#postgresql) and +that to achieve full High Availability a third party PostgreSQL database solution will be required. +We hope to offer a built in solutions for these restrictions in the future but in the meantime a non HA PostgreSQL server +can be set up via Omnibus GitLab, which the above specs reflect. Refer to the following issues for more information: [`omnibus-gitlab#5919`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5919) & [`gitaly#3398`](https://gitlab.com/gitlab-org/gitaly/-/issues/3398) + ## Setup components To set up GitLab and its components to accommodate up to 3,000 users: -1. [Configure the external load balancing node](#configure-the-external-load-balancer) +1. [Configure the external load balancer](#configure-the-external-load-balancer) to handle the load balancing of the GitLab application services nodes. +1. [Configure the internal load balancer](#configure-the-internal-load-balancer). + to handle the load balancing of GitLab application internal connections. 1. [Configure Redis](#configure-redis). 1. [Configure Consul and Sentinel](#configure-consul-and-sentinel). 1. [Configure PostgreSQL](#configure-postgresql), the database for GitLab. 1. [Configure PgBouncer](#configure-pgbouncer). -1. [Configure the internal load balancing node](#configure-the-internal-load-balancer). -1. [Configure Gitaly](#configure-gitaly), - which provides access to the Git repositories. +1. [Configure Gitaly Cluster](#configure-gitaly-cluster), + provides access to the Git repositories. 1. [Configure Sidekiq](#configure-sidekiq). 1. [Configure the main GitLab Rails application](#configure-gitlab-rails) to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend @@ -155,6 +190,11 @@ The following list includes descriptions of each server and its assigned IP: - `10.6.0.20`: Internal Load Balancer - `10.6.0.51`: Gitaly 1 - `10.6.0.52`: Gitaly 2 +- `10.6.0.93`: Gitaly 3 +- `10.6.0.131`: Praefect 1 +- `10.6.0.132`: Praefect 2 +- `10.6.0.133`: Praefect 3 +- `10.6.0.141`: Praefect PostgreSQL 1 (non HA) - `10.6.0.71`: Sidekiq 1 - `10.6.0.72`: Sidekiq 2 - `10.6.0.73`: Sidekiq 3 @@ -285,6 +325,71 @@ Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. </a> </div> +## Configure the internal load balancer + +The Internal Load Balancer is used to balance any internal connections the GitLab environment requires +such as connections to [PgBouncer](#configure-pgbouncer) and [Praefect](#configure-praefect) (Gitaly Cluster). + +Note that it's a separate node from the External Load Balancer and shouldn't have any access externally. + +The following IP will be used as an example: + +- `10.6.0.40`: Internal Load Balancer + +Here's how you could do it with [HAProxy](https://www.haproxy.org/): + +```plaintext +global + log /dev/log local0 + log localhost local1 notice + log stdout format raw local0 + +defaults + log global + default-server inter 10s fall 3 rise 2 + balance leastconn + +frontend internal-pgbouncer-tcp-in + bind *:6432 + mode tcp + option tcplog + + default_backend pgbouncer + +frontend internal-praefect-tcp-in + bind *:2305 + mode tcp + option tcplog + option clitcpka + + default_backend praefect + +backend pgbouncer + mode tcp + option tcp-check + + server pgbouncer1 10.6.0.21:6432 check + server pgbouncer2 10.6.0.22:6432 check + server pgbouncer3 10.6.0.23:6432 check + +backend praefect + mode tcp + option tcp-check + option srvtcpka + + server praefect1 10.6.0.131:2305 check + server praefect2 10.6.0.132:2305 check + server praefect3 10.6.0.133:2305 check +``` + +Refer to your preferred Load Balancer's documentation for further guidance. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + ## Configure Redis Using [Redis](https://redis.io/) in scalable environment is possible using a **Primary** x **Replica** @@ -925,45 +1030,96 @@ The following IPs will be used as an example: </a> </div> -### Configure the internal load balancer +## Configure Gitaly Cluster -If you're running more than one PgBouncer node as recommended, then at this time you'll need to set -up a TCP internal load balancer to serve each correctly. +[Gitaly Cluster](../gitaly/praefect.md) is a GitLab provided and recommended fault tolerant solution for storing Git repositories. +In this configuration, every Git repository is stored on every Gitaly node in the cluster, with one being designated the primary, and failover occurs automatically if the primary node goes down. -The following IP will be used as an example: +The recommended cluster setup includes the following components: -- `10.6.0.20`: Internal Load Balancer +- 3 Gitaly nodes: Replicated storage of Git repositories. +- 3 Praefect nodes: Router and transaction manager for Gitaly Cluster. +- 1 Praefect PostgreSQL node: Database server for Praefect. A third-party solution + is required for Praefect database connections to be made highly available. +- 1 load balancer: A load balancer is required for Praefect. The + [internal load balancer](#configure-the-internal-load-balancer) will be used. -Here's how you could do it with [HAProxy](https://www.haproxy.org/): +This section will detail how to configure the recommended standard setup in order. +For more advanced setups refer to the [standalone Gitaly Cluster documentation](../gitaly/praefect.md). -```plaintext -global - log /dev/log local0 - log localhost local1 notice - log stdout format raw local0 +### Configure Praefect PostgreSQL -defaults - log global - default-server inter 10s fall 3 rise 2 - balance leastconn +Praefect, the routing and transaction manager for Gitaly Cluster, requires its own database server to store data on Gitaly Cluster status. -frontend internal-pgbouncer-tcp-in - bind *:6432 - mode tcp - option tcplog +If you want to have a highly available setup, Praefect requires a third-party PostgreSQL database. +A built-in solution is being [worked on](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5919). - default_backend pgbouncer +#### Praefect non-HA PostgreSQL standalone using Omnibus GitLab -backend pgbouncer - mode tcp - option tcp-check +The following IPs will be used as an example: - server pgbouncer1 10.6.0.21:6432 check - server pgbouncer2 10.6.0.22:6432 check - server pgbouncer3 10.6.0.23:6432 check -``` +- `10.6.0.141`: Praefect PostgreSQL -Refer to your preferred Load Balancer's documentation for further guidance. +First, make sure to [install](https://about.gitlab.com/install/) +the Linux GitLab package in the Praefect PostgreSQL node. Following the steps, +install the necessary dependencies from step 1, and add the +GitLab package repository from step 2. When installing GitLab +in the second step, do not supply the `EXTERNAL_URL` value. + +1. SSH in to the Praefect PostgreSQL node. +1. Create a strong password to be used for the Praefect PostgreSQL user. Take note of this password as `<praefect_postgresql_password>`. +1. Generate the password hash for the Praefect PostgreSQL username/password pair. This assumes you will use the default + username of `praefect` (recommended). The command will request the password `<praefect_postgresql_password>` + and confirmation. Use the value that is output by this command in the next + step as the value of `<praefect_postgresql_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 praefect + ``` + +1. Edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: + + ```ruby + # Disable all components except PostgreSQL and Consul + roles ['postgres_role'] + repmgr['enable'] = false + patroni['enable'] = false + + # PostgreSQL configuration + postgresql['listen_address'] = '0.0.0.0' + postgresql['max_connections'] = 200 + + gitlab_rails['auto_migrate'] = false + + # Configure the Consul agent + consul['enable'] = true + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # START user configuration + # Please set the real values as explained in Required Information section + # + # Replace PRAEFECT_POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = "<praefect_postgresql_password_hash>" + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) + + # Set the network addresses that the exporters will listen on for monitoring + node_exporter['listen_address'] = '0.0.0.0:9100' + postgres_exporter['listen_address'] = '0.0.0.0:9187' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + # + # END user configuration + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Follow the [post configuration](#praefect-postgresql-post-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -971,19 +1127,186 @@ Refer to your preferred Load Balancer's documentation for further guidance. </a> </div> -## Configure Gitaly +#### Praefect HA PostgreSQL third-party solution -NOTE: -[Gitaly Cluster](../gitaly/praefect.md) support -for the Reference Architectures is being -worked on as a [collaborative effort](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/1) between the Quality Engineering and Gitaly teams. When this component has been verified -some Architecture specs will likely change as a result to support the new -and improved designed. +[As noted](#configure-praefect-postgresql), a third-party PostgreSQL solution for +Praefect's database is recommended if aiming for full High Availability. + +There are many third-party solutions for PostgreSQL HA. The solution selected must have the following to work with Praefect: + +- A static IP for all connections that doesn't change on failover. +- [`LISTEN`](https://www.postgresql.org/docs/12/sql-listen.html) SQL functionality must be supported. + +Examples of the above could include [Google's Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) or [Amazon RDS](https://aws.amazon.com/rds/). + +Once the database is set up, follow the [post configuration](#praefect-postgresql-post-configuration). + +#### Praefect PostgreSQL post-configuration + +After the Praefect PostgreSQL server has been set up, you'll then need to configure the user and database for Praefect to use. + +We recommend the user be named `praefect` and the database `praefect_production`, and these can be configured as standard in PostgreSQL. +The password for the user is the same as the one you configured earlier as `<praefect_postgresql_password>`. + +This is how this would work with a Omnibus GitLab PostgreSQL setup: + +1. SSH in to the Praefect PostgreSQL node. +1. Connect to the PostgreSQL server with administrative access. + The `gitlab-psql` user should be used here for this as it's added by default in Omnibus. + The database `template1` is used because it is created by default on all PostgreSQL servers. -[Gitaly](../gitaly/index.md) server node requirements are dependent on data, -specifically the number of projects and those projects' sizes. It's recommended -that a Gitaly server node stores no more than 5 TB of data. Depending on your -repository storage requirements, you may require additional Gitaly server nodes. + ```shell + /opt/gitlab/embedded/bin/psql -U gitlab-psql -d template1 -h POSTGRESQL_SERVER_ADDRESS + ``` + +1. Create the new user `praefect`, replacing `<praefect_postgresql_password>`: + + ```shell + CREATE ROLE praefect WITH LOGIN CREATEDB PASSWORD <praefect_postgresql_password>; + ``` + +1. Reconnect to the PostgreSQL server, this time as the `praefect` user: + + ```shell + /opt/gitlab/embedded/bin/psql -U praefect -d template1 -h POSTGRESQL_SERVER_ADDRESS + ``` + +1. Create a new database `praefect_production`: + + ```shell + CREATE DATABASE praefect_production WITH ENCODING=UTF8; + ``` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +### Configure Praefect + +Praefect is the router and transaction manager for Gitaly Cluster and all connections to Gitaly go through +it. This section details how to configure it. + +Praefect requires several secret tokens to secure communications across the Cluster: + +- `<praefect_external_token>`: Used for repositories hosted on your Gitaly cluster and can only be accessed by Gitaly clients that carry this token. +- `<praefect_internal_token>`: Used for replication traffic inside your Gitaly cluster. This is distinct from `praefect_external_token` because Gitaly clients must not be able to access internal nodes of the Praefect cluster directly; that could lead to data loss. +- `<praefect_postgresql_password>`: The Praefect PostgreSQL password defined in the previous section is also required as part of this setup. + +Gitaly Cluster nodes are configured in Praefect via a `virtual storage`. Each storage contains +the details of each Gitaly node that makes up the cluster. Each storage is also given a name +and this name is used in several areas of the config. In this guide, the name of the storage will be +`default`. Also, this guide is geared towards new installs, if upgrading an existing environment +to use Gitaly Cluster, you may need to use a different name. +Refer to the [Praefect documentation](../gitaly/praefect.md#praefect) for more info. + +The following IPs will be used as an example: + +- `10.6.0.131`: Praefect 1 +- `10.6.0.132`: Praefect 2 +- `10.6.0.133`: Praefect 3 + +To configure the Praefect nodes, on each one: + +1. SSH in to the Praefect server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. +1. Edit the `/etc/gitlab/gitlab.rb` file to configure Praefect: + + ```ruby + # Avoid running unnecessary services on the Gitaly server + postgresql['enable'] = false + redis['enable'] = false + nginx['enable'] = false + puma['enable'] = false + unicorn['enable'] = false + sidekiq['enable'] = false + gitlab_workhorse['enable'] = false + grafana['enable'] = false + + # If you run a separate monitoring node you can disable these services + alertmanager['enable'] = false + prometheus['enable'] = false + + # Praefect Configuration + praefect['enable'] = true + praefect['listen_addr'] = '0.0.0.0:2305' + + gitlab_rails['rake_cache_clear'] = false + gitlab_rails['auto_migrate'] = false + + # Configure the Consul agent + consul['enable'] = true + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # START user configuration + # Please set the real values as explained in Required Information section + # + + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + praefect['auth_token'] = '<praefect_external_token>' + + # Praefect Database Settings + praefect['database_host'] = '10.6.0.141' + praefect['database_port'] = 5432 + # `no_proxy` settings must always be a direct connection for caching + praefect['database_host_no_proxy'] = '10.6.0.141' + praefect['database_port_no_proxy'] = 5432 + praefect['database_dbname'] = 'praefect_production' + praefect['database_user'] = 'praefect' + praefect['database_password'] = '<praefect_postgresql_password>' + + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + praefect['virtual_storages'] = { + 'default' => { + 'gitaly-1' => { + 'address' => 'tcp://10.6.0.91:8075', + 'token' => '<praefect_internal_token>', + 'primary' => true + }, + 'gitaly-2' => { + 'address' => 'tcp://10.6.0.92:8075', + 'token' => '<praefect_internal_token>' + }, + 'gitaly-3' => { + 'address' => 'tcp://10.6.0.93:8075', + 'token' => '<praefect_internal_token>' + }, + } + } + + # Set the network addresses that the exporters will listen on for monitoring + node_exporter['listen_address'] = '0.0.0.0:9100' + praefect['prometheus_listen_addr'] = '0.0.0.0:9652' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + # + # END user configuration + ``` + + 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and + then replace the file of the same name on this server. If that file isn't on + this server, add the file from your Consul server to this server. + + 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +### Configure Gitaly + +The [Gitaly](../gitaly/index.md) server nodes that make up the cluster have +requirements that are dependent on data, specifically the number of projects +and those projects' sizes. It's recommended that a Gitaly Cluster stores +no more than 5 TB of data on each node. Depending on your +repository storage requirements, you may require additional Gitaly Clusters. Due to Gitaly having notable input and output requirements, we strongly recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs @@ -994,36 +1317,21 @@ adjusted to greater or lesser values depending on the scale of your environment's workload. If you're running the environment on a Cloud provider, refer to their documentation about how to configure IOPS correctly. -Be sure to note the following items: +Gitaly servers must not be exposed to the public internet, as Gitaly's network +traffic is unencrypted by default. The use of a firewall is highly recommended +to restrict access to the Gitaly server. Another option is to +[use TLS](#gitaly-cluster-tls-support). -- The GitLab Rails application shards repositories into - [repository storage paths](../repository_storage_paths.md). -- A Gitaly server can host one or more storage paths. -- A GitLab server can use one or more Gitaly server nodes. -- Gitaly addresses must be specified to be correctly resolvable for all Gitaly - clients. -- Gitaly servers must not be exposed to the public internet, as Gitaly's network - traffic is unencrypted by default. The use of a firewall is highly recommended - to restrict access to the Gitaly server. Another option is to - [use TLS](#gitaly-tls-support). - -NOTE: -The token referred to throughout the Gitaly documentation is an arbitrary -password selected by the administrator. This token is unrelated to tokens -created for the GitLab API or other similar web API tokens. +For configuring Gitaly you should note the following: -This section describes how to configure two Gitaly servers, with the following -IPs and domain names: +- `git_data_dirs` should be configured to reflect the storage path for the specific Gitaly node +- `auth_token` should be the same as `praefect_internal_token` -- `10.6.0.51`: Gitaly 1 (`gitaly1.internal`) -- `10.6.0.52`: Gitaly 2 (`gitaly2.internal`) - -Assumptions about your servers include having the secret token be `gitalysecret`, -and that your GitLab installation has three repository storages: +The following IPs will be used as an example: -- `default` on Gitaly 1 -- `storage1` on Gitaly 1 -- `storage2` on Gitaly 2 +- `10.6.0.91`: Gitaly 1 +- `10.6.0.92`: Gitaly 2 +- `10.6.0.93`: Gitaly 3 On each node: @@ -1033,21 +1341,9 @@ On each node: 1. Edit the Gitaly server node's `/etc/gitlab/gitlab.rb` file to configure storage paths, enable the network listener, and to configure the token: - <!-- - updates to following example must also be made at - https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab - --> - ```ruby # /etc/gitlab/gitlab.rb - # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests - # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. - # The following two values must be the same as their respective values - # of the GitLab Rails application setup - gitaly['auth_token'] = 'gitalysecret' - gitlab_shell['secret_token'] = 'shellsecret' - # Avoid running unnecessary services on the Gitaly server postgresql['enable'] = false redis['enable'] = false @@ -1057,7 +1353,6 @@ On each node: sidekiq['enable'] = false gitlab_workhorse['enable'] = false grafana['enable'] = false - gitlab_exporter['enable'] = false # If you run a separate monitoring node you can disable these services alertmanager['enable'] = false @@ -1078,101 +1373,86 @@ On each node: # Comment out following line if you only want to support TLS connections gitaly['listen_addr'] = "0.0.0.0:8075" - ## Enable service discovery for Prometheus - consul['enable'] = true - consul['monitoring_service_discovery'] = true - - # Set the network addresses that the exporters will listen on for monitoring - gitaly['prometheus_listen_addr'] = "0.0.0.0:9236" - node_exporter['listen_address'] = '0.0.0.0:9100' - gitlab_rails['prometheus_address'] = '10.6.0.81:9090' - - ## The IPs of the Consul server nodes - ## You can also use FQDNs and intermix them with IPs - consul['configuration'] = { - retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), - } + # Gitaly Auth Token + # Should be the same as praefect_internal_token + gitaly['auth_token'] = '<praefect_internal_token>' ``` 1. Append the following to `/etc/gitlab/gitlab.rb` for each respective server: - - On `gitaly1.internal`: + - On Gitaly node 1: ```ruby git_data_dirs({ - 'default' => { - 'path' => '/var/opt/gitlab/git-data' - }, - 'storage1' => { - 'path' => '/mnt/gitlab/git-data' - }, + "gitaly-1" => { + "path" => "/var/opt/gitlab/git-data" + } }) ``` - - On `gitaly2.internal`: + - On Gitaly node 2: ```ruby git_data_dirs({ - 'storage2' => { - 'path' => '/mnt/gitlab/git-data' - }, + "gitaly-2" => { + "path" => "/var/opt/gitlab/git-data" + } }) ``` - <!-- - updates to following example must also be made at - https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab - --> + - On Gitaly node 3: + + ```ruby + git_data_dirs({ + "gitaly-3" => { + "path" => "/var/opt/gitlab/git-data" + } + }) + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and + then replace the file of the same name on this server. If that file isn't on + this server, add the file from your Consul server to this server. 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -1. Confirm that Gitaly can perform callbacks to the internal API: - ```shell - sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml - ``` +### Gitaly Cluster TLS support -1. Verify the GitLab services are running: +Praefect supports TLS encryption. To communicate with a Praefect instance that listens +for secure connections, you must: - ```shell - sudo gitlab-ctl status - ``` +- Use a `tls://` URL scheme in the `gitaly_address` of the corresponding storage entry + in the GitLab configuration. +- Bring your own certificates because this isn't provided automatically. The certificate + corresponding to each Praefect server must be installed on that Praefect server. - The output should be similar to the following: +Additionally the certificate, or its certificate authority, must be installed on all Gitaly servers +and on all Praefect clients that communicate with it following the procedure described in +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) (and repeated below). - ```plaintext - run: consul: (pid 30339) 77006s; run: log: (pid 29878) 77020s - run: gitaly: (pid 30351) 77005s; run: log: (pid 29660) 77040s - run: logrotate: (pid 7760) 3213s; run: log: (pid 29782) 77032s - run: node-exporter: (pid 30378) 77004s; run: log: (pid 29812) 77026s - ``` +Note the following: -### Gitaly TLS support +- The certificate must specify the address you use to access the Praefect server. If + addressing the Praefect server by: -Gitaly supports TLS encryption. To be able to communicate -with a Gitaly instance that listens for secure connections you will need to use `tls://` URL -scheme in the `gitaly_address` of the corresponding storage entry in the GitLab configuration. + - Hostname, you can either use the Common Name field for this, or add it as a Subject + Alternative Name. + - IP address, you must add it as a Subject Alternative Name to the certificate. -You will need to bring your own certificates as this isn't provided automatically. -The certificate, or its certificate authority, must be installed on all Gitaly -nodes (including the Gitaly node using the certificate) and on all client nodes -that communicate with it following the procedure described in -[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). +- You can configure Praefect servers with both an unencrypted listening address + `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. + This allows you to do a gradual transition from unencrypted to encrypted traffic, if + necessary. -NOTE: -The self-signed certificate must specify the address you use to access the -Gitaly server. If you are addressing the Gitaly server by a hostname, you can -either use the Common Name field for this, or add it as a Subject Alternative -Name. If you are addressing the Gitaly server by its IP address, you must add it -as a Subject Alternative Name to the certificate. -[gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). +- The Internal Load Balancer will also access to the certificates and need to be configured + to allow for TLS passthrough. + Refer to the load balancers documentation on how to configure this. -It's possible to configure Gitaly servers with both an unencrypted listening -address (`listen_addr`) and an encrypted listening address (`tls_listen_addr`) -at the same time. This allows you to do a gradual transition from unencrypted to -encrypted traffic, if necessary. +To configure Praefect with TLS: -To configure Gitaly with TLS: +1. Create certificates for Praefect servers. -1. Create the `/etc/gitlab/ssl` directory and copy your key and certificate there: +1. On the Praefect servers, create the `/etc/gitlab/ssl` directory and copy your key + and certificate there: ```shell sudo mkdir -p /etc/gitlab/ssl @@ -1181,27 +1461,35 @@ To configure Gitaly with TLS: sudo chmod 644 key.pem cert.pem ``` -1. Copy the cert to `/etc/gitlab/trusted-certs` so Gitaly will trust the cert when - calling into itself: +1. Edit `/etc/gitlab/gitlab.rb` and add: - ```shell - sudo cp /etc/gitlab/ssl/cert.pem /etc/gitlab/trusted-certs/ + ```ruby + praefect['tls_listen_addr'] = "0.0.0.0:3305" + praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" + praefect['key_path'] = "/etc/gitlab/ssl/key.pem" ``` -1. Edit `/etc/gitlab/gitlab.rb` and add: +1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). - <!-- - updates to following example must also be made at - https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab - --> +1. On the Praefect clients (including each Gitaly server), copy the certificates, + or their certificate authority, into `/etc/gitlab/trusted-certs`: + + ```shell + sudo cp cert.pem /etc/gitlab/trusted-certs/ + ``` + +1. On the Praefect clients (except Gitaly servers), edit `git_data_dirs` in + `/etc/gitlab/gitlab.rb` as follows: ```ruby - gitaly['tls_listen_addr'] = "0.0.0.0:9999" - gitaly['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - gitaly['key_path'] = "/etc/gitlab/ssl/key.pem" + git_data_dirs({ + "default" => { + "gitaly_address" => 'tls://LOAD_BALANCER_SERVER_ADDRESS:2305', + "gitaly_token" => 'PRAEFECT_EXTERNAL_TOKEN' + } + }) ``` -1. Delete `gitaly['listen_addr']` to allow only encrypted connections. 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). <div align="right"> @@ -1272,17 +1560,20 @@ To configure the Sidekiq nodes, one each one: ### Gitaly ### ####################################### + # git_data_dirs get configured for the Praefect virtual storage + # Address is Internal Load Balancer for Praefect + # Token is praefect_external_token git_data_dirs({ - 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, - 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, - 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + "default" => { + "gitaly_address" => "tcp://10.6.0.40:2305", # internal load balancer IP + "gitaly_token" => '<praefect_external_token>' + } }) - gitlab_rails['gitaly_token'] = 'YOUR_TOKEN' ####################################### ### Postgres ### ####################################### - gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP + gitlab_rails['db_host'] = '10.6.0.40' # internal load balancer IP gitlab_rails['db_port'] = 6432 gitlab_rails['db_password'] = '<postgresql_user_password>' gitlab_rails['db_adapter'] = 'postgresql' @@ -1401,17 +1692,14 @@ On each node perform the following: ```ruby external_url 'https://gitlab.example.com' - # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests - # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. - # The following two values must be the same as their respective values - # of the Gitaly setup - gitlab_rails['gitaly_token'] = 'gitalysecret' - gitlab_shell['secret_token'] = 'shellsecret' - + # git_data_dirs get configured for the Praefect virtual storage + # Address is Interal Load Balancer for Praefect + # Token is praefect_external_token git_data_dirs({ - 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, - 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, - 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + "default" => { + "gitaly_address" => "tcp://10.6.0.40:2305", # internal load balancer IP + "gitaly_token" => '<praefect_external_token>' + } }) ## Disable components that will not be on the GitLab application server @@ -1499,14 +1787,15 @@ On each node perform the following: gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = "<gcp-terraform-state-bucket-name>" ``` -1. If you're using [Gitaly with TLS support](#gitaly-tls-support), make sure the +1. If you're using [Gitaly with TLS support](#gitaly-cluster-tls-support), make sure the `git_data_dirs` entry is configured with `tls` instead of `tcp`: ```ruby git_data_dirs({ - 'default' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, - 'storage1' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, - 'storage2' => { 'gitaly_address' => 'tls://gitaly2.internal:9999' }, + "default" => { + "gitaly_address" => "tls://10.6.0.40:2305", # internal load balancer IP + "gitaly_token" => '<praefect_external_token>' + } }) ``` diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index 04ce39046d8..475a4944492 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -12,100 +12,110 @@ full list of reference architectures, see [Available reference architectures](index.md#available-reference-architectures). > - **Supported users (approximate):** 50,000 -> - **High Availability:** Yes +> - **High Availability:** Yes ([Praefect](#configure-praefect-postgresql) needs a third-party PostgreSQL solution for HA) > - **Test requests per second (RPS) rates:** API: 1000 RPS, Web: 100 RPS, Git (Pull): 100 RPS, Git (Push): 20 RPS | Service | Nodes | Configuration | GCP | AWS | Azure | |-----------------------------------------|-------------|-------------------------|-----------------|--------------|----------| -| External load balancing node | 1 | 8 vCPU, 7.2 GB memory | n1-highcpu-8 | `c5.2xlarge` | F8s v2 | -| Consul | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | -| PostgreSQL | 3 | 16 vCPU, 60 GB memory | n1-standard-16 | `m5.4xlarge` | D16s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | -| Internal load balancing node | 1 | 8 vCPU, 7.2 GB memory | n1-highcpu-8 | `c5.2xlarge` | F8s v2 | -| Redis - Cache | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | `m5.xlarge` | D4s v3 | -| Redis - Queues / Shared State | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | `m5.xlarge` | D4s v3 | -| Redis Sentinel - Cache | 3 | 1 vCPU, 1.7 GB memory | g1-small | `t3.small` | B1MS | -| Redis Sentinel - Queues / Shared State | 3 | 1 vCPU, 1.7 GB memory | g1-small | `t3.small` | B1MS | -| Gitaly | 2 (minimum) | 64 vCPU, 240 GB memory | n1-standard-64 | `m5.16xlarge` | D64s v3 | -| Sidekiq | 4 | 4 vCPU, 15 GB memory | n1-standard-4 | `m5.xlarge` | D4s v3 | -| GitLab Rails | 12 | 32 vCPU, 28.8 GB memory | n1-highcpu-32 | `c5.9xlarge` | F32s v2 | -| Monitoring node | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | `c5.xlarge` | F4s v2 | +| External load balancing node | 1 | 8 vCPU, 7.2 GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | +| Consul | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| PostgreSQL | 3 | 32 vCPU, 120 GB memory | n1-standard-32 | m5.8xlarge | D32s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node | 1 | 8 vCPU, 7.2 GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | +| Redis - Cache | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis - Queues / Shared State | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis Sentinel - Cache | 3 | 1 vCPU, 1.7 GB memory | g1-small | t3.small | B1MS | +| Redis Sentinel - Queues / Shared State | 3 | 1 vCPU, 1.7 GB memory | g1-small | t3.small | B1MS | +| Gitaly | 3 | 64 vCPU, 240 GB memory | n1-standard-64 | m5.16xlarge | D64s v3 | +| Praefect | 3 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Praefect PostgreSQL | 1+* | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Sidekiq | 4 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| GitLab Rails | 12 | 32 vCPU, 28.8 GB memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | +| Monitoring node | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | | Object storage | n/a | n/a | n/a | n/a | n/a | -| NFS server | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | `c5.xlarge` | F4s v2 | - -```mermaid -stateDiagram-v2 - [*] --> LoadBalancer - LoadBalancer --> ApplicationServer - - ApplicationServer --> BackgroundJobs - ApplicationServer --> Gitaly - ApplicationServer --> Redis_Cache - ApplicationServer --> Redis_Queues - ApplicationServer --> PgBouncer - PgBouncer --> Database - ApplicationServer --> ObjectStorage - BackgroundJobs --> ObjectStorage - - ApplicationMonitoring -->ApplicationServer - ApplicationMonitoring -->PgBouncer - ApplicationMonitoring -->Database - ApplicationMonitoring -->BackgroundJobs - - ApplicationServer --> Consul - - Consul --> Database - Consul --> PgBouncer - Redis_Cache --> Consul - Redis_Queues --> Consul - BackgroundJobs --> Consul - - state Consul { - "Consul_1..3" - } - - state Database { - "PG_Primary_Node" - "PG_Secondary_Node_1..2" - } - - state Redis_Cache { - "R_Cache_Primary_Node" - "R_Cache_Replica_Node_1..2" - "R_Cache_Sentinel_1..3" - } - - state Redis_Queues { - "R_Queues_Primary_Node" - "R_Queues_Replica_Node_1..2" - "R_Queues_Sentinel_1..3" - } - - state Gitaly { - "Gitaly_1..2" - } - - state BackgroundJobs { - "Sidekiq_1..4" - } - - state ApplicationServer { - "GitLab_Rails_1..12" - } - - state LoadBalancer { - "LoadBalancer_1" - } - - state ApplicationMonitoring { - "Prometheus" - "Grafana" - } - - state PgBouncer { - "Internal_Load_Balancer" - "PgBouncer_1..3" - } +| NFS server | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | + +```plantuml +@startuml 50k +card "**External Load Balancer**" as elb #6a9be7 +card "**Internal Load Balancer**" as ilb #9370DB + +together { + collections "**GitLab Rails** x12" as gitlab #32CD32 + collections "**Sidekiq** x4" as sidekiq #ff8dd1 +} + +together { + card "**Prometheus + Grafana**" as monitor #7FFFD4 + collections "**Consul** x3" as consul #e76a9b +} + +card "Gitaly Cluster" as gitaly_cluster { + collections "**Praefect** x3" as praefect #FF8C00 + collections "**Gitaly** x3" as gitaly #FF8C00 + card "**Praefect PostgreSQL***\n//Non fault-tolerant//" as praefect_postgres #FF8C00 + + praefect -[#FF8C00]-> gitaly + praefect -[#FF8C00]> praefect_postgres +} + +card "Database" as database { + collections "**PGBouncer** x3" as pgbouncer #4EA7FF + card "**PostgreSQL** (Primary)" as postgres_primary #4EA7FF + collections "**PostgreSQL** (Secondary) x2" as postgres_secondary #4EA7FF + + pgbouncer -[#4EA7FF]-> postgres_primary + postgres_primary .[#4EA7FF]> postgres_secondary +} + +card "redis" as redis { + collections "**Redis Persistent** x3" as redis_persistent #FF6347 + collections "**Redis Cache** x3" as redis_cache #FF6347 + collections "**Redis Persistent Sentinel** x3" as redis_persistent_sentinel #FF6347 + collections "**Redis Cache Sentinel** x3"as redis_cache_sentinel #FF6347 + + redis_persistent <.[#FF6347]- redis_persistent_sentinel + redis_cache <.[#FF6347]- redis_cache_sentinel +} + +cloud "**Object Storage**" as object_storage #white + +elb -[#6a9be7]-> gitlab +elb -[#6a9be7]--> monitor + +gitlab -[#32CD32]> sidekiq +gitlab -[#32CD32]--> ilb +gitlab -[#32CD32]-> object_storage +gitlab -[#32CD32]---> redis +gitlab -[hidden]-> monitor +gitlab -[hidden]-> consul + +sidekiq -[#ff8dd1]--> ilb +sidekiq -[#ff8dd1]-> object_storage +sidekiq -[#ff8dd1]---> redis +sidekiq -[hidden]-> monitor +sidekiq -[hidden]-> consul + +ilb -[#9370DB]-> gitaly_cluster +ilb -[#9370DB]-> database + +consul .[#e76a9b]u-> gitlab +consul .[#e76a9b]u-> sidekiq +consul .[#e76a9b]> monitor +consul .[#e76a9b]-> database +consul .[#e76a9b]-> gitaly_cluster +consul .[#e76a9b,norank]--> redis + +monitor .[#7FFFD4]u-> gitlab +monitor .[#7FFFD4]u-> sidekiq +monitor .[#7FFFD4]> consul +monitor .[#7FFFD4]-> database +monitor .[#7FFFD4]-> gitaly_cluster +monitor .[#7FFFD4,norank]--> redis +monitor .[#7FFFD4]> ilb +monitor .[#7FFFD4,norank]u--> elb + +@enduml ``` The Google Cloud Platform (GCP) architectures were built and tested using the @@ -120,19 +130,25 @@ uploads, or artifacts), using an [object storage service](#configure-the-object- is recommended instead of using NFS. Using an object storage service also doesn't require you to provision and maintain a node. +It's also worth noting that at this time [Praefect requires its own database server](../gitaly/praefect.md#postgresql) and +that to achieve full High Availability a third party PostgreSQL database solution will be required. +We hope to offer a built in solutions for these restrictions in the future but in the meantime a non HA PostgreSQL server +can be set up via Omnibus GitLab, which the above specs reflect. Refer to the following issues for more information: [`omnibus-gitlab#5919`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5919) & [`gitaly#3398`](https://gitlab.com/gitlab-org/gitaly/-/issues/3398) + ## Setup components To set up GitLab and its components to accommodate up to 50,000 users: -1. [Configure the external load balancing node](#configure-the-external-load-balancer) +1. [Configure the external load balancer](#configure-the-external-load-balancer) to handle the load balancing of the GitLab application services nodes. +1. [Configure the internal load balancer](#configure-the-internal-load-balancer). + to handle the loa 1. [Configure Consul](#configure-consul). 1. [Configure PostgreSQL](#configure-postgresql), the database for GitLab. 1. [Configure PgBouncer](#configure-pgbouncer). -1. [Configure the internal load balancing node](#configure-the-internal-load-balancer). 1. [Configure Redis](#configure-redis). -1. [Configure Gitaly](#configure-gitaly), - which provides access to the Git repositories. +1. [Configure Gitaly Cluster](#configure-gitaly-cluster), + provides access to the Git repositories. 1. [Configure Sidekiq](#configure-sidekiq). 1. [Configure the main GitLab Rails application](#configure-gitlab-rails) to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend @@ -178,6 +194,11 @@ The following list includes descriptions of each server and its assigned IP: - `10.6.0.83`: Sentinel - Queues 3 - `10.6.0.91`: Gitaly 1 - `10.6.0.92`: Gitaly 2 +- `10.6.0.93`: Gitaly 3 +- `10.6.0.131`: Praefect 1 +- `10.6.0.132`: Praefect 2 +- `10.6.0.133`: Praefect 3 +- `10.6.0.141`: Praefect PostgreSQL 1 (non HA) - `10.6.0.101`: Sidekiq 1 - `10.6.0.102`: Sidekiq 2 - `10.6.0.103`: Sidekiq 3 @@ -185,7 +206,16 @@ The following list includes descriptions of each server and its assigned IP: - `10.6.0.111`: GitLab application 1 - `10.6.0.112`: GitLab application 2 - `10.6.0.113`: GitLab application 3 -- `10.6.0.121`: Prometheus +- `10.6.0.114`: GitLab application 4 +- `10.6.0.115`: GitLab application 5 +- `10.6.0.116`: GitLab application 6 +- `10.6.0.117`: GitLab application 7 +- `10.6.0.118`: GitLab application 8 +- `10.6.0.119`: GitLab application 9 +- `10.6.0.120`: GitLab application 10 +- `10.6.0.121`: GitLab application 11 +- `10.6.0.122`: GitLab application 12 +- `10.6.0.151`: Prometheus ## Configure the external load balancer @@ -308,6 +338,71 @@ Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. </a> </div> +## Configure the internal load balancer + +The Internal Load Balancer is used to balance any internal connections the GitLab environment requires +such as connections to [PgBouncer](#configure-pgbouncer) and [Praefect](#configure-praefect) (Gitaly Cluster). + +Note that it's a separate node from the External Load Balancer and shouldn't have any access externally. + +The following IP will be used as an example: + +- `10.6.0.40`: Internal Load Balancer + +Here's how you could do it with [HAProxy](https://www.haproxy.org/): + +```plaintext +global + log /dev/log local0 + log localhost local1 notice + log stdout format raw local0 + +defaults + log global + default-server inter 10s fall 3 rise 2 + balance leastconn + +frontend internal-pgbouncer-tcp-in + bind *:6432 + mode tcp + option tcplog + + default_backend pgbouncer + +frontend internal-praefect-tcp-in + bind *:2305 + mode tcp + option tcplog + option clitcpka + + default_backend praefect + +backend pgbouncer + mode tcp + option tcp-check + + server pgbouncer1 10.6.0.21:6432 check + server pgbouncer2 10.6.0.22:6432 check + server pgbouncer3 10.6.0.23:6432 check + +backend praefect + mode tcp + option tcp-check + option srvtcpka + + server praefect1 10.6.0.131:2305 check + server praefect2 10.6.0.132:2305 check + server praefect3 10.6.0.133:2305 check +``` + +Refer to your preferred Load Balancer's documentation for further guidance. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + ## Configure Consul The following IPs will be used as an example: @@ -662,52 +757,6 @@ The following IPs will be used as an example: </a> </div> -### Configure the internal load balancer - -If you're running more than one PgBouncer node as recommended, then at this time you'll need to set -up a TCP internal load balancer to serve each correctly. - -The following IP will be used as an example: - -- `10.6.0.40`: Internal Load Balancer - -Here's how you could do it with [HAProxy](https://www.haproxy.org/): - -```plaintext -global - log /dev/log local0 - log localhost local1 notice - log stdout format raw local0 - -defaults - log global - default-server inter 10s fall 3 rise 2 - balance leastconn - -frontend internal-pgbouncer-tcp-in - bind *:6432 - mode tcp - option tcplog - - default_backend pgbouncer - -backend pgbouncer - mode tcp - option tcp-check - - server pgbouncer1 10.6.0.21:6432 check - server pgbouncer2 10.6.0.22:6432 check - server pgbouncer3 10.6.0.23:6432 check -``` - -Refer to your preferred Load Balancer's documentation for further guidance. - -<div align="right"> - <a type="button" class="btn btn-default" href="#setup-components"> - Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> - </a> -</div> - ## Configure Redis Using [Redis](https://redis.io/) in scalable environment is possible using a **Primary** x **Replica** @@ -1302,19 +1351,283 @@ To configure the Sentinel Queues server: </a> </div> -## Configure Gitaly +## Configure Gitaly Cluster -NOTE: -[Gitaly Cluster](../gitaly/praefect.md) support -for the Reference Architectures is being -worked on as a [collaborative effort](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/1) between the Quality Engineering and Gitaly teams. When this component has been verified -some Architecture specs will likely change as a result to support the new -and improved designed. +[Gitaly Cluster](../gitaly/praefect.md) is a GitLab provided and recommended fault tolerant solution for storing Git repositories. +In this configuration, every Git repository is stored on every Gitaly node in the cluster, with one being designated the primary, and failover occurs automatically if the primary node goes down. + +The recommended cluster setup includes the following components: + +- 3 Gitaly nodes: Replicated storage of Git repositories. +- 3 Praefect nodes: Router and transaction manager for Gitaly Cluster. +- 1 Praefect PostgreSQL node: Database server for Praefect. A third-party solution + is required for Praefect database connections to be made highly available. +- 1 load balancer: A load balancer is required for Praefect. The + [internal load balancer](#configure-the-internal-load-balancer) will be used. + +This section will detail how to configure the recommended standard setup in order. +For more advanced setups refer to the [standalone Gitaly Cluster documentation](../gitaly/praefect.md). + +### Configure Praefect PostgreSQL + +Praefect, the routing and transaction manager for Gitaly Cluster, requires its own database server to store data on Gitaly Cluster status. + +If you want to have a highly available setup, Praefect requires a third-party PostgreSQL database. +A built-in solution is being [worked on](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5919). + +#### Praefect non-HA PostgreSQL standalone using Omnibus GitLab + +The following IPs will be used as an example: + +- `10.6.0.141`: Praefect PostgreSQL + +First, make sure to [install](https://about.gitlab.com/install/) +the Linux GitLab package in the Praefect PostgreSQL node. Following the steps, +install the necessary dependencies from step 1, and add the +GitLab package repository from step 2. When installing GitLab +in the second step, do not supply the `EXTERNAL_URL` value. + +1. SSH in to the Praefect PostgreSQL node. +1. Create a strong password to be used for the Praefect PostgreSQL user. Take note of this password as `<praefect_postgresql_password>`. +1. Generate the password hash for the Praefect PostgreSQL username/password pair. This assumes you will use the default + username of `praefect` (recommended). The command will request the password `<praefect_postgresql_password>` + and confirmation. Use the value that is output by this command in the next + step as the value of `<praefect_postgresql_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 praefect + ``` + +1. Edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: + + ```ruby + # Disable all components except PostgreSQL and Consul + roles ['postgres_role'] + repmgr['enable'] = false + patroni['enable'] = false + + # PostgreSQL configuration + postgresql['listen_address'] = '0.0.0.0' + postgresql['max_connections'] = 200 + + gitlab_rails['auto_migrate'] = false -[Gitaly](../gitaly/index.md) server node requirements are dependent on data, -specifically the number of projects and those projects' sizes. It's recommended -that a Gitaly server node stores no more than 5 TB of data. Depending on your -repository storage requirements, you may require additional Gitaly server nodes. + # Configure the Consul agent + consul['enable'] = true + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # START user configuration + # Please set the real values as explained in Required Information section + # + # Replace PRAEFECT_POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = "<praefect_postgresql_password_hash>" + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) + + # Set the network addresses that the exporters will listen on for monitoring + node_exporter['listen_address'] = '0.0.0.0:9100' + postgres_exporter['listen_address'] = '0.0.0.0:9187' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + # + # END user configuration + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Follow the [post configuration](#praefect-postgresql-post-configuration). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +#### Praefect HA PostgreSQL third-party solution + +[As noted](#configure-praefect-postgresql), a third-party PostgreSQL solution for +Praefect's database is recommended if aiming for full High Availability. + +There are many third-party solutions for PostgreSQL HA. The solution selected must have the following to work with Praefect: + +- A static IP for all connections that doesn't change on failover. +- [`LISTEN`](https://www.postgresql.org/docs/12/sql-listen.html) SQL functionality must be supported. + +Examples of the above could include [Google's Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) or [Amazon RDS](https://aws.amazon.com/rds/). + +Once the database is set up, follow the [post configuration](#praefect-postgresql-post-configuration). + +#### Praefect PostgreSQL post-configuration + +After the Praefect PostgreSQL server has been set up, you'll then need to configure the user and database for Praefect to use. + +We recommend the user be named `praefect` and the database `praefect_production`, and these can be configured as standard in PostgreSQL. +The password for the user is the same as the one you configured earlier as `<praefect_postgresql_password>`. + +This is how this would work with a Omnibus GitLab PostgreSQL setup: + +1. SSH in to the Praefect PostgreSQL node. +1. Connect to the PostgreSQL server with administrative access. + The `gitlab-psql` user should be used here for this as it's added by default in Omnibus. + The database `template1` is used because it is created by default on all PostgreSQL servers. + + ```shell + /opt/gitlab/embedded/bin/psql -U gitlab-psql -d template1 -h POSTGRESQL_SERVER_ADDRESS + ``` + +1. Create the new user `praefect`, replacing `<praefect_postgresql_password>`: + + ```shell + CREATE ROLE praefect WITH LOGIN CREATEDB PASSWORD <praefect_postgresql_password>; + ``` + +1. Reconnect to the PostgreSQL server, this time as the `praefect` user: + + ```shell + /opt/gitlab/embedded/bin/psql -U praefect -d template1 -h POSTGRESQL_SERVER_ADDRESS + ``` + +1. Create a new database `praefect_production`: + + ```shell + CREATE DATABASE praefect_production WITH ENCODING=UTF8; + ``` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +### Configure Praefect + +Praefect is the router and transaction manager for Gitaly Cluster and all connections to Gitaly go through +it. This section details how to configure it. + +Praefect requires several secret tokens to secure communications across the Cluster: + +- `<praefect_external_token>`: Used for repositories hosted on your Gitaly cluster and can only be accessed by Gitaly clients that carry this token. +- `<praefect_internal_token>`: Used for replication traffic inside your Gitaly cluster. This is distinct from `praefect_external_token` because Gitaly clients must not be able to access internal nodes of the Praefect cluster directly; that could lead to data loss. +- `<praefect_postgresql_password>`: The Praefect PostgreSQL password defined in the previous section is also required as part of this setup. + +Gitaly Cluster nodes are configured in Praefect via a `virtual storage`. Each storage contains +the details of each Gitaly node that makes up the cluster. Each storage is also given a name +and this name is used in several areas of the config. In this guide, the name of the storage will be +`default`. Also, this guide is geared towards new installs, if upgrading an existing environment +to use Gitaly Cluster, you may need to use a different name. +Refer to the [Praefect documentation](../gitaly/praefect.md#praefect) for more info. + +The following IPs will be used as an example: + +- `10.6.0.131`: Praefect 1 +- `10.6.0.132`: Praefect 2 +- `10.6.0.133`: Praefect 3 + +To configure the Praefect nodes, on each one: + +1. SSH in to the Praefect server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. +1. Edit the `/etc/gitlab/gitlab.rb` file to configure Praefect: + + ```ruby + # Avoid running unnecessary services on the Gitaly server + postgresql['enable'] = false + redis['enable'] = false + nginx['enable'] = false + puma['enable'] = false + unicorn['enable'] = false + sidekiq['enable'] = false + gitlab_workhorse['enable'] = false + grafana['enable'] = false + + # If you run a separate monitoring node you can disable these services + alertmanager['enable'] = false + prometheus['enable'] = false + + # Praefect Configuration + praefect['enable'] = true + praefect['listen_addr'] = '0.0.0.0:2305' + + gitlab_rails['rake_cache_clear'] = false + gitlab_rails['auto_migrate'] = false + + # Configure the Consul agent + consul['enable'] = true + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # START user configuration + # Please set the real values as explained in Required Information section + # + + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + praefect['auth_token'] = '<praefect_external_token>' + + # Praefect Database Settings + praefect['database_host'] = '10.6.0.141' + praefect['database_port'] = 5432 + # `no_proxy` settings must always be a direct connection for caching + praefect['database_host_no_proxy'] = '10.6.0.141' + praefect['database_port_no_proxy'] = 5432 + praefect['database_dbname'] = 'praefect_production' + praefect['database_user'] = 'praefect' + praefect['database_password'] = '<praefect_postgresql_password>' + + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + praefect['virtual_storages'] = { + 'default' => { + 'gitaly-1' => { + 'address' => 'tcp://10.6.0.91:8075', + 'token' => '<praefect_internal_token>', + 'primary' => true + }, + 'gitaly-2' => { + 'address' => 'tcp://10.6.0.92:8075', + 'token' => '<praefect_internal_token>' + }, + 'gitaly-3' => { + 'address' => 'tcp://10.6.0.93:8075', + 'token' => '<praefect_internal_token>' + }, + } + } + + # Set the network addresses that the exporters will listen on for monitoring + node_exporter['listen_address'] = '0.0.0.0:9100' + praefect['prometheus_listen_addr'] = '0.0.0.0:9652' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + # + # END user configuration + ``` + + 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and + then replace the file of the same name on this server. If that file isn't on + this server, add the file from your Consul server to this server. + + 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +### Configure Gitaly + +The [Gitaly](../gitaly/index.md) server nodes that make up the cluster have +requirements that are dependent on data, specifically the number of projects +and those projects' sizes. It's recommended that a Gitaly Cluster stores +no more than 5 TB of data on each node. Depending on your +repository storage requirements, you may require additional Gitaly Clusters. Due to Gitaly having notable input and output requirements, we strongly recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs @@ -1325,36 +1638,21 @@ adjusted to greater or lesser values depending on the scale of your environment's workload. If you're running the environment on a Cloud provider, refer to their documentation about how to configure IOPS correctly. -Be sure to note the following items: +Gitaly servers must not be exposed to the public internet, as Gitaly's network +traffic is unencrypted by default. The use of a firewall is highly recommended +to restrict access to the Gitaly server. Another option is to +[use TLS](#gitaly-cluster-tls-support). -- The GitLab Rails application shards repositories into - [repository storage paths](../repository_storage_paths.md). -- A Gitaly server can host one or more storage paths. -- A GitLab server can use one or more Gitaly server nodes. -- Gitaly addresses must be specified to be correctly resolvable for all Gitaly - clients. -- Gitaly servers must not be exposed to the public internet, as Gitaly's network - traffic is unencrypted by default. The use of a firewall is highly recommended - to restrict access to the Gitaly server. Another option is to - [use TLS](#gitaly-tls-support). +For configuring Gitaly you should note the following: -NOTE: -The token referred to throughout the Gitaly documentation is an arbitrary -password selected by the administrator. This token is unrelated to tokens -created for the GitLab API or other similar web API tokens. - -This section describes how to configure two Gitaly servers, with the following -IPs and domain names: - -- `10.6.0.91`: Gitaly 1 (`gitaly1.internal`) -- `10.6.0.92`: Gitaly 2 (`gitaly2.internal`) +- `git_data_dirs` should be configured to reflect the storage path for the specific Gitaly node +- `auth_token` should be the same as `praefect_internal_token` -Assumptions about your servers include having the secret token be `gitalysecret`, -and that your GitLab installation has three repository storages: +The following IPs will be used as an example: -- `default` on Gitaly 1 -- `storage1` on Gitaly 1 -- `storage2` on Gitaly 2 +- `10.6.0.91`: Gitaly 1 +- `10.6.0.92`: Gitaly 2 +- `10.6.0.93`: Gitaly 3 On each node: @@ -1364,21 +1662,9 @@ On each node: 1. Edit the Gitaly server node's `/etc/gitlab/gitlab.rb` file to configure storage paths, enable the network listener, and to configure the token: - <!-- - updates to following example must also be made at - https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab - --> - ```ruby # /etc/gitlab/gitlab.rb - # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests - # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. - # The following two values must be the same as their respective values - # of the GitLab Rails application setup - gitaly['auth_token'] = 'gitalysecret' - gitlab_shell['secret_token'] = 'shellsecret' - # Avoid running unnecessary services on the Gitaly server postgresql['enable'] = false redis['enable'] = false @@ -1407,36 +1693,42 @@ On each node: # firewalls to restrict access to this address/port. # Comment out following line if you only want to support TLS connections gitaly['listen_addr'] = "0.0.0.0:8075" + + # Gitaly Auth Token + # Should be the same as praefect_internal_token + gitaly['auth_token'] = '<praefect_internal_token>' ``` 1. Append the following to `/etc/gitlab/gitlab.rb` for each respective server: - - On `gitaly1.internal`: + - On Gitaly node 1: ```ruby git_data_dirs({ - 'default' => { - 'path' => '/var/opt/gitlab/git-data' - }, - 'storage1' => { - 'path' => '/mnt/gitlab/git-data' - }, + "gitaly-1" => { + "path" => "/var/opt/gitlab/git-data" + } }) ``` - - On `gitaly2.internal`: + - On Gitaly node 2: ```ruby git_data_dirs({ - 'storage2' => { - 'path' => '/mnt/gitlab/git-data' - }, + "gitaly-2" => { + "path" => "/var/opt/gitlab/git-data" + } }) ``` - <!-- - updates to following example must also be made at - https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab - --> + - On Gitaly node 3: + + ```ruby + git_data_dirs({ + "gitaly-3" => { + "path" => "/var/opt/gitlab/git-data" + } + }) + ``` 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and then replace the file of the same name on this server. If that file isn't on @@ -1444,34 +1736,44 @@ On each node: 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -### Gitaly TLS support +### Gitaly Cluster TLS support -Gitaly supports TLS encryption. To be able to communicate -with a Gitaly instance that listens for secure connections you will need to use `tls://` URL -scheme in the `gitaly_address` of the corresponding storage entry in the GitLab configuration. +Praefect supports TLS encryption. To communicate with a Praefect instance that listens +for secure connections, you must: -You will need to bring your own certificates as this isn't provided automatically. -The certificate, or its certificate authority, must be installed on all Gitaly -nodes (including the Gitaly node using the certificate) and on all client nodes -that communicate with it following the procedure described in -[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). +- Use a `tls://` URL scheme in the `gitaly_address` of the corresponding storage entry + in the GitLab configuration. +- Bring your own certificates because this isn't provided automatically. The certificate + corresponding to each Praefect server must be installed on that Praefect server. -NOTE: -The self-signed certificate must specify the address you use to access the -Gitaly server. If you are addressing the Gitaly server by a hostname, you can -either use the Common Name field for this, or add it as a Subject Alternative -Name. If you are addressing the Gitaly server by its IP address, you must add it -as a Subject Alternative Name to the certificate. -[gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). +Additionally the certificate, or its certificate authority, must be installed on all Gitaly servers +and on all Praefect clients that communicate with it following the procedure described in +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) (and repeated below). -It's possible to configure Gitaly servers with both an unencrypted listening -address (`listen_addr`) and an encrypted listening address (`tls_listen_addr`) -at the same time. This allows you to do a gradual transition from unencrypted to -encrypted traffic, if necessary. +Note the following: -To configure Gitaly with TLS: +- The certificate must specify the address you use to access the Praefect server. If + addressing the Praefect server by: -1. Create the `/etc/gitlab/ssl` directory and copy your key and certificate there: + - Hostname, you can either use the Common Name field for this, or add it as a Subject + Alternative Name. + - IP address, you must add it as a Subject Alternative Name to the certificate. + +- You can configure Praefect servers with both an unencrypted listening address + `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. + This allows you to do a gradual transition from unencrypted to encrypted traffic, if + necessary. + +- The Internal Load Balancer will also access to the certificates and need to be configured + to allow for TLS passthrough. + Refer to the load balancers documentation on how to configure this. + +To configure Praefect with TLS: + +1. Create certificates for Praefect servers. + +1. On the Praefect servers, create the `/etc/gitlab/ssl` directory and copy your key + and certificate there: ```shell sudo mkdir -p /etc/gitlab/ssl @@ -1480,27 +1782,34 @@ To configure Gitaly with TLS: sudo chmod 644 key.pem cert.pem ``` -1. Copy the cert to `/etc/gitlab/trusted-certs` so Gitaly will trust the cert when - calling into itself: +1. Edit `/etc/gitlab/gitlab.rb` and add: - ```shell - sudo cp /etc/gitlab/ssl/cert.pem /etc/gitlab/trusted-certs/ + ```ruby + praefect['tls_listen_addr'] = "0.0.0.0:3305" + praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" + praefect['key_path'] = "/etc/gitlab/ssl/key.pem" ``` -1. Edit `/etc/gitlab/gitlab.rb` and add: +1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). - <!-- - updates to following example must also be made at - https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab - --> +1. On the Praefect clients (including each Gitaly server), copy the certificates, + or their certificate authority, into `/etc/gitlab/trusted-certs`: - ```ruby - gitaly['tls_listen_addr'] = "0.0.0.0:9999" - gitaly['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - gitaly['key_path'] = "/etc/gitlab/ssl/key.pem" + ```shell + sudo cp cert.pem /etc/gitlab/trusted-certs/ ``` -1. Delete `gitaly['listen_addr']` to allow only encrypted connections. +1. On the Praefect clients (except Gitaly servers), edit `git_data_dirs` in + `/etc/gitlab/gitlab.rb` as follows: + + ```ruby + git_data_dirs({ + "default" => { + "gitaly_address" => 'tls://LOAD_BALANCER_SERVER_ADDRESS:2305', + "gitaly_token" => 'PRAEFECT_EXTERNAL_TOKEN' + } + }) + ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -1587,12 +1896,15 @@ To configure the Sidekiq nodes, on each one: ### Gitaly ### ####################################### + # git_data_dirs get configured for the Praefect virtual storage + # Address is Internal Load Balancer for Praefect + # Token is praefect_external_token git_data_dirs({ - 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, - 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, - 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + "default" => { + "gitaly_address" => "tcp://10.6.0.40:2305", # internal load balancer IP + "gitaly_token" => '<praefect_external_token>' + } }) - gitlab_rails['gitaly_token'] = 'YOUR_TOKEN' ####################################### ### Postgres ### @@ -1624,7 +1936,7 @@ To configure the Sidekiq nodes, on each one: node_exporter['listen_address'] = '0.0.0.0:9100' # Rails Status for prometheus - gitlab_rails['monitoring_whitelist'] = ['10.6.0.121/32', '127.0.0.0/8'] + gitlab_rails['monitoring_whitelist'] = ['10.6.0.151/32', '127.0.0.0/8'] ############################# ### Object storage ### @@ -1671,6 +1983,15 @@ The following IPs will be used as an example: - `10.6.0.111`: GitLab application 1 - `10.6.0.112`: GitLab application 2 - `10.6.0.113`: GitLab application 3 +- `10.6.0.114`: GitLab application 4 +- `10.6.0.115`: GitLab application 5 +- `10.6.0.116`: GitLab application 6 +- `10.6.0.117`: GitLab application 7 +- `10.6.0.118`: GitLab application 8 +- `10.6.0.119`: GitLab application 9 +- `10.6.0.120`: GitLab application 10 +- `10.6.0.121`: GitLab application 11 +- `10.6.0.122`: GitLab application 12 On each node perform the following: @@ -1690,17 +2011,14 @@ On each node perform the following: ```ruby external_url 'https://gitlab.example.com' - # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests - # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. - # The following two values must be the same as their respective values - # of the Gitaly setup - gitlab_rails['gitaly_token'] = 'gitalysecret' - gitlab_shell['secret_token'] = 'shellsecret' - + # git_data_dirs get configured for the Praefect virtual storage + # Address is Interal Load Balancer for Praefect + # Token is praefect_external_token git_data_dirs({ - 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, - 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, - 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + "default" => { + "gitaly_address" => "tcp://10.6.0.40:2305", # internal load balancer IP + "gitaly_token" => '<praefect_external_token>' + } }) ## Disable components that will not be on the GitLab application server @@ -1755,8 +2073,8 @@ On each node perform the following: # Add the monitoring node's IP address to the monitoring whitelist and allow it to # scrape the NGINX metrics - gitlab_rails['monitoring_whitelist'] = ['10.6.0.121/32', '127.0.0.0/8'] - nginx['status']['options']['allow'] = ['10.6.0.121/32', '127.0.0.0/8'] + gitlab_rails['monitoring_whitelist'] = ['10.6.0.151/32', '127.0.0.0/8'] + nginx['status']['options']['allow'] = ['10.6.0.151/32', '127.0.0.0/8'] ############################# ### Object storage ### @@ -1779,14 +2097,15 @@ On each node perform the following: ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -1. If you're using [Gitaly with TLS support](#gitaly-tls-support), make sure the +1. If you're using [Gitaly with TLS support](#gitaly-cluster-tls-support), make sure the `git_data_dirs` entry is configured with `tls` instead of `tcp`: ```ruby git_data_dirs({ - 'default' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, - 'storage1' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, - 'storage2' => { 'gitaly_address' => 'tls://gitaly2.internal:9999' }, + "default" => { + "gitaly_address" => "tls://10.6.0.40:2305", # internal load balancer IP + "gitaly_token" => '<praefect_external_token>' + } }) ``` @@ -1891,7 +2210,7 @@ running [Prometheus](../monitoring/prometheus/index.md) and The following IP will be used as an example: -- `10.6.0.121`: Prometheus +- `10.6.0.151`: Prometheus To configure the Monitoring node: diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 37e67b0ab73..fb9fdfcc3f1 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -19,79 +19,107 @@ costly-to-operate environment by using the [2,000-user reference architecture](2k_users.md). > - **Supported users (approximate):** 5,000 -> - **High Availability:** Yes +> - **High Availability:** Yes ([Praefect](#configure-praefect-postgresql) needs a third-party PostgreSQL solution for HA) > - **Test requests per second (RPS) rates:** API: 100 RPS, Web: 10 RPS, Git (Pull): 10 RPS, Git (Push): 2 RPS | Service | Nodes | Configuration | GCP | AWS | Azure | |--------------------------------------------|-------------|-------------------------|----------------|-------------|----------| -| External load balancing node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | -| Redis | 3 | 2 vCPU, 7.5 GB memory | n1-standard-2 | `m5.large` | D2s v3 | -| Consul + Sentinel | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | -| PostgreSQL | 3 | 2 vCPU, 7.5 GB memory | n1-standard-2 | `m5.large` | D2s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | -| Internal load balancing node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | -| Gitaly | 2 (minimum) | 8 vCPU, 30 GB memory | n1-standard-8 | `m5.2xlarge` | D8s v3 | -| Sidekiq | 4 | 2 vCPU, 7.5 GB memory | n1-standard-2 | `m5.large` | D2s v3 | -| GitLab Rails | 3 | 16 vCPU, 14.4 GB memory | n1-highcpu-16 | `c5.4xlarge` | F16s v2 | -| Monitoring node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | `c5.large` | F2s v2 | +| External load balancing node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Redis | 3 | 2 vCPU, 7.5 GB memory | n1-standard-2 | m5.large | D2s v3 | +| Consul + Sentinel | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| PostgreSQL | 3 | 4 vCPU, 15 GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Gitaly | 3 | 8 vCPU, 30 GB memory | n1-standard-8 | m5.2xlarge | D8s v3 | +| Praefect | 3 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Praefect PostgreSQL | 1+* | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Sidekiq | 4 | 2 vCPU, 7.5 GB memory | n1-standard-2 | m5.large | D2s v3 | +| GitLab Rails | 3 | 16 vCPU, 14.4 GB memory | n1-highcpu-16 | c5.4xlarge | F16s v2 | +| Monitoring node | 1 | 2 vCPU, 1.8 GB memory | n1-highcpu-2 | c5.large | F2s v2 | | Object storage | n/a | n/a | n/a | n/a | n/a | -| NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | `c5.xlarge` | F4s v2 | - -```mermaid -stateDiagram-v2 - [*] --> LoadBalancer - LoadBalancer --> ApplicationServer - - ApplicationServer --> BackgroundJobs - ApplicationServer --> Gitaly - ApplicationServer --> Redis - ApplicationServer --> PgBouncer - PgBouncer --> Database - ApplicationServer --> ObjectStorage - BackgroundJobs --> ObjectStorage - - ApplicationMonitoring -->ApplicationServer - ApplicationMonitoring -->Redis - ApplicationMonitoring -->PgBouncer - ApplicationMonitoring -->Database - ApplicationMonitoring -->BackgroundJobs - - state Database { - "PG_Primary_Node" - "PG_Secondary_Node_1..2" - } - - state Redis { - "R_Primary_Node" - "R_Replica_Node_1..2" - "R_Consul/Sentinel_1..3" - } - - state Gitaly { - "Gitaly_1..2" - } - - state BackgroundJobs { - "Sidekiq_1..4" - } - - state ApplicationServer { - "GitLab_Rails_1..3" - } - - state LoadBalancer { - "LoadBalancer_1" - } - - state ApplicationMonitoring { - "Prometheus" - "Grafana" - } - - state PgBouncer { - "Internal_Load_Balancer" - "PgBouncer_1..3" - } +| NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | + +```plantuml +@startuml 5k +card "**External Load Balancer**" as elb #6a9be7 +card "**Internal Load Balancer**" as ilb #9370DB + +together { + collections "**GitLab Rails** x3" as gitlab #32CD32 + collections "**Sidekiq** x4" as sidekiq #ff8dd1 +} + +together { + card "**Prometheus + Grafana**" as monitor #7FFFD4 + collections "**Consul** x3" as consul #e76a9b +} + +card "Gitaly Cluster" as gitaly_cluster { + collections "**Praefect** x3" as praefect #FF8C00 + collections "**Gitaly** x3" as gitaly #FF8C00 + card "**Praefect PostgreSQL***\n//Non fault-tolerant//" as praefect_postgres #FF8C00 + + praefect -[#FF8C00]-> gitaly + praefect -[#FF8C00]> praefect_postgres +} + +card "Database" as database { + collections "**PGBouncer** x3" as pgbouncer #4EA7FF + card "**PostgreSQL** (Primary)" as postgres_primary #4EA7FF + collections "**PostgreSQL** (Secondary) x2" as postgres_secondary #4EA7FF + + pgbouncer -[#4EA7FF]-> postgres_primary + postgres_primary .[#4EA7FF]> postgres_secondary +} + +card "redis" as redis { + collections "**Redis Persistent** x3" as redis_persistent #FF6347 + collections "**Redis Cache** x3" as redis_cache #FF6347 + collections "**Redis Persistent Sentinel** x3" as redis_persistent_sentinel #FF6347 + collections "**Redis Cache Sentinel** x3"as redis_cache_sentinel #FF6347 + + redis_persistent <.[#FF6347]- redis_persistent_sentinel + redis_cache <.[#FF6347]- redis_cache_sentinel +} + +cloud "**Object Storage**" as object_storage #white + +elb -[#6a9be7]-> gitlab +elb -[#6a9be7]--> monitor + +gitlab -[#32CD32]> sidekiq +gitlab -[#32CD32]--> ilb +gitlab -[#32CD32]-> object_storage +gitlab -[#32CD32]---> redis +gitlab -[hidden]-> monitor +gitlab -[hidden]-> consul + +sidekiq -[#ff8dd1]--> ilb +sidekiq -[#ff8dd1]-> object_storage +sidekiq -[#ff8dd1]---> redis +sidekiq -[hidden]-> monitor +sidekiq -[hidden]-> consul + +ilb -[#9370DB]-> gitaly_cluster +ilb -[#9370DB]-> database + +consul .[#e76a9b]u-> gitlab +consul .[#e76a9b]u-> sidekiq +consul .[#e76a9b]> monitor +consul .[#e76a9b]-> database +consul .[#e76a9b]-> gitaly_cluster +consul .[#e76a9b,norank]--> redis + +monitor .[#7FFFD4]u-> gitlab +monitor .[#7FFFD4]u-> sidekiq +monitor .[#7FFFD4]> consul +monitor .[#7FFFD4]-> database +monitor .[#7FFFD4]-> gitaly_cluster +monitor .[#7FFFD4,norank]--> redis +monitor .[#7FFFD4]> ilb +monitor .[#7FFFD4,norank]u--> elb + +@enduml ``` The Google Cloud Platform (GCP) architectures were built and tested using the @@ -106,19 +134,25 @@ uploads, or artifacts), using an [object storage service](#configure-the-object- is recommended instead of using NFS. Using an object storage service also doesn't require you to provision and maintain a node. +It's also worth noting that at this time [Praefect requires its own database server](../gitaly/praefect.md#postgresql) and +that to achieve full High Availability a third party PostgreSQL database solution will be required. +We hope to offer a built in solutions for these restrictions in the future but in the meantime a non HA PostgreSQL server +can be set up via Omnibus GitLab, which the above specs reflect. Refer to the following issues for more information: [`omnibus-gitlab#5919`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5919) & [`gitaly#3398`](https://gitlab.com/gitlab-org/gitaly/-/issues/3398) + ## Setup components To set up GitLab and its components to accommodate up to 5,000 users: -1. [Configure the external load balancing node](#configure-the-external-load-balancer) +1. [Configure the external load balancer](#configure-the-external-load-balancer) to handle the load balancing of the GitLab application services nodes. +1. [Configure the internal load balancer](#configure-the-internal-load-balancer). + to handle the load balancing of GitLab application internal connections. 1. [Configure Redis](#configure-redis). 1. [Configure Consul and Sentinel](#configure-consul-and-sentinel). 1. [Configure PostgreSQL](#configure-postgresql), the database for GitLab. 1. [Configure PgBouncer](#configure-pgbouncer). -1. [Configure the internal load balancing node](#configure-the-internal-load-balancer). -1. [Configure Gitaly](#configure-gitaly), - which provides access to the Git repositories. +1. [Configure Gitaly Cluster](#configure-gitaly-cluster), + provides access to the Git repositories. 1. [Configure Sidekiq](#configure-sidekiq). 1. [Configure the main GitLab Rails application](#configure-gitlab-rails) to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend @@ -155,6 +189,11 @@ The following list includes descriptions of each server and its assigned IP: - `10.6.0.20`: Internal Load Balancer - `10.6.0.51`: Gitaly 1 - `10.6.0.52`: Gitaly 2 +- `10.6.0.93`: Gitaly 3 +- `10.6.0.131`: Praefect 1 +- `10.6.0.132`: Praefect 2 +- `10.6.0.133`: Praefect 3 +- `10.6.0.141`: Praefect PostgreSQL 1 (non HA) - `10.6.0.71`: Sidekiq 1 - `10.6.0.72`: Sidekiq 2 - `10.6.0.73`: Sidekiq 3 @@ -285,6 +324,71 @@ Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. </a> </div> +## Configure the internal load balancer + +The Internal Load Balancer is used to balance any internal connections the GitLab environment requires +such as connections to [PgBouncer](#configure-pgbouncer) and [Praefect](#configure-praefect) (Gitaly Cluster). + +Note that it's a separate node from the External Load Balancer and shouldn't have any access externally. + +The following IP will be used as an example: + +- `10.6.0.40`: Internal Load Balancer + +Here's how you could do it with [HAProxy](https://www.haproxy.org/): + +```plaintext +global + log /dev/log local0 + log localhost local1 notice + log stdout format raw local0 + +defaults + log global + default-server inter 10s fall 3 rise 2 + balance leastconn + +frontend internal-pgbouncer-tcp-in + bind *:6432 + mode tcp + option tcplog + + default_backend pgbouncer + +frontend internal-praefect-tcp-in + bind *:2305 + mode tcp + option tcplog + option clitcpka + + default_backend praefect + +backend pgbouncer + mode tcp + option tcp-check + + server pgbouncer1 10.6.0.21:6432 check + server pgbouncer2 10.6.0.22:6432 check + server pgbouncer3 10.6.0.23:6432 check + +backend praefect + mode tcp + option tcp-check + option srvtcpka + + server praefect1 10.6.0.131:2305 check + server praefect2 10.6.0.132:2305 check + server praefect3 10.6.0.133:2305 check +``` + +Refer to your preferred Load Balancer's documentation for further guidance. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + ## Configure Redis Using [Redis](https://redis.io/) in scalable environment is possible using a **Primary** x **Replica** @@ -924,45 +1028,96 @@ The following IPs will be used as an example: </a> </div> -### Configure the internal load balancer +## Configure Gitaly Cluster -If you're running more than one PgBouncer node as recommended, then at this time you'll need to set -up a TCP internal load balancer to serve each correctly. +[Gitaly Cluster](../gitaly/praefect.md) is a GitLab provided and recommended fault tolerant solution for storing Git repositories. +In this configuration, every Git repository is stored on every Gitaly node in the cluster, with one being designated the primary, and failover occurs automatically if the primary node goes down. -The following IP will be used as an example: +The recommended cluster setup includes the following components: -- `10.6.0.20`: Internal Load Balancer +- 3 Gitaly nodes: Replicated storage of Git repositories. +- 3 Praefect nodes: Router and transaction manager for Gitaly Cluster. +- 1 Praefect PostgreSQL node: Database server for Praefect. A third-party solution + is required for Praefect database connections to be made highly available. +- 1 load balancer: A load balancer is required for Praefect. The + [internal load balancer](#configure-the-internal-load-balancer) will be used. -Here's how you could do it with [HAProxy](https://www.haproxy.org/): +This section will detail how to configure the recommended standard setup in order. +For more advanced setups refer to the [standalone Gitaly Cluster documentation](../gitaly/praefect.md). -```plaintext -global - log /dev/log local0 - log localhost local1 notice - log stdout format raw local0 +### Configure Praefect PostgreSQL -defaults - log global - default-server inter 10s fall 3 rise 2 - balance leastconn +Praefect, the routing and transaction manager for Gitaly Cluster, requires its own database server to store data on Gitaly Cluster status. -frontend internal-pgbouncer-tcp-in - bind *:6432 - mode tcp - option tcplog +If you want to have a highly available setup, Praefect requires a third-party PostgreSQL database. +A built-in solution is being [worked on](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5919). - default_backend pgbouncer +#### Praefect non-HA PostgreSQL standalone using Omnibus GitLab -backend pgbouncer - mode tcp - option tcp-check +The following IPs will be used as an example: - server pgbouncer1 10.6.0.21:6432 check - server pgbouncer2 10.6.0.22:6432 check - server pgbouncer3 10.6.0.23:6432 check -``` +- `10.6.0.141`: Praefect PostgreSQL -Refer to your preferred Load Balancer's documentation for further guidance. +First, make sure to [install](https://about.gitlab.com/install/) +the Linux GitLab package in the Praefect PostgreSQL node. Following the steps, +install the necessary dependencies from step 1, and add the +GitLab package repository from step 2. When installing GitLab +in the second step, do not supply the `EXTERNAL_URL` value. + +1. SSH in to the Praefect PostgreSQL node. +1. Create a strong password to be used for the Praefect PostgreSQL user. Take note of this password as `<praefect_postgresql_password>`. +1. Generate the password hash for the Praefect PostgreSQL username/password pair. This assumes you will use the default + username of `praefect` (recommended). The command will request the password `<praefect_postgresql_password>` + and confirmation. Use the value that is output by this command in the next + step as the value of `<praefect_postgresql_password_hash>`: + + ```shell + sudo gitlab-ctl pg-password-md5 praefect + ``` + +1. Edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: + + ```ruby + # Disable all components except PostgreSQL and Consul + roles ['postgres_role'] + repmgr['enable'] = false + patroni['enable'] = false + + # PostgreSQL configuration + postgresql['listen_address'] = '0.0.0.0' + postgresql['max_connections'] = 200 + + gitlab_rails['auto_migrate'] = false + + # Configure the Consul agent + consul['enable'] = true + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # START user configuration + # Please set the real values as explained in Required Information section + # + # Replace PRAEFECT_POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = "<praefect_postgresql_password_hash>" + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) + + # Set the network addresses that the exporters will listen on for monitoring + node_exporter['listen_address'] = '0.0.0.0:9100' + postgres_exporter['listen_address'] = '0.0.0.0:9187' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + # + # END user configuration + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Follow the [post configuration](#praefect-postgresql-post-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -970,19 +1125,186 @@ Refer to your preferred Load Balancer's documentation for further guidance. </a> </div> -## Configure Gitaly +#### Praefect HA PostgreSQL third-party solution -NOTE: -[Gitaly Cluster](../gitaly/praefect.md) support -for the Reference Architectures is being -worked on as a [collaborative effort](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/1) between the Quality Engineering and Gitaly teams. When this component has been verified -some Architecture specs will likely change as a result to support the new -and improved designed. +[As noted](#configure-praefect-postgresql), a third-party PostgreSQL solution for +Praefect's database is recommended if aiming for full High Availability. + +There are many third-party solutions for PostgreSQL HA. The solution selected must have the following to work with Praefect: + +- A static IP for all connections that doesn't change on failover. +- [`LISTEN`](https://www.postgresql.org/docs/12/sql-listen.html) SQL functionality must be supported. + +Examples of the above could include [Google's Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) or [Amazon RDS](https://aws.amazon.com/rds/). + +Once the database is set up, follow the [post configuration](#praefect-postgresql-post-configuration). + +#### Praefect PostgreSQL post-configuration + +After the Praefect PostgreSQL server has been set up, you'll then need to configure the user and database for Praefect to use. + +We recommend the user be named `praefect` and the database `praefect_production`, and these can be configured as standard in PostgreSQL. +The password for the user is the same as the one you configured earlier as `<praefect_postgresql_password>`. + +This is how this would work with a Omnibus GitLab PostgreSQL setup: + +1. SSH in to the Praefect PostgreSQL node. +1. Connect to the PostgreSQL server with administrative access. + The `gitlab-psql` user should be used here for this as it's added by default in Omnibus. + The database `template1` is used because it is created by default on all PostgreSQL servers. -[Gitaly](../gitaly/index.md) server node requirements are dependent on data, -specifically the number of projects and those projects' sizes. It's recommended -that a Gitaly server node stores no more than 5 TB of data. Depending on your -repository storage requirements, you may require additional Gitaly server nodes. + ```shell + /opt/gitlab/embedded/bin/psql -U gitlab-psql -d template1 -h POSTGRESQL_SERVER_ADDRESS + ``` + +1. Create the new user `praefect`, replacing `<praefect_postgresql_password>`: + + ```shell + CREATE ROLE praefect WITH LOGIN CREATEDB PASSWORD <praefect_postgresql_password>; + ``` + +1. Reconnect to the PostgreSQL server, this time as the `praefect` user: + + ```shell + /opt/gitlab/embedded/bin/psql -U praefect -d template1 -h POSTGRESQL_SERVER_ADDRESS + ``` + +1. Create a new database `praefect_production`: + + ```shell + CREATE DATABASE praefect_production WITH ENCODING=UTF8; + ``` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +### Configure Praefect + +Praefect is the router and transaction manager for Gitaly Cluster and all connections to Gitaly go through +it. This section details how to configure it. + +Praefect requires several secret tokens to secure communications across the Cluster: + +- `<praefect_external_token>`: Used for repositories hosted on your Gitaly cluster and can only be accessed by Gitaly clients that carry this token. +- `<praefect_internal_token>`: Used for replication traffic inside your Gitaly cluster. This is distinct from `praefect_external_token` because Gitaly clients must not be able to access internal nodes of the Praefect cluster directly; that could lead to data loss. +- `<praefect_postgresql_password>`: The Praefect PostgreSQL password defined in the previous section is also required as part of this setup. + +Gitaly Cluster nodes are configured in Praefect via a `virtual storage`. Each storage contains +the details of each Gitaly node that makes up the cluster. Each storage is also given a name +and this name is used in several areas of the config. In this guide, the name of the storage will be +`default`. Also, this guide is geared towards new installs, if upgrading an existing environment +to use Gitaly Cluster, you may need to use a different name. +Refer to the [Praefect documentation](../gitaly/praefect.md#praefect) for more info. + +The following IPs will be used as an example: + +- `10.6.0.131`: Praefect 1 +- `10.6.0.132`: Praefect 2 +- `10.6.0.133`: Praefect 3 + +To configure the Praefect nodes, on each one: + +1. SSH in to the Praefect server. +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package of your choice. Be sure to follow _only_ installation steps 1 and 2 + on the page. +1. Edit the `/etc/gitlab/gitlab.rb` file to configure Praefect: + + ```ruby + # Avoid running unnecessary services on the Gitaly server + postgresql['enable'] = false + redis['enable'] = false + nginx['enable'] = false + puma['enable'] = false + unicorn['enable'] = false + sidekiq['enable'] = false + gitlab_workhorse['enable'] = false + grafana['enable'] = false + + # If you run a separate monitoring node you can disable these services + alertmanager['enable'] = false + prometheus['enable'] = false + + # Praefect Configuration + praefect['enable'] = true + praefect['listen_addr'] = '0.0.0.0:2305' + + gitlab_rails['rake_cache_clear'] = false + gitlab_rails['auto_migrate'] = false + + # Configure the Consul agent + consul['enable'] = true + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # START user configuration + # Please set the real values as explained in Required Information section + # + + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + praefect['auth_token'] = '<praefect_external_token>' + + # Praefect Database Settings + praefect['database_host'] = '10.6.0.141' + praefect['database_port'] = 5432 + # `no_proxy` settings must always be a direct connection for caching + praefect['database_host_no_proxy'] = '10.6.0.141' + praefect['database_port_no_proxy'] = 5432 + praefect['database_dbname'] = 'praefect_production' + praefect['database_user'] = 'praefect' + praefect['database_password'] = '<praefect_postgresql_password>' + + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + praefect['virtual_storages'] = { + 'default' => { + 'gitaly-1' => { + 'address' => 'tcp://10.6.0.91:8075', + 'token' => '<praefect_internal_token>', + 'primary' => true + }, + 'gitaly-2' => { + 'address' => 'tcp://10.6.0.92:8075', + 'token' => '<praefect_internal_token>' + }, + 'gitaly-3' => { + 'address' => 'tcp://10.6.0.93:8075', + 'token' => '<praefect_internal_token>' + }, + } + } + + # Set the network addresses that the exporters will listen on for monitoring + node_exporter['listen_address'] = '0.0.0.0:9100' + praefect['prometheus_listen_addr'] = '0.0.0.0:9652' + + ## The IPs of the Consul server nodes + ## You can also use FQDNs and intermix them with IPs + consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), + } + # + # END user configuration + ``` + + 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and + then replace the file of the same name on this server. If that file isn't on + this server, add the file from your Consul server to this server. + + 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +### Configure Gitaly + +The [Gitaly](../gitaly/index.md) server nodes that make up the cluster have +requirements that are dependent on data, specifically the number of projects +and those projects' sizes. It's recommended that a Gitaly Cluster stores +no more than 5 TB of data on each node. Depending on your +repository storage requirements, you may require additional Gitaly Clusters. Due to Gitaly having notable input and output requirements, we strongly recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs @@ -993,36 +1315,21 @@ adjusted to greater or lesser values depending on the scale of your environment's workload. If you're running the environment on a Cloud provider, refer to their documentation about how to configure IOPS correctly. -Be sure to note the following items: +Gitaly servers must not be exposed to the public internet, as Gitaly's network +traffic is unencrypted by default. The use of a firewall is highly recommended +to restrict access to the Gitaly server. Another option is to +[use TLS](#gitaly-cluster-tls-support). -- The GitLab Rails application shards repositories into - [repository storage paths](../repository_storage_paths.md). -- A Gitaly server can host one or more storage paths. -- A GitLab server can use one or more Gitaly server nodes. -- Gitaly addresses must be specified to be correctly resolvable for all Gitaly - clients. -- Gitaly servers must not be exposed to the public internet, as Gitaly's network - traffic is unencrypted by default. The use of a firewall is highly recommended - to restrict access to the Gitaly server. Another option is to - [use TLS](#gitaly-tls-support). - -NOTE: -The token referred to throughout the Gitaly documentation is an arbitrary -password selected by the administrator. This token is unrelated to tokens -created for the GitLab API or other similar web API tokens. +For configuring Gitaly you should note the following: -This section describes how to configure two Gitaly servers, with the following -IPs and domain names: +- `git_data_dirs` should be configured to reflect the storage path for the specific Gitaly node +- `auth_token` should be the same as `praefect_internal_token` -- `10.6.0.51`: Gitaly 1 (`gitaly1.internal`) -- `10.6.0.52`: Gitaly 2 (`gitaly2.internal`) - -Assumptions about your servers include having the secret token be `gitalysecret`, -and that your GitLab installation has three repository storages: +The following IPs will be used as an example: -- `default` on Gitaly 1 -- `storage1` on Gitaly 1 -- `storage2` on Gitaly 2 +- `10.6.0.91`: Gitaly 1 +- `10.6.0.92`: Gitaly 2 +- `10.6.0.93`: Gitaly 3 On each node: @@ -1032,21 +1339,9 @@ On each node: 1. Edit the Gitaly server node's `/etc/gitlab/gitlab.rb` file to configure storage paths, enable the network listener, and to configure the token: - <!-- - updates to following example must also be made at - https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab - --> - ```ruby # /etc/gitlab/gitlab.rb - # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests - # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. - # The following two values must be the same as their respective values - # of the GitLab Rails application setup - gitaly['auth_token'] = 'gitalysecret' - gitlab_shell['secret_token'] = 'shellsecret' - # Avoid running unnecessary services on the Gitaly server postgresql['enable'] = false redis['enable'] = false @@ -1056,7 +1351,6 @@ On each node: sidekiq['enable'] = false gitlab_workhorse['enable'] = false grafana['enable'] = false - gitlab_exporter['enable'] = false # If you run a separate monitoring node you can disable these services alertmanager['enable'] = false @@ -1077,101 +1371,86 @@ On each node: # Comment out following line if you only want to support TLS connections gitaly['listen_addr'] = "0.0.0.0:8075" - ## Enable service discovery for Prometheus - consul['enable'] = true - consul['monitoring_service_discovery'] = true - - # Set the network addresses that the exporters will listen on for monitoring - gitaly['prometheus_listen_addr'] = "0.0.0.0:9236" - node_exporter['listen_address'] = '0.0.0.0:9100' - gitlab_rails['prometheus_address'] = '10.6.0.81:9090' - - ## The IPs of the Consul server nodes - ## You can also use FQDNs and intermix them with IPs - consul['configuration'] = { - retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), - } + # Gitaly Auth Token + # Should be the same as praefect_internal_token + gitaly['auth_token'] = '<praefect_internal_token>' ``` 1. Append the following to `/etc/gitlab/gitlab.rb` for each respective server: - - On `gitaly1.internal`: + - On Gitaly node 1: ```ruby git_data_dirs({ - 'default' => { - 'path' => '/var/opt/gitlab/git-data' - }, - 'storage1' => { - 'path' => '/mnt/gitlab/git-data' - }, + "gitaly-1" => { + "path" => "/var/opt/gitlab/git-data" + } }) ``` - - On `gitaly2.internal`: + - On Gitaly node 2: ```ruby git_data_dirs({ - 'storage2' => { - 'path' => '/mnt/gitlab/git-data' - }, + "gitaly-2" => { + "path" => "/var/opt/gitlab/git-data" + } }) ``` - <!-- - updates to following example must also be made at - https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab - --> + - On Gitaly node 3: + + ```ruby + git_data_dirs({ + "gitaly-3" => { + "path" => "/var/opt/gitlab/git-data" + } + }) + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and + then replace the file of the same name on this server. If that file isn't on + this server, add the file from your Consul server to this server. 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -1. Confirm that Gitaly can perform callbacks to the internal API: - ```shell - sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml - ``` +### Gitaly Cluster TLS support -1. Verify the GitLab services are running: +Praefect supports TLS encryption. To communicate with a Praefect instance that listens +for secure connections, you must: - ```shell - sudo gitlab-ctl status - ``` +- Use a `tls://` URL scheme in the `gitaly_address` of the corresponding storage entry + in the GitLab configuration. +- Bring your own certificates because this isn't provided automatically. The certificate + corresponding to each Praefect server must be installed on that Praefect server. - The output should be similar to the following: +Additionally the certificate, or its certificate authority, must be installed on all Gitaly servers +and on all Praefect clients that communicate with it following the procedure described in +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) (and repeated below). - ```plaintext - run: consul: (pid 30339) 77006s; run: log: (pid 29878) 77020s - run: gitaly: (pid 30351) 77005s; run: log: (pid 29660) 77040s - run: logrotate: (pid 7760) 3213s; run: log: (pid 29782) 77032s - run: node-exporter: (pid 30378) 77004s; run: log: (pid 29812) 77026s - ``` +Note the following: -### Gitaly TLS support +- The certificate must specify the address you use to access the Praefect server. If + addressing the Praefect server by: -Gitaly supports TLS encryption. To be able to communicate -with a Gitaly instance that listens for secure connections you will need to use `tls://` URL -scheme in the `gitaly_address` of the corresponding storage entry in the GitLab configuration. + - Hostname, you can either use the Common Name field for this, or add it as a Subject + Alternative Name. + - IP address, you must add it as a Subject Alternative Name to the certificate. -You will need to bring your own certificates as this isn't provided automatically. -The certificate, or its certificate authority, must be installed on all Gitaly -nodes (including the Gitaly node using the certificate) and on all client nodes -that communicate with it following the procedure described in -[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). +- You can configure Praefect servers with both an unencrypted listening address + `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. + This allows you to do a gradual transition from unencrypted to encrypted traffic, if + necessary. -NOTE: -The self-signed certificate must specify the address you use to access the -Gitaly server. If you are addressing the Gitaly server by a hostname, you can -either use the Common Name field for this, or add it as a Subject Alternative -Name. If you are addressing the Gitaly server by its IP address, you must add it -as a Subject Alternative Name to the certificate. -[gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). +- The Internal Load Balancer will also access to the certificates and need to be configured + to allow for TLS passthrough. + Refer to the load balancers documentation on how to configure this. -It's possible to configure Gitaly servers with both an unencrypted listening -address (`listen_addr`) and an encrypted listening address (`tls_listen_addr`) -at the same time. This allows you to do a gradual transition from unencrypted to -encrypted traffic, if necessary. +To configure Praefect with TLS: -To configure Gitaly with TLS: +1. Create certificates for Praefect servers. -1. Create the `/etc/gitlab/ssl` directory and copy your key and certificate there: +1. On the Praefect servers, create the `/etc/gitlab/ssl` directory and copy your key + and certificate there: ```shell sudo mkdir -p /etc/gitlab/ssl @@ -1180,27 +1459,35 @@ To configure Gitaly with TLS: sudo chmod 644 key.pem cert.pem ``` -1. Copy the cert to `/etc/gitlab/trusted-certs` so Gitaly will trust the cert when - calling into itself: +1. Edit `/etc/gitlab/gitlab.rb` and add: - ```shell - sudo cp /etc/gitlab/ssl/cert.pem /etc/gitlab/trusted-certs/ + ```ruby + praefect['tls_listen_addr'] = "0.0.0.0:3305" + praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" + praefect['key_path'] = "/etc/gitlab/ssl/key.pem" ``` -1. Edit `/etc/gitlab/gitlab.rb` and add: +1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). - <!-- - updates to following example must also be made at - https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab - --> +1. On the Praefect clients (including each Gitaly server), copy the certificates, + or their certificate authority, into `/etc/gitlab/trusted-certs`: + + ```shell + sudo cp cert.pem /etc/gitlab/trusted-certs/ + ``` + +1. On the Praefect clients (except Gitaly servers), edit `git_data_dirs` in + `/etc/gitlab/gitlab.rb` as follows: ```ruby - gitaly['tls_listen_addr'] = "0.0.0.0:9999" - gitaly['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - gitaly['key_path'] = "/etc/gitlab/ssl/key.pem" + git_data_dirs({ + "default" => { + "gitaly_address" => 'tls://LOAD_BALANCER_SERVER_ADDRESS:2305', + "gitaly_token" => 'PRAEFECT_EXTERNAL_TOKEN' + } + }) ``` -1. Delete `gitaly['listen_addr']` to allow only encrypted connections. 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). <div align="right"> @@ -1269,17 +1556,20 @@ To configure the Sidekiq nodes, one each one: ### Gitaly ### ####################################### + # git_data_dirs get configured for the Praefect virtual storage + # Address is Internal Load Balancer for Praefect + # Token is praefect_external_token git_data_dirs({ - 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, - 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, - 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + "default" => { + "gitaly_address" => "tcp://10.6.0.40:2305", # internal load balancer IP + "gitaly_token" => '<praefect_external_token>' + } }) - gitlab_rails['gitaly_token'] = 'YOUR_TOKEN' ####################################### ### Postgres ### ####################################### - gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP + gitlab_rails['db_host'] = '10.6.0.40' # internal load balancer IP gitlab_rails['db_port'] = 6432 gitlab_rails['db_password'] = '<postgresql_user_password>' gitlab_rails['db_adapter'] = 'postgresql' @@ -1397,17 +1687,14 @@ On each node perform the following: ```ruby external_url 'https://gitlab.example.com' - # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests - # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. - # The following two values must be the same as their respective values - # of the Gitaly setup - gitlab_rails['gitaly_token'] = 'gitalysecret' - gitlab_shell['secret_token'] = 'shellsecret' - + # git_data_dirs get configured for the Praefect virtual storage + # Address is Interal Load Balancer for Praefect + # Token is praefect_external_token git_data_dirs({ - 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, - 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, - 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + "default" => { + "gitaly_address" => "tcp://10.6.0.40:2305", # internal load balancer IP + "gitaly_token" => '<praefect_external_token>' + } }) ## Disable components that will not be on the GitLab application server @@ -1495,14 +1782,15 @@ On each node perform the following: #registry['gid'] = 9002 ``` -1. If you're using [Gitaly with TLS support](#gitaly-tls-support), make sure the +1. If you're using [Gitaly with TLS support](#gitaly-cluster-tls-support), make sure the `git_data_dirs` entry is configured with `tls` instead of `tcp`: ```ruby git_data_dirs({ - 'default' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, - 'storage1' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, - 'storage2' => { 'gitaly_address' => 'tls://gitaly2.internal:9999' }, + "default" => { + "gitaly_address" => "tls://10.6.0.40:2305", # internal load balancer IP + "gitaly_token" => '<praefect_external_token>' + } }) ``` diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json index fc89798a131..800866e9eca 100644 --- a/doc/api/graphql/reference/gitlab_schema.json +++ b/doc/api/graphql/reference/gitlab_schema.json @@ -85890,4 +85890,4 @@ ] } } -}
\ No newline at end of file +} diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 8fbd425d118..e329da46509 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -757,6 +757,16 @@ Autogenerated return type of BoardListUpdateLimitMetrics. | `commit` | Commit | Commit for the branch. | | `name` | String! | Name of the branch. | +### BulkFindOrCreateDevopsAdoptionSegmentsPayload + +Autogenerated return type of BulkFindOrCreateDevopsAdoptionSegments. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `segments` | DevopsAdoptionSegment! => Array | Created segments after mutation. | + ### BurnupChartDailyTotals Represents the total number of issues and their weights for a particular day. @@ -1906,6 +1916,7 @@ Represents an epic board. | `hideBacklogList` | Boolean | Whether or not backlog list is hidden. | | `hideClosedList` | Boolean | Whether or not closed list is hidden. | | `id` | BoardsEpicBoardID! | Global ID of the epic board. | +| `labels` | LabelConnection | Labels of the board. | | `lists` | EpicListConnection | Epic board lists. | | `name` | String | Name of the epic board. | | `webPath` | String! | Web path of the epic board. | @@ -1931,6 +1942,16 @@ Autogenerated return type of EpicBoardListCreate. | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `list` | EpicList | Epic list in the epic board. | +### EpicBoardUpdatePayload + +Autogenerated return type of EpicBoardUpdate. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `epicBoard` | EpicBoard | The updated epic board. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | + ### EpicDescendantCount Counts of descendent epics. @@ -2715,6 +2736,7 @@ Autogenerated return type of MarkAsSpamSnippet. | `reference` | String! | Internal reference of the merge request. Returned in shortened format by default. | | `reviewers` | UserConnection | Users from whom a review has been requested. | | `securityAutoFix` | Boolean | Indicates if the merge request is created by @GitLab-Security-Bot. | +| `securityReportsUpToDateOnTargetBranch` | Boolean! | Indicates if the target branch security reports are out of date. | | `shouldBeRebased` | Boolean! | Indicates if the merge request will be rebased. | | `shouldRemoveSourceBranch` | Boolean | Indicates if the source branch of the merge request will be deleted after merge. | | `sourceBranch` | String! | Source branch of the merge request. | diff --git a/doc/api/vulnerability_findings.md b/doc/api/vulnerability_findings.md index 95e4774ae96..4144a912617 100644 --- a/doc/api/vulnerability_findings.md +++ b/doc/api/vulnerability_findings.md @@ -49,7 +49,6 @@ GET /projects/:id/vulnerability_findings?scope=all GET /projects/:id/vulnerability_findings?scope=dismissed GET /projects/:id/vulnerability_findings?severity=high GET /projects/:id/vulnerability_findings?confidence=unknown,experimental -GET /projects/:id/vulnerability_findings?scanner=bandit,find_sec_bugs GET /projects/:id/vulnerability_findings?pipeline_id=42 ``` @@ -63,7 +62,6 @@ Beginning with GitLab 12.9, the `undefined` severity and confidence level is no | `scope` | string | no | Returns vulnerability findings for the given scope: `all` or `dismissed`. Defaults to `dismissed`. | | `severity` | string array | no | Returns vulnerability findings belonging to specified severity level: `info`, `unknown`, `low`, `medium`, `high`, or `critical`. Defaults to all. | | `confidence` | string array | no | Returns vulnerability findings belonging to specified confidence level: `ignore`, `unknown`, `experimental`, `low`, `medium`, `high`, or `confirmed`. Defaults to all. | -| `scanner` | string array | no | Returns vulnerability findings detected by specified scanner. | `pipeline_id` | integer/string | no | Returns vulnerability findings belonging to specified pipeline. | ```shell diff --git a/doc/ci/pipelines/index.md b/doc/ci/pipelines/index.md index c8e82de651c..d5ccb8d020c 100644 --- a/doc/ci/pipelines/index.md +++ b/doc/ci/pipelines/index.md @@ -137,10 +137,10 @@ To execute a pipeline manually: 1. Navigate to your project's **CI/CD > Pipelines**. 1. Select the **Run Pipeline** button. 1. On the **Run Pipeline** page: - 1. Select the branch to run the pipeline for in the **Create for** field. + 1. Select the branch or tag to run the pipeline for in the **Run for branch name or tag** field. 1. Enter any [environment variables](../variables/README.md) required for the pipeline run. You can set specific variables to have their [values prefilled in the form](#prefill-variables-in-manual-pipelines). - 1. Click the **Create pipeline** button. + 1. Click the **Run pipeline** button. The pipeline now executes the jobs as configured. diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 7c42233fdef..ed829b7f0ce 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -46,7 +46,7 @@ The keywords available for jobs are: | [`only`](#onlyexcept-basic) | Limit when jobs are created. | | [`pages`](#pages) | Upload the result of a job to use with GitLab Pages. | | [`parallel`](#parallel) | How many instances of a job should be run in parallel. | -| [`release`](#release) | Instructs the runner to generate a [Release](../../user/project/releases/index.md) object. | +| [`release`](#release) | Instructs the runner to generate a [release](../../user/project/releases/index.md) object. | | [`resource_group`](#resource_group) | Limit job concurrency. | | [`retry`](#retry) | When and how many times a job can be auto-retried in case of a failure. | | [`rules`](#rules) | List of conditions to evaluate and determine selected attributes of a job, and whether or not it's created. | @@ -79,7 +79,7 @@ You can't use these keywords as job names: You can set global defaults for some keywords. Jobs that do not define one or more of the listed keywords use the value defined in the `default:` section. -The following job keywords can be defined inside a `default:` section: +These job keywords can be defined inside a `default:` section: - [`after_script`](#after_script) - [`artifacts`](#artifacts) @@ -92,7 +92,7 @@ The following job keywords can be defined inside a `default:` section: - [`tags`](#tags) - [`timeout`](#timeout) -This example sets the `ruby:2.5` image as the default for all jobs in the pipeline. +The following example sets the `ruby:2.5` image as the default for all jobs in the pipeline. The `rspec 2.6` job does not use the default, because it overrides the default with a job-specific `image:` section: @@ -159,9 +159,9 @@ the [`needs`](#needs) keyword. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/29654) in GitLab 12.5 -The top-level `workflow:` keyword determines whether or not a pipeline is created. -It accepts a single `rules:` keyword that is similar to [`rules:` defined in jobs](#rules). -Use it to define what can trigger a new pipeline. +Use `workflow:` to determine whether or not a pipeline is created. +Define this keyword at the top level, with a single `rules:` keyword that +is similar to [`rules:` defined in jobs](#rules). You can use the [`workflow:rules` templates](#workflowrules-templates) to import a preconfigured `workflow: rules` entry. @@ -186,7 +186,7 @@ Some example `if` clauses for `workflow: rules`: See the [common `if` clauses for `rules`](#common-if-clauses-for-rules) for more examples. -For example, in the following configuration, pipelines run for all `push` events (changes to +In the following example, pipelines run for all `push` events (changes to branches and new tags). Pipelines for push events with `-draft` in the commit message don't run, because they are set to `when: never`. Pipelines for schedules or merge requests don't run either, because no rules evaluate to true for them: @@ -226,7 +226,7 @@ If your rules match both branch pipelines and merge request pipelines, > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217732) in GitLab 13.0. -We provide templates that set up `workflow: rules` +GitLab provides templates that set up `workflow: rules` for common scenarios. These templates help prevent duplicate pipelines. The [`Branch-Pipelines` template](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml) @@ -234,12 +234,12 @@ makes your pipelines run for branches and tags. Branch pipeline status is displayed in merge requests that use the branch as a source. However, this pipeline type does not support any features offered by -[Merge Request Pipelines](../merge_request_pipelines/), like -[Pipelines for Merge Results](../merge_request_pipelines/#pipelines-for-merged-results) -or [Merge Trains](../merge_request_pipelines/pipelines_for_merged_results/merge_trains/). -Use this template if you are intentionally avoiding those features. +[merge request pipelines](../merge_request_pipelines/), like +[pipelines for merge results](../merge_request_pipelines/#pipelines-for-merged-results) +or [merge trains](../merge_request_pipelines/pipelines_for_merged_results/merge_trains/). +This template intentionally avoids those features. -It is [included](#include) as follows: +To [include](#include) it: ```yaml include: @@ -249,10 +249,9 @@ include: The [`MergeRequest-Pipelines` template](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml) makes your pipelines run for the default branch, tags, and all types of merge request pipelines. Use this template if you use any of the -the [Pipelines for Merge Requests features](../merge_request_pipelines/), as mentioned -above. +the [pipelines for merge requests features](../merge_request_pipelines/). -It is [included](#include) as follows: +To [include](#include) it: ```yaml include: @@ -317,7 +316,7 @@ does not block triggered pipelines. > - Available for Starter, Premium, and Ultimate in GitLab 10.6 and later. > - [Moved](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/42861) to GitLab Free in 11.4. -Use the `include` keyword to include external YAML files in your CI/CD configuration. +Use `include` to include external YAML files in your CI/CD configuration. You can break down one long `gitlab-ci.yml` file into multiple files to increase readability, or reduce duplication of the same configuration in multiple places. @@ -339,11 +338,11 @@ YAML files, use [`!reference` tags](#reference-tags) or the [`extends` keyword]( | [`remote`](#includeremote) | Include a file from a remote URL. Must be publicly accessible. | | [`template`](#includetemplate) | Include templates that are provided by GitLab. | -The `.gitlab-ci.yml` file configuration included by all methods is evaluated when the pipeline is created. -The configuration is a snapshot in time and persisted in the database. Any changes to -the referenced `.gitlab-ci.yml` file configuration is not reflected in GitLab until the next pipeline is created. +When the pipeline starts, the `.gitlab-ci.yml` file configuration included by all methods is evaluated. +The configuration is a snapshot in time and persists in the database. GitLab does not reflect any changes to +the referenced `.gitlab-ci.yml` file configuration until the next pipeline starts. -The files defined by `include` are: +The `include` files are: - Deep merged with those in the `.gitlab-ci.yml` file. - Always evaluated first and merged with the content of the `.gitlab-ci.yml` file, @@ -367,13 +366,13 @@ include: file: '.compliance-gitlab-ci.yml' ``` -For an example of how you can include these predefined variables, and their impact on CI jobs, -see the following [CI/CD variable demo](https://youtu.be/4XR8gw3Pkos). +For an example of how you can include these predefined variables, and the variables' impact on CI/CD jobs, +see this [CI/CD variable demo](https://youtu.be/4XR8gw3Pkos). #### `include:local` -`include:local` includes a file that is in the same repository as the `.gitlab-ci.yml` file. -It's referenced with full paths relative to the root directory (`/`). +Use `include:local` to include a file that is in the same repository as the `.gitlab-ci.yml` file. +Use a full path relative to the root directory (`/`). If you use `include:local`, make sure that both the `.gitlab-ci.yml` file and the local file are on the same branch. @@ -390,7 +389,7 @@ include: - local: '/templates/.gitlab-ci-template.yml' ``` -This can be defined as a short local include: +You can also use shorter syntax to define the path: ```yaml include: '.gitlab-ci-production.yml' @@ -404,8 +403,9 @@ Use local includes instead of symbolic links. To include files from another private project on the same GitLab instance, use `include:file`. You can use `include:file` in combination with `include:project` only. +Use a full path, relative to the root directory (`/`). -The included file is referenced with a full path, relative to the root directory (`/`). For example: +For example: ```yaml include: @@ -413,7 +413,7 @@ include: file: '/templates/.gitlab-ci-template.yml' ``` -You can also specify a `ref`. If not specified, it defaults to the `HEAD` of the project: +You can also specify a `ref`. If you do not specify a value, the ref defaults to the `HEAD` of the project: ```yaml include: @@ -502,15 +502,15 @@ to resolve all files is 30 seconds. #### Additional `includes` examples -There is a list of [additional `includes` examples](includes.md) available. +View [additional `includes` examples](includes.md). ## Keyword details -The following are detailed explanations for keywords used to configure CI/CD pipelines. +The following topics explain how to use keywords to configure CI/CD pipelines. ### `image` -Used to specify [a Docker image](../docker/using_docker_images.md#what-is-an-image) to use for the job. +Use `image` to specify [a Docker image](../docker/using_docker_images.md#what-is-an-image) to use for the job. For: @@ -531,13 +531,13 @@ For more information, see [Available settings for `image`](../docker/using_docke #### `services` -Used to specify a [service Docker image](../docker/using_docker_images.md#what-is-a-service), linked to a base image specified in [`image`](#image). +Use `services` to specify a [service Docker image](../docker/using_docker_images.md#what-is-a-service), linked to a base image specified in [`image`](#image). For: - Usage examples, see [Define `image` and `services` from `.gitlab-ci.yml`](../docker/using_docker_images.md#define-image-and-services-from-gitlab-ciyml). - Detailed usage information, refer to [Docker integration](../docker/index.md) documentation. -- For example services, see [GitLab CI/CD Services](../services/index.md). +- Example services, see [GitLab CI/CD Services](../services/index.md). ##### `services:name` @@ -565,8 +565,11 @@ For more information, see [Available settings for `services`](../docker/using_do ### `script` -`script` is the only required keyword that a job needs. It's a shell script -that is executed by the runner. For example: +Use `script` to specify a shell script for the runner to execute. + +All jobs except [trigger jobs](#trigger) require a `script` keyword. + +For example: ```yaml job: @@ -575,7 +578,7 @@ job: You can use [YAML anchors with `script`](#yaml-anchors-for-scripts). -This keyword can also contain several commands in an array: +The `script` keyword can also contain several commands in an array: ```yaml job: @@ -609,7 +612,7 @@ job: You can verify the syntax is valid with the [CI Lint](../lint.md) tool. -Be careful when using these special characters as well: +Be careful when using these characters as well: - `{`, `}`, `[`, `]`, `,`, `&`, `*`, `#`, `?`, `|`, `-`, `<`, `>`, `=`, `!`, `%`, `@`, `` ` ``. @@ -629,10 +632,10 @@ job: Use `before_script` to define an array of commands that should run before each job, but after [artifacts](#artifacts) are restored. -Scripts specified in `before_script` are concatenated with any scripts specified -in the main [`script`](#script), and executed together in a single shell. +Scripts you specify in `before_script` are concatenated with any scripts you specify +in the main [`script`](#script). The combine scripts execute together in a single shell. -It's possible to overwrite a globally defined `before_script` if you define it in a job: +You can overwrite a globally-defined `before_script` if you define it in a job: ```yaml default: @@ -657,11 +660,11 @@ You can use [YAML anchors with `before_script`](#yaml-anchors-for-scripts). Use `after_script` to define an array of commands that run after each job, including failed jobs. -If a job times out or is cancelled, the `after_script` commands are not executed. -Support for executing `after_script` commands for timed-out or cancelled jobs -[is planned](https://gitlab.com/gitlab-org/gitlab/-/issues/15603). +If a job times out or is cancelled, the `after_script` commands do not execute. +An [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/15603) exists to support +executing `after_script` commands for timed-out or cancelled jobs. -Scripts specified in `after_script` are executed in a new shell, separate from any +Scripts you specify in `after_script` execute in a new shell, separate from any `before_script` or `script` scripts. As a result, they: - Have a current working directory set back to the default. @@ -694,7 +697,7 @@ You can use [YAML anchors with `after_script`](#yaml-anchors-for-scripts). #### Script syntax -You can use special syntax in [`script`](README.md#script) sections to: +You can use syntax in [`script`](README.md#script) sections to: - [Split long commands](script.md#split-long-commands) into multiline commands. - [Use color codes](script.md#add-color-codes-to-script-output) to make job logs easier to review. @@ -703,9 +706,19 @@ You can use special syntax in [`script`](README.md#script) sections to: ### `stage` -`stage` is defined per-job and relies on [`stages`](#stages), which is defined -globally. Use `stage` to define which stage a job runs in, and jobs of the same -`stage` are executed in parallel (subject to [certain conditions](#use-your-own-runners)). For example: +Use `stage` to define which stage a job runs in. Jobs in the same +`stage` can execute in parallel (subject to [certain conditions](#use-your-own-runners)). + +Jobs without a `stage` entry use the `test` stage by default. If [`stages`](#stages) +is not defined in the pipeline, you can use the 5 default stages, which execute in +this order: + +- [`.pre`](#pre-and-post) +- `build` +- `test` +- `deploy` +- [`.post`](#pre-and-post) +For example: ```yaml stages: @@ -751,45 +764,39 @@ is greater than `1`. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31441) in GitLab 12.4. -The following stages are available to every pipeline: +Use `pre` and `post` for jobs that need to run first or last in a pipeline. -- `.pre`, which is guaranteed to always be the first stage in a pipeline. -- `.post`, which is guaranteed to always be the last stage in a pipeline. +- `.pre` is guaranteed to always be the first stage in a pipeline. +- `.post` is guaranteed to always be the last stage in a pipeline. User-defined stages are executed after `.pre` and before `.post`. -A pipeline is not created if all jobs are in `.pre` or `.post` stages. - -The order of `.pre` and `.post` can't be changed, even if defined out of order in the `.gitlab-ci.yml` file. -For example, the following are equivalent configuration: - -- Configured in order: +You must have a job in at least one stage other than `.pre` or `.post`. - ```yaml - stages: - - .pre - - a - - b - - .post - ``` - -- Configured out of order: +You can't change the order of `.pre` and `.post`, even if you define them out of order in the `.gitlab-ci.yml` file. +For example, the following configurations are equivalent: - ```yaml - stages: - - a - - .pre - - b - - .post - ``` +```yaml +stages: + - .pre + - a + - b + - .post +``` -- Not explicitly configured: +```yaml +stages: + - a + - .pre + - b + - .post +``` - ```yaml - stages: - - a - - b - ``` +```yaml +stages: + - a + - b +``` ### `extends` @@ -799,7 +806,7 @@ Use `extends` to reuse configuration sections. It's an alternative to [YAML anch and is a little more flexible and readable. You can use `extends` to reuse configuration from [included configuration files](#use-extends-and-include-together). -In this example, the `rspec` job uses the configuration from the `.tests` template job. +In the following example, the `rspec` job uses the configuration from the `.tests` template job. GitLab: - Performs a reverse deep merge based on the keys. @@ -870,8 +877,8 @@ In GitLab 12.0 and later, it's also possible to use multiple parents for #### Merge details -`extends` is able to merge hashes but not arrays. -The algorithm used for merge is "closest scope wins", so +You can use `extends` to merge hashes but not arrays. +The algorithm used for merge is "closest scope wins," so keys from the last member always override anything defined on other levels. For example: @@ -923,7 +930,7 @@ rspec: - rake rspec ``` -Note that in the example above: +In this example: - The `variables` sections merge, but `URL: "http://docker-url.internal"` overwrites `URL: "http://my-url.internal"`. - `tags: ['docker']` overwrites `tags: ['production']`. @@ -935,8 +942,8 @@ Note that in the example above: To reuse configuration from different configuration files, combine `extends` and [`include`](#include). -In this example, a `script` is defined in the `included.yml` file. -Then, in the `.gitlab-ci.yml` file, you use `extends` to refer +In the following example, a `script` is defined in the `included.yml` file. +Then, in the `.gitlab-ci.yml` file, `extends` refers to the contents of the `script`: - `included.yml`: @@ -961,11 +968,11 @@ to the contents of the `script`: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27863) in GitLab 12.3. -Use the `rules` keyword to include or exclude jobs in pipelines. +Use `rules` to include or exclude jobs in pipelines. -Rules are evaluated *in order* until the first match. When matched, the job +Rules are evaluated *in order* until the first match. When a match is found, the job is either included or excluded from the pipeline, depending on the configuration. -If included, the job also has [certain attributes](#rules-attributes) +The job can also have [certain attributes](#rules-attributes) added to it. `rules` replaces [`only/except`](#onlyexcept-basic) and they can't be used together @@ -1024,7 +1031,7 @@ The job is not added to the pipeline: `when: always`. - If a rule matches, and has `when: never` as the attribute. -This example uses `if` to strictly limit when jobs run: +The following example uses `if` to strictly limit when jobs run: ```yaml job: @@ -1101,7 +1108,7 @@ other pipelines, including **both** push (branch) and merge request pipelines. W this configuration, every push to an open merge request's source branch causes duplicated pipelines. -There are multiple ways to avoid duplicate pipelines: +To avoid duplicate pipelines, you can: - Use [`workflow`](#workflow) to specify which types of pipelines can run. @@ -1164,12 +1171,11 @@ runs in all cases except merge requests. #### `rules:if` -`rules:if` clauses determine whether or not jobs are added to a pipeline by evaluating -an `if` statement. If the `if` statement is true, the job is either included -or excluded from a pipeline. In plain English, `if` rules can be interpreted as one of: +Use `rules:if` clauses to specify when to add a job to a pipeline: -- "If this rule evaluates to true, add the job" (default). -- "If this rule evaluates to true, do not add the job" (by adding `when: never`). +- If an `if` statement is true, add the job to the pipeline. +- If an `if` statement is true, but it's combined with `when: never`, do not add the job to the pipeline. +- If no `if` statements are true, do not add the job to the pipeline. `rules:if` differs slightly from `only:variables` by accepting only a single expression string per rule, rather than an array of them. Any set of expressions to be @@ -1227,7 +1233,9 @@ check the value of the `$CI_PIPELINE_SOURCE` variable: | `web` | For pipelines created by using **Run pipeline** button in the GitLab UI, from the project's **CI/CD > Pipelines** section. | | `webide` | For pipelines created by using the [WebIDE](../../user/project/web_ide/index.md). | -For example: +The following example runs the job as a manual job in scheduled pipelines or in push +pipelines (to branches or tags), with `when: on_success` (default). It does not +add the job to any other pipeline type. ```yaml job: @@ -1239,11 +1247,8 @@ job: - if: '$CI_PIPELINE_SOURCE == "push"' ``` -This example runs the job as a manual job in scheduled pipelines or in push -pipelines (to branches or tags), with `when: on_success` (default). It does not -add the job to any other pipeline type. - -Another example: +The following example runs the job as a `when: on_success` job in [merge request pipelines](../merge_request_pipelines/index.md) +and scheduled pipelines. It does not run in any other pipeline type. ```yaml job: @@ -1253,9 +1258,6 @@ job: - if: '$CI_PIPELINE_SOURCE == "schedule"' ``` -This example runs the job as a `when: on_success` job in [merge request pipelines](../merge_request_pipelines/index.md) -and scheduled pipelines. It does not run in any other pipeline type. - Other commonly used variables for `if` clauses: - `if: $CI_COMMIT_TAG`: If changes are pushed for a tag. @@ -1272,11 +1274,11 @@ Other commonly used variables for `if` clauses: #### `rules:changes` -`rules:changes` determines whether or not to add jobs to a pipeline by checking for +Use `rules:changes` to specify when to add a job to a pipeline by checking for changes to specific files. -`rules: changes` works exactly the same way as [`only: changes` and `except: changes`](#onlychangesexceptchanges), -accepting an array of paths. It's recommended to only use `rules: changes` with branch +`rules: changes` works the same way as [`only: changes` and `except: changes`](#onlychangesexceptchanges). +It accepts an array of paths. You should use `rules: changes` only with branch pipelines or merge request pipelines. For example, it's common to use `rules: changes` with merge request pipelines: @@ -1299,7 +1301,7 @@ In this example: - If `Dockerfile` has not changed, do not add job to any pipeline (same as `when: never`). To use `rules: changes` with branch pipelines instead of merge request pipelines, -change the `if:` clause in the example above to: +change the `if:` clause in the previous example to: ```yaml rules: @@ -1321,7 +1323,7 @@ if there is no `if:` statement that limits the job to branch or merge request pi > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34272) in GitLab 13.6. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/267192) in GitLab 13.7. -CI/CD variables can be used in `rules:changes` expressions to determine when +You can use CI/CD variables in `rules:changes` expressions to determine when to add jobs to a pipeline: ```yaml @@ -1334,7 +1336,7 @@ docker build: - $DOCKERFILES_DIR/* ``` -You can use The `$` character for both variables and paths. For example, if the +You can use the `$` character for both variables and paths. For example, if the `$DOCKERFILES_DIR` variable exists, its value is used. If it does not exist, the `$` is interpreted as being part of a path. @@ -1342,10 +1344,10 @@ You can use The `$` character for both variables and paths. For example, if the > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24021) in GitLab 12.4. -`exists` accepts an array of paths and matches if any of these paths exist -as files in the repository. +Use `exists` to run a job when certain files exist in the repository. +You can use an array of paths. -In this example, `job` runs if a `Dockerfile` exists anywhere in the repository: +In the following example, `job` runs if a `Dockerfile` exists anywhere in the repository: ```yaml job: @@ -1378,11 +1380,11 @@ For performance reasons, GitLab matches a maximum of 10,000 `exists` patterns. A You can use [`allow_failure: true`](#allow_failure) in `rules:` to allow a job to fail, or a manual job to wait for action, without stopping the pipeline itself. All jobs that use `rules:` default to `allow_failure: false` -if `allow_failure:` is not defined. +if you do not define `allow_failure:`. The rule-level `rules:allow_failure` option overrides the job-level -[`allow_failure`](#allow_failure) option, and is only applied when the job is -triggered by the particular rule. +[`allow_failure`](#allow_failure) option, and is only applied when +the particular rule triggers the job. ```yaml job: @@ -1400,7 +1402,7 @@ In this example, if the first rule matches, then the job has `when: manual` and > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/209864) in GitLab 13.7. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/289803) in GitLab 13.10. -You can use [`variables`](#variables) in `rules:` to define variables for specific conditions. +Use [`variables`](#variables) in `rules:` to define variables for specific conditions. For example: @@ -1476,14 +1478,14 @@ to add jobs to pipelines, use [`rules`](#rules). 1. `except` defines the names of branches and tags the job does **not** run for. -There are a few rules that apply to the usage of job policy: +A few rules apply to the usage of job policy: - `only` and `except` are inclusive. If both `only` and `except` are defined in a job specification, the ref is filtered by `only` and `except`. - `only` and `except` can use regular expressions ([supported regexp syntax](#supported-onlyexcept-regexp-syntax)). - `only` and `except` can specify a repository path to filter jobs for forks. -In addition, `only` and `except` can use special keywords: +In addition, `only` and `except` can use these keywords: | **Value** | **Description** | |--------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -1504,8 +1506,8 @@ Scheduled pipelines run on specific branches, so jobs configured with `only: bra run on scheduled pipelines too. Add `except: schedules` to prevent jobs with `only: branches` from running on scheduled pipelines. -In the example below, `job` runs only for refs that start with `issue-`, -whereas all branches are skipped: +In the following example, `job` runs only for refs that start with `issue-`. +All branches are skipped: ```yaml job: @@ -1517,8 +1519,8 @@ job: - branches ``` -Pattern matching is case-sensitive by default. Use `i` flag modifier, like -`/pattern/i` to make a pattern case-insensitive: +Pattern matching is case-sensitive by default. Use the `i` flag modifier, like +`/pattern/i`, to make a pattern case-insensitive: ```yaml job: @@ -1530,8 +1532,8 @@ job: - branches ``` -In this example, `job` runs only for refs that are tagged, or if a build is -explicitly requested by an API trigger or a [Pipeline Schedule](../pipelines/schedules.md): +In the following example, `job` runs only for refs that are tagged, or if a build is +explicitly requested by an API trigger or a [pipeline schedule](../pipelines/schedules.md): ```yaml job: @@ -1542,8 +1544,7 @@ job: - schedules ``` -Use the repository path to have jobs executed only for the parent -repository and not forks: +To execute jobs only for the parent repository and not forks: ```yaml job: @@ -1554,11 +1555,11 @@ job: - /^release/.*$/@gitlab-org/gitlab ``` -The above example runs `job` for all branches on `gitlab-org/gitlab`, -except `master` and those with names prefixed with `release/`. +This example runs `job` for all branches on `gitlab-org/gitlab`, +except `master` and branches that start with `release/`. If a job does not have an `only` rule, `only: ['branches', 'tags']` is set by -default. If it does not have an `except` rule, it's empty. +default. If the job does not have an `except` rule, it's empty. For example, `job1` and `job2` are essentially the same: @@ -1634,7 +1635,7 @@ the pipeline if the following is true: - `(any listed refs are true) AND (any listed variables are true) AND (any listed changes are true) AND (any chosen Kubernetes status matches)` -In the example below, the `test` job is `only` created when **all** of the following are true: +In the following example, the `test` job is `only` created when **all** of the following are true: - The pipeline is [scheduled](../pipelines/schedules.md) **or** runs for `master`. - The `variables` keyword matches. @@ -1657,7 +1658,7 @@ added if the following is true: - `(any listed refs are true) OR (any listed variables are true) OR (any listed changes are true) OR (a chosen Kubernetes status matches)` -In the example below, the `test` job is **not** created when **any** of the following are true: +In the following example, the `test` job is **not** created when **any** of the following are true: - The pipeline runs for the `master` branch. - There are changes to the `README.md` file in the root directory of the repository. @@ -1679,7 +1680,7 @@ test: The `refs` strategy can take the same values as the [simplified only/except configuration](#onlyexcept-basic). -In the example below, the `deploy` job is created only when the +In the following example, the `deploy` job is created only when the pipeline is [scheduled](../pipelines/schedules.md) or runs for the `master` branch: ```yaml @@ -1696,7 +1697,7 @@ deploy: The `kubernetes` strategy accepts only the `active` keyword. -In the example below, the `deploy` job is created only when the +In the following example, the `deploy` job is created only when the Kubernetes service is active in the project: ```yaml @@ -1767,7 +1768,7 @@ In pipelines with [sources other than the three above](../variables/predefined_v You can configure jobs to use `only: changes` with other `only: refs` keywords. However, those jobs ignore the changes and always run. -In this example, when you push commits to an existing branch, the `docker build` job +In the following example, when you push commits to an existing branch, the `docker build` job runs only if any of these files change: - The `Dockerfile` file. @@ -1864,7 +1865,7 @@ docker build service one: - service-one/**/* ``` -In the example above, the pipeline might fail because of changes to a file in `service-one/**/*`. +In this example, the pipeline might fail because of changes to a file in `service-one/**/*`. A later commit that doesn't have changes in `service-one/**/*` but does have changes to the `Dockerfile` can pass. The job @@ -1898,13 +1899,22 @@ All files are considered to have changed when a scheduled pipeline runs. > - In GitLab 12.3, maximum number of jobs in `needs` array raised from five to 50. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30631) in GitLab 12.8, `needs: []` lets jobs start immediately. -Use the `needs:` keyword to execute jobs out-of-order. Relationships between jobs +Use `needs:` to execute jobs out-of-order. Relationships between jobs that use `needs` can be visualized as a [directed acyclic graph](../directed_acyclic_graph/index.md). You can ignore stage ordering and run some jobs without waiting for others to complete. Jobs in multiple stages can run concurrently. -Let's consider the following example: +The following example creates four paths of execution: + +- Linter: the `lint` job runs immediately without waiting for the `build` stage + to complete because it has no needs (`needs: []`). +- Linux path: the `linux:rspec` and `linux:rubocop` jobs runs as soon as the `linux:build` + job finishes without waiting for `mac:build` to finish. +- macOS path: the `mac:rspec` and `mac:rubocop` jobs runs as soon as the `mac:build` + job finishes, without waiting for `linux:build` to finish. +- The `production` job runs as soon as all previous jobs finish; in this case: + `linux:build`, `linux:rspec`, `linux:rubocop`, `mac:build`, `mac:rspec`, `mac:rubocop`. ```yaml linux:build: @@ -1937,20 +1947,6 @@ production: stage: deploy ``` -This example creates four paths of execution: - -- Linter: the `lint` job runs immediately without waiting for the `build` stage to complete because it has no needs (`needs: []`). - -- Linux path: the `linux:rspec` and `linux:rubocop` jobs runs as soon - as the `linux:build` job finishes without waiting for `mac:build` to finish. - -- macOS path: the `mac:rspec` and `mac:rubocop` jobs runs as soon - as the `mac:build` job finishes, without waiting for `linux:build` to finish. - -- The `production` job runs as soon as all previous jobs - finish; in this case: `linux:build`, `linux:rspec`, `linux:rubocop`, - `mac:build`, `mac:rspec`, `mac:rubocop`. - #### Requirements and limitations - If `needs:` is set to point to a job that is not instantiated @@ -1967,7 +1963,7 @@ This example creates four paths of execution: - `needs:` is similar to `dependencies:` in that it must use jobs from prior stages, meaning it's impossible to create circular dependencies. Depending on jobs in the current stage is not possible either, but support [is planned](https://gitlab.com/gitlab-org/gitlab/-/issues/30632). -- Related to the above, stages must be explicitly defined for all jobs +- Stages must be explicitly defined for all jobs that have the keyword `needs:` or are referred to by one. ##### Changing the `needs:` job limit **(FREE SELF)** @@ -1990,7 +1986,7 @@ To disable directed acyclic graphs (DAG), set the limit to `0`. Use `artifacts: true` (default) or `artifacts: false` to control when artifacts are downloaded in jobs that use `needs`. -In this example, the `rspec` job downloads the `build_job` artifacts, but the +In the following example, the `rspec` job downloads the `build_job` artifacts, but the `rubocop` job does not: ```yaml @@ -2013,7 +2009,7 @@ rubocop: artifacts: false ``` -In this example, the `rspec` job downloads the artifacts from all three `build_jobs`. +In the following example, the `rspec` job downloads the artifacts from all three `build_jobs`. `artifacts` is: - Set to true for `build_job_1`. @@ -2064,7 +2060,7 @@ The user running the pipeline must have at least `reporter` access to the group Use `needs` to download artifacts from different pipelines in the current project. Set the `project` keyword as the current project's name, and specify a ref. -In this example, `build_job` downloads the artifacts for the latest successful +In the following example, `build_job` downloads the artifacts for the latest successful `build-1` job with the `other-ref` ref: ```yaml @@ -2153,7 +2149,7 @@ available for the project. When you register a runner, you can specify the runner's tags, for example `ruby`, `postgres`, `development`. -In this example, the job is run by a runner that +In the following example, the job is run by a runner that has both `ruby` and `postgres` tags defined. ```yaml @@ -2202,7 +2198,7 @@ Assuming all other jobs are successful, the job's stage and its pipeline show the same orange warning. However, the associated commit is marked as "passed", without warnings. -In the example below, `job1` and `job2` run in parallel, but if `job1` +In the following example, `job1` and `job2` run in parallel. If `job1` fails, it doesn't stop the next stage from running, because it's marked with `allow_failure: true`: @@ -2269,7 +2265,12 @@ The valid values of `when` are: - With [`rules`](#rules), don't execute job. - With [`workflow`](#workflow), don't run pipeline. -For example: +In the following example, the script: + +1. Executes `cleanup_build_job` only when `build_job` fails. +1. Always executes `cleanup_job` as the last step in pipeline regardless of + success or failure. +1. Executes `deploy_job` when you run it manually in the GitLab UI. ```yaml stages: @@ -2308,13 +2309,6 @@ cleanup_job: when: always ``` -The above script: - -1. Executes `cleanup_build_job` only when `build_job` fails. -1. Always executes `cleanup_job` as the last step in pipeline regardless of - success or failure. -1. Executes `deploy_job` when you run it manually in the GitLab UI. - #### `when:manual` A manual job is a type of job that is not executed automatically and must be explicitly @@ -2377,7 +2371,7 @@ To protect a manual job: ``` 1. In the [protected environments settings](../environments/protected_environments.md#protecting-environments), - select the environment (`production` in the example above) and add the users, roles or groups + select the environment (`production` in this example) and add the users, roles or groups that are authorized to trigger the manual job to the **Allowed to Deploy** list. Only those in this list can trigger this manual job, as well as GitLab administrators who are always able to use protected environments. @@ -2670,7 +2664,7 @@ as Review Apps. You can see an example that uses Review Apps at ### `cache` -Use the `cache` keyword to specify a list of files and directories to +Use `cache` to specify a list of files and directories to cache between jobs. You can only use paths that are in the local working copy. If `cache` is defined outside the scope of jobs, it's set @@ -2771,7 +2765,7 @@ to download cache that's tagged with `test`. If a cache with this tag is not found, you can use `CACHE_FALLBACK_KEY` to specify a cache to use when none exists. -In this example, if the `$CI_COMMIT_REF_SLUG` is not found, the job uses the key defined +In the following example, if the `$CI_COMMIT_REF_SLUG` is not found, the job uses the key defined by the `CACHE_FALLBACK_KEY` variable: ```yaml @@ -2808,7 +2802,7 @@ cache: - node_modules ``` -In this example we're creating a cache for Ruby and Node.js dependencies that +This example creates a cache for Ruby and Node.js dependencies that is tied to current versions of the `Gemfile.lock` and `package.json` files. Whenever one of these files changes, a new cache key is computed and a new cache is created. Any future job runs that use the same `Gemfile.lock` and `package.json` with `cache:key:files` @@ -2942,7 +2936,7 @@ To do so, add `policy: push` to the job. ### `artifacts` -Use the `artifacts` keyword to specify a list of files and directories that are +Use `artifacts` to specify a list of files and directories that are attached to the job when it [succeeds, fails, or always](#artifactswhen). The artifacts are sent to GitLab after the job finishes. They are @@ -3422,7 +3416,7 @@ If `retry` is set to `2`, and a job succeeds in a second run (first retry), it i The `retry` value must be a positive integer, from `0` to `2` (two retries maximum, three runs in total). -This example retries all failure cases: +The following example retries all failure cases: ```yaml test: @@ -3584,7 +3578,7 @@ deploystacks: STACK: [data, processing] ``` -This example generates 10 parallel `deploystacks` jobs, each with different values +The following example generates 10 parallel `deploystacks` jobs, each with different values for `PROVIDER` and `STACK`: ```plaintext @@ -3769,6 +3763,10 @@ To force the `trigger` job to wait for the downstream (multi-project or child) p pipeline completes. At that point, the `trigger` job completes and displays the same status as the downstream job. +This setting can help keep your pipeline execution linear. In the following example, jobs from +subsequent stages wait for the triggered pipeline to successfully complete before +starting, which reduces parallelization. + ```yaml trigger_job: trigger: @@ -3776,10 +3774,6 @@ trigger_job: strategy: depend ``` -This setting can help keep your pipeline execution linear. In the example above, jobs from -subsequent stages wait for the triggered pipeline to successfully complete before -starting, which reduces parallelization. - #### Trigger a pipeline by API call To force a rebuild of a specific branch, tag, or commit, you can use an API call @@ -3806,7 +3800,12 @@ When enabled, a pipeline is immediately canceled when a new pipeline starts on t Set jobs as interruptible that can be safely canceled once started (for instance, a build job). -For example: +In the following example, a new pipeline run causes an existing running pipeline to be: + +- Canceled, if only `step-1` is running or pending. +- Not canceled, once `step-2` starts running. + +After an uninterruptible job starts running, the pipeline cannot be canceled. ```yaml stages: @@ -3832,13 +3831,6 @@ step-3: interruptible: true ``` -In the example above, a new pipeline run causes an existing running pipeline to be: - -- Canceled, if only `step-1` is running or pending. -- Not canceled, once `step-2` starts running. - -When an uninterruptible job is running, the pipeline cannot be canceled, regardless of the final job's state. - ### `resource_group` > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15536) in GitLab 12.7. @@ -3885,7 +3877,7 @@ executions. The [`trigger` keyword](#trigger) can trigger downstream pipelines. [`resource_group` keyword](#resource_group) can co-exist with it. This is useful to control the concurrency for deployment pipelines, while running non-sensitive jobs concurrently. -This example has two pipeline configurations in a project. When a pipeline starts running, +The following example has two pipeline configurations in a project. When a pipeline starts running, non-sensitive jobs are executed first and aren't affected by concurrent executions in other pipelines. However, GitLab ensures that there are no other deployment pipelines running before triggering a deployment (child) pipeline. If other deployment pipelines are running, GitLab waits @@ -3926,7 +3918,7 @@ deployment: script: echo "Deploying..." ``` -Note that you must define [`strategy: depend`](#linking-pipelines-with-triggerstrategy) +You must define [`strategy: depend`](#linking-pipelines-with-triggerstrategy) with the `trigger` keyword. This ensures that the lock isn't released until the downstream pipeline finishes. @@ -3934,7 +3926,7 @@ finishes. > [Introduced](https://gitlab.com/gitlab-org/gitlab/merge_requests/19298) in GitLab 13.2. -`release` indicates that the job creates a [Release](../../user/project/releases/index.md). +Use `release` to create a [release](../../user/project/releases/index.md). These keywords are supported: @@ -3945,18 +3937,18 @@ These keywords are supported: - [`milestones`](#releasemilestones) (optional) - [`released_at`](#releasereleased_at) (optional) -The Release is created only if the job processes without error. If the Rails API -returns an error during Release creation, the `release` job fails. +The release is created only if the job processes without error. If the Rails API +returns an error during release creation, the `release` job fails. #### `release-cli` Docker image -The Docker image to use for the `release-cli` must be specified, using the following directive: +You must specify the Docker image to use for the `release-cli`: ```yaml image: registry.gitlab.com/gitlab-org/release-cli:latest ``` -#### Script +#### `script` All jobs except [trigger](#trigger) jobs must have the `script` keyword. A `release` job can use the output from script commands, but you can use a placeholder script if @@ -3989,11 +3981,12 @@ android-release: #### `release:tag_name` -The `tag_name` must be specified. It can refer to an existing Git tag or can be specified by the user. +You must specify a `tag_name` for the release. The tag can refer to an existing Git tag or +you can specify a new tag. When the specified tag doesn't exist in the repository, a new tag is created from the associated SHA of the pipeline. -For example, when creating a Release from a Git tag: +For example, when creating a release from a Git tag: ```yaml job: @@ -4012,17 +4005,17 @@ job: description: 'Release description' ``` -- The Release is created only if the job's main script succeeds. -- If the Release already exists, it is not updated and the job with the `release` keyword fails. +- The release is created only if the job's main script succeeds. +- If the release already exists, it is not updated and the job with the `release` keyword fails. - The `release` section executes after the `script` tag and before the `after_script`. #### `release:name` -The Release name. If omitted, it is populated with the value of `release: tag_name`. +The release name. If omitted, it is populated with the value of `release: tag_name`. #### `release:description` -Specifies the long description of the Release. You can also specify a file that contains the +Specifies the long description of the release. You can also specify a file that contains the description. ##### Read description from a file @@ -4060,8 +4053,7 @@ released_at: '2021-03-15T08:00:00Z' #### Complete example for `release` -Combining the individual examples given above for `release` results in the following -code snippets. There are two options, depending on how you generate the +If you combine the previous examples for `release`, you get two options, depending on how you generate the tags. You can't use these options together, so choose one: - To create a release when you push a Git tag, or when you add a Git tag @@ -4145,7 +4137,7 @@ The entries under the `release` node are transformed into a `bash` command line to the Docker container, which contains the [release-cli](https://gitlab.com/gitlab-org/release-cli). You can also call the `release-cli` directly from a `script` entry. -For example, using the YAML described above: +For example, if you use the YAML described previously: ```shell release-cli create --name "Release $CI_COMMIT_SHA" --description "Created using the release-cli $EXTRA_DESCRIPTION" --tag-name "v${MAJOR}.${MINOR}.${REVISION}" --ref "$CI_COMMIT_SHA" --released-at "2020-07-15T08:00:00Z" --milestone "m1" --milestone "m2" --milestone "m3" @@ -4155,7 +4147,7 @@ release-cli create --name "Release $CI_COMMIT_SHA" --description "Created using > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/33014) in GitLab 13.4. -`secrets` indicates the [CI/CD Secrets](../secrets/index.md) this job needs. It should be a hash, +Use `secrets` to specify the [CI/CD Secrets](../secrets/index.md) the job needs. It should be a hash, and the keys should be the names of the variables that are made available to the job. The value of each secret is saved in a temporary file. This file's path is stored in these variables. @@ -4164,7 +4156,8 @@ variables. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/28321) in GitLab 13.4. -`vault` keyword specifies secrets provided by [Hashicorp's Vault](https://www.vaultproject.io/). +Use `vault` to specify secrets provided by [Hashicorp's Vault](https://www.vaultproject.io/). + This syntax has multiple forms. The shortest form assumes the use of the [KV-V2](https://www.vaultproject.io/docs/secrets/kv/kv-v2) secrets engine, mounted at the default path `kv-v2`. The last part of the secret's path is the @@ -4202,14 +4195,13 @@ job: ### `pages` -`pages` is a special job that uploads static content to GitLab that -is then published as a website. It has a special syntax, so the two -requirements below must be met: +Use `pages` to upload static content to GitLab. The content +is then published as a website. You must: -- Any static content must be placed under a `public/` directory. -- `artifacts` with a path to the `public/` directory must be defined. +- Place any static content in a `public/` directory. +- Define [`artifacts`](#artifacts) with a path to the `public/` directory. -The example below moves all files from the root of the project to the +The following example moves all files from the root of the project to the `public/` directory. The `.public` workaround is so `cp` does not also copy `public/` to itself in an infinite loop: @@ -4227,14 +4219,14 @@ pages: - master ``` -Read more on [GitLab Pages user documentation](../../user/project/pages/index.md). +View the [GitLab Pages user documentation](../../user/project/pages/index.md). ### `inherit` > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207484) in GitLab 12.9. -You can disable inheritance of globally defined defaults -and variables with the `inherit:` keyword. +Use `inherit:` to control inheritance of globally-defined defaults +and variables. To enable or disable the inheritance of all `default:` or `variables:` keywords, use: @@ -4263,7 +4255,7 @@ inherit: - VARIABLE2 ``` -In the example below: +In the following example: - `rubocop`: - inherits: Nothing. @@ -4367,7 +4359,7 @@ You can use [YAML anchors for variables](#yaml-anchors-for-variables). > [Introduced in](https://gitlab.com/gitlab-org/gitlab/-/issues/30101) GitLab 13.7. -You can use the `value` and `description` keywords to define [variables that are prefilled](../pipelines/index.md#prefill-variables-in-manual-pipelines) +Use the `value` and `description` keywords to define [variables that are prefilled](../pipelines/index.md#prefill-variables-in-manual-pipelines) when [running a pipeline manually](../pipelines/index.md#run-a-pipeline-manually): ```yaml @@ -4379,7 +4371,7 @@ variables: ### Configure runner behavior with variables -You can use [CI/CD variables](../variables/README.md) to configure runner Git behavior: +You can use [CI/CD variables](../variables/README.md) to configure how the runner processes Git requests: - [`GIT_STRATEGY`](../runners/README.md#git-strategy) - [`GIT_SUBMODULE_STRATEGY`](../runners/README.md#git-submodule-strategy) @@ -4395,16 +4387,16 @@ You can use [CI/CD variables](../variables/README.md) to configure runner Git be You can also use variables to configure how many times a runner [attempts certain stages of job execution](../runners/README.md#job-stages-attempts). -## Special YAML features +## YAML-specific features -It's possible to use special YAML features like anchors (`&`), aliases (`*`) +In your `.gitlab-ci.yml` file, you can use YAML-specific features like anchors (`&`), aliases (`*`), and map merging (`<<`). Use these features to reduce the complexity of the code in the `.gitlab-ci.yml` file. Read more about the various [YAML features](https://learnxinyminutes.com/docs/yaml/). -In most cases, the [`extends` keyword](#extends) is more user friendly and should -be used over these special YAML features. +In most cases, the [`extends` keyword](#extends) is more user friendly and you should +use it when possible. You can use YAML anchors to merge YAML arrays. @@ -4445,8 +4437,8 @@ test2: ``` `&` sets up the name of the anchor (`job_configuration`), `<<` means "merge the -given hash into the current one", and `*` includes the named anchor -(`job_configuration` again). The expanded version of the example above is: +given hash into the current one," and `*` includes the named anchor +(`job_configuration` again). The expanded version of this example is: ```yaml .job_template: @@ -4586,7 +4578,7 @@ Use [YAML anchors](#anchors) with `variables` to repeat assignment of variables across multiple jobs. You can also use YAML anchors when a job requires a specific `variables` block that would otherwise override the global variables. -In the example below, we override the `GIT_STRATEGY` variable without affecting +The following example shows how override the `GIT_STRATEGY` variable without affecting the use of the `SAMPLE_VARIABLE` variable: ```yaml @@ -4606,7 +4598,7 @@ job_no_git_strategy: ### Hide jobs -If you want to temporarily 'disable' a job, rather than commenting out all the +If you want to temporarily disable a job, rather than commenting out all the lines where the job is defined: ```yaml @@ -4625,7 +4617,7 @@ GitLab CI/CD. In the following example, `.hidden_job` is ignored: ``` Use this feature to ignore jobs, or use the -[special YAML features](#special-yaml-features) and transform the hidden jobs +[YAML-specific features](#yaml-specific-features) and transform the hidden jobs into templates. ### `!reference` tags @@ -4637,7 +4629,7 @@ sections and reuse it in the current section. Unlike [YAML anchors](#anchors), y use `!reference` tags to reuse configuration from [included](#include) configuration files as well. -In this example, a `script` and an `after_script` from two different locations are +In the following example, a `script` and an `after_script` from two different locations are reused in the `test` job: - `setup.yml`: @@ -4666,7 +4658,7 @@ reused in the `test` job: - !reference [.teardown, after_script] ``` -In this example, `test-vars-1` reuses the all the variables in `.vars`, while `test-vars-2` +In the following example, `test-vars-1` reuses the all the variables in `.vars`, while `test-vars-2` selects a specific variable and reuses it as a new `MY_VAR` variable. ```yaml diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index 164ec2d32bc..01477c33011 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -38,7 +38,7 @@ each node should have: - [Memory](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_memory): 8 GiB (minimum). - [CPU](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_cpus): Modern processor with multiple cores. -- [Storage](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_disks): Use SSD storage. You will need enough storage for 50% of the total size of your Git repositories. +- [Storage](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_disks): Use SSD storage. The total storage size of all Elasticsearch nodes is about 50% of the total size of your Git repositories. It includes one primary and one replica. A few notes on CPU and storage: diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index 20131a795c5..00cf23d8ffd 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -1,6 +1,6 @@ --- stage: Enablement -group: Distribution +group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/user/admin_area/settings/continuous_integration.md b/doc/user/admin_area/settings/continuous_integration.md index f14241bb663..0756086af73 100644 --- a/doc/user/admin_area/settings/continuous_integration.md +++ b/doc/user/admin_area/settings/continuous_integration.md @@ -99,6 +99,13 @@ are allowed to expire. This setting takes precedence over the [project level setting](../../../ci/pipelines/job_artifacts.md#keep-artifacts-from-most-recent-successful-jobs). If disabled at the instance level, you cannot enable this per-project. +To disable the setting: + +1. Go to **Admin Area > Settings > CI/CD**. +1. Expand **Continuous Integration and Deployment**. +1. Clear the **Keep the latest artifacts for all jobs in the latest successful pipelines** checkbox. +1. Click **Save changes** + When you disable the feature, the latest artifacts do not immediately expire. A new pipeline must run before the latest artifacts can expire and be deleted. diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb index dc4f1413907..2d25e76626a 100644 --- a/lib/api/v3/github.rb +++ b/lib/api/v3/github.rb @@ -199,10 +199,7 @@ module API user_project = find_project_with_access(params) merge_requests = authorized_merge_requests_for_project(user_project) - - if Feature.enabled?(:api_v3_repos_events_optimization, user_project) - merge_requests = merge_requests.preload(:author, :assignees, :metrics, source_project: :namespace, target_project: :namespace) - end + merge_requests = merge_requests.preload(:author, :assignees, :metrics, source_project: :namespace, target_project: :namespace) present paginate(merge_requests), with: ::API::Github::Entities::PullRequestEvent end diff --git a/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb b/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb index 0a8c203421b..f6b36571c90 100644 --- a/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb +++ b/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb @@ -6,9 +6,9 @@ module Gitlab class BackfillArtifactExpiryDate include Gitlab::Utils::StrongMemoize - BATCH_SIZE = 1_000 - DEFAULT_EXPIRATION_SWITCH_DATE = Date.new(2020, 6, 22).freeze + SWITCH_DATE = Date.new(2020, 06, 22).freeze OLD_ARTIFACT_AGE = 15.months + BATCH_SIZE = 1_000 OLD_ARTIFACT_EXPIRY_OFFSET = 3.months RECENT_ARTIFACT_EXPIRY_OFFSET = 1.year @@ -18,16 +18,17 @@ module Gitlab self.table_name = 'ci_job_artifacts' - scope :between, -> (start_id, end_id) { where(id: start_id..end_id) } - scope :before_default_expiration_switch, -> { where('created_at < ?', DEFAULT_EXPIRATION_SWITCH_DATE) } scope :without_expiry_date, -> { where(expire_at: nil) } + scope :before_switch, -> { where("date(created_at AT TIME ZONE 'UTC') < ?::date", SWITCH_DATE) } + scope :between, -> (start_id, end_id) { where(id: start_id..end_id) } scope :old, -> { where(self.arel_table[:created_at].lt(OLD_ARTIFACT_AGE.ago)) } scope :recent, -> { where(self.arel_table[:created_at].gt(OLD_ARTIFACT_AGE.ago)) } end def perform(start_id, end_id) - Ci::JobArtifact.between(start_id, end_id) - .without_expiry_date.before_default_expiration_switch + Ci::JobArtifact + .without_expiry_date.before_switch + .between(start_id, end_id) .each_batch(of: BATCH_SIZE) do |batch| batch.old.update_all(expire_at: old_artifact_expiry_date) batch.recent.update_all(expire_at: recent_artifact_expiry_date) diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index dd7d4716de5..387eaba9861 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -54,10 +54,6 @@ module Gitlab tracking_category: 'Growth::Conversion::Experiment::ContactSalesInApp', use_backwards_compatible_subject_index: true }, - group_only_trials: { - tracking_category: 'Growth::Conversion::Experiment::GroupOnlyTrials', - use_backwards_compatible_subject_index: true - }, remove_known_trial_form_fields: { tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFields' }, diff --git a/lib/gitlab/marginalia/comment.rb b/lib/gitlab/marginalia/comment.rb index 7b4e4b06f00..ee15d3b1812 100644 --- a/lib/gitlab/marginalia/comment.rb +++ b/lib/gitlab/marginalia/comment.rb @@ -37,6 +37,10 @@ module Gitlab job end end + + def endpoint_id + Labkit::Context.current&.get_attribute(:caller_id) + end end end end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 0f9160ca50e..7c8f5624147 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -1031,9 +1031,6 @@ msgstr "" msgid "(deleted)" msgstr "" -msgid "(line: %{startLine})" -msgstr "" - msgid "(max size 15 MB)" msgstr "" @@ -5202,6 +5199,9 @@ msgstr "" msgid "Busy" msgstr "" +msgid "Buy CI Minutes" +msgstr "" + msgid "Buy License" msgstr "" @@ -11876,9 +11876,6 @@ msgstr "" msgid "Epics|To schedule your epic's %{epicDateType} date based on milestones, assign a milestone with a %{epicDateType} date to any issue in the epic." msgstr "" -msgid "Epics|Unable to perform this action" -msgstr "" - msgid "Epics|Unable to save epic. Please try again" msgstr "" @@ -16546,9 +16543,6 @@ msgstr "" msgid "Invite members" msgstr "" -msgid "Invite team members" -msgstr "" - msgid "Invite your team" msgstr "" @@ -22031,9 +22025,6 @@ msgstr "" msgid "Pipeline Schedules" msgstr "" -msgid "Pipeline cannot be run." -msgstr "" - msgid "Pipeline minutes quota" msgstr "" @@ -22292,6 +22283,9 @@ msgstr "" msgid "Pipeline|Branch name" msgstr "" +msgid "Pipeline|Branches or tags could not be loaded." +msgstr "" + msgid "Pipeline|Canceled" msgstr "" @@ -22337,6 +22331,9 @@ msgstr "" msgid "Pipeline|Merge train pipeline" msgstr "" +msgid "Pipeline|Merge train pipeline jobs can not be retried" +msgstr "" + msgid "Pipeline|Merged result pipeline" msgstr "" @@ -22352,6 +22349,9 @@ msgstr "" msgid "Pipeline|Pipeline %{idStart}#%{idEnd} %{statusStart}%{statusEnd} for %{commitStart}%{commitEnd}" msgstr "" +msgid "Pipeline|Pipeline cannot be run." +msgstr "" + msgid "Pipeline|Pipelines" msgstr "" @@ -27879,9 +27879,6 @@ msgstr "" msgid "Something went wrong on our end. Please try again." msgstr "" -msgid "Something went wrong trying to change the confidentiality of this issue" -msgstr "" - msgid "Something went wrong trying to change the locked state of this %{issuableDisplayName}" msgstr "" @@ -31573,12 +31570,6 @@ msgstr "" msgid "Trials|You can apply your trial to a new group or an existing group." msgstr "" -msgid "Trials|You can apply your trial to a new group or your personal account." -msgstr "" - -msgid "Trials|You can apply your trial to a new group, an existing group, or your personal account." -msgstr "" - msgid "Trials|You won't get a free trial right now but you can always resume this process by clicking on your avatar and choosing 'Start a free trial'" msgstr "" @@ -31708,12 +31699,6 @@ msgstr "" msgid "Tuning settings" msgstr "" -msgid "Turn Off" -msgstr "" - -msgid "Turn On" -msgstr "" - msgid "Turn off" msgstr "" @@ -34033,9 +34018,6 @@ msgstr "" msgid "You are going to turn on confidentiality. Only team members with %{strongStart}at least Reporter access%{strongEnd} will be able to see and leave comments on the %{issuableType}." msgstr "" -msgid "You are going to turn on the confidentiality. This means that only team members with %{strongStart}at least Reporter access%{strongEnd} are able to see and leave comments on the %{issuableType}." -msgstr "" - msgid "You are not allowed to approve a user" msgstr "" diff --git a/spec/frontend/notes/components/comment_form_spec.js b/spec/frontend/notes/components/comment_form_spec.js index 672eccb026b..6393394b68c 100644 --- a/spec/frontend/notes/components/comment_form_spec.js +++ b/spec/frontend/notes/components/comment_form_spec.js @@ -474,16 +474,4 @@ describe('issue_comment_form component', () => { expect(findTextArea().exists()).toBe(false); }); }); - - describe('close/reopen button variants', () => { - it.each([ - [constants.OPENED, 'warning'], - [constants.REOPENED, 'warning'], - [constants.CLOSED, 'default'], - ])('when %s, the variant of the btn is %s', (state, expected) => { - mountComponent({ noteableData: { ...noteableDataMock, state } }); - - expect(findCloseReopenButton().props('variant')).toBe(expected); - }); - }); }); diff --git a/spec/frontend/pipeline_new/components/pipeline_new_form_spec.js b/spec/frontend/pipeline_new/components/pipeline_new_form_spec.js index 51bb0ecee9c..7ec5818010a 100644 --- a/spec/frontend/pipeline_new/components/pipeline_new_form_spec.js +++ b/spec/frontend/pipeline_new/components/pipeline_new_form_spec.js @@ -1,4 +1,4 @@ -import { GlDropdown, GlDropdownItem, GlForm, GlSprintf, GlLoadingIcon } from '@gitlab/ui'; +import { GlForm, GlSprintf, GlLoadingIcon } from '@gitlab/ui'; import { mount, shallowMount } from '@vue/test-utils'; import MockAdapter from 'axios-mock-adapter'; import waitForPromises from 'helpers/wait_for_promises'; @@ -6,34 +6,26 @@ import axios from '~/lib/utils/axios_utils'; import httpStatusCodes from '~/lib/utils/http_status'; import { redirectTo } from '~/lib/utils/url_utility'; import PipelineNewForm from '~/pipeline_new/components/pipeline_new_form.vue'; -import { - mockBranches, - mockTags, - mockParams, - mockPostParams, - mockProjectId, - mockError, -} from '../mock_data'; +import RefsDropdown from '~/pipeline_new/components/refs_dropdown.vue'; +import { mockQueryParams, mockPostParams, mockProjectId, mockError, mockRefs } from '../mock_data'; jest.mock('~/lib/utils/url_utility', () => ({ redirectTo: jest.fn(), })); +const projectRefsEndpoint = '/root/project/refs'; const pipelinesPath = '/root/project/-/pipelines'; const configVariablesPath = '/root/project/-/pipelines/config_variables'; -const postResponse = { id: 1 }; +const newPipelinePostResponse = { id: 1 }; +const defaultBranch = 'master'; describe('Pipeline New Form', () => { let wrapper; let mock; - - const dummySubmitEvent = { - preventDefault() {}, - }; + let dummySubmitEvent; const findForm = () => wrapper.find(GlForm); - const findDropdown = () => wrapper.find(GlDropdown); - const findDropdownItems = () => wrapper.findAll(GlDropdownItem); + const findRefsDropdown = () => wrapper.findComponent(RefsDropdown); const findSubmitButton = () => wrapper.find('[data-testid="run_pipeline_button"]'); const findVariableRows = () => wrapper.findAll('[data-testid="ci-variable-row"]'); const findRemoveIcons = () => wrapper.findAll('[data-testid="remove-ci-variable-row"]'); @@ -44,33 +36,42 @@ describe('Pipeline New Form', () => { const findWarningAlertSummary = () => findWarningAlert().find(GlSprintf); const findWarnings = () => wrapper.findAll('[data-testid="run-pipeline-warning"]'); const findLoadingIcon = () => wrapper.find(GlLoadingIcon); - const getExpectedPostParams = () => JSON.parse(mock.history.post[0].data); - const changeRef = (i) => findDropdownItems().at(i).vm.$emit('click'); + const getFormPostParams = () => JSON.parse(mock.history.post[0].data); + + const selectBranch = (branch) => { + // Select a branch in the dropdown + findRefsDropdown().vm.$emit('input', { + shortName: branch, + fullName: `refs/heads/${branch}`, + }); + }; - const createComponent = (term = '', props = {}, method = shallowMount) => { + const createComponent = (props = {}, method = shallowMount) => { wrapper = method(PipelineNewForm, { + provide: { + projectRefsEndpoint, + }, propsData: { projectId: mockProjectId, pipelinesPath, configVariablesPath, - branches: mockBranches, - tags: mockTags, - defaultBranch: 'master', + defaultBranch, + refParam: defaultBranch, settingsLink: '', maxWarnings: 25, ...props, }, - data() { - return { - searchTerm: term, - }; - }, }); }; beforeEach(() => { mock = new MockAdapter(axios); mock.onGet(configVariablesPath).reply(httpStatusCodes.OK, {}); + mock.onGet(projectRefsEndpoint).reply(httpStatusCodes.OK, mockRefs); + + dummySubmitEvent = { + preventDefault: jest.fn(), + }; }); afterEach(() => { @@ -80,38 +81,17 @@ describe('Pipeline New Form', () => { mock.restore(); }); - describe('Dropdown with branches and tags', () => { - beforeEach(() => { - mock.onPost(pipelinesPath).reply(httpStatusCodes.OK, postResponse); - }); - - it('displays dropdown with all branches and tags', () => { - const refLength = mockBranches.length + mockTags.length; - - createComponent(); - - expect(findDropdownItems()).toHaveLength(refLength); - }); - - it('when user enters search term the list is filtered', () => { - createComponent('master'); - - expect(findDropdownItems()).toHaveLength(1); - expect(findDropdownItems().at(0).text()).toBe('master'); - }); - }); - describe('Form', () => { beforeEach(async () => { - createComponent('', mockParams, mount); + createComponent(mockQueryParams, mount); - mock.onPost(pipelinesPath).reply(httpStatusCodes.OK, postResponse); + mock.onPost(pipelinesPath).reply(httpStatusCodes.OK, newPipelinePostResponse); await waitForPromises(); }); it('displays the correct values for the provided query params', async () => { - expect(findDropdown().props('text')).toBe('tag-1'); + expect(findRefsDropdown().props('value')).toEqual({ shortName: 'tag-1' }); expect(findVariableRows()).toHaveLength(3); }); @@ -152,11 +132,19 @@ describe('Pipeline New Form', () => { describe('Pipeline creation', () => { beforeEach(async () => { - mock.onPost(pipelinesPath).reply(httpStatusCodes.OK, postResponse); + mock.onPost(pipelinesPath).reply(httpStatusCodes.OK, newPipelinePostResponse); await waitForPromises(); }); + it('does not submit the native HTML form', async () => { + createComponent(); + + findForm().vm.$emit('submit', dummySubmitEvent); + + expect(dummySubmitEvent.preventDefault).toHaveBeenCalled(); + }); + it('disables the submit button immediately after submitting', async () => { createComponent(); @@ -171,19 +159,15 @@ describe('Pipeline New Form', () => { it('creates pipeline with full ref and variables', async () => { createComponent(); - changeRef(0); - findForm().vm.$emit('submit', dummySubmitEvent); - await waitForPromises(); - expect(getExpectedPostParams().ref).toEqual(wrapper.vm.$data.refValue.fullName); - expect(redirectTo).toHaveBeenCalledWith(`${pipelinesPath}/${postResponse.id}`); + expect(getFormPostParams().ref).toEqual(`refs/heads/${defaultBranch}`); + expect(redirectTo).toHaveBeenCalledWith(`${pipelinesPath}/${newPipelinePostResponse.id}`); }); - it('creates a pipeline with short ref and variables', async () => { - // query params are used - createComponent('', mockParams); + it('creates a pipeline with short ref and variables from the query params', async () => { + createComponent(mockQueryParams); await waitForPromises(); @@ -191,19 +175,19 @@ describe('Pipeline New Form', () => { await waitForPromises(); - expect(getExpectedPostParams()).toEqual(mockPostParams); - expect(redirectTo).toHaveBeenCalledWith(`${pipelinesPath}/${postResponse.id}`); + expect(getFormPostParams()).toEqual(mockPostParams); + expect(redirectTo).toHaveBeenCalledWith(`${pipelinesPath}/${newPipelinePostResponse.id}`); }); }); describe('When the ref has been changed', () => { beforeEach(async () => { - createComponent('', {}, mount); + createComponent({}, mount); await waitForPromises(); }); it('variables persist between ref changes', async () => { - changeRef(0); // change to master + selectBranch('master'); await waitForPromises(); @@ -213,7 +197,7 @@ describe('Pipeline New Form', () => { await wrapper.vm.$nextTick(); - changeRef(1); // change to branch-1 + selectBranch('branch-1'); await waitForPromises(); @@ -223,14 +207,14 @@ describe('Pipeline New Form', () => { await wrapper.vm.$nextTick(); - changeRef(0); // change back to master + selectBranch('master'); await waitForPromises(); expect(findKeyInputs().at(0).element.value).toBe('build_var'); expect(findVariableRows().length).toBe(2); - changeRef(1); // change back to branch-1 + selectBranch('branch-1'); await waitForPromises(); @@ -248,7 +232,7 @@ describe('Pipeline New Form', () => { const mockYmlDesc = 'A var from yml.'; it('loading icon is shown when content is requested and hidden when received', async () => { - createComponent('', mockParams, mount); + createComponent(mockQueryParams, mount); mock.onGet(configVariablesPath).reply(httpStatusCodes.OK, { [mockYmlKey]: { @@ -265,7 +249,7 @@ describe('Pipeline New Form', () => { }); it('multi-line strings are added to the value field without removing line breaks', async () => { - createComponent('', mockParams, mount); + createComponent(mockQueryParams, mount); mock.onGet(configVariablesPath).reply(httpStatusCodes.OK, { [mockYmlKey]: { @@ -281,7 +265,7 @@ describe('Pipeline New Form', () => { describe('with description', () => { beforeEach(async () => { - createComponent('', mockParams, mount); + createComponent(mockQueryParams, mount); mock.onGet(configVariablesPath).reply(httpStatusCodes.OK, { [mockYmlKey]: { @@ -323,7 +307,7 @@ describe('Pipeline New Form', () => { describe('without description', () => { beforeEach(async () => { - createComponent('', mockParams, mount); + createComponent(mockQueryParams, mount); mock.onGet(configVariablesPath).reply(httpStatusCodes.OK, { [mockYmlKey]: { @@ -346,6 +330,21 @@ describe('Pipeline New Form', () => { createComponent(); }); + describe('when the refs cannot be loaded', () => { + beforeEach(() => { + mock + .onGet(projectRefsEndpoint, { params: { search: '' } }) + .reply(httpStatusCodes.INTERNAL_SERVER_ERROR); + + findRefsDropdown().vm.$emit('loadingError'); + }); + + it('shows both an error alert', () => { + expect(findErrorAlert().exists()).toBe(true); + expect(findWarningAlert().exists()).toBe(false); + }); + }); + describe('when the error response can be handled', () => { beforeEach(async () => { mock.onPost(pipelinesPath).reply(httpStatusCodes.BAD_REQUEST, mockError); diff --git a/spec/frontend/pipeline_new/components/refs_dropdown_spec.js b/spec/frontend/pipeline_new/components/refs_dropdown_spec.js new file mode 100644 index 00000000000..8dafbf230f9 --- /dev/null +++ b/spec/frontend/pipeline_new/components/refs_dropdown_spec.js @@ -0,0 +1,182 @@ +import { GlDropdown, GlDropdownItem, GlSearchBoxByType } from '@gitlab/ui'; +import { shallowMount } from '@vue/test-utils'; +import MockAdapter from 'axios-mock-adapter'; +import waitForPromises from 'helpers/wait_for_promises'; +import axios from '~/lib/utils/axios_utils'; +import httpStatusCodes from '~/lib/utils/http_status'; + +import RefsDropdown from '~/pipeline_new/components/refs_dropdown.vue'; + +import { mockRefs, mockFilteredRefs } from '../mock_data'; + +const projectRefsEndpoint = '/root/project/refs'; +const refShortName = 'master'; +const refFullName = 'refs/heads/master'; + +jest.mock('~/flash'); + +describe('Pipeline New Form', () => { + let wrapper; + let mock; + + const findDropdown = () => wrapper.find(GlDropdown); + const findRefsDropdownItems = () => wrapper.findAll(GlDropdownItem); + const findSearchBox = () => wrapper.findComponent(GlSearchBoxByType); + + const createComponent = (props = {}, mountFn = shallowMount) => { + wrapper = mountFn(RefsDropdown, { + provide: { + projectRefsEndpoint, + }, + propsData: { + value: { + shortName: refShortName, + fullName: refFullName, + }, + ...props, + }, + }); + }; + + beforeEach(() => { + mock = new MockAdapter(axios); + mock.onGet(projectRefsEndpoint, { params: { search: '' } }).reply(httpStatusCodes.OK, mockRefs); + }); + + afterEach(() => { + wrapper.destroy(); + wrapper = null; + + mock.restore(); + }); + + beforeEach(() => { + createComponent(); + }); + + it('displays empty dropdown initially', async () => { + await findDropdown().vm.$emit('show'); + + expect(findRefsDropdownItems()).toHaveLength(0); + }); + + it('does not make requests immediately', async () => { + expect(mock.history.get).toHaveLength(0); + }); + + describe('when user opens dropdown', () => { + beforeEach(async () => { + await findDropdown().vm.$emit('show'); + await waitForPromises(); + }); + + it('requests unfiltered tags and branches', async () => { + expect(mock.history.get).toHaveLength(1); + expect(mock.history.get[0].url).toBe(projectRefsEndpoint); + expect(mock.history.get[0].params).toEqual({ search: '' }); + }); + + it('displays dropdown with branches and tags', async () => { + const refLength = mockRefs.Tags.length + mockRefs.Branches.length; + + expect(findRefsDropdownItems()).toHaveLength(refLength); + }); + + it('displays the names of refs', () => { + // Branches + expect(findRefsDropdownItems().at(0).text()).toBe(mockRefs.Branches[0]); + + // Tags (appear after branches) + const firstTag = mockRefs.Branches.length; + expect(findRefsDropdownItems().at(firstTag).text()).toBe(mockRefs.Tags[0]); + }); + + it('when user shows dropdown a second time, only one request is done', () => { + expect(mock.history.get).toHaveLength(1); + }); + + describe('when user selects a value', () => { + const selectedIndex = 1; + + beforeEach(async () => { + await findRefsDropdownItems().at(selectedIndex).vm.$emit('click'); + }); + + it('component emits @input', () => { + const inputs = wrapper.emitted('input'); + + expect(inputs).toHaveLength(1); + expect(inputs[0]).toEqual([{ shortName: 'branch-1', fullName: 'refs/heads/branch-1' }]); + }); + }); + + describe('when user types searches for a tag', () => { + const mockSearchTerm = 'my-search'; + + beforeEach(async () => { + mock + .onGet(projectRefsEndpoint, { params: { search: mockSearchTerm } }) + .reply(httpStatusCodes.OK, mockFilteredRefs); + + await findSearchBox().vm.$emit('input', mockSearchTerm); + await waitForPromises(); + }); + + it('requests filtered tags and branches', async () => { + expect(mock.history.get).toHaveLength(2); + expect(mock.history.get[1].params).toEqual({ + search: mockSearchTerm, + }); + }); + + it('displays dropdown with branches and tags', async () => { + const filteredRefLength = mockFilteredRefs.Tags.length + mockFilteredRefs.Branches.length; + + expect(findRefsDropdownItems()).toHaveLength(filteredRefLength); + }); + }); + }); + + describe('when user has selected a value', () => { + const selectedIndex = 1; + const mockShortName = mockRefs.Branches[selectedIndex]; + const mockFullName = `refs/heads/${mockShortName}`; + + beforeEach(async () => { + mock + .onGet(projectRefsEndpoint, { + params: { ref: mockFullName }, + }) + .reply(httpStatusCodes.OK, mockRefs); + + createComponent({ + value: { + shortName: mockShortName, + fullName: mockFullName, + }, + }); + await findDropdown().vm.$emit('show'); + await waitForPromises(); + }); + + it('branch is checked', () => { + expect(findRefsDropdownItems().at(selectedIndex).props('isChecked')).toBe(true); + }); + }); + + describe('when server returns an error', () => { + beforeEach(async () => { + mock + .onGet(projectRefsEndpoint, { params: { search: '' } }) + .reply(httpStatusCodes.INTERNAL_SERVER_ERROR); + + await findDropdown().vm.$emit('show'); + await waitForPromises(); + }); + + it('loading error event is emitted', () => { + expect(wrapper.emitted('loadingError')).toHaveLength(1); + expect(wrapper.emitted('loadingError')[0]).toEqual([expect.any(Error)]); + }); + }); +}); diff --git a/spec/frontend/pipeline_new/mock_data.js b/spec/frontend/pipeline_new/mock_data.js index feb24ec602d..4fb58cb8e62 100644 --- a/spec/frontend/pipeline_new/mock_data.js +++ b/spec/frontend/pipeline_new/mock_data.js @@ -1,16 +1,14 @@ -export const mockBranches = [ - { shortName: 'master', fullName: 'refs/heads/master' }, - { shortName: 'branch-1', fullName: 'refs/heads/branch-1' }, - { shortName: 'branch-2', fullName: 'refs/heads/branch-2' }, -]; +export const mockRefs = { + Branches: ['master', 'branch-1', 'branch-2'], + Tags: ['1.0.0', '1.1.0', '1.2.0'], +}; -export const mockTags = [ - { shortName: '1.0.0', fullName: 'refs/tags/1.0.0' }, - { shortName: '1.1.0', fullName: 'refs/tags/1.1.0' }, - { shortName: '1.2.0', fullName: 'refs/tags/1.2.0' }, -]; +export const mockFilteredRefs = { + Branches: ['branch-1'], + Tags: ['1.0.0', '1.1.0'], +}; -export const mockParams = { +export const mockQueryParams = { refParam: 'tag-1', variableParams: { test_var: 'test_var_val', diff --git a/spec/frontend/pipelines/stage_spec.js b/spec/frontend/pipelines/stage_spec.js index db15f6520ca..7820e26e2c2 100644 --- a/spec/frontend/pipelines/stage_spec.js +++ b/spec/frontend/pipelines/stage_spec.js @@ -48,6 +48,7 @@ describe('Pipelines stage component', () => { const findDropdownMenu = () => wrapper.find('[data-testid="mini-pipeline-graph-dropdown-menu-list"]'); const findCiActionBtn = () => wrapper.find('.js-ci-action'); + const findMergeTrainWarning = () => wrapper.find('[data-testid="warning-message-merge-trains"]'); const openStageDropdown = () => { findDropdownToggle().trigger('click'); @@ -172,4 +173,40 @@ describe('Pipelines stage component', () => { expect(wrapper.emitted('pipelineActionRequestComplete')).toHaveLength(1); }); }); + + describe('With merge trains enabled', () => { + beforeEach(async () => { + mock.onGet(dropdownPath).reply(200, stageReply); + createComponent({ + isMergeTrain: true, + }); + + await openStageDropdown(); + await axios.waitForAll(); + }); + + it('shows a warning on the dropdown', () => { + const warning = findMergeTrainWarning(); + + expect(warning.text()).toBe('Merge train pipeline jobs can not be retried'); + }); + }); + + describe('With merge trains disabled', () => { + beforeEach(async () => { + mock.onGet(dropdownPath).reply(200, stageReply); + createComponent({ + isMergeTrain: false, + }); + + await openStageDropdown(); + await axios.waitForAll(); + }); + + it('does not show a warning on the dropdown', () => { + const warning = findMergeTrainWarning(); + + expect(warning.exists()).toBe(false); + }); + }); }); diff --git a/spec/frontend/sidebar/__snapshots__/confidential_issue_sidebar_spec.js.snap b/spec/frontend/sidebar/__snapshots__/confidential_issue_sidebar_spec.js.snap deleted file mode 100644 index 2367667544d..00000000000 --- a/spec/frontend/sidebar/__snapshots__/confidential_issue_sidebar_spec.js.snap +++ /dev/null @@ -1,199 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`Confidential Issue Sidebar Block renders for confidential = false and isEditable = false 1`] = ` -<div - class="block issuable-sidebar-item confidentiality" - iid="" -> - <div - class="sidebar-collapsed-icon" - title="Not confidential" - > - <gl-icon-stub - name="eye" - size="16" - /> - </div> - - <div - class="title hide-collapsed" - > - - Confidentiality - - <!----> - </div> - - <div - class="value sidebar-item-value hide-collapsed" - > - <!----> - - <div - class="no-value sidebar-item-value" - data-testid="not-confidential" - > - <gl-icon-stub - class="sidebar-item-icon inline" - name="eye" - size="16" - /> - - Not confidential - - </div> - </div> -</div> -`; - -exports[`Confidential Issue Sidebar Block renders for confidential = false and isEditable = true 1`] = ` -<div - class="block issuable-sidebar-item confidentiality" - iid="" -> - <div - class="sidebar-collapsed-icon" - title="Not confidential" - > - <gl-icon-stub - name="eye" - size="16" - /> - </div> - - <div - class="title hide-collapsed" - > - - Confidentiality - - <a - class="float-right confidential-edit" - data-track-event="click_edit_button" - data-track-label="right_sidebar" - data-track-property="confidentiality" - href="#" - > - Edit - </a> - </div> - - <div - class="value sidebar-item-value hide-collapsed" - > - <!----> - - <div - class="no-value sidebar-item-value" - data-testid="not-confidential" - > - <gl-icon-stub - class="sidebar-item-icon inline" - name="eye" - size="16" - /> - - Not confidential - - </div> - </div> -</div> -`; - -exports[`Confidential Issue Sidebar Block renders for confidential = true and isEditable = false 1`] = ` -<div - class="block issuable-sidebar-item confidentiality" - iid="" -> - <div - class="sidebar-collapsed-icon" - title="Confidential" - > - <gl-icon-stub - name="eye-slash" - size="16" - /> - </div> - - <div - class="title hide-collapsed" - > - - Confidentiality - - <!----> - </div> - - <div - class="value sidebar-item-value hide-collapsed" - > - <!----> - - <div - class="value sidebar-item-value hide-collapsed" - > - <gl-icon-stub - class="sidebar-item-icon inline is-active" - name="eye-slash" - size="16" - /> - - This issue is confidential - - </div> - </div> -</div> -`; - -exports[`Confidential Issue Sidebar Block renders for confidential = true and isEditable = true 1`] = ` -<div - class="block issuable-sidebar-item confidentiality" - iid="" -> - <div - class="sidebar-collapsed-icon" - title="Confidential" - > - <gl-icon-stub - name="eye-slash" - size="16" - /> - </div> - - <div - class="title hide-collapsed" - > - - Confidentiality - - <a - class="float-right confidential-edit" - data-track-event="click_edit_button" - data-track-label="right_sidebar" - data-track-property="confidentiality" - href="#" - > - Edit - </a> - </div> - - <div - class="value sidebar-item-value hide-collapsed" - > - <!----> - - <div - class="value sidebar-item-value hide-collapsed" - > - <gl-icon-stub - class="sidebar-item-icon inline is-active" - name="eye-slash" - size="16" - /> - - This issue is confidential - - </div> - </div> -</div> -`; diff --git a/spec/frontend/sidebar/components/confidential/__snapshots__/edit_form_spec.js.snap b/spec/frontend/sidebar/components/confidential/__snapshots__/edit_form_spec.js.snap deleted file mode 100644 index d33f6c7f389..00000000000 --- a/spec/frontend/sidebar/components/confidential/__snapshots__/edit_form_spec.js.snap +++ /dev/null @@ -1,50 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`Edit Form Dropdown when confidential renders on or off text based on confidentiality 1`] = ` -<div - class="dropdown show" - toggleform="function () {}" - updateconfidentialattribute="function () {}" -> - <div - class="dropdown-menu sidebar-item-warning-message" - > - <div> - <p> - <gl-sprintf-stub - message="You are going to turn off the confidentiality. This means %{strongStart}everyone%{strongEnd} will be able to see and leave a comment on this %{issuableType}." - /> - </p> - - <edit-form-buttons-stub - confidential="true" - fullpath="" - /> - </div> - </div> -</div> -`; - -exports[`Edit Form Dropdown when not confidential renders "You are going to turn on the confidentiality." in the 1`] = ` -<div - class="dropdown show" - toggleform="function () {}" - updateconfidentialattribute="function () {}" -> - <div - class="dropdown-menu sidebar-item-warning-message" - > - <div> - <p> - <gl-sprintf-stub - message="You are going to turn on the confidentiality. This means that only team members with %{strongStart}at least Reporter access%{strongEnd} are able to see and leave comments on the %{issuableType}." - /> - </p> - - <edit-form-buttons-stub - fullpath="" - /> - </div> - </div> -</div> -`; diff --git a/spec/frontend/sidebar/components/confidential/edit_form_buttons_spec.js b/spec/frontend/sidebar/components/confidential/edit_form_buttons_spec.js deleted file mode 100644 index 427e3a89c29..00000000000 --- a/spec/frontend/sidebar/components/confidential/edit_form_buttons_spec.js +++ /dev/null @@ -1,146 +0,0 @@ -import { shallowMount } from '@vue/test-utils'; -import waitForPromises from 'helpers/wait_for_promises'; -import { deprecatedCreateFlash as flash } from '~/flash'; -import createStore from '~/notes/stores'; -import EditFormButtons from '~/sidebar/components/confidential/edit_form_buttons.vue'; -import eventHub from '~/sidebar/event_hub'; - -jest.mock('~/sidebar/event_hub', () => ({ $emit: jest.fn() })); -jest.mock('~/flash'); - -describe('Edit Form Buttons', () => { - let wrapper; - let store; - const findConfidentialToggle = () => wrapper.find('[data-testid="confidential-toggle"]'); - - const createComponent = ({ props = {}, data = {}, resolved = true }) => { - store = createStore(); - if (resolved) { - jest.spyOn(store, 'dispatch').mockResolvedValue(); - } else { - jest.spyOn(store, 'dispatch').mockRejectedValue(); - } - - wrapper = shallowMount(EditFormButtons, { - propsData: { - fullPath: '', - ...props, - }, - data() { - return { - isLoading: true, - ...data, - }; - }, - store, - }); - }; - - afterEach(() => { - wrapper.destroy(); - wrapper = null; - }); - - describe('when isLoading', () => { - beforeEach(() => { - createComponent({ - props: { - confidential: false, - }, - }); - }); - - it('renders "Applying" in the toggle button', () => { - expect(findConfidentialToggle().text()).toBe('Applying'); - }); - - it('disables the toggle button', () => { - expect(findConfidentialToggle().props('disabled')).toBe(true); - }); - - it('sets loading on the toggle button', () => { - expect(findConfidentialToggle().props('loading')).toBe(true); - }); - }); - - describe('when not confidential', () => { - it('renders Turn On in the toggle button', () => { - createComponent({ - data: { - isLoading: false, - }, - props: { - confidential: false, - }, - }); - - expect(findConfidentialToggle().text()).toBe('Turn On'); - }); - }); - - describe('when confidential', () => { - beforeEach(() => { - createComponent({ - data: { - isLoading: false, - }, - props: { - confidential: true, - }, - }); - }); - - it('renders on or off text based on confidentiality', () => { - expect(findConfidentialToggle().text()).toBe('Turn Off'); - }); - }); - - describe('when succeeds', () => { - beforeEach(() => { - createComponent({ data: { isLoading: false }, props: { confidential: true } }); - findConfidentialToggle().vm.$emit('click', new Event('click')); - }); - - it('dispatches the correct action', () => { - expect(store.dispatch).toHaveBeenCalledWith('updateConfidentialityOnIssuable', { - confidential: false, - fullPath: '', - }); - }); - - it('resets loading on the toggle button', () => { - return waitForPromises().then(() => { - expect(findConfidentialToggle().props('loading')).toBe(false); - }); - }); - - it('emits close form', () => { - return waitForPromises().then(() => { - expect(eventHub.$emit).toHaveBeenCalledWith('closeConfidentialityForm'); - }); - }); - - it('emits updateOnConfidentiality event', () => { - return waitForPromises().then(() => { - expect(eventHub.$emit).toHaveBeenCalledWith('updateIssuableConfidentiality', false); - }); - }); - }); - - describe('when fails', () => { - beforeEach(() => { - createComponent({ - data: { isLoading: false }, - props: { confidential: true }, - resolved: false, - }); - findConfidentialToggle().vm.$emit('click', new Event('click')); - }); - - it('calls flash with the correct message', () => { - expect(flash).toHaveBeenCalledWith( - 'Something went wrong trying to change the confidentiality of this issue', - ); - }); - }); -}); diff --git a/spec/frontend/sidebar/components/confidential/edit_form_spec.js b/spec/frontend/sidebar/components/confidential/edit_form_spec.js deleted file mode 100644 index 6b571df10ae..00000000000 --- a/spec/frontend/sidebar/components/confidential/edit_form_spec.js +++ /dev/null @@ -1,48 +0,0 @@ -import { shallowMount } from '@vue/test-utils'; -import EditForm from '~/sidebar/components/confidential/edit_form.vue'; - -describe('Edit Form Dropdown', () => { - let wrapper; - const toggleForm = () => {}; - const updateConfidentialAttribute = () => {}; - - const createComponent = (props) => { - wrapper = shallowMount(EditForm, { - propsData: { - ...props, - isLoading: false, - fullPath: '', - issuableType: 'issue', - }, - }); - }; - - afterEach(() => { - wrapper.destroy(); - wrapper = null; - }); - - describe('when not confidential', () => { - it('renders "You are going to turn on the confidentiality." in the ', () => { - createComponent({ - confidential: false, - toggleForm, - updateConfidentialAttribute, - }); - - expect(wrapper.element).toMatchSnapshot(); - }); - }); - - describe('when confidential', () => { - it('renders on or off text based on confidentiality', () => { - createComponent({ - confidential: true, - toggleForm, - updateConfidentialAttribute, - }); - - expect(wrapper.element).toMatchSnapshot(); - }); - }); -}); diff --git a/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_content_spec.js b/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_content_spec.js index 5887afb703b..8844e1626cd 100644 --- a/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_content_spec.js +++ b/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_content_spec.js @@ -7,11 +7,13 @@ describe('Sidebar Confidentiality Content', () => { const findIcon = () => wrapper.findComponent(GlIcon); const findText = () => wrapper.find('[data-testid="confidential-text"]'); + const findCollapsedIcon = () => wrapper.find('[data-testid="sidebar-collapsed-icon"]'); - const createComponent = (confidential = false) => { + const createComponent = ({ confidential = false, issuableType = 'issue' } = {}) => { wrapper = shallowMount(SidebarConfidentialityContent, { propsData: { confidential, + issuableType, }, }); }; @@ -20,6 +22,13 @@ describe('Sidebar Confidentiality Content', () => { wrapper.destroy(); }); + it('emits `expandSidebar` event on collapsed icon click', () => { + createComponent(); + findCollapsedIcon().trigger('click'); + + expect(wrapper.emitted('expandSidebar')).toHaveLength(1); + }); + describe('when issue is non-confidential', () => { beforeEach(() => { createComponent(); @@ -39,20 +48,24 @@ describe('Sidebar Confidentiality Content', () => { }); describe('when issue is confidential', () => { - beforeEach(() => { - createComponent(true); - }); - - it('renders a non-confidential icon', () => { + it('renders a confidential icon', () => { + createComponent({ confidential: true }); expect(findIcon().props('name')).toBe('eye-slash'); }); - it('does not add `is-active` class to the icon', () => { + it('adds `is-active` class to the icon', () => { + createComponent({ confidential: true }); expect(findIcon().classes()).toContain('is-active'); }); - it('displays a non-confidential text', () => { - expect(findText().text()).toBe('This is confidential'); + it('displays a correct confidential text for issue', () => { + createComponent({ confidential: true }); + expect(findText().text()).toBe('This issue is confidential'); + }); + + it('displays a correct confidential text for epic', () => { + createComponent({ confidential: true, issuableType: 'epic' }); + expect(findText().text()).toBe('This epic is confidential'); }); }); }); diff --git a/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_form_spec.js b/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_form_spec.js index 2d102331272..d5e6310ed38 100644 --- a/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_form_spec.js +++ b/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_form_spec.js @@ -143,4 +143,31 @@ describe('Sidebar Confidentiality Form', () => { }); }); }); + + describe('when issuable type is `epic`', () => { + beforeEach(() => { + createComponent({ props: { confidential: true, issuableType: 'epic' } }); + }); + + it('renders a message about making an epic non-confidential', () => { + expect(findWarningMessage().text()).toBe( + 'You are going to turn off the confidentiality. This means everyone will be able to see and leave a comment on this epic.', + ); + }); + + it('calls a mutation to set epic confidentiality with correct parameters', () => { + findConfidentialToggle().vm.$emit('click', new MouseEvent('click')); + + expect(wrapper.vm.$apollo.mutate).toHaveBeenCalledWith({ + mutation: confidentialityQueries[wrapper.vm.issuableType].mutation, + variables: { + input: { + confidential: false, + iid: '1', + groupPath: 'group/project', + }, + }, + }); + }); + }); }); diff --git a/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_widget_spec.js b/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_widget_spec.js index 7dc1fe62cfd..20a5be9b518 100644 --- a/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_widget_spec.js +++ b/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_widget_spec.js @@ -86,6 +86,10 @@ describe('Sidebar Confidentiality Widget', () => { expect(findConfidentialityForm().props('confidential')).toBe(true); expect(findConfidentialityContent().props('confidential')).toBe(true); }); + + it('emits `confidentialityUpdated` event with a `false` payload', () => { + expect(wrapper.emitted('confidentialityUpdated')).toEqual([[false]]); + }); }); describe('when issue is confidential', () => { @@ -111,6 +115,10 @@ describe('Sidebar Confidentiality Widget', () => { expect(findConfidentialityForm().props('confidential')).toBe(false); expect(findConfidentialityContent().props('confidential')).toBe(false); }); + + it('emits `confidentialityUpdated` event with a `true` payload', () => { + expect(wrapper.emitted('confidentialityUpdated')).toEqual([[true]]); + }); }); it('displays a flash message when query is rejected', async () => { @@ -138,5 +146,14 @@ describe('Sidebar Confidentiality Widget', () => { expect(findConfidentialityForm().isVisible()).toBe(false); expect(el.dispatchEvent).toHaveBeenCalled(); + expect(wrapper.emitted('closeForm')).toHaveLength(1); + }); + + it('emits `expandSidebar` event when it is emitted from child component', async () => { + createComponent(); + await waitForPromises(); + findConfidentialityContent().vm.$emit('expandSidebar'); + + expect(wrapper.emitted('expandSidebar')).toHaveLength(1); }); }); diff --git a/spec/frontend/sidebar/confidential_issue_sidebar_spec.js b/spec/frontend/sidebar/confidential_issue_sidebar_spec.js deleted file mode 100644 index 93a6401b1fc..00000000000 --- a/spec/frontend/sidebar/confidential_issue_sidebar_spec.js +++ /dev/null @@ -1,159 +0,0 @@ -import { shallowMount } from '@vue/test-utils'; -import { useMockLocationHelper } from 'helpers/mock_window_location_helper'; -import { mockTracking, triggerEvent } from 'helpers/tracking_helper'; -import createStore from '~/notes/stores'; -import * as types from '~/notes/stores/mutation_types'; -import ConfidentialIssueSidebar from '~/sidebar/components/confidential/confidential_issue_sidebar.vue'; -import EditForm from '~/sidebar/components/confidential/edit_form.vue'; - -jest.mock('~/flash'); -jest.mock('~/sidebar/services/sidebar_service'); - -describe('Confidential Issue Sidebar Block', () => { - useMockLocationHelper(); - - let wrapper; - const mutate = jest - .fn() - .mockResolvedValue({ data: { issueSetConfidential: { issue: { confidential: true } } } }); - - const createComponent = ({ propsData, data = {} }) => { - const store = createStore(); - wrapper = shallowMount(ConfidentialIssueSidebar, { - store, - data() { - return data; - }, - propsData: { - iid: '', - fullPath: '', - ...propsData, - }, - mocks: { - $apollo: { - mutate, - }, - }, - }); - }; - - afterEach(() => { - wrapper.destroy(); - }); - - it.each` - confidential | isEditable - ${false} | ${false} - ${false} | ${true} - ${true} | ${false} - ${true} | ${true} - `( - 'renders for confidential = $confidential and isEditable = $isEditable', - ({ confidential, isEditable }) => { - createComponent({ - propsData: { - isEditable, - }, - }); - wrapper.vm.$store.state.noteableData.confidential = confidential; - - return wrapper.vm.$nextTick().then(() => { - expect(wrapper.element).toMatchSnapshot(); - }); - }, - ); - - describe('if editable', () => { - beforeEach(() => { - createComponent({ - propsData: { - isEditable: true, - }, - }); - wrapper.vm.$store.state.noteableData.confidential = true; - }); - - it('displays the edit form when editable', () => { - wrapper.setData({ edit: false }); - - return wrapper.vm - .$nextTick() - .then(() => { - wrapper.find({ ref: 'editLink' }).trigger('click'); - return wrapper.vm.$nextTick(); - }) - .then(() => { - expect(wrapper.find(EditForm).exists()).toBe(true); - }); - }); - - it('displays the edit form when opened from collapsed state', () => { - wrapper.setData({ edit: false }); - - return wrapper.vm - .$nextTick() - .then(() => { - wrapper.find({ ref: 'collapseIcon' }).trigger('click'); - return wrapper.vm.$nextTick(); - }) - .then(() => { - expect(wrapper.find(EditForm).exists()).toBe(true); - }); - }); - - it('tracks the event when "Edit" is clicked', () => { - const spy = mockTracking('_category_', wrapper.element, jest.spyOn); - - const editLink = wrapper.find({ ref: 'editLink' }); - triggerEvent(editLink.element); - - expect(spy).toHaveBeenCalledWith('_category_', 'click_edit_button', { - label: 'right_sidebar', - property: 'confidentiality', - }); - }); - }); - describe('computed confidential', () => { - beforeEach(() => { - createComponent({ - propsData: { - isEditable: true, - }, - }); - }); - - it('returns false when noteableData is not present', () => { - wrapper.vm.$store.commit(types.SET_NOTEABLE_DATA, null); - - expect(wrapper.vm.confidential).toBe(false); - }); - - it('returns true when noteableData has confidential attr as true', () => { - wrapper.vm.$store.commit(types.SET_NOTEABLE_DATA, {}); - wrapper.vm.$store.commit(types.SET_ISSUE_CONFIDENTIAL, true); - - expect(wrapper.vm.confidential).toBe(true); - }); - - it('returns false when noteableData has confidential attr as false', () => { - wrapper.vm.$store.commit(types.SET_NOTEABLE_DATA, {}); - wrapper.vm.$store.commit(types.SET_ISSUE_CONFIDENTIAL, false); - - expect(wrapper.vm.confidential).toBe(false); - }); - - it('returns true when confidential attr is true', () => { - wrapper.vm.$store.commit(types.SET_NOTEABLE_DATA, {}); - wrapper.vm.$store.commit(types.SET_ISSUE_CONFIDENTIAL, true); - - expect(wrapper.vm.confidential).toBe(true); - }); - - it('returns false when confidential attr is false', () => { - wrapper.vm.$store.commit(types.SET_NOTEABLE_DATA, {}); - wrapper.vm.$store.commit(types.SET_ISSUE_CONFIDENTIAL, false); - - expect(wrapper.vm.confidential).toBe(false); - }); - }); -}); diff --git a/spec/lib/banzai/filter/custom_emoji_filter_spec.rb b/spec/lib/banzai/filter/custom_emoji_filter_spec.rb index ca8c9750e7f..82e399fff9b 100644 --- a/spec/lib/banzai/filter/custom_emoji_filter_spec.rb +++ b/spec/lib/banzai/filter/custom_emoji_filter_spec.rb @@ -18,10 +18,10 @@ RSpec.describe Banzai::Filter::CustomEmojiFilter do end it 'ignores non existent custom emoji' do - exp = act = '<p>:foo:</p>' - doc = filter(act) + exp = '<p>:foo:</p>' + doc = filter(exp) - expect(doc.to_html).to match Regexp.escape(exp) + expect(doc.to_html).to eq(exp) end it 'correctly uses the custom emoji URL' do @@ -77,4 +77,10 @@ RSpec.describe Banzai::Filter::CustomEmojiFilter do filter('<p>:tanuki: :party-parrot:</p>') end.not_to exceed_all_query_limit(control_count.count) end + + context 'ancestor tags' do + let(:emoji_name) { ':tanuki:' } + + it_behaves_like 'ignored ancestor tags' + end end diff --git a/spec/lib/banzai/filter/emoji_filter_spec.rb b/spec/lib/banzai/filter/emoji_filter_spec.rb index 9005b4401b7..d76a36b24e1 100644 --- a/spec/lib/banzai/filter/emoji_filter_spec.rb +++ b/spec/lib/banzai/filter/emoji_filter_spec.rb @@ -117,4 +117,10 @@ RSpec.describe Banzai::Filter::EmojiFilter do expect(doc.to_html).to match(/^This deserves a <gl-emoji.+>, big time\.\z/) end + + context 'ancestor tags' do + let(:emoji_name) { ':see_no_evil:' } + + it_behaves_like 'ignored ancestor tags' + end end diff --git a/spec/lib/gitlab/experimentation_spec.rb b/spec/lib/gitlab/experimentation_spec.rb index 1cd9411f475..3df82924092 100644 --- a/spec/lib/gitlab/experimentation_spec.rb +++ b/spec/lib/gitlab/experimentation_spec.rb @@ -11,8 +11,7 @@ RSpec.describe Gitlab::Experimentation::EXPERIMENTS do :invite_members_version_a, :invite_members_version_b, :invite_members_empty_group_version_a, - :contact_sales_btn_in_app, - :group_only_trials + :contact_sales_btn_in_app ] backwards_compatible_experiment_keys = described_class.filter { |_, v| v[:use_backwards_compatible_subject_index] }.keys diff --git a/spec/requests/api/api_spec.rb b/spec/requests/api/api_spec.rb index 8bd6049e6fa..e1050d0b5f7 100644 --- a/spec/requests/api/api_spec.rb +++ b/spec/requests/api/api_spec.rb @@ -133,6 +133,28 @@ RSpec.describe API::API do end end + describe 'Marginalia comments' do + context 'GET /user/:id' do + let_it_be(:user) { create(:user) } + let(:component_map) do + { + "application" => "test", + "endpoint_id" => "/api/:version/users/:id" + } + end + + subject { ActiveRecord::QueryRecorder.new { get api("/users/#{user.id}", user) } } + + it 'generates a query that includes the expected annotations' do + expect(subject.log.last).to match(/correlation_id:.*/) + + component_map.each do |component, value| + expect(subject.log.last).to include("#{component}:#{value}") + end + end + end + end + describe 'supported content-types' do context 'GET /user/:id.txt' do let_it_be(:user) { create(:user) } diff --git a/spec/requests/api/graphql/project/merge_requests_spec.rb b/spec/requests/api/graphql/project/merge_requests_spec.rb index 12060eb51e9..ba40eec9b69 100644 --- a/spec/requests/api/graphql/project/merge_requests_spec.rb +++ b/spec/requests/api/graphql/project/merge_requests_spec.rb @@ -427,63 +427,37 @@ RSpec.describe 'getting merge request listings nested in a project' do QUERY end - shared_examples 'count examples' do - it 'returns the correct count' do - post_graphql(query, current_user: current_user) + it 'does not query the merge requests table for the count' do + query_recorder = ActiveRecord::QueryRecorder.new { post_graphql(query, current_user: current_user) } - count = graphql_data.dig('project', 'mergeRequests', 'count') - expect(count).to eq(1) - end + queries = query_recorder.data.each_value.first[:occurrences] + expect(queries).not_to include(match(/SELECT COUNT\(\*\) FROM "merge_requests"/)) + expect(queries).to include(match(/SELECT COUNT\(\*\) FROM "merge_request_metrics"/)) end - context 'when "optimized_merge_request_count_with_merged_at_filter" feature flag is enabled' do - before do - stub_feature_flags(optimized_merge_request_count_with_merged_at_filter: true) + context 'when total_time_to_merge and count is queried' do + let(:query) do + graphql_query_for(:project, { full_path: project.full_path }, <<~QUERY) + mergeRequests(mergedAfter: "2020-01-01", mergedBefore: "2020-01-05", first: 0) { + totalTimeToMerge + count + } + QUERY end - it 'does not query the merge requests table for the count' do + it 'does not query the merge requests table for the total_time_to_merge' do query_recorder = ActiveRecord::QueryRecorder.new { post_graphql(query, current_user: current_user) } queries = query_recorder.data.each_value.first[:occurrences] - expect(queries).not_to include(match(/SELECT COUNT\(\*\) FROM "merge_requests"/)) - expect(queries).to include(match(/SELECT COUNT\(\*\) FROM "merge_request_metrics"/)) - end - - context 'when total_time_to_merge and count is queried' do - let(:query) do - graphql_query_for(:project, { full_path: project.full_path }, <<~QUERY) - mergeRequests(mergedAfter: "2020-01-01", mergedBefore: "2020-01-05", first: 0) { - totalTimeToMerge - count - } - QUERY - end - - it 'does not query the merge requests table for the total_time_to_merge' do - query_recorder = ActiveRecord::QueryRecorder.new { post_graphql(query, current_user: current_user) } - - queries = query_recorder.data.each_value.first[:occurrences] - expect(queries).to include(match(/SELECT.+SUM.+FROM "merge_request_metrics" WHERE/)) - end + expect(queries).to include(match(/SELECT.+SUM.+FROM "merge_request_metrics" WHERE/)) end + end - it_behaves_like 'count examples' - - context 'when "optimized_merge_request_count_with_merged_at_filter" feature flag is disabled' do - before do - stub_feature_flags(optimized_merge_request_count_with_merged_at_filter: false) - end - - it 'queries the merge requests table for the count' do - query_recorder = ActiveRecord::QueryRecorder.new { post_graphql(query, current_user: current_user) } - - queries = query_recorder.data.each_value.first[:occurrences] - expect(queries).to include(match(/SELECT COUNT\(\*\) FROM "merge_requests"/)) - expect(queries).not_to include(match(/SELECT COUNT\(\*\) FROM "merge_request_metrics"/)) - end + it 'returns the correct count' do + post_graphql(query, current_user: current_user) - it_behaves_like 'count examples' - end + count = graphql_data.dig('project', 'mergeRequests', 'count') + expect(count).to eq(1) end end end diff --git a/spec/requests/api/v3/github_spec.rb b/spec/requests/api/v3/github_spec.rb index bcfd97d5cfc..197c6cbb0eb 100644 --- a/spec/requests/api/v3/github_spec.rb +++ b/spec/requests/api/v3/github_spec.rb @@ -187,23 +187,6 @@ RSpec.describe API::V3::Github do expect { jira_get v3_api(events_path, user) }.not_to exceed_all_query_limit(control_count) end - context 'with `api_v3_repos_events_optimization` feature flag disabled' do - before do - stub_feature_flags(api_v3_repos_events_optimization: false) - end - - it 'falls back to less optimal query performance' do - create(:merge_request, source_project: project) - source_project = fork_project(project, nil, repository: true) - - control_count = ActiveRecord::QueryRecorder.new(skip_cached: false) { jira_get v3_api(events_path, user) }.count - - create_list(:merge_request, 2, :unique_branches, source_project: source_project, target_project: project) - - expect { jira_get v3_api(events_path, user) }.to exceed_all_query_limit(control_count) - end - end - context 'if there are more merge requests' do let!(:merge_request) { create(:merge_request, id: 10000, source_project: project, target_project: project, author: user) } let!(:merge_request2) { create(:merge_request, id: 10001, source_project: project, source_branch: generate(:branch), target_project: project, author: user) } diff --git a/spec/services/issues/clone_service_spec.rb b/spec/services/issues/clone_service_spec.rb index 512a60b1382..9ceb4ffeec5 100644 --- a/spec/services/issues/clone_service_spec.rb +++ b/spec/services/issues/clone_service_spec.rb @@ -280,6 +280,12 @@ RSpec.describe Issues::CloneService do expect(new_issue.designs.first.notes.size).to eq(1) end end + + context 'issue relative position' do + let(:subject) { clone_service.execute(old_issue, new_project) } + + it_behaves_like 'copy or reset relative position' + end end describe 'clone permissions' do diff --git a/spec/services/issues/move_service_spec.rb b/spec/services/issues/move_service_spec.rb index 9b8d21bb8eb..eb124f07900 100644 --- a/spec/services/issues/move_service_spec.rb +++ b/spec/services/issues/move_service_spec.rb @@ -244,6 +244,12 @@ RSpec.describe Issues::MoveService do expect(new_issue.designs.first.notes.size).to eq(1) end end + + context 'issue relative position' do + let(:subject) { move_service.execute(old_issue, new_project) } + + it_behaves_like 'copy or reset relative position' + end end describe 'move permissions' do diff --git a/spec/support/services/issues/move_and_clone_services_shared_examples.rb b/spec/support/services/issues/move_and_clone_services_shared_examples.rb new file mode 100644 index 00000000000..2b2e90c0461 --- /dev/null +++ b/spec/support/services/issues/move_and_clone_services_shared_examples.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +RSpec.shared_examples 'copy or reset relative position' do + before do + # ensure we have a relative position and it is known + old_issue.update!(relative_position: 1000) + end + + context 'when moved to a project within same group hierarchy' do + it 'does not reset the relative_position' do + expect(subject.relative_position).to eq(1000) + end + end + + context 'when moved to a project in a different group hierarchy' do + let_it_be(:new_project) { create(:project, group: create(:group)) } + + it 'does reset the relative_position' do + expect(subject.relative_position).to be_nil + end + end +end diff --git a/spec/support/shared_examples/banzai/filters/ignored_tags_emoji_shared_examples.rb b/spec/support/shared_examples/banzai/filters/ignored_tags_emoji_shared_examples.rb new file mode 100644 index 00000000000..89d7cdc4bba --- /dev/null +++ b/spec/support/shared_examples/banzai/filters/ignored_tags_emoji_shared_examples.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +RSpec.shared_examples 'ignored ancestor tags' do + it 'does not match emoji in a pre tag' do + doc = filter("<p><pre>#{emoji_name}</pre></p>") + + expect(doc.css('img').size).to eq 0 + end + + it 'does not match emoji in code tag' do + doc = filter("<p><code>#{emoji_name} wow</code></p>") + + expect(doc.css('img').size).to eq 0 + end + + it 'does not match emoji in tt tag' do + doc = filter("<p><tt>#{emoji_name} yes!</tt></p>") + + expect(doc.css('img').size).to eq 0 + end +end diff --git a/spec/support/shared_examples/services/boards/update_boards_shared_examples.rb b/spec/support/shared_examples/services/boards/update_boards_shared_examples.rb new file mode 100644 index 00000000000..cd773a2a04a --- /dev/null +++ b/spec/support/shared_examples/services/boards/update_boards_shared_examples.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +RSpec.shared_examples 'board update service' do + subject(:service) { described_class.new(board.resource_parent, user, all_params) } + + it 'updates the board with valid params' do + result = described_class.new(group, user, name: 'Engineering').execute(board) + + expect(result).to eq(true) + expect(board.reload.name).to eq('Engineering') + end + + it 'does not update the board with invalid params' do + orig_name = board.name + + result = described_class.new(group, user, name: nil).execute(board) + + expect(result).to eq(false) + expect(board.reload.name).to eq(orig_name) + end + + context 'with scoped_issue_board available' do + before do + stub_licensed_features(scoped_issue_board: true) + end + + context 'user is member of the board parent' do + before do + board.resource_parent.add_reporter(user) + end + + it 'updates the configuration params when scoped issue board is enabled' do + service.execute(board) + + labels = updated_scoped_params.delete(:labels) + expect(board.reload).to have_attributes(updated_scoped_params) + expect(board.labels).to match_array(labels) + end + end + + context 'when labels param is used' do + let(:params) { { labels: [label.name, parent_label.name, 'new label'].join(',') } } + + subject(:service) { described_class.new(board.resource_parent, user, params) } + + context 'when user can create new labels' do + before do + board.resource_parent.add_reporter(user) + end + + it 'adds labels to the board' do + service.execute(board) + + expect(board.reload.labels.map(&:name)).to match_array([label.name, parent_label.name, 'new label']) + end + end + + context 'when user can not create new labels' do + before do + board.resource_parent.add_guest(user) + end + + it 'adds only existing labels to the board' do + service.execute(board) + + expect(board.reload.labels.map(&:name)).to match_array([label.name, parent_label.name]) + end + end + end + end + + context 'without scoped_issue_board available' do + before do + stub_licensed_features(scoped_issue_board: false) + end + + it 'filters unpermitted params when scoped issue board is not enabled' do + service.execute(board) + + expect(board.reload).to have_attributes(updated_without_scoped_params) + end + end +end diff --git a/spec/views/groups/show.html.haml_spec.rb b/spec/views/groups/show.html.haml_spec.rb deleted file mode 100644 index a53aab43c18..00000000000 --- a/spec/views/groups/show.html.haml_spec.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe 'groups/show.html.haml' do - let_it_be(:user) { build(:user) } - let_it_be(:group) { create(:group) } - - before do - assign(:group, group) - end - - context 'when rendering with the layout' do - subject(:render_page) { render template: 'groups/show.html.haml', layout: 'layouts/group' } - - describe 'invite team members' do - before do - allow(view).to receive(:session).and_return({}) - allow(view).to receive(:current_user_mode).and_return(Gitlab::Auth::CurrentUserMode.new(user)) - allow(view).to receive(:current_user).and_return(user) - allow(view).to receive(:experiment_enabled?).and_return(false) - allow(view).to receive(:group_path).and_return('') - allow(view).to receive(:group_shared_path).and_return('') - allow(view).to receive(:group_archived_path).and_return('') - end - - context 'when invite team members is not available in sidebar' do - before do - allow(view).to receive(:can_invite_members_for_group?).and_return(false) - end - - it 'does not display the js-invite-members-trigger' do - render_page - - expect(rendered).not_to have_selector('.js-invite-members-trigger') - end - end - - context 'when invite team members is available' do - before do - allow(view).to receive(:can_invite_members_for_group?).and_return(true) - end - - it 'includes the div for js-invite-members-trigger' do - render_page - - expect(rendered).to have_selector('.js-invite-members-trigger') - end - end - end - end -end diff --git a/spec/views/projects/empty.html.haml_spec.rb b/spec/views/projects/empty.html.haml_spec.rb index 6762dcd22d5..de83722160e 100644 --- a/spec/views/projects/empty.html.haml_spec.rb +++ b/spec/views/projects/empty.html.haml_spec.rb @@ -79,41 +79,4 @@ RSpec.describe 'projects/empty' do it_behaves_like 'no invite member info' end end - - context 'when rendering with the layout' do - subject(:render_page) { render template: 'projects/empty.html.haml', layout: 'layouts/project' } - - describe 'invite team members' do - before do - allow(view).to receive(:session).and_return({}) - allow(view).to receive(:current_user_mode).and_return(Gitlab::Auth::CurrentUserMode.new(user)) - allow(view).to receive(:current_user).and_return(user) - allow(view).to receive(:experiment_enabled?).and_return(false) - end - - context 'when invite team members is not available in sidebar' do - before do - allow(view).to receive(:can_invite_members_for_project?).and_return(false) - end - - it 'does not display the js-invite-members-trigger' do - render_page - - expect(rendered).not_to have_selector('.js-invite-members-trigger') - end - end - - context 'when invite team members is available' do - before do - allow(view).to receive(:can_invite_members_for_project?).and_return(true) - end - - it 'includes the div for js-invite-members-trigger' do - render_page - - expect(rendered).to have_selector('.js-invite-members-trigger') - end - end - end - end end diff --git a/spec/views/projects/show.html.haml_spec.rb b/spec/views/projects/show.html.haml_spec.rb deleted file mode 100644 index 995e31e83af..00000000000 --- a/spec/views/projects/show.html.haml_spec.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe 'projects/show.html.haml' do - let_it_be(:user) { build(:user) } - let_it_be(:project) { ProjectPresenter.new(create(:project, :repository), current_user: user) } - - before do - assign(:project, project) - end - - context 'when rendering with the layout' do - subject(:render_page) { render template: 'projects/show.html.haml', layout: 'layouts/project' } - - describe 'invite team members' do - before do - allow(view).to receive(:event_filter_link) - allow(view).to receive(:session).and_return({}) - allow(view).to receive(:current_user_mode).and_return(Gitlab::Auth::CurrentUserMode.new(user)) - allow(view).to receive(:current_user).and_return(user) - allow(view).to receive(:experiment_enabled?).and_return(false) - allow(view).to receive(:add_page_startup_graphql_call) - end - - context 'when invite team members is not available in sidebar' do - before do - allow(view).to receive(:can_invite_members_for_project?).and_return(false) - end - - it 'does not display the js-invite-members-trigger' do - render_page - - expect(rendered).not_to have_selector('.js-invite-members-trigger') - end - end - - context 'when invite team members is available' do - before do - allow(view).to receive(:can_invite_members_for_project?).and_return(true) - end - - it 'includes the div for js-invite-members-trigger' do - render_page - - expect(rendered).to have_selector('.js-invite-members-trigger') - end - end - end - end -end |