diff options
126 files changed, 2386 insertions, 611 deletions
diff --git a/.rubocop.yml b/.rubocop.yml index cd2d865d9be..bf5e3c0fb56 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -289,11 +289,6 @@ Rails/InverseOf: Rails/UniqueValidationWithoutIndex: Enabled: false -Rails/HasManyOrHasOneDependent: - Include: - - app/models/**/*.rb - - ee/app/models/**/*.rb - Rails/HelperInstanceVariable: Include: - app/helpers/**/*.rb diff --git a/.rubocop_todo/gitlab/json.yml b/.rubocop_todo/gitlab/json.yml index 5b893710725..190778a3de7 100644 --- a/.rubocop_todo/gitlab/json.yml +++ b/.rubocop_todo/gitlab/json.yml @@ -17,6 +17,31 @@ Gitlab/Json: - 'app/controllers/projects/templates_controller.rb' - 'app/controllers/projects_controller.rb' - 'app/controllers/search_controller.rb' + - 'app/helpers/access_tokens_helper.rb' + - 'app/helpers/application_settings_helper.rb' + - 'app/helpers/breadcrumbs_helper.rb' + - 'app/helpers/ci/builds_helper.rb' + - 'app/helpers/ci/pipelines_helper.rb' + - 'app/helpers/compare_helper.rb' + - 'app/helpers/emails_helper.rb' + - 'app/helpers/environment_helper.rb' + - 'app/helpers/groups_helper.rb' + - 'app/helpers/ide_helper.rb' + - 'app/helpers/integrations_helper.rb' + - 'app/helpers/invite_members_helper.rb' + - 'app/helpers/issuables_description_templates_helper.rb' + - 'app/helpers/issuables_helper.rb' + - 'app/helpers/jira_connect_helper.rb' + - 'app/helpers/learn_gitlab_helper.rb' + - 'app/helpers/namespaces_helper.rb' + - 'app/helpers/notes_helper.rb' + - 'app/helpers/operations_helper.rb' + - 'app/helpers/packages_helper.rb' + - 'app/helpers/projects/project_members_helper.rb' + - 'app/helpers/projects_helper.rb' + - 'app/helpers/search_helper.rb' + - 'app/helpers/terms_helper.rb' + - 'app/helpers/users_helper.rb' - 'app/mailers/emails/members.rb' - 'app/presenters/packages/composer/packages_presenter.rb' - 'app/presenters/projects/security/configuration_presenter.rb' diff --git a/app/assets/javascripts/boards/components/board_filtered_search.vue b/app/assets/javascripts/boards/components/board_filtered_search.vue index 11a5d89cc8c..f6601898c99 100644 --- a/app/assets/javascripts/boards/components/board_filtered_search.vue +++ b/app/assets/javascripts/boards/components/board_filtered_search.vue @@ -17,8 +17,6 @@ import eventHub from '../eventhub'; export default { i18n: { search: __('Search'), - label: __('Label'), - author: __('Author'), }, components: { FilteredSearch }, inject: ['initialFilterParams'], diff --git a/app/assets/javascripts/boards/components/issue_board_filtered_search.vue b/app/assets/javascripts/boards/components/issue_board_filtered_search.vue index bab6fe26978..84fb9a019a7 100644 --- a/app/assets/javascripts/boards/components/issue_board_filtered_search.vue +++ b/app/assets/javascripts/boards/components/issue_board_filtered_search.vue @@ -15,6 +15,13 @@ import { TOKEN_TITLE_MY_REACTION, OPERATOR_IS_AND_IS_NOT, OPERATOR_IS_ONLY, + TOKEN_TITLE_AUTHOR, + TOKEN_TITLE_ASSIGNEE, + TOKEN_TITLE_CONFIDENTIAL, + TOKEN_TITLE_LABEL, + TOKEN_TITLE_MILESTONE, + TOKEN_TITLE_RELEASE, + TOKEN_TITLE_TYPE, } from '~/vue_shared/components/filtered_search_bar/constants'; import AuthorToken from '~/vue_shared/components/filtered_search_bar/tokens/author_token.vue'; import EmojiToken from '~/vue_shared/components/filtered_search_bar/tokens/emoji_token.vue'; @@ -28,17 +35,8 @@ export default { INCIDENT: 'INCIDENT', }, i18n: { - search: __('Search'), - epic: __('Epic'), - label: __('Label'), - author: __('Author'), - assignee: __('Assignee'), - type: __('Type'), incident: __('Incident'), issue: __('Issue'), - milestone: __('Milestone'), - release: __('Release'), - confidential: __('Confidential'), }, components: { BoardFilteredSearch }, inject: ['isSignedIn', 'releasesFetchPath', 'fullPath', 'boardType'], @@ -52,17 +50,7 @@ export default { : this.fullPath.slice(0, this.fullPath.lastIndexOf('/')); }, tokensCE() { - const { - label, - author, - assignee, - issue, - incident, - type, - milestone, - release, - confidential, - } = this.$options.i18n; + const { issue, incident } = this.$options.i18n; const { types } = this.$options; const { fetchAuthors, fetchLabels } = issueBoardFilters( this.$apollo, @@ -73,7 +61,7 @@ export default { const tokens = [ { icon: 'user', - title: assignee, + title: TOKEN_TITLE_ASSIGNEE, type: 'assignee', operators: OPERATOR_IS_AND_IS_NOT, token: AuthorToken, @@ -83,7 +71,7 @@ export default { }, { icon: 'pencil', - title: author, + title: TOKEN_TITLE_AUTHOR, type: 'author', operators: OPERATOR_IS_AND_IS_NOT, symbol: '@', @@ -94,7 +82,7 @@ export default { }, { icon: 'labels', - title: label, + title: TOKEN_TITLE_LABEL, type: 'label', operators: OPERATOR_IS_AND_IS_NOT, token: LabelToken, @@ -129,7 +117,7 @@ export default { { type: 'confidential', icon: 'eye-slash', - title: confidential, + title: TOKEN_TITLE_CONFIDENTIAL, unique: true, token: GlFilteredSearchToken, operators: OPERATOR_IS_ONLY, @@ -142,7 +130,7 @@ export default { : []), { type: 'milestone', - title: milestone, + title: TOKEN_TITLE_MILESTONE, icon: 'clock', symbol: '%', token: MilestoneToken, @@ -152,7 +140,7 @@ export default { }, { icon: 'issues', - title: type, + title: TOKEN_TITLE_TYPE, type: 'type', token: GlFilteredSearchToken, unique: true, @@ -163,7 +151,7 @@ export default { }, { type: 'release', - title: release, + title: TOKEN_TITLE_RELEASE, icon: 'rocket', token: ReleaseToken, fetchReleases: (search) => { diff --git a/app/assets/javascripts/ci/runner/components/search_tokens/status_token_config.js b/app/assets/javascripts/ci/runner/components/search_tokens/status_token_config.js index f5c42d120fb..117a630719e 100644 --- a/app/assets/javascripts/ci/runner/components/search_tokens/status_token_config.js +++ b/app/assets/javascripts/ci/runner/components/search_tokens/status_token_config.js @@ -1,5 +1,7 @@ -import { __ } from '~/locale'; -import { OPERATOR_IS_ONLY } from '~/vue_shared/components/filtered_search_bar/constants'; +import { + OPERATOR_IS_ONLY, + TOKEN_TITLE_STATUS, +} from '~/vue_shared/components/filtered_search_bar/constants'; import BaseToken from '~/vue_shared/components/filtered_search_bar/tokens/base_token.vue'; import { I18N_STATUS_ONLINE, @@ -22,7 +24,7 @@ const options = [ export const statusTokenConfig = { icon: 'status', - title: __('Status'), + title: TOKEN_TITLE_STATUS, type: PARAM_KEY_STATUS, token: BaseToken, unique: true, diff --git a/app/assets/javascripts/cycle_analytics/components/filter_bar.vue b/app/assets/javascripts/cycle_analytics/components/filter_bar.vue index 016fea354fe..0ad325a8523 100644 --- a/app/assets/javascripts/cycle_analytics/components/filter_bar.vue +++ b/app/assets/javascripts/cycle_analytics/components/filter_bar.vue @@ -1,9 +1,12 @@ <script> import { mapActions, mapState } from 'vuex'; -import { __ } from '~/locale'; import { OPERATOR_IS_ONLY, DEFAULT_NONE_ANY, + TOKEN_TITLE_ASSIGNEE, + TOKEN_TITLE_AUTHOR, + TOKEN_TITLE_LABEL, + TOKEN_TITLE_MILESTONE, } from '~/vue_shared/components/filtered_search_bar/constants'; import FilteredSearchBar from '~/vue_shared/components/filtered_search_bar/filtered_search_bar_root.vue'; import { @@ -43,7 +46,7 @@ export default { return [ { icon: 'clock', - title: __('Milestone'), + title: TOKEN_TITLE_MILESTONE, type: 'milestone', token: MilestoneToken, initialMilestones: this.milestonesData, @@ -54,7 +57,7 @@ export default { }, { icon: 'labels', - title: __('Label'), + title: TOKEN_TITLE_LABEL, type: 'labels', token: LabelToken, defaultLabels: DEFAULT_NONE_ANY, @@ -66,7 +69,7 @@ export default { }, { icon: 'pencil', - title: __('Author'), + title: TOKEN_TITLE_AUTHOR, type: 'author', token: AuthorToken, initialAuthors: this.authorsData, @@ -76,7 +79,7 @@ export default { }, { icon: 'user', - title: __('Assignees'), + title: TOKEN_TITLE_ASSIGNEE, type: 'assignees', token: AuthorToken, initialAuthors: this.assigneesData, diff --git a/app/assets/javascripts/filtered_search/issuable_filtered_search_token_keys.js b/app/assets/javascripts/filtered_search/issuable_filtered_search_token_keys.js index acb7449f830..d6e7887f93f 100644 --- a/app/assets/javascripts/filtered_search/issuable_filtered_search_token_keys.js +++ b/app/assets/javascripts/filtered_search/issuable_filtered_search_token_keys.js @@ -1,10 +1,18 @@ import { flattenDeep } from 'lodash'; -import { __, s__ } from '~/locale'; +import { __ } from '~/locale'; +import { + TOKEN_TITLE_ASSIGNEE, + TOKEN_TITLE_AUTHOR, + TOKEN_TITLE_LABEL, + TOKEN_TITLE_MILESTONE, + TOKEN_TITLE_MY_REACTION, + TOKEN_TITLE_RELEASE, +} from '~/vue_shared/components/filtered_search_bar/constants'; import FilteredSearchTokenKeys from './filtered_search_token_keys'; export const tokenKeys = [ { - formattedKey: __('Author'), + formattedKey: TOKEN_TITLE_AUTHOR, key: 'author', type: 'string', param: 'username', @@ -13,7 +21,7 @@ export const tokenKeys = [ tag: '@author', }, { - formattedKey: s__('SearchToken|Assignee'), + formattedKey: TOKEN_TITLE_ASSIGNEE, key: 'assignee', type: 'string', param: 'username', @@ -22,7 +30,7 @@ export const tokenKeys = [ tag: '@assignee', }, { - formattedKey: __('Milestone'), + formattedKey: TOKEN_TITLE_MILESTONE, key: 'milestone', type: 'string', param: 'title', @@ -31,7 +39,7 @@ export const tokenKeys = [ tag: '%milestone', }, { - formattedKey: __('Release'), + formattedKey: TOKEN_TITLE_RELEASE, key: 'release', type: 'string', param: 'tag', @@ -40,7 +48,7 @@ export const tokenKeys = [ tag: __('tag name'), }, { - formattedKey: __('Label'), + formattedKey: TOKEN_TITLE_LABEL, key: 'label', type: 'array', param: 'name[]', @@ -53,7 +61,7 @@ export const tokenKeys = [ if (gon.current_user_id) { // Appending tokenkeys only logged-in tokenKeys.push({ - formattedKey: __('My-Reaction'), + formattedKey: TOKEN_TITLE_MY_REACTION, key: 'my-reaction', type: 'string', param: 'emoji', @@ -65,7 +73,7 @@ if (gon.current_user_id) { export const alternativeTokenKeys = [ { - formattedKey: __('Label'), + formattedKey: TOKEN_TITLE_LABEL, key: 'label', type: 'string', param: 'name', diff --git a/app/assets/javascripts/packages_and_registries/infrastructure_registry/list/components/packages_list.vue b/app/assets/javascripts/packages_and_registries/infrastructure_registry/list/components/packages_list.vue index 325fd8c1946..707e8f09045 100644 --- a/app/assets/javascripts/packages_and_registries/infrastructure_registry/list/components/packages_list.vue +++ b/app/assets/javascripts/packages_and_registries/infrastructure_registry/list/components/packages_list.vue @@ -1,7 +1,6 @@ <script> import { GlPagination } from '@gitlab/ui'; import { mapState, mapGetters } from 'vuex'; -import { s__ } from '~/locale'; import Tracking from '~/tracking'; import DeletePackageModal from '~/packages_and_registries/shared/components/delete_package_modal.vue'; import PackagesListRow from '~/packages_and_registries/infrastructure_registry/shared/package_list_row.vue'; @@ -63,11 +62,6 @@ export default { this.itemToBeDeleted = null; }, }, - i18n: { - deleteModalContent: s__('PackageRegistry|You are about to delete %{name}, are you sure?'), - modalTitle: s__('PackageRegistry|Delete package'), - modalAction: s__('PackageRegistry|Permanently delete'), - }, }; </script> diff --git a/app/assets/javascripts/vue_merge_request_widget/components/states/merge_checks_failed.vue b/app/assets/javascripts/vue_merge_request_widget/components/states/merge_checks_failed.vue index 2b22033514f..38b99dae264 100644 --- a/app/assets/javascripts/vue_merge_request_widget/components/states/merge_checks_failed.vue +++ b/app/assets/javascripts/vue_merge_request_widget/components/states/merge_checks_failed.vue @@ -9,6 +9,7 @@ export default { blockingMergeRequests: s__( 'mrWidget|Merge blocked: you can only merge after the above items are resolved.', ), + externalStatusChecksFailed: s__('mrWidget|Merge blocked: all status checks must pass.'), }, components: { StatusIcon, @@ -25,6 +26,8 @@ export default { return this.$options.i18n.approvalNeeded; } else if (this.mr.detailedMergeStatus === DETAILED_MERGE_STATUS.BLOCKED_STATUS) { return this.$options.i18n.blockingMergeRequests; + } else if (this.mr.detailedMergeStatus === DETAILED_MERGE_STATUS.EXTERNAL_STATUS_CHECKS) { + return this.$options.i18n.externalStatusChecksFailed; } return null; diff --git a/app/assets/javascripts/vue_merge_request_widget/constants.js b/app/assets/javascripts/vue_merge_request_widget/constants.js index c6baf3b46ff..7109bed7743 100644 --- a/app/assets/javascripts/vue_merge_request_widget/constants.js +++ b/app/assets/javascripts/vue_merge_request_widget/constants.js @@ -192,4 +192,5 @@ export const DETAILED_MERGE_STATUS = { POLICIES_DENIED: 'POLICIES_DENIED', CI_MUST_PASS: 'CI_MUST_PASS', CI_STILL_RUNNING: 'CI_STILL_RUNNING', + EXTERNAL_STATUS_CHECKS: 'EXTERNAL_STATUS_CHECKS', }; diff --git a/app/assets/javascripts/vue_shared/components/filtered_search_bar/constants.js b/app/assets/javascripts/vue_shared/components/filtered_search_bar/constants.js index 4873996d357..e23a2dc91f1 100644 --- a/app/assets/javascripts/vue_shared/components/filtered_search_bar/constants.js +++ b/app/assets/javascripts/vue_shared/components/filtered_search_bar/constants.js @@ -55,6 +55,9 @@ export const TOKEN_TITLE_MILESTONE = __('Milestone'); export const TOKEN_TITLE_MY_REACTION = __('My-Reaction'); export const TOKEN_TITLE_ORGANIZATION = s__('Crm|Organization'); export const TOKEN_TITLE_RELEASE = __('Release'); +export const TOKEN_TITLE_SOURCE_BRANCH = __('Source Branch'); +export const TOKEN_TITLE_STATUS = __('Status'); +export const TOKEN_TITLE_TARGET_BRANCH = __('Target Branch'); export const TOKEN_TITLE_TYPE = __('Type'); // As health status gets reused between issue lists and boards diff --git a/app/assets/javascripts/vue_shared/components/paginated_table_with_search_and_tabs/paginated_table_with_search_and_tabs.vue b/app/assets/javascripts/vue_shared/components/paginated_table_with_search_and_tabs/paginated_table_with_search_and_tabs.vue index a5027d2ca5c..867222279b2 100644 --- a/app/assets/javascripts/vue_shared/components/paginated_table_with_search_and_tabs/paginated_table_with_search_and_tabs.vue +++ b/app/assets/javascripts/vue_shared/components/paginated_table_with_search_and_tabs/paginated_table_with_search_and_tabs.vue @@ -9,9 +9,12 @@ import { } from '@gitlab/ui'; import Api from '~/api'; import { updateHistory, setUrlParams } from '~/lib/utils/url_utility'; -import { __ } from '~/locale'; import Tracking from '~/tracking'; -import { OPERATOR_IS_ONLY } from '~/vue_shared/components/filtered_search_bar/constants'; +import { + OPERATOR_IS_ONLY, + TOKEN_TITLE_ASSIGNEE, + TOKEN_TITLE_AUTHOR, +} from '~/vue_shared/components/filtered_search_bar/constants'; import FilteredSearchBar from '~/vue_shared/components/filtered_search_bar/filtered_search_bar_root.vue'; import AuthorToken from '~/vue_shared/components/filtered_search_bar/tokens/author_token.vue'; import { initialPaginationState, defaultI18n, defaultPageSize } from './constants'; @@ -112,7 +115,7 @@ export default { { type: 'author_username', icon: 'user', - title: __('Author'), + title: TOKEN_TITLE_AUTHOR, unique: true, symbol: '@', token: AuthorToken, @@ -123,7 +126,7 @@ export default { { type: 'assignee_username', icon: 'user', - title: __('Assignee'), + title: TOKEN_TITLE_ASSIGNEE, unique: true, symbol: '@', token: AuthorToken, diff --git a/app/helpers/access_tokens_helper.rb b/app/helpers/access_tokens_helper.rb index 07e61b7f552..44200e84afb 100644 --- a/app/helpers/access_tokens_helper.rb +++ b/app/helpers/access_tokens_helper.rb @@ -9,7 +9,7 @@ module AccessTokensHelper end def tokens_app_data - data = { + { feed_token: { enabled: !Gitlab::CurrentSettings.disable_feed_token, token: current_user.feed_token, @@ -25,9 +25,7 @@ module AccessTokensHelper token: current_user.enabled_static_object_token, reset_path: reset_static_object_token_profile_path } - } - - Gitlab::Json.dump(data) + }.to_json end def expires_at_field_data diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index 42aad60d1ba..7f13f609353 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -137,7 +137,7 @@ module ApplicationSettingsHelper } end - Gitlab::Json.dump(options) + options.to_json end def external_authorization_description diff --git a/app/helpers/breadcrumbs_helper.rb b/app/helpers/breadcrumbs_helper.rb index a14bc8e00bf..38ed6e95a44 100644 --- a/app/helpers/breadcrumbs_helper.rb +++ b/app/helpers/breadcrumbs_helper.rb @@ -40,11 +40,11 @@ module BreadcrumbsHelper end def schema_breadcrumb_json - Gitlab::Json.dump({ + { '@context': 'https://schema.org', '@type': 'BreadcrumbList', 'itemListElement': build_item_list_elements - }) + }.to_json end private diff --git a/app/helpers/ci/builds_helper.rb b/app/helpers/ci/builds_helper.rb index 265969a6370..afd0af18ba7 100644 --- a/app/helpers/ci/builds_helper.rb +++ b/app/helpers/ci/builds_helper.rb @@ -38,13 +38,13 @@ module Ci end def prepare_failed_jobs_summary_data(failed_builds) - Gitlab::Json.dump(failed_builds.map do |build| + failed_builds.map do |build| { id: build.id, failure: build.present.callout_failure_message, failure_summary: build_summary(build) } - end) + end.to_json end end end diff --git a/app/helpers/ci/pipelines_helper.rb b/app/helpers/ci/pipelines_helper.rb index 80632f2ea10..c93c8dd8d76 100644 --- a/app/helpers/ci/pipelines_helper.rb +++ b/app/helpers/ci/pipelines_helper.rb @@ -87,7 +87,7 @@ module Ci endpoint: list_url, project_id: project.id, default_branch_name: project.default_branch, - params: Gitlab::Json.dump(params), + params: params.to_json, artifacts_endpoint: downloadable_artifacts_project_pipeline_path(project, artifacts_endpoint_placeholder, format: :json), artifacts_endpoint_placeholder: artifacts_endpoint_placeholder, pipeline_schedule_url: pipeline_schedules_path(project), @@ -100,7 +100,7 @@ module Ci reset_cache_path: can?(current_user, :admin_pipeline, project) && reset_cache_project_settings_ci_cd_path(project), has_gitlab_ci: has_gitlab_ci?(project).to_s, pipeline_editor_path: can?(current_user, :create_pipeline, project) && project_ci_pipeline_editor_path(project), - suggested_ci_templates: Gitlab::Json.dump(suggested_ci_templates), + suggested_ci_templates: suggested_ci_templates.to_json, ci_runner_settings_path: project_settings_ci_cd_path(project, ci_runner_templates: true, anchor: 'js-runners-settings') } diff --git a/app/helpers/compare_helper.rb b/app/helpers/compare_helper.rb index 91f8567aa2d..9ecf780f55b 100644 --- a/app/helpers/compare_helper.rb +++ b/app/helpers/compare_helper.rb @@ -37,17 +37,17 @@ module CompareHelper def project_compare_selector_data(project, merge_request, params) { project_compare_index_path: project_compare_index_path(project), - source_project: Gitlab::Json.dump({ id: project.id, name: project.full_path }), - target_project: Gitlab::Json.dump({ id: @target_project.id, name: @target_project.full_path }), + source_project: { id: project.id, name: project.full_path }.to_json, + target_project: { id: @target_project.id, name: @target_project.full_path }.to_json, source_project_refs_path: refs_project_path(project), target_project_refs_path: refs_project_path(@target_project), params_from: params[:from], params_to: params[:to], straight: params[:straight] }.tap do |data| - data[:projects_from] = Gitlab::Json.dump(target_projects(project).map do |target_project| + data[:projects_from] = target_projects(project).map do |target_project| { id: target_project.id, name: target_project.full_path } - end) + end.to_json data[:project_merge_request_path] = if merge_request.present? diff --git a/app/helpers/emails_helper.rb b/app/helpers/emails_helper.rb index fe3324120eb..54733fa9101 100644 --- a/app/helpers/emails_helper.rb +++ b/app/helpers/emails_helper.rb @@ -36,7 +36,7 @@ module EmailsHelper } content_tag :script, type: 'application/ld+json' do - Gitlab::Json.dump(data).html_safe + data.to_json.html_safe end end diff --git a/app/helpers/environment_helper.rb b/app/helpers/environment_helper.rb index 677892e7d32..b6997b6fb70 100644 --- a/app/helpers/environment_helper.rb +++ b/app/helpers/environment_helper.rb @@ -91,6 +91,6 @@ module EnvironmentHelper end def environments_detail_data_json(user, project, environment) - Gitlab::Json.dump(environments_detail_data(user, project, environment)) + environments_detail_data(user, project, environment).to_json end end diff --git a/app/helpers/ide_helper.rb b/app/helpers/ide_helper.rb index 69a9519291f..34f4749c42a 100644 --- a/app/helpers/ide_helper.rb +++ b/app/helpers/ide_helper.rb @@ -58,7 +58,7 @@ module IdeHelper def convert_to_project_entity_json(project) return unless project - Gitlab::Json.dump(API::Entities::Project.represent(project, current_user: current_user)) + API::Entities::Project.represent(project, current_user: current_user).to_json end def enable_environments_guidance? diff --git a/app/helpers/integrations_helper.rb b/app/helpers/integrations_helper.rb index 14eec407d15..abfa55cff24 100644 --- a/app/helpers/integrations_helper.rb +++ b/app/helpers/integrations_helper.rb @@ -114,7 +114,7 @@ module IntegrationsHelper learn_more_path: integrations_help_page_path, about_pricing_url: Gitlab::Saas.about_pricing_url, trigger_events: trigger_events_for_integration(integration), - sections: Gitlab::Json.dump(integration.sections), + sections: integration.sections.to_json, fields: fields_for_integration(integration), inherit_from_id: integration.inherit_from_id, integration_level: integration_level(integration), @@ -144,7 +144,7 @@ module IntegrationsHelper def integration_list_data(integrations, group: nil, project: nil) { - integrations: Gitlab::Json.dump(integrations.map { |i| serialize_integration(i, group: group, project: project) }) + integrations: integrations.map { |i| serialize_integration(i, group: group, project: project) }.to_json } end @@ -237,15 +237,11 @@ module IntegrationsHelper end def trigger_events_for_integration(integration) - serializer = Integrations::EventSerializer.new(integration: integration).represent(integration.configurable_events) - - Gitlab::Json.dump(serializer) + Integrations::EventSerializer.new(integration: integration).represent(integration.configurable_events).to_json end def fields_for_integration(integration) - serializer = Integrations::FieldSerializer.new(integration: integration).represent(integration.form_fields) - - Gitlab::Json.dump(serializer) + Integrations::FieldSerializer.new(integration: integration).represent(integration.form_fields).to_json end def integration_level(integration) diff --git a/app/helpers/invite_members_helper.rb b/app/helpers/invite_members_helper.rb index 8672dfb39dc..5d537767eaf 100644 --- a/app/helpers/invite_members_helper.rb +++ b/app/helpers/invite_members_helper.rb @@ -29,7 +29,7 @@ module InviteMembersHelper invalid_groups: source.related_group_ids, help_link: help_page_url('user/permissions'), is_project: is_project, - access_levels: Gitlab::Json.dump(member_class.permissible_access_level_roles(current_user, source)) + access_levels: member_class.permissible_access_level_roles(current_user, source).to_json }.merge(group_select_data(source)) end @@ -44,8 +44,8 @@ module InviteMembersHelper if show_invite_members_for_task?(source) dataset.merge!( - tasks_to_be_done_options: Gitlab::Json.dump(tasks_to_be_done_options), - projects: Gitlab::Json.dump(projects_for_source(source)), + tasks_to_be_done_options: tasks_to_be_done_options.to_json, + projects: projects_for_source(source).to_json, new_project_path: source.is_a?(Group) ? new_project_path(namespace_id: source.id) : '' ) end diff --git a/app/helpers/issuables_description_templates_helper.rb b/app/helpers/issuables_description_templates_helper.rb index 075b2710722..58b86dca1e0 100644 --- a/app/helpers/issuables_description_templates_helper.rb +++ b/app/helpers/issuables_description_templates_helper.rb @@ -53,7 +53,7 @@ module IssuablesDescriptionTemplatesHelper end def available_service_desk_templates_for(project) - Gitlab::Json.dump(issuable_templates(project, 'issue').flatten) + issuable_templates(project, 'issue').flatten.to_json end def template_names_path(parent, issuable) diff --git a/app/helpers/issuables_helper.rb b/app/helpers/issuables_helper.rb index 0a5d02cbcc7..e9e6241a6a7 100644 --- a/app/helpers/issuables_helper.rb +++ b/app/helpers/issuables_helper.rb @@ -73,9 +73,10 @@ module IssuablesHelper MergeRequestSerializer end - Gitlab::Json.dump(serializer_klass + serializer_klass .new(current_user: current_user, project: issuable.project) - .represent(issuable, opts)) + .represent(issuable, opts) + .to_json end def users_dropdown_label(selected_users) @@ -440,7 +441,7 @@ module IssuablesHelper labels_manage_path: project_labels_path(project), project_issues_path: issuable_sidebar[:project_issuables_path], project_path: project.full_path, - selected_labels: Gitlab::Json.dump(issuable_sidebar[:labels]) + selected_labels: issuable_sidebar[:labels].to_json } end diff --git a/app/helpers/jira_connect_helper.rb b/app/helpers/jira_connect_helper.rb index 70e1f7ca4f6..0971fdae8dd 100644 --- a/app/helpers/jira_connect_helper.rb +++ b/app/helpers/jira_connect_helper.rb @@ -6,12 +6,12 @@ module JiraConnectHelper { groups_path: api_v4_groups_path(params: { min_access_level: Gitlab::Access::MAINTAINER, skip_groups: skip_groups }), - subscriptions: Gitlab::Json.dump(subscriptions.map { |s| serialize_subscription(s) }), + subscriptions: subscriptions.map { |s| serialize_subscription(s) }.to_json, add_subscriptions_path: jira_connect_subscriptions_path, subscriptions_path: jira_connect_subscriptions_path(format: :json), users_path: current_user ? nil : jira_connect_users_path, # users_path is used to determine if user is signed in gitlab_user_path: current_user ? user_path(current_user) : nil, - oauth_metadata: Feature.enabled?(:jira_connect_oauth, current_user) ? Gitlab::Json.dump(jira_connect_oauth_data(installation)) : nil + oauth_metadata: Feature.enabled?(:jira_connect_oauth, current_user) ? jira_connect_oauth_data(installation).to_json : nil } end diff --git a/app/helpers/learn_gitlab_helper.rb b/app/helpers/learn_gitlab_helper.rb index 485a85cc73f..a07922e451a 100644 --- a/app/helpers/learn_gitlab_helper.rb +++ b/app/helpers/learn_gitlab_helper.rb @@ -14,9 +14,9 @@ module LearnGitlabHelper def learn_gitlab_data(project) { - actions: Gitlab::Json.dump(onboarding_actions_data(project)), - sections: Gitlab::Json.dump(onboarding_sections_data), - project: Gitlab::Json.dump(onboarding_project_data(project)) + actions: onboarding_actions_data(project).to_json, + sections: onboarding_sections_data.to_json, + project: onboarding_project_data(project).to_json } end diff --git a/app/helpers/namespaces_helper.rb b/app/helpers/namespaces_helper.rb index 3d4f7254c18..60796e628a3 100644 --- a/app/helpers/namespaces_helper.rb +++ b/app/helpers/namespaces_helper.rb @@ -67,7 +67,7 @@ module NamespacesHelper end { - popover_data: Gitlab::Json.dump(popover_data), + popover_data: popover_data.to_json, testid: 'cascading-settings-lock-icon' } end diff --git a/app/helpers/notes_helper.rb b/app/helpers/notes_helper.rb index c0d26d4759d..b47f4633348 100644 --- a/app/helpers/notes_helper.rb +++ b/app/helpers/notes_helper.rb @@ -22,14 +22,12 @@ module NotesHelper end def noteable_json(noteable) - data = { + { id: noteable.id, class: noteable.class.name, resources: noteable.class.table_name, project_id: noteable.project.id - } - - Gitlab::Json.dump(data) + }.to_json end def diff_view_data diff --git a/app/helpers/operations_helper.rb b/app/helpers/operations_helper.rb index 542b8260d20..baeb9a477c3 100644 --- a/app/helpers/operations_helper.rb +++ b/app/helpers/operations_helper.rb @@ -26,7 +26,7 @@ module OperationsHelper 'disabled' => disabled.to_s, 'project_path' => @project.full_path, 'multi_integrations' => 'false', - 'templates' => Gitlab::Json.dump(templates), + 'templates' => templates.to_json, 'create_issue' => setting.create_issue.to_s, 'issue_template_key' => setting.issue_template_key.to_s, 'send_email' => setting.send_email.to_s, diff --git a/app/helpers/packages_helper.rb b/app/helpers/packages_helper.rb index 9f72acf40b5..f9ec20bdd01 100644 --- a/app/helpers/packages_helper.rb +++ b/app/helpers/packages_helper.rb @@ -24,7 +24,7 @@ module PackagesHelper def package_from_presenter(package) presenter = ::Packages::Detail::PackagePresenter.new(package) - Gitlab::Json.dump(presenter.detail_view) + presenter.detail_view.to_json end def pypi_registry_url(project_id) @@ -68,9 +68,9 @@ module PackagesHelper { project_id: @project.id, project_path: @project.full_path, - cadence_options: Gitlab::Json.dump(cadence_options), - keep_n_options: Gitlab::Json.dump(keep_n_options), - older_than_options: Gitlab::Json.dump(older_than_options), + cadence_options: cadence_options.to_json, + keep_n_options: keep_n_options.to_json, + older_than_options: older_than_options.to_json, is_admin: current_user&.admin.to_s, admin_settings_path: ci_cd_admin_application_settings_path(anchor: 'js-registry-settings'), project_settings_path: project_settings_packages_and_registries_path(@project), diff --git a/app/helpers/projects/project_members_helper.rb b/app/helpers/projects/project_members_helper.rb index 540877e3fb8..6026124abb9 100644 --- a/app/helpers/projects/project_members_helper.rb +++ b/app/helpers/projects/project_members_helper.rb @@ -2,7 +2,7 @@ module Projects::ProjectMembersHelper def project_members_app_data_json(project, members:, invited:, access_requests:, include_relations:, search:) - Gitlab::Json.dump({ + { user: project_members_list_data(project, members, { param_name: :page, params: { search_groups: nil } }), group: project_group_links_list_data(project, include_relations, search), invite: project_members_list_data(project, invited.nil? ? [] : invited), @@ -10,7 +10,7 @@ module Projects::ProjectMembersHelper source_id: project.id, can_manage_members: Ability.allowed?(current_user, :admin_project_member, project), can_manage_access_requests: Ability.allowed?(current_user, :admin_member_access_request, project) - }) + }.to_json end def project_member_header_subtext(project) diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index 4448dd6caac..e41a3fa5091 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -300,15 +300,13 @@ module ProjectsHelper return if setting.blank? || setting.project_slug.blank? || setting.organization_slug.blank? - data = { + { sentry_project_id: setting.sentry_project_id, name: setting.project_name, organization_name: setting.organization_name, organization_slug: setting.organization_slug, slug: setting.project_slug - } - - Gitlab::Json.dump(data) + }.to_json end def directory? diff --git a/app/helpers/search_helper.rb b/app/helpers/search_helper.rb index f244a6bc17f..1ac219d4819 100644 --- a/app/helpers/search_helper.rb +++ b/app/helpers/search_helper.rb @@ -447,9 +447,9 @@ module SearchHelper def search_navigation_json sorted_navigation = search_navigation.sort_by { |_, h| h[:sort] } - Gitlab::Json.dump(sorted_navigation.each_with_object({}) do |(key, value), hash| + sorted_navigation.each_with_object({}) do |(key, value), hash| hash[key] = search_filter_link_json(key, value[:label], value[:data], value[:search]) if value[:condition] - end) + end.to_json end def search_filter_input_options(type, placeholder = _('Search or filter results...')) diff --git a/app/helpers/terms_helper.rb b/app/helpers/terms_helper.rb index 1dc5e4f1974..5f321551413 100644 --- a/app/helpers/terms_helper.rb +++ b/app/helpers/terms_helper.rb @@ -4,7 +4,7 @@ module TermsHelper def terms_data(terms, redirect) redirect_params = { redirect: redirect } if redirect - data = { + { terms: markdown_field(terms, :terms), permissions: { can_accept: can?(current_user, :accept_terms, terms), @@ -15,8 +15,6 @@ module TermsHelper decline: decline_term_path(terms, redirect_params), root: root_path } - } - - Gitlab::Json.dump(data) + }.to_json end end diff --git a/app/helpers/users_helper.rb b/app/helpers/users_helper.rb index d66942c2884..4f345fdeb9c 100644 --- a/app/helpers/users_helper.rb +++ b/app/helpers/users_helper.rb @@ -3,8 +3,8 @@ module UsersHelper def admin_users_data_attributes(users) { - users: Gitlab::Json.dump(Admin::UserSerializer.new.represent(users, { current_user: current_user })), - paths: Gitlab::Json.dump(admin_users_paths) + users: Admin::UserSerializer.new.represent(users, { current_user: current_user }).to_json, + paths: admin_users_paths.to_json } end @@ -163,8 +163,8 @@ module UsersHelper def admin_user_actions_data_attributes(user) { - user: Gitlab::Json.dump(Admin::UserEntity.represent(user, { current_user: current_user })), - paths: Gitlab::Json.dump(admin_users_paths) + user: Admin::UserEntity.represent(user, { current_user: current_user }).to_json, + paths: admin_users_paths.to_json } end diff --git a/app/models/clusters/applications/cert_manager.rb b/app/models/clusters/applications/cert_manager.rb index 2a051233de2..11f84940c38 100644 --- a/app/models/clusters/applications/cert_manager.rb +++ b/app/models/clusters/applications/cert_manager.rb @@ -15,11 +15,8 @@ module Clusters include ::Clusters::Concerns::ApplicationVersion include ::Clusters::Concerns::ApplicationData - default_value_for :version, VERSION - - default_value_for :email do |cert_manager| - cert_manager.cluster&.user&.email - end + attribute :version, default: VERSION + after_initialize :set_default_email, if: :new_record? validates :email, presence: true @@ -55,6 +52,10 @@ module Clusters private + def set_default_email + self.email ||= self.cluster&.user&.email + end + def pre_install_script [ apply_file("https://raw.githubusercontent.com/jetstack/cert-manager/release-#{CRD_VERSION}/deploy/manifests/00-crds.yaml"), diff --git a/app/models/clusters/applications/crossplane.rb b/app/models/clusters/applications/crossplane.rb index 07378b4e8dc..a7b4fb57149 100644 --- a/app/models/clusters/applications/crossplane.rb +++ b/app/models/clusters/applications/crossplane.rb @@ -14,11 +14,8 @@ module Clusters include ::Clusters::Concerns::ApplicationVersion include ::Clusters::Concerns::ApplicationData - default_value_for :version, VERSION - - default_value_for :stack do |crossplane| - '' - end + attribute :version, default: VERSION + attribute :stack, default: "" validates :stack, presence: true diff --git a/app/models/clusters/applications/helm.rb b/app/models/clusters/applications/helm.rb index e89cb8be1e7..9fac852ed5b 100644 --- a/app/models/clusters/applications/helm.rb +++ b/app/models/clusters/applications/helm.rb @@ -18,7 +18,7 @@ module Clusters include ::Clusters::Concerns::ApplicationStatus include ::Gitlab::Utils::StrongMemoize - default_value_for :version, Gitlab::Kubernetes::Helm::V2::BaseCommand::HELM_VERSION + attribute :version, default: Gitlab::Kubernetes::Helm::V2::BaseCommand::HELM_VERSION before_create :create_keys_and_certs diff --git a/app/models/clusters/applications/ingress.rb b/app/models/clusters/applications/ingress.rb index 27550616002..034b178d67d 100644 --- a/app/models/clusters/applications/ingress.rb +++ b/app/models/clusters/applications/ingress.rb @@ -17,12 +17,11 @@ module Clusters include AfterCommitQueue include UsageStatistics - default_value_for :ingress_type, :nginx - default_value_for :version, VERSION + attribute :version, default: VERSION enum ingress_type: { nginx: 1 - } + }, _default: :nginx FETCH_IP_ADDRESS_DELAY = 30.seconds diff --git a/app/models/clusters/applications/jupyter.rb b/app/models/clusters/applications/jupyter.rb index 8d7d9c20bfa..9c0e90d59ed 100644 --- a/app/models/clusters/applications/jupyter.rb +++ b/app/models/clusters/applications/jupyter.rb @@ -18,7 +18,7 @@ module Clusters belongs_to :oauth_application, class_name: 'Doorkeeper::Application' - default_value_for :version, VERSION + attribute :version, default: VERSION def set_initial_status return unless not_installable? diff --git a/app/models/clusters/applications/knative.rb b/app/models/clusters/applications/knative.rb index 0e7cbb35e47..64366594583 100644 --- a/app/models/clusters/applications/knative.rb +++ b/app/models/clusters/applications/knative.rb @@ -43,7 +43,7 @@ module Clusters end end - default_value_for :version, VERSION + attribute :version, default: VERSION validates :hostname, presence: true, hostname: true diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb index d1e169a1f78..a076c871824 100644 --- a/app/models/clusters/applications/prometheus.rb +++ b/app/models/clusters/applications/prometheus.rb @@ -15,7 +15,7 @@ module Clusters include ::Clusters::Concerns::ApplicationData include AfterCommitQueue - default_value_for :version, VERSION + attribute :version, default: VERSION scope :preload_cluster_platform, -> { preload(cluster: [:platform_kubernetes]) } @@ -24,7 +24,7 @@ module Clusters key: Settings.attr_encrypted_db_key_base_32, algorithm: 'aes-256-gcm' - default_value_for(:alert_manager_token) { SecureRandom.hex } + after_initialize :set_alert_manager_token, if: :new_record? after_destroy do cluster.find_or_build_integration_prometheus.destroy @@ -101,6 +101,10 @@ module Clusters private + def set_alert_manager_token + self.alert_manager_token = SecureRandom.hex + end + def install_knative_metrics return [] unless cluster.application_knative_available? diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index 1ac4cbac1da..b8ed33828bc 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -15,7 +15,7 @@ module Clusters belongs_to :runner, class_name: 'Ci::Runner', foreign_key: :runner_id delegate :project, :group, to: :cluster - default_value_for :version, VERSION + attribute :version, default: VERSION def chart "#{name}/gitlab-runner" diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index ad1e7dc305f..25d41d68b9e 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -79,7 +79,7 @@ module Clusters validates :namespace_per_environment, inclusion: { in: [true, false] } validates :helm_major_version, inclusion: { in: [2, 3] } - default_value_for :helm_major_version, 3 + attribute :helm_major_version, default: 3 validate :restrict_modification, on: :update validate :no_groups, unless: :group_type? diff --git a/app/models/clusters/integrations/prometheus.rb b/app/models/clusters/integrations/prometheus.rb index 899529ff49f..935d6238dba 100644 --- a/app/models/clusters/integrations/prometheus.rb +++ b/app/models/clusters/integrations/prometheus.rb @@ -26,7 +26,7 @@ module Clusters key: Settings.attr_encrypted_db_key_base_32, algorithm: 'aes-256-gcm' - default_value_for(:alert_manager_token) { SecureRandom.hex } + after_initialize :set_alert_manager_token, if: :new_record? scope :enabled, -> { where(enabled: true) } @@ -54,6 +54,10 @@ module Clusters private + def set_alert_manager_token + self.alert_manager_token = SecureRandom.hex + end + def activate_project_integrations ::Clusters::Applications::ActivateIntegrationWorker .perform_async(cluster_id, ::Integrations::Prometheus.to_param) diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index 9d4f0a89403..6003b60a23f 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -64,9 +64,7 @@ module Clusters unknown_authorization: nil, rbac: 1, abac: 2 - } - - default_value_for :authorization_type, :rbac + }, _default: :rbac nullify_if_blank :namespace diff --git a/app/models/clusters/providers/aws.rb b/app/models/clusters/providers/aws.rb index af2eba42721..f0f56d9ebd9 100644 --- a/app/models/clusters/providers/aws.rb +++ b/app/models/clusters/providers/aws.rb @@ -12,9 +12,9 @@ module Clusters belongs_to :cluster, inverse_of: :provider_aws, class_name: 'Clusters::Cluster' - default_value_for :region, DEFAULT_REGION - default_value_for :num_nodes, 3 - default_value_for :instance_type, 'm5.large' + attribute :region, default: DEFAULT_REGION + attribute :num_nodes, default: 3 + attribute :instance_type, default: "m5.large" attr_encrypted :secret_access_key, mode: :per_attribute_iv, diff --git a/app/models/clusters/providers/gcp.rb b/app/models/clusters/providers/gcp.rb index 2ca7d0249dc..fde5ed592cb 100644 --- a/app/models/clusters/providers/gcp.rb +++ b/app/models/clusters/providers/gcp.rb @@ -9,10 +9,10 @@ module Clusters belongs_to :cluster, inverse_of: :provider_gcp, class_name: 'Clusters::Cluster' - default_value_for :zone, 'us-central1-a' - default_value_for :num_nodes, 3 - default_value_for :machine_type, 'n1-standard-2' - default_value_for :cloud_run, false + attribute :zone, default: 'us-central1-a' + attribute :num_nodes, default: 3 + attribute :machine_type, default: 'n1-standard-2' + attribute :cloud_run, default: false scope :cloud_run, -> { where(cloud_run: true) } diff --git a/app/models/repository.rb b/app/models/repository.rb index 3413b3e3424..95d1b815e74 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -98,6 +98,10 @@ class Repository alias_method :raw, :raw_repository + def flipper_id + raw_repository.flipper_id + end + # Don't use this! It's going away. Use Gitaly to read or write from repos. def path_to_repo @path_to_repo ||= @@ -1232,7 +1236,8 @@ class Repository Gitlab::Git::Repository.new(shard, disk_path + '.git', repo_type.identifier_for_container(container), - container.full_path) + container.full_path, + container: container) end end diff --git a/app/services/projects/unlink_fork_service.rb b/app/services/projects/unlink_fork_service.rb index 9eccc16a8b2..898421364db 100644 --- a/app/services/projects/unlink_fork_service.rb +++ b/app/services/projects/unlink_fork_service.rb @@ -60,3 +60,5 @@ module Projects end end end + +Projects::UnlinkForkService.prepend_mod_with('Projects::UnlinkForkService') diff --git a/config/feature_flags/development/actors_aware_gitaly_calls.yml b/config/feature_flags/development/actors_aware_gitaly_calls.yml new file mode 100644 index 00000000000..0ae6140c579 --- /dev/null +++ b/config/feature_flags/development/actors_aware_gitaly_calls.yml @@ -0,0 +1,8 @@ +--- +name: actors_aware_gitaly_calls +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/101218 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/381516 +milestone: '15.6' +type: development +group: group::gitaly +default_enabled: false diff --git a/config/feature_flags/development/incident_declare_slash_command.yml b/config/feature_flags/development/incident_declare_slash_command.yml new file mode 100644 index 00000000000..9a35dd3274c --- /dev/null +++ b/config/feature_flags/development/incident_declare_slash_command.yml @@ -0,0 +1,8 @@ +--- +name: incident_declare_slash_command +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/101177 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/378072 +milestone: '15.6' +type: development +group: group::respond +default_enabled: false diff --git a/db/migrate/20221010184839_add_new_amount_used_to_ci_project_monthly_usages.rb b/db/migrate/20221010184839_add_new_amount_used_to_ci_project_monthly_usages.rb new file mode 100644 index 00000000000..5c77dfe9334 --- /dev/null +++ b/db/migrate/20221010184839_add_new_amount_used_to_ci_project_monthly_usages.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class AddNewAmountUsedToCiProjectMonthlyUsages < Gitlab::Database::Migration[2.0] + TABLE = :ci_project_monthly_usages + OLD_COLUMN = :amount_used + NEW_COLUMN = :new_amount_used + TRIGGER_NAME = 'sync_projects_amount_used_columns' + + disable_ddl_transaction! + + def up + check_trigger_permissions!(TABLE) + + add_column(TABLE, NEW_COLUMN, :decimal, default: 0.0, precision: 18, scale: 4, null: false, if_not_exists: true) + + install_rename_triggers(TABLE, OLD_COLUMN, NEW_COLUMN, trigger_name: TRIGGER_NAME) + end + + def down + remove_rename_triggers(TABLE, TRIGGER_NAME) + + remove_column(TABLE, NEW_COLUMN) + end +end diff --git a/db/migrate/20221107184542_add_new_amount_used_to_ci_namespace_monthly_usages.rb b/db/migrate/20221107184542_add_new_amount_used_to_ci_namespace_monthly_usages.rb new file mode 100644 index 00000000000..23a5a716164 --- /dev/null +++ b/db/migrate/20221107184542_add_new_amount_used_to_ci_namespace_monthly_usages.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class AddNewAmountUsedToCiNamespaceMonthlyUsages < Gitlab::Database::Migration[2.0] + TABLE = :ci_namespace_monthly_usages + OLD_COLUMN = :amount_used + NEW_COLUMN = :new_amount_used + TRIGGER_NAME = 'sync_namespaces_amount_used_columns' + + disable_ddl_transaction! + + def up + check_trigger_permissions!(TABLE) + + add_column(TABLE, NEW_COLUMN, :decimal, default: 0.0, precision: 18, scale: 4, null: false, if_not_exists: true) + + install_rename_triggers(TABLE, OLD_COLUMN, NEW_COLUMN, trigger_name: TRIGGER_NAME) + end + + def down + remove_rename_triggers(TABLE, TRIGGER_NAME) + + remove_column(TABLE, NEW_COLUMN) + end +end diff --git a/db/post_migrate/20221011062254_sync_new_amount_used_for_ci_project_monthly_usages.rb b/db/post_migrate/20221011062254_sync_new_amount_used_for_ci_project_monthly_usages.rb new file mode 100644 index 00000000000..32943f10fcf --- /dev/null +++ b/db/post_migrate/20221011062254_sync_new_amount_used_for_ci_project_monthly_usages.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class SyncNewAmountUsedForCiProjectMonthlyUsages < Gitlab::Database::Migration[2.0] + restrict_gitlab_migration gitlab_schema: :gitlab_ci + + def up + project_usages = define_batchable_model('ci_project_monthly_usages') + + project_usages.each_batch(of: 500) do |batch| + batch.where('amount_used > 0').update_all('new_amount_used = amount_used') + end + end + + def down + # Non reversible migration. + # This data migration keeps `new_amount_used` in sync with the old `amount_used`. + # In case of failure or interruption the migration can be retried. + end +end diff --git a/db/post_migrate/20221107184758_sync_new_amount_used_for_ci_namespace_monthly_usages.rb b/db/post_migrate/20221107184758_sync_new_amount_used_for_ci_namespace_monthly_usages.rb new file mode 100644 index 00000000000..9a6014b5c94 --- /dev/null +++ b/db/post_migrate/20221107184758_sync_new_amount_used_for_ci_namespace_monthly_usages.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class SyncNewAmountUsedForCiNamespaceMonthlyUsages < Gitlab::Database::Migration[2.0] + restrict_gitlab_migration gitlab_schema: :gitlab_ci + + def up + namespace_usages = define_batchable_model('ci_namespace_monthly_usages') + + namespace_usages.each_batch(of: 500) do |batch| + batch.where('amount_used > 0').update_all('new_amount_used = amount_used') + end + end + + def down + # Non reversible migration. + # This data migration keeps `new_amount_used` in sync with the old `amount_used`. + # In case of failure or interruption the migration can be retried. + end +end diff --git a/db/schema_migrations/20221010184839 b/db/schema_migrations/20221010184839 new file mode 100644 index 00000000000..de24fc0a4f0 --- /dev/null +++ b/db/schema_migrations/20221010184839 @@ -0,0 +1 @@ +062f807c0bf823b00e19cc55323faedd506b19d6492d1f23f09aaa662eaddcfb
\ No newline at end of file diff --git a/db/schema_migrations/20221011062254 b/db/schema_migrations/20221011062254 new file mode 100644 index 00000000000..a64a2dd4bfe --- /dev/null +++ b/db/schema_migrations/20221011062254 @@ -0,0 +1 @@ +ae4f3c26acd69aa0c701651a246a1359c91a0daad0daad64ba8dc186268880a4
\ No newline at end of file diff --git a/db/schema_migrations/20221107184542 b/db/schema_migrations/20221107184542 new file mode 100644 index 00000000000..ac7a4c78f66 --- /dev/null +++ b/db/schema_migrations/20221107184542 @@ -0,0 +1 @@ +876dce84bbc5ee00cd7ee837be6c549c677ec7eb7ec8db39d7711877d57dd7be
\ No newline at end of file diff --git a/db/schema_migrations/20221107184758 b/db/schema_migrations/20221107184758 new file mode 100644 index 00000000000..f327e45183b --- /dev/null +++ b/db/schema_migrations/20221107184758 @@ -0,0 +1 @@ +1aa75f334b1b56e46e8e0985febba567b0eb55af0f1ced0e198bfc979e5e262c
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 9ab594b0646..349af43dc91 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -234,6 +234,24 @@ BEGIN END; $$; +CREATE FUNCTION sync_namespaces_amount_used_columns() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN + NEW."new_amount_used" := NEW."amount_used"; + RETURN NEW; +END; +$$; + +CREATE FUNCTION sync_projects_amount_used_columns() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN + NEW."new_amount_used" := NEW."amount_used"; + RETURN NEW; +END; +$$; + CREATE FUNCTION unset_has_issues_on_vulnerability_reads() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -12976,6 +12994,7 @@ CREATE TABLE ci_namespace_monthly_usages ( notification_level smallint DEFAULT 100 NOT NULL, shared_runners_duration integer DEFAULT 0 NOT NULL, created_at timestamp with time zone, + new_amount_used numeric(18,4) DEFAULT 0.0 NOT NULL, CONSTRAINT ci_namespace_monthly_usages_year_month_constraint CHECK ((date = date_trunc('month'::text, (date)::timestamp with time zone))) ); @@ -13256,6 +13275,7 @@ CREATE TABLE ci_project_monthly_usages ( amount_used numeric(18,2) DEFAULT 0.0 NOT NULL, shared_runners_duration integer DEFAULT 0 NOT NULL, created_at timestamp with time zone, + new_amount_used numeric(18,4) DEFAULT 0.0 NOT NULL, CONSTRAINT ci_project_monthly_usages_year_month_constraint CHECK ((date = date_trunc('month'::text, (date)::timestamp with time zone))) ); @@ -32579,6 +32599,10 @@ CREATE TRIGGER projects_loose_fk_trigger AFTER DELETE ON projects REFERENCING OL CREATE TRIGGER trigger_1a857e8db6cd BEFORE INSERT OR UPDATE ON vulnerability_occurrences FOR EACH ROW EXECUTE FUNCTION trigger_1a857e8db6cd(); +CREATE TRIGGER sync_namespaces_amount_used_columns BEFORE INSERT OR UPDATE ON ci_namespace_monthly_usages FOR EACH ROW EXECUTE FUNCTION sync_namespaces_amount_used_columns(); + +CREATE TRIGGER sync_projects_amount_used_columns BEFORE INSERT OR UPDATE ON ci_project_monthly_usages FOR EACH ROW EXECUTE FUNCTION sync_projects_amount_used_columns(); + CREATE TRIGGER trigger_delete_project_namespace_on_project_delete AFTER DELETE ON projects FOR EACH ROW WHEN ((old.project_namespace_id IS NOT NULL)) EXECUTE FUNCTION delete_associated_project_namespace(); CREATE TRIGGER trigger_has_external_issue_tracker_on_delete AFTER DELETE ON integrations FOR EACH ROW WHEN ((((old.category)::text = 'issue_tracker'::text) AND (old.active = true) AND (old.project_id IS NOT NULL))) EXECUTE FUNCTION set_has_external_issue_tracker(); diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index ce019976049..588a00eb20c 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -28,7 +28,7 @@ full list of reference architectures, see | Internal load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | | Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | -| Gitaly<sup>5</sup> | 3 | 16 vCPU, 60 GB memory | `n1-standard-16` | `m5.4xlarge` | +| Gitaly<sup>5</sup><sup>6</sup> | 3 | 16 vCPU, 60 GB memory | `n1-standard-16` | `m5.4xlarge` | | Praefect<sup>5</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | @@ -50,6 +50,7 @@ full list of reference architectures, see - [Google Cloud Load Balancing](https://cloud.google.com/load-balancing) and [Amazon Elastic Load Balancing](https://aws.amazon.com/elasticloadbalancing/) are known to work. 4. Should be run on reputable Cloud Provider or Self Managed solutions. More information can be found in the [Configure the object storage](#configure-the-object-storage) section. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. +6. Gitaly has been designed and tested with repositories of varying sizes that follow best practices. However, large repositories or monorepos that don't follow these practices can significantly impact Gitaly requirements. Refer to the [Large Repositories](#large-repositories) for more info. <!-- markdownlint-enable MD029 --> NOTE: @@ -158,10 +159,45 @@ Any "burstable" instance types are not recommended due to inconsistent performan ### Supported infrastructure -As a general guidance, GitLab should run on most infrastructure such as reputable Cloud Providers (AWS, GCP) and their services, or self managed (ESXi) that meet both the specs detailed above, as well as any requirements in this section. However, this does not constitute a guarantee for every potential permutation. +As a general guidance, GitLab should run on most infrastructure such as reputable Cloud Providers (AWS, GCP) and their services, +or self managed (ESXi) that meet both the specs detailed above, as well as any requirements in this section. +However, this does not constitute a guarantee for every potential permutation. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. +### Additional workloads + +The Reference Architectures have been [designed and tested](index.md#validation-and-test-results) for standard GitLab setups with +good headroom in mind to cover most scenarios. However, if any additional workloads are being added on the nodes, +such as security software, you may still need to adjust the specs accordingly to compensate. + +This also applies for some GitLab features where it's possible to run custom scripts, for example [server hooks](../server_hooks.md). + +As a general rule, it's recommended to have robust monitoring in place to measure the impact of +any additional workloads to inform any changes needed to be made. + +### Large repositories + +The Reference Architectures were tested with repositories of varying sizes that follow best practices. + +However, large repositories or monorepos (several gigabytes or more) can **significantly** impact the performance +of Git and in turn the environment itself if best practices aren't being followed such as not storing +binary or blob files in LFS. Repositories are at the core of any environment the consequences can be wide-ranging +when they are not optimized. Some examples of this impact include [Git packing operations](https://git-scm.com/book/en/v2/Git-Internals-Packfiles) +taking longer and consuming high CPU / Memory resources or Git checkouts taking longer that affect both users and +CI pipelines alike. + +As such, large repositories come with notable cost and typically will require more resources to handle, +significantly so in some cases. It's therefore **strongly** recommended then to review large repositories +to ensure they maintain good repo health and reduce their size wherever possible. + +NOTE: +If best practices aren't followed and large repositories are present on the environment, +increased Gitaly specs may be required to ensure stable performance. + +Refer to the [Managing large repositories documentation](../../user/project/repository/managing_large_repositories.md) +for more information and guidance. + ### Praefect PostgreSQL It's worth noting that at this time [Praefect requires its own database server](../gitaly/praefect.md#postgresql) and @@ -241,8 +277,7 @@ In a multi-node GitLab configuration, you'll need a load balancer to route traffic to the application servers. The specifics on which load balancer to use or its exact configuration is beyond the scope of GitLab documentation. We assume that if you're managing multi-node systems like GitLab, you already have a load -balancer of choice and that the routing methods used are distributing calls evenly -between all nodes. Some load balancer examples include HAProxy (open-source), +balancer of choice. Some load balancer examples include HAProxy (open-source), F5 Big-IP LTM, and Citrix Net Scaler. This documentation outline the ports and protocols needed for use with GitLab. @@ -250,47 +285,13 @@ This architecture has been tested and validated with [HAProxy](https://www.hapro as the load balancer. Although other load balancers with similar feature sets could also be used, those load balancers have not been validated. -The next question is how you will handle SSL in your environment. -There are several different options: - -- [The application node terminates SSL](#application-node-terminates-ssl). -- [The load balancer terminates SSL without backend SSL](#load-balancer-terminates-ssl-without-backend-ssl) - and communication is not secure between the load balancer and the application node. -- [The load balancer terminates SSL with backend SSL](#load-balancer-terminates-ssl-with-backend-ssl) - and communication is *secure* between the load balancer and the application node. - -### Application node terminates SSL +### Balancing algorithm -Configure your load balancer to pass connections on port 443 as `TCP` rather -than `HTTP(S)` protocol. This will pass the connection to the application node's -NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. +We recommend that a least-connection load balancing algorithm or equivalent +is used wherever possible to ensure equal spread of calls to the nodes and good performance. -See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) -for details on managing SSL certificates and configuring NGINX. - -### Load balancer terminates SSL without backend SSL - -Configure your load balancer to use the `HTTP(S)` protocol rather than `TCP`. -The load balancer will then be responsible for managing SSL certificates and -terminating SSL. - -Since communication between the load balancer and GitLab will not be secure, -there is some additional configuration needed. See the -[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) -for details. - -### Load balancer terminates SSL with backend SSL - -Configure your load balancers to use the 'HTTP(S)' protocol rather than 'TCP'. -The load balancers will be responsible for managing SSL certificates that -end users will see. - -Traffic will also be secure between the load balancers and NGINX in this -scenario. There is no need to add configuration for proxied SSL since the -connection will be secure all the way. However, configuration will need to be -added to GitLab to configure SSL certificates. See -the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) -for details on managing SSL certificates and configuring NGINX. +We don't recommend the use of round-robin algorithms as they are known to not +spread connections equally in practice. ### Readiness checks @@ -351,6 +352,50 @@ Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. | ------- | ------------ | -------- | | 443 | 22 | TCP | +### SSL + +The next question is how you will handle SSL in your environment. +There are several different options: + +- [The application node terminates SSL](#application-node-terminates-ssl). +- [The load balancer terminates SSL without backend SSL](#load-balancer-terminates-ssl-without-backend-ssl) + and communication is not secure between the load balancer and the application node. +- [The load balancer terminates SSL with backend SSL](#load-balancer-terminates-ssl-with-backend-ssl) + and communication is *secure* between the load balancer and the application node. + +#### Application node terminates SSL + +Configure your load balancer to pass connections on port 443 as `TCP` rather +than `HTTP(S)` protocol. This will pass the connection to the application node's +NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. + +See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +for details on managing SSL certificates and configuring NGINX. + +#### Load balancer terminates SSL without backend SSL + +Configure your load balancer to use the `HTTP(S)` protocol rather than `TCP`. +The load balancer will then be responsible for managing SSL certificates and +terminating SSL. + +Since communication between the load balancer and GitLab will not be secure, +there is some additional configuration needed. See the +[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) +for details. + +#### Load balancer terminates SSL with backend SSL + +Configure your load balancers to use the 'HTTP(S)' protocol rather than 'TCP'. +The load balancers will be responsible for managing SSL certificates that +end users will see. + +Traffic will also be secure between the load balancers and NGINX in this +scenario. There is no need to add configuration for proxied SSL since the +connection will be secure all the way. However, configuration will need to be +added to GitLab to configure SSL certificates. See +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +for details on managing SSL certificates and configuring NGINX. + <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> @@ -415,8 +460,14 @@ backend praefect ``` Refer to your preferred Load Balancer's documentation for further guidance. -Also ensure that the routing methods used are distributing calls evenly across -all nodes. + +### Balancing algorithm + +We recommend that a least-connection-based load balancing algorithm or equivalent +is used wherever possible to ensure equal spread of calls to the nodes and good performance. + +We don't recommend the use of round-robin algorithms as they are known to not +spread connections equally in practice. <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -1168,6 +1219,12 @@ NOTE: Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). For implementations with sharded Gitaly, use the same Gitaly specs. Follow the [separate Gitaly documentation](../gitaly/configure_gitaly.md) instead of this section. +NOTE: +Gitaly has been designed and tested with repositories of varying sizes that follow best practices. +However, large repositories or monorepos not following these practices can significantly +impact Gitaly performance and requirements. +Refer to the [Large Repositories](#large-repositories) for more info. + The recommended cluster setup includes the following components: - 3 Gitaly nodes: Replicated storage of Git repositories. @@ -1475,9 +1532,15 @@ The [Gitaly](../gitaly/index.md) server nodes that make up the cluster have requirements that are dependent on data and load. NOTE: -The Reference Architecture specs have been designed with good headroom in mind -but for Gitaly, increased specs or additional -Gitaly Cluster arrays may be required for notably large data sets or load. +Increased specs for Gitaly nodes may be required in some circumstances such as +significantly large repositories or if any [additional workloads](#additional-workloads), +such as [server hooks](../server_hooks.md), have been added. + +NOTE: +Gitaly has been designed and tested with repositories of varying sizes that follow best practices. +However, large repositories or monorepos not following these practices can significantly +impact Gitaly performance and requirements. +Refer to the [Large Repositories](#large-repositories) for more info. Due to Gitaly having notable input and output requirements, we strongly recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs @@ -2279,7 +2342,7 @@ services where applicable): | Internal load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | | Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | -| Gitaly<sup>5</sup> | 3 | 16 vCPU, 60 GB memory | `n1-standard-16` | `m5.4xlarge` | +| Gitaly<sup>5</sup><sup>6</sup> | 3 | 16 vCPU, 60 GB memory | `n1-standard-16` | `m5.4xlarge` | | Praefect<sup>5</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Object storage<sup>4</sup> | - | - | - | - | @@ -2297,6 +2360,7 @@ services where applicable): - [Google Cloud Load Balancing](https://cloud.google.com/load-balancing) and [Amazon Elastic Load Balancing](https://aws.amazon.com/elasticloadbalancing/) are known to work. 4. Should be run on reputable Cloud Provider or Self Managed solutions. More information can be found in the [Configure the object storage](#configure-the-object-storage) section. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. +6. Gitaly has been designed and tested with repositories of varying sizes that follow best practices. However, large repositories or monorepos that don't follow these practices can significantly impact Gitaly requirements. Refer to the [Large Repositories](#large-repositories) for more info. <!-- markdownlint-enable MD029 --> NOTE: diff --git a/doc/administration/reference_architectures/1k_users.md b/doc/administration/reference_architectures/1k_users.md index a8e0e23512f..2a9636b6e05 100644 --- a/doc/administration/reference_architectures/1k_users.md +++ b/doc/administration/reference_architectures/1k_users.md @@ -82,10 +82,23 @@ Any "burstable" instance types are not recommended due to inconsistent performan ### Supported infrastructure -As a general guidance, GitLab should run on most infrastructure such as reputable Cloud Providers (AWS, GCP, Azure) and their services, or self managed (ESXi) that meet both the specs detailed above, as well as any requirements in this section. However, this does not constitute a guarantee for every potential permutation. +As a general guidance, GitLab should run on most infrastructure such as reputable Cloud Providers (AWS, GCP) and their services, +or self managed (ESXi) that meet both the specs detailed above, as well as any requirements in this section. +However, this does not constitute a guarantee for every potential permutation. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. +### Additional workloads + +The Reference Architectures have been [designed and tested](index.md#validation-and-test-results) for standard GitLab setups with +good headroom in mind to cover most scenarios. However, if any additional workloads are being added on the nodes, +such as security software, you may still need to adjust the specs accordingly to compensate. + +This also applies for some GitLab features where it's possible to run custom scripts, for example [server hooks](../server_hooks.md). + +As a general rule, it's recommended to have robust monitoring in place to measure the impact of +any additional workloads to inform any changes needed to be made. + ### Swap In addition to the stated configurations, we recommend having at least 2 GB of diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index fba5e5ec6ae..a071bc94bf0 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -28,7 +28,7 @@ full list of reference architectures, see | Internal load balancing node<sup>3</sup> | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | | Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | | Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | -| Gitaly<sup>5</sup> | 3 | 32 vCPU, 120 GB memory | `n1-standard-32` | `m5.8xlarge` | +| Gitaly<sup>5</sup><sup>6</sup> | 3 | 32 vCPU, 120 GB memory | `n1-standard-32` | `m5.8xlarge` | | Praefect<sup>5</sup> | 3 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | | Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | @@ -50,6 +50,7 @@ full list of reference architectures, see - [Google Cloud Load Balancing](https://cloud.google.com/load-balancing) and [Amazon Elastic Load Balancing](https://aws.amazon.com/elasticloadbalancing/) are known to work. 4. Should be run on reputable Cloud Provider or Self Managed solutions. More information can be found in the [Configure the object storage](#configure-the-object-storage) section. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. +6. Gitaly has been designed and tested with repositories of varying sizes that follow best practices. However, large repositories or monorepos that don't follow these practices can significantly impact Gitaly requirements. Refer to the [Large Repositories](#large-repositories) for more info. <!-- markdownlint-enable MD029 --> NOTE: @@ -158,10 +159,45 @@ Any "burstable" instance types are not recommended due to inconsistent performan ### Supported infrastructure -As a general guidance, GitLab should run on most infrastructure such as reputable Cloud Providers (AWS, GCP) and their services, or self managed (ESXi) that meet both the specs detailed above, as well as any requirements in this section. However, this does not constitute a guarantee for every potential permutation. +As a general guidance, GitLab should run on most infrastructure such as reputable Cloud Providers (AWS, GCP, Azure) and their services, +or self managed (ESXi) that meet both the specs detailed above, as well as any requirements in this section. +However, this does not constitute a guarantee for every potential permutation. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. +### Additional workloads + +The Reference Architectures have been [designed and tested](index.md#validation-and-test-results) for standard GitLab setups with +good headroom in mind to cover most scenarios. However, if any additional workloads are being added on the nodes, +such as security software, you may still need to adjust the specs accordingly to compensate. + +This also applies for some GitLab features where it's possible to run custom scripts, for example [server hooks](../server_hooks.md). + +As a general rule, it's recommended to have robust monitoring in place to measure the impact of +any additional workloads to inform any changes needed to be made. + +### Large repositories + +The Reference Architectures were tested with repositories of varying sizes that follow best practices. + +However, large repositories or monorepos (several gigabytes or more) can **significantly** impact the performance +of Git and in turn the environment itself if best practices aren't being followed such as not storing +binary or blob files in LFS. Repositories are at the core of any environment the consequences can be wide-ranging +when they are not optimized. Some examples of this impact include [Git packing operations](https://git-scm.com/book/en/v2/Git-Internals-Packfiles) +taking longer and consuming high CPU / Memory resources or Git checkouts taking longer that affect both users and +CI pipelines alike. + +As such, large repositories come with notable cost and typically will require more resources to handle, +significantly so in some cases. It's therefore **strongly** recommended then to review large repositories +to ensure they maintain good repo health and reduce their size wherever possible. + +NOTE: +If best practices aren't followed and large repositories are present on the environment, +increased Gitaly specs may be required to ensure stable performance. + +Refer to the [Managing large repositories documentation](../../user/project/repository/managing_large_repositories.md) +for more information and guidance. + ### Praefect PostgreSQL It's worth noting that at this time [Praefect requires its own database server](../gitaly/praefect.md#postgresql) and @@ -243,8 +279,7 @@ In a multi-node GitLab configuration, you'll need a load balancer to route traffic to the application servers. The specifics on which load balancer to use or its exact configuration is beyond the scope of GitLab documentation. We assume that if you're managing multi-node systems like GitLab, you already have a load -balancer of choice and that the routing methods used are distributing calls evenly -between all nodes. Some load balancer examples include HAProxy (open-source), +balancer of choice. Some load balancer examples include HAProxy (open-source), F5 Big-IP LTM, and Citrix Net Scaler. This documentation outline the ports and protocols needed for use with GitLab. @@ -261,38 +296,13 @@ There are several different options: - [The load balancer terminates SSL with backend SSL](#load-balancer-terminates-ssl-with-backend-ssl) and communication is *secure* between the load balancer and the application node. -### Application node terminates SSL - -Configure your load balancer to pass connections on port 443 as `TCP` rather -than `HTTP(S)` protocol. This will pass the connection to the application node's -NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. - -See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) -for details on managing SSL certificates and configuring NGINX. - -### Load balancer terminates SSL without backend SSL - -Configure your load balancer to use the `HTTP(S)` protocol rather than `TCP`. -The load balancer will then be responsible for managing SSL certificates and -terminating SSL. - -Since communication between the load balancer and GitLab will not be secure, -there is some additional configuration needed. See the -[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) -for details. +### Balancing algorithm -### Load balancer terminates SSL with backend SSL +We recommend that a least-connection load balancing algorithm or equivalent +is used wherever possible to ensure equal spread of calls to the nodes and good performance. -Configure your load balancers to use the 'HTTP(S)' protocol rather than 'TCP'. -The load balancers will be responsible for managing SSL certificates that -end users will see. - -Traffic will also be secure between the load balancers and NGINX in this -scenario. There is no need to add configuration for proxied SSL since the -connection will be secure all the way. However, configuration will need to be -added to GitLab to configure SSL certificates. See the -[HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) -for details on managing SSL certificates and configuring NGINX. +We don't recommend the use of round-robin algorithms as they are known to not +spread connections equally in practice. ### Readiness checks @@ -353,6 +363,50 @@ Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. | ------- | ------------ | -------- | | 443 | 22 | TCP | +### SSL + +The next question is how you will handle SSL in your environment. +There are several different options: + +- [The application node terminates SSL](#application-node-terminates-ssl). +- [The load balancer terminates SSL without backend SSL](#load-balancer-terminates-ssl-without-backend-ssl) + and communication is not secure between the load balancer and the application node. +- [The load balancer terminates SSL with backend SSL](#load-balancer-terminates-ssl-with-backend-ssl) + and communication is *secure* between the load balancer and the application node. + +#### Application node terminates SSL + +Configure your load balancer to pass connections on port 443 as `TCP` rather +than `HTTP(S)` protocol. This will pass the connection to the application node's +NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. + +See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +for details on managing SSL certificates and configuring NGINX. + +#### Load balancer terminates SSL without backend SSL + +Configure your load balancer to use the `HTTP(S)` protocol rather than `TCP`. +The load balancer will then be responsible for managing SSL certificates and +terminating SSL. + +Since communication between the load balancer and GitLab will not be secure, +there is some additional configuration needed. See the +[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) +for details. + +#### Load balancer terminates SSL with backend SSL + +Configure your load balancers to use the 'HTTP(S)' protocol rather than 'TCP'. +The load balancers will be responsible for managing SSL certificates that +end users will see. + +Traffic will also be secure between the load balancers and NGINX in this +scenario. There is no need to add configuration for proxied SSL since the +connection will be secure all the way. However, configuration will need to be +added to GitLab to configure SSL certificates. See +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +for details on managing SSL certificates and configuring NGINX. + <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> @@ -417,8 +471,20 @@ backend praefect ``` Refer to your preferred Load Balancer's documentation for further guidance. -Also ensure that the routing methods used are distributing calls evenly across -all nodes. + +### Balancing algorithm + +We recommend that a least-connection load balancing algorithm or equivalent +is used wherever possible to ensure equal spread of calls to the nodes and good performance. + +We don't recommend the use of round-robin algorithms as they are known to not +spread connections equally in practice. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -1173,6 +1239,12 @@ NOTE: Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). For implementations with sharded Gitaly, use the same Gitaly specs. Follow the [separate Gitaly documentation](../gitaly/configure_gitaly.md) instead of this section. +NOTE: +Gitaly has been designed and tested with repositories of varying sizes that follow best practices. +However, large repositories or monorepos not following these practices can significantly +impact Gitaly performance and requirements. +Refer to the [Large Repositories](#large-repositories) for more info. + The recommended cluster setup includes the following components: - 3 Gitaly nodes: Replicated storage of Git repositories. @@ -1478,9 +1550,15 @@ The [Gitaly](../gitaly/index.md) server nodes that make up the cluster have requirements that are dependent on data and load. NOTE: -The Reference Architecture specs have been designed with good headroom in mind -but for Gitaly, increased specs or additional -Gitaly Cluster arrays may be required for notably large data sets or load. +Increased specs for Gitaly nodes may be required in some circumstances such as +significantly large repositories or if any [additional workloads](#additional-workloads), +such as [server hooks](../server_hooks.md), have been added. + +NOTE: +Gitaly has been designed and tested with repositories of varying sizes that follow best practices. +However, large repositories or monorepos not following these practices can significantly +impact Gitaly performance and requirements. +Refer to the [Large Repositories](#large-repositories) for more info. Due to Gitaly having notable input and output requirements, we strongly recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs @@ -2283,7 +2361,7 @@ services where applicable): | Internal load balancing node<sup>3</sup> | 1 | 4 vCPU, 3.6GB memory | `n1-highcpu-4` | `c5.xlarge` | | Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | | Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | -| Gitaly<sup>5</sup> | 3 | 32 vCPU, 120 GB memory | `n1-standard-32` | `m5.8xlarge` | +| Gitaly<sup>5</sup><sup>6</sup> | 3 | 32 vCPU, 120 GB memory | `n1-standard-32` | `m5.8xlarge` | | Praefect<sup>5</sup> | 3 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | | Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Object storage<sup>4</sup> | - | - | - | - | @@ -2301,6 +2379,7 @@ services where applicable): - [Google Cloud Load Balancing](https://cloud.google.com/load-balancing) and [Amazon Elastic Load Balancing](https://aws.amazon.com/elasticloadbalancing/) are known to work. 4. Should be run on reputable Cloud Provider or Self Managed solutions. More information can be found in the [Configure the object storage](#configure-the-object-storage) section. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. +6. Gitaly has been designed and tested with repositories of varying sizes that follow best practices. However, large repositories or monorepos that don't follow these practices can significantly impact Gitaly requirements. Refer to the [Large Repositories](#large-repositories) for more info. <!-- markdownlint-enable MD029 --> NOTE: diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index 26a8d0e038a..07f857c7f3e 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -25,7 +25,7 @@ For a full list of reference architectures, see | Load balancer<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | | PostgreSQL<sup>1</sup> | 1 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | | Redis<sup>2</sup> | 1 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `m5.large` | `D2s v3` | -| Gitaly | 1 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| Gitaly<sup>5</sup> | 1 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | | GitLab Rails | 2 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | | Monitoring node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | | Object storage<sup>4</sup> | - | - | - | - | - | @@ -42,6 +42,7 @@ For a full list of reference architectures, see 3. Can be optionally run on reputable third-party load balancing services (LB PaaS). See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. - [Google Cloud Load Balancing](https://cloud.google.com/load-balancing) and [Amazon Elastic Load Balancing](https://aws.amazon.com/elasticloadbalancing/) are known to work. 4. Should be run on reputable Cloud Provider or Self Managed solutions. More information can be found in the [Configure the object storage](#configure-the-object-storage) section. +5. Gitaly has been designed and tested with repositories of varying sizes that follow best practices. However, large repositories or monorepos that don't follow these practices can significantly impact Gitaly requirements. Refer to the [Large Repositories](#large-repositories) for more info. <!-- markdownlint-enable MD029 --> NOTE: @@ -94,10 +95,45 @@ Any "burstable" instance types are not recommended due to inconsistent performan ### Supported infrastructure -As a general guidance, GitLab should run on most infrastructure such as reputable Cloud Providers (AWS, GCP, Azure) and their services, or self managed (ESXi) that meet both the specs detailed above, as well as any requirements in this section. However, this does not constitute a guarantee for every potential permutation. +As a general guidance, GitLab should run on most infrastructure such as reputable Cloud Providers (AWS, GCP) and their services, +or self managed (ESXi) that meet both the specs detailed above, as well as any requirements in this section. +However, this does not constitute a guarantee for every potential permutation. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. +### Additional workloads + +The Reference Architectures have been [designed and tested](index.md#validation-and-test-results) for standard GitLab setups with +good headroom in mind to cover most scenarios. However, if any additional workloads are being added on the nodes, +such as security software, you may still need to adjust the specs accordingly to compensate. + +This also applies for some GitLab features where it's possible to run custom scripts, for example [server hooks](../server_hooks.md). + +As a general rule, it's recommended to have robust monitoring in place to measure the impact of +any additional workloads to inform any changes needed to be made. + +### Large repositories + +The Reference Architectures were tested with repositories of varying sizes that follow best practices. + +However, large repositories or monorepos (several gigabytes or more) can **significantly** impact the performance +of Git and in turn the environment itself if best practices aren't being followed such as not storing +binary or blob files in LFS. Repositories are at the core of any environment the consequences can be wide-ranging +when they are not optimized. Some examples of this impact include [Git packing operations](https://git-scm.com/book/en/v2/Git-Internals-Packfiles) +taking longer and consuming high CPU / Memory resources or Git checkouts taking longer that affect both users and +CI pipelines alike. + +As such, large repositories come with notable cost and typically will require more resources to handle, +significantly so in some cases. It's therefore **strongly** recommended then to review large repositories +to ensure they maintain good repo health and reduce their size wherever possible. + +NOTE: +If best practices aren't followed and large repositories are present on the environment, +increased Gitaly specs may be required to ensure stable performance. + +Refer to the [Managing large repositories documentation](../../user/project/repository/managing_large_repositories.md) +for more information and guidance. + ## Setup components To set up GitLab and its components to accommodate up to 2,000 users: @@ -127,8 +163,7 @@ In a multi-node GitLab configuration, you'll need a load balancer to route traffic to the application servers. The specifics on which load balancer to use or its exact configuration is beyond the scope of GitLab documentation. We assume that if you're managing multi-node systems like GitLab, you already have a load -balancer of choice and that the routing methods used are distributing calls evenly -between all nodes. Some load balancer examples include HAProxy (open-source), +balancer of choice. Some load balancer examples include HAProxy (open-source), F5 Big-IP LTM, and Citrix Net Scaler. This documentation outline the ports and protocols needed for use with GitLab. @@ -145,36 +180,13 @@ several different options: - [The load balancer terminates SSL with backend SSL](#load-balancer-terminates-ssl-with-backend-ssl) and communication is *secure* between the load balancer and the application node. -### Application node terminates SSL - -Configure your load balancer to pass connections on port 443 as `TCP` instead -of `HTTP(S)`. This will pass the connection unaltered to the application node's -NGINX service, which has the SSL certificate and listens to port 443. - -For details about managing SSL certificates and configuring NGINX, see the -[HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +### Balancing algorithm -### Load balancer terminates SSL without backend SSL - -Configure your load balancer to use the `HTTP(S)` protocol instead of `TCP`. -The load balancer will be responsible for both managing SSL certificates and -terminating SSL. +We recommend that a least-connection load balancing algorithm or equivalent +is used wherever possible to ensure equal spread of calls to the nodes and good performance. -Due to communication between the load balancer and GitLab not being secure, -you'll need to complete some additional configuration. For details, see the -[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination). - -### Load balancer terminates SSL with backend SSL - -Configure your load balancers (or single balancer, if you have only one) to use -the `HTTP(S)` protocol rather than `TCP`. The load balancers will be -responsible for the managing SSL certificates for end users. - -Traffic will be secure between the load balancers and NGINX in this scenario, -and there's no need to add a configuration for proxied SSL. However, you'll -need to add a configuration to GitLab to configure SSL certificates. For -details about managing SSL certificates and configuring NGINX, see the -[HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html). +We don't recommend the use of round-robin algorithms as they are known to not +spread connections equally in practice. ### Readiness checks @@ -186,56 +198,99 @@ connect. ### Ports -The basic load balancer ports you should use are described in the following -table: +The basic ports to be used are shown in the table below. -| Port | Backend Port | Protocol | +| LB Port | Backend Port | Protocol | | ------- | ------------ | ------------------------ | | 80 | 80 | HTTP (*1*) | | 443 | 443 | TCP or HTTPS (*1*) (*2*) | | 22 | 22 | TCP | -- (*1*): [Web terminal](../../ci/environments/index.md#web-terminals-deprecated) support - requires your load balancer to correctly handle WebSocket connections. - When using HTTP or HTTPS proxying, your load balancer must be configured - to pass through the `Connection` and `Upgrade` hop-by-hop headers. For - details, see the [web terminal](../integration/terminal.md) integration guide. -- (*2*): When using the HTTPS protocol for port 443, you'll need to add an SSL - certificate to the load balancers. If you need to terminate SSL at the - GitLab application server, use the TCP protocol. +- (*1*): [Web terminal](../../ci/environments/index.md#web-terminals-deprecated) support requires + your load balancer to correctly handle WebSocket connections. When using + HTTP or HTTPS proxying, this means your load balancer must be configured + to pass through the `Connection` and `Upgrade` hop-by-hop headers. See the + [web terminal](../integration/terminal.md) integration guide for + more details. +- (*2*): When using HTTPS protocol for port 443, you will need to add an SSL + certificate to the load balancers. If you wish to terminate SSL at the + GitLab application server instead, use TCP protocol. If you're using GitLab Pages with custom domain support you will need some -additional port configurations. GitLab Pages requires a separate virtual IP -address. Configure DNS to point the `pages_external_url` from -`/etc/gitlab/gitlab.rb` to the new virtual IP address. For more information, -see the [GitLab Pages documentation](../pages/index.md). +additional port configurations. +GitLab Pages requires a separate virtual IP address. Configure DNS to point the +`pages_external_url` from `/etc/gitlab/gitlab.rb` at the new virtual IP address. See the +[GitLab Pages documentation](../pages/index.md) for more information. -| Port | Backend Port | Protocol | +| LB Port | Backend Port | Protocol | | ------- | ------------- | --------- | | 80 | Varies (*1*) | HTTP | | 443 | Varies (*1*) | TCP (*2*) | - (*1*): The backend port for GitLab Pages depends on the `gitlab_pages['external_http']` and `gitlab_pages['external_https']` - settings. For details, see the [GitLab Pages documentation](../pages/index.md). -- (*2*): Port 443 for GitLab Pages must use the TCP protocol. Users can - configure custom domains with custom SSL, which wouldn't be possible if SSL - was terminated at the load balancer. + setting. See [GitLab Pages documentation](../pages/index.md) for more details. +- (*2*): Port 443 for GitLab Pages should always use the TCP protocol. Users can + configure custom domains with custom SSL, which would not be possible + if SSL was terminated at the load balancer. #### Alternate SSH Port Some organizations have policies against opening SSH port 22. In this case, -it may be helpful to configure an alternate SSH hostname that instead allows -users to use SSH over port 443. An alternate SSH hostname requires a new -virtual IP address compared to the previously described GitLab HTTP -configuration. +it may be helpful to configure an alternate SSH hostname that allows users +to use SSH on port 443. An alternate SSH hostname will require a new virtual IP address +compared to the other GitLab HTTP configuration above. -Configure DNS for an alternate SSH hostname, such as `altssh.gitlab.example.com`: +Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. | LB Port | Backend Port | Protocol | | ------- | ------------ | -------- | | 443 | 22 | TCP | +### SSL + +The next question is how you will handle SSL in your environment. +There are several different options: + +- [The application node terminates SSL](#application-node-terminates-ssl). +- [The load balancer terminates SSL without backend SSL](#load-balancer-terminates-ssl-without-backend-ssl) + and communication is not secure between the load balancer and the application node. +- [The load balancer terminates SSL with backend SSL](#load-balancer-terminates-ssl-with-backend-ssl) + and communication is *secure* between the load balancer and the application node. + +#### Application node terminates SSL + +Configure your load balancer to pass connections on port 443 as `TCP` rather +than `HTTP(S)` protocol. This will pass the connection to the application node's +NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. + +See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +for details on managing SSL certificates and configuring NGINX. + +#### Load balancer terminates SSL without backend SSL + +Configure your load balancer to use the `HTTP(S)` protocol rather than `TCP`. +The load balancer will then be responsible for managing SSL certificates and +terminating SSL. + +Since communication between the load balancer and GitLab will not be secure, +there is some additional configuration needed. See the +[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) +for details. + +#### Load balancer terminates SSL with backend SSL + +Configure your load balancers to use the 'HTTP(S)' protocol rather than 'TCP'. +The load balancers will be responsible for managing SSL certificates that +end users will see. + +Traffic will also be secure between the load balancers and NGINX in this +scenario. There is no need to add configuration for proxied SSL since the +connection will be secure all the way. However, configuration will need to be +added to GitLab to configure SSL certificates. See +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +for details on managing SSL certificates and configuring NGINX. + <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> @@ -407,9 +462,15 @@ are supported and can be added if needed. specifically the number of projects and those projects' sizes. NOTE: -The Reference Architecture specs have been designed with good headroom in mind -but for Gitaly, increased specs or switching to Gitaly Cluster -may be required for notably large data sets or load. +Increased specs for Gitaly nodes may be required in some circumstances such as +significantly large repositories or if any [additional workloads](#additional-workloads), +such as [server hooks](../server_hooks.md), have been added. + +NOTE: +Gitaly has been designed and tested with repositories of varying sizes that follow best practices. +However, large repositories or monorepos not following these practices can significantly +impact Gitaly performance and requirements. +Refer to the [Large Repositories](#large-repositories) for more info. Due to Gitaly having notable input and output requirements, we strongly recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 1ce19877580..23b5f59f674 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -37,7 +37,7 @@ For a full list of reference architectures, see | PostgreSQL<sup>1</sup> | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | | PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Internal load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Gitaly<sup>5</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | +| Gitaly<sup>5</sup><sup>6</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | | Praefect<sup>5</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Sidekiq | 4 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | @@ -59,6 +59,7 @@ For a full list of reference architectures, see - [Google Cloud Load Balancing](https://cloud.google.com/load-balancing) and [Amazon Elastic Load Balancing](https://aws.amazon.com/elasticloadbalancing/) are known to work. 4. Should be run on reputable Cloud Provider or Self Managed solutions. More information can be found in the [Configure the object storage](#configure-the-object-storage) section. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. +6. Gitaly has been designed and tested with repositories of varying sizes that follow best practices. However, large repositories or monorepos that don't follow these practices can significantly impact Gitaly requirements. Refer to the [Large Repositories](#large-repositories) for more info. <!-- markdownlint-enable MD029 --> NOTE: @@ -164,10 +165,45 @@ Any "burstable" instance types are not recommended due to inconsistent performan ### Supported infrastructure -As a general guidance, GitLab should run on most infrastructure such as reputable Cloud Providers (AWS, GCP) and their services, or self managed (ESXi) that meet both the specs detailed above, as well as any requirements in this section. However, this does not constitute a guarantee for every potential permutation. +As a general guidance, GitLab should run on most infrastructure such as reputable Cloud Providers (AWS, GCP) and their services, +or self managed (ESXi) that meet both the specs detailed above, as well as any requirements in this section. +However, this does not constitute a guarantee for every potential permutation. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. +### Additional workloads + +The Reference Architectures have been [designed and tested](index.md#validation-and-test-results) for standard GitLab setups with +good headroom in mind to cover most scenarios. However, if any additional workloads are being added on the nodes, +such as security software, you may still need to adjust the specs accordingly to compensate. + +This also applies for some GitLab features where it's possible to run custom scripts, for example [server hooks](../server_hooks.md). + +As a general rule, it's recommended to have robust monitoring in place to measure the impact of +any additional workloads to inform any changes needed to be made. + +### Large repositories + +The Reference Architectures were tested with repositories of varying sizes that follow best practices. + +However, large repositories or monorepos (several gigabytes or more) can **significantly** impact the performance +of Git and in turn the environment itself if best practices aren't being followed such as not storing +binary or blob files in LFS. Repositories are at the core of any environment the consequences can be wide-ranging +when they are not optimized. Some examples of this impact include [Git packing operations](https://git-scm.com/book/en/v2/Git-Internals-Packfiles) +taking longer and consuming high CPU / Memory resources or Git checkouts taking longer that affect both users and +CI pipelines alike. + +As such, large repositories come with notable cost and typically will require more resources to handle, +significantly so in some cases. It's therefore **strongly** recommended then to review large repositories +to ensure they maintain good repo health and reduce their size wherever possible. + +NOTE: +If best practices aren't followed and large repositories are present on the environment, +increased Gitaly specs may be required to ensure stable performance. + +Refer to the [Managing large repositories documentation](../../user/project/repository/managing_large_repositories.md) +for more information and guidance. + ### Praefect PostgreSQL It's worth noting that at this time [Praefect requires its own database server](../gitaly/praefect.md#postgresql) and @@ -244,8 +280,7 @@ In a multi-node GitLab configuration, you'll need a load balancer to route traffic to the application servers. The specifics on which load balancer to use or its exact configuration is beyond the scope of GitLab documentation. We assume that if you're managing multi-node systems like GitLab, you already have a load -balancer of choice and that the routing methods used are distributing calls evenly -between all nodes. Some load balancer examples include HAProxy (open-source), +balancer of choice. Some load balancer examples include HAProxy (open-source), F5 Big-IP LTM, and Citrix Net Scaler. This documentation outline the ports and protocols needed for use with GitLab. @@ -262,38 +297,13 @@ There are several different options: - [The load balancer terminates SSL with backend SSL](#load-balancer-terminates-ssl-with-backend-ssl) and communication is *secure* between the load balancer and the application node. -### Application node terminates SSL +### Balancing algorithm -Configure your load balancer to pass connections on port 443 as `TCP` rather -than `HTTP(S)` protocol. This will pass the connection to the application node's -NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. - -See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) -for details on managing SSL certificates and configuring NGINX. - -### Load balancer terminates SSL without backend SSL +We recommend that a least-connection load balancing algorithm or equivalent +is used wherever possible to ensure equal spread of calls to the nodes and good performance. -Configure your load balancer to use the `HTTP(S)` protocol rather than `TCP`. -The load balancer will then be responsible for managing SSL certificates and -terminating SSL. - -Since communication between the load balancer and GitLab will not be secure, -there is some additional configuration needed. See the -[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) -for details. - -### Load balancer terminates SSL with backend SSL - -Configure your load balancers to use the 'HTTP(S)' protocol rather than 'TCP'. -The load balancers will be responsible for managing SSL certificates that -end users will see. - -Traffic will also be secure between the load balancers and NGINX in this -scenario. There is no need to add configuration for proxied SSL since the -connection will be secure all the way. However, configuration will need to be -added to GitLab to configure SSL certificates. See the -[HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) -for details on managing SSL certificates and configuring NGINX. +We don't recommend the use of round-robin algorithms as they are known to not +spread connections equally in practice. ### Readiness checks @@ -354,6 +364,50 @@ Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. | ------- | ------------ | -------- | | 443 | 22 | TCP | +### SSL + +The next question is how you will handle SSL in your environment. +There are several different options: + +- [The application node terminates SSL](#application-node-terminates-ssl). +- [The load balancer terminates SSL without backend SSL](#load-balancer-terminates-ssl-without-backend-ssl) + and communication is not secure between the load balancer and the application node. +- [The load balancer terminates SSL with backend SSL](#load-balancer-terminates-ssl-with-backend-ssl) + and communication is *secure* between the load balancer and the application node. + +#### Application node terminates SSL + +Configure your load balancer to pass connections on port 443 as `TCP` rather +than `HTTP(S)` protocol. This will pass the connection to the application node's +NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. + +See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +for details on managing SSL certificates and configuring NGINX. + +#### Load balancer terminates SSL without backend SSL + +Configure your load balancer to use the `HTTP(S)` protocol rather than `TCP`. +The load balancer will then be responsible for managing SSL certificates and +terminating SSL. + +Since communication between the load balancer and GitLab will not be secure, +there is some additional configuration needed. See the +[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) +for details. + +#### Load balancer terminates SSL with backend SSL + +Configure your load balancers to use the 'HTTP(S)' protocol rather than 'TCP'. +The load balancers will be responsible for managing SSL certificates that +end users will see. + +Traffic will also be secure between the load balancers and NGINX in this +scenario. There is no need to add configuration for proxied SSL since the +connection will be secure all the way. However, configuration will need to be +added to GitLab to configure SSL certificates. See +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +for details on managing SSL certificates and configuring NGINX. + <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> @@ -418,8 +472,14 @@ backend praefect ``` Refer to your preferred Load Balancer's documentation for further guidance. -Also ensure that the routing methods used are distributing calls evenly across -all nodes. + +### Balancing algorithm + +We recommend that a least-connection load balancing algorithm or equivalent +is used wherever possible to ensure equal spread of calls to the nodes and good performance. + +We don't recommend the use of round-robin algorithms as they are known to not +spread connections equally in practice. <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -1114,6 +1174,12 @@ NOTE: Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). For implementations with sharded Gitaly, use the same Gitaly specs. Follow the [separate Gitaly documentation](../gitaly/configure_gitaly.md) instead of this section. +NOTE: +Gitaly has been designed and tested with repositories of varying sizes that follow best practices. +However, large repositories or monorepos not following these practices can significantly +impact Gitaly performance and requirements. +Refer to the [Large Repositories](#large-repositories) for more info. + The recommended cluster setup includes the following components: - 3 Gitaly nodes: Replicated storage of Git repositories. @@ -1418,9 +1484,15 @@ The [Gitaly](../gitaly/index.md) server nodes that make up the cluster have requirements that are dependent on data and load. NOTE: -The Reference Architecture specs have been designed with good headroom in mind -but for Gitaly, increased specs or additional -Gitaly Cluster arrays may be required for notably large data sets or load. +Increased specs for Gitaly nodes may be required in some circumstances such as +significantly large repositories or if any [additional workloads](#additional-workloads), +such as [server hooks](../server_hooks.md), have been added. + +NOTE: +Gitaly has been designed and tested with repositories of varying sizes that follow best practices. +However, large repositories or monorepos not following these practices can significantly +impact Gitaly performance and requirements. +Refer to the [Large Repositories](#large-repositories) for more info. Due to Gitaly having notable input and output requirements, we strongly recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs @@ -2260,7 +2332,7 @@ services where applicable): | PostgreSQL<sup>1</sup> | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | | PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Internal load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Gitaly<sup>5</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | +| Gitaly<sup>5</sup><sup>6</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | | Praefect<sup>5</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Object storage<sup>4</sup> | - | - | - | - | @@ -2278,6 +2350,7 @@ services where applicable): - [Google Cloud Load Balancing](https://cloud.google.com/load-balancing) and [Amazon Elastic Load Balancing](https://aws.amazon.com/elasticloadbalancing/) are known to work. 4. Should be run on reputable Cloud Provider or Self Managed solutions. More information can be found in the [Configure the object storage](#configure-the-object-storage) section. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. +6. Gitaly has been designed and tested with repositories of varying sizes that follow best practices. However, large repositories or monorepos that don't follow these practices can significantly impact Gitaly requirements. Refer to the [Large Repositories](#large-repositories) for more info. <!-- markdownlint-enable MD029 --> NOTE: diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index b87af8b27ac..06889644223 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -28,7 +28,7 @@ full list of reference architectures, see | Internal load balancing node<sup>3</sup> | 1 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | | Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | | Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | -| Gitaly<sup>5</sup> | 3 | 64 vCPU, 240 GB memory | `n1-standard-64` | `m5.16xlarge` | +| Gitaly<sup>5</sup><sup>6</sup> | 3 | 64 vCPU, 240 GB memory | `n1-standard-64` | `m5.16xlarge` | | Praefect<sup>5</sup> | 3 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | | Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | @@ -50,6 +50,7 @@ full list of reference architectures, see - [Google Cloud Load Balancing](https://cloud.google.com/load-balancing) and [Amazon Elastic Load Balancing](https://aws.amazon.com/elasticloadbalancing/) are known to work. 4. Should be run on reputable Cloud Provider or Self Managed solutions. More information can be found in the [Configure the object storage](#configure-the-object-storage) section. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. +6. Gitaly has been designed and tested with repositories of varying sizes that follow best practices. However, large repositories or monorepos that don't follow these practices can significantly impact Gitaly requirements. Refer to the [Large Repositories](#large-repositories) for more info. <!-- markdownlint-enable MD029 --> NOTE: @@ -158,10 +159,45 @@ Any "burstable" instance types are not recommended due to inconsistent performan ### Supported infrastructure -As a general guidance, GitLab should run on most infrastructure such as reputable Cloud Providers (AWS, GCP) and their services, or self managed (ESXi) that meet both the specs detailed above, as well as any requirements in this section. However, this does not constitute a guarantee for every potential permutation. +As a general guidance, GitLab should run on most infrastructure such as reputable Cloud Providers (AWS, GCP, Azure) and their services, +or self managed (ESXi) that meet both the specs detailed above, as well as any requirements in this section. +However, this does not constitute a guarantee for every potential permutation. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. +### Additional workloads + +The Reference Architectures have been [designed and tested](index.md#validation-and-test-results) for standard GitLab setups with +good headroom in mind to cover most scenarios. However, if any additional workloads are being added on the nodes, +such as security software, you may still need to adjust the specs accordingly to compensate. + +This also applies for some GitLab features where it's possible to run custom scripts, for example [server hooks](../server_hooks.md). + +As a general rule, it's recommended to have robust monitoring in place to measure the impact of +any additional workloads to inform any changes needed to be made. + +### Large repositories + +The Reference Architectures were tested with repositories of varying sizes that follow best practices. + +However, large repositories or monorepos (Several gigabytes or more) can **significantly** impact the performance +of Git and in turn the environment itself if best practices aren't being followed such as not storing +binary or blob files in LFS. Repositories are at the core of any environment the consequences can be wide-ranging +when they are not optimized. Some examples of this impact include [Git packing operations](https://git-scm.com/book/en/v2/Git-Internals-Packfiles) +taking longer and consuming high CPU / Memory resources or Git checkouts taking longer that affect both users and +CI pipelines alike. + +As such, large repositories come with notable cost and typically will require more resources to handle, +significantly so in some cases. It's therefore **strongly** recommended then to review large repositories +to ensure they maintain good repo health and reduce their size wherever possible. + +NOTE: +If best practices aren't followed and large repositories are present on the environment, +increased Gitaly specs may be required to ensure stable performance. + +Refer to the [Managing large repositories documentation](../../user/project/repository/managing_large_repositories.md) +for more information and guidance. + ### Praefect PostgreSQL It's worth noting that at this time [Praefect requires its own database server](../gitaly/praefect.md#postgresql) and @@ -250,8 +286,7 @@ In a multi-node GitLab configuration, you'll need a load balancer to route traffic to the application servers. The specifics on which load balancer to use or its exact configuration is beyond the scope of GitLab documentation. We assume that if you're managing multi-node systems like GitLab, you already have a load -balancer of choice and that the routing methods used are distributing calls evenly -between all nodes. Some load balancer examples include HAProxy (open-source), +balancer of choice. Some load balancer examples include HAProxy (open-source), F5 Big-IP LTM, and Citrix Net Scaler. This documentation outline the ports and protocols needed for use with GitLab. @@ -259,47 +294,13 @@ This architecture has been tested and validated with [HAProxy](https://www.hapro as the load balancer. Although other load balancers with similar feature sets could also be used, those load balancers have not been validated. -The next question is how you will handle SSL in your environment. -There are several different options: +### Balancing algorithm -- [The application node terminates SSL](#application-node-terminates-ssl). -- [The load balancer terminates SSL without backend SSL](#load-balancer-terminates-ssl-without-backend-ssl) - and communication is not secure between the load balancer and the application node. -- [The load balancer terminates SSL with backend SSL](#load-balancer-terminates-ssl-with-backend-ssl) - and communication is *secure* between the load balancer and the application node. - -### Application node terminates SSL - -Configure your load balancer to pass connections on port 443 as `TCP` rather -than `HTTP(S)` protocol. This will pass the connection to the application node's -NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. +We recommend that a least-connection load balancing algorithm or equivalent +is used wherever possible to ensure equal spread of calls to the nodes and good performance. -See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) -for details on managing SSL certificates and configuring NGINX. - -### Load balancer terminates SSL without backend SSL - -Configure your load balancer to use the `HTTP(S)` protocol rather than `TCP`. -The load balancer will then be responsible for managing SSL certificates and -terminating SSL. - -Since communication between the load balancer and GitLab will not be secure, -there is some additional configuration needed. See the -[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) -for details. - -### Load balancer terminates SSL with backend SSL - -Configure your load balancers to use the 'HTTP(S)' protocol rather than 'TCP'. -The load balancers will be responsible for managing SSL certificates that -end users will see. - -Traffic will also be secure between the load balancers and NGINX in this -scenario. There is no need to add configuration for proxied SSL since the -connection will be secure all the way. However, configuration will need to be -added to GitLab to configure SSL certificates. See the -[HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) -for details on managing SSL certificates and configuring NGINX. +We don't recommend the use of round-robin algorithms as they are known to not +spread connections equally in practice. ### Readiness checks @@ -360,6 +361,50 @@ Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. | ------- | ------------ | -------- | | 443 | 22 | TCP | +### SSL + +The next question is how you will handle SSL in your environment. +There are several different options: + +- [The application node terminates SSL](#application-node-terminates-ssl). +- [The load balancer terminates SSL without backend SSL](#load-balancer-terminates-ssl-without-backend-ssl) + and communication is not secure between the load balancer and the application node. +- [The load balancer terminates SSL with backend SSL](#load-balancer-terminates-ssl-with-backend-ssl) + and communication is *secure* between the load balancer and the application node. + +#### Application node terminates SSL + +Configure your load balancer to pass connections on port 443 as `TCP` rather +than `HTTP(S)` protocol. This will pass the connection to the application node's +NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. + +See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +for details on managing SSL certificates and configuring NGINX. + +#### Load balancer terminates SSL without backend SSL + +Configure your load balancer to use the `HTTP(S)` protocol rather than `TCP`. +The load balancer will then be responsible for managing SSL certificates and +terminating SSL. + +Since communication between the load balancer and GitLab will not be secure, +there is some additional configuration needed. See the +[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) +for details. + +#### Load balancer terminates SSL with backend SSL + +Configure your load balancers to use the 'HTTP(S)' protocol rather than 'TCP'. +The load balancers will be responsible for managing SSL certificates that +end users will see. + +Traffic will also be secure between the load balancers and NGINX in this +scenario. There is no need to add configuration for proxied SSL since the +connection will be secure all the way. However, configuration will need to be +added to GitLab to configure SSL certificates. See +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +for details on managing SSL certificates and configuring NGINX. + <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> @@ -424,8 +469,14 @@ backend praefect ``` Refer to your preferred Load Balancer's documentation for further guidance. -Also ensure that the routing methods used are distributing calls evenly across -all nodes. + +### Balancing algorithm + +We recommend that a least-connection load balancing algorithm or equivalent +is used wherever possible to ensure equal spread of calls to the nodes and good performance. + +We don't recommend the use of round-robin algorithms as they are known to not +spread connections equally in practice. <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -1181,6 +1232,12 @@ NOTE: Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). For implementations with sharded Gitaly, use the same Gitaly specs. Follow the [separate Gitaly documentation](../gitaly/configure_gitaly.md) instead of this section. +NOTE: +Gitaly has been designed and tested with repositories of varying sizes that follow best practices. +However, large repositories or monorepos not following these practices can significantly +impact Gitaly performance and requirements. +Refer to the [Large Repositories](#large-repositories) for more info. + The recommended cluster setup includes the following components: - 3 Gitaly nodes: Replicated storage of Git repositories. @@ -1488,9 +1545,15 @@ The [Gitaly](../gitaly/index.md) server nodes that make up the cluster have requirements that are dependent on data and load. NOTE: -The Reference Architecture specs have been designed with good headroom in mind -but for Gitaly, increased specs or additional -Gitaly Cluster arrays may be required for notably large data sets or load. +Increased specs for Gitaly nodes may be required in some circumstances such as +significantly large repositories or if any [additional workloads](#additional-workloads), +such as [server hooks](../server_hooks.md), have been added. + +NOTE: +Gitaly has been designed and tested with repositories of varying sizes that follow best practices. +However, large repositories or monorepos not following these practices can significantly +impact Gitaly performance and requirements. +Refer to the [Large Repositories](#large-repositories) for more info. Due to Gitaly having notable input and output requirements, we strongly recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs @@ -2300,7 +2363,7 @@ services where applicable): | Internal load balancing node<sup>3</sup> | 1 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | | Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | | Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | -| Gitaly<sup>5</sup> | 3 | 64 vCPU, 240 GB memory | `n1-standard-64` | `m5.16xlarge` | +| Gitaly<sup>5</sup><sup>6</sup> | 3 | 64 vCPU, 240 GB memory | `n1-standard-64` | `m5.16xlarge` | | Praefect<sup>5</sup> | 3 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | | Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Object storage<sup>4</sup> | - | - | - | - | @@ -2318,6 +2381,7 @@ services where applicable): - [Google Cloud Load Balancing](https://cloud.google.com/load-balancing) and [Amazon Elastic Load Balancing](https://aws.amazon.com/elasticloadbalancing/) are known to work. 4. Should be run on reputable Cloud Provider or Self Managed solutions. More information can be found in the [Configure the object storage](#configure-the-object-storage) section. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. +6. Gitaly has been designed and tested with repositories of varying sizes that follow best practices. However, large repositories or monorepos that don't follow these practices can significantly impact Gitaly requirements. Refer to the [Large Repositories](#large-repositories) for more info. <!-- markdownlint-enable MD029 --> NOTE: diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 510fa7c99b7..b27d8ce4847 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -34,7 +34,7 @@ costly-to-operate environment by using the | PostgreSQL<sup>1</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | | PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Internal load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Gitaly<sup>5</sup> | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | `m5.2xlarge` | +| Gitaly<sup>5</sup><sup>6</sup> | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | `m5.2xlarge` | | Praefect<sup>5</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Sidekiq | 4 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | @@ -56,6 +56,7 @@ costly-to-operate environment by using the - [Google Cloud Load Balancing](https://cloud.google.com/load-balancing) and [Amazon Elastic Load Balancing](https://aws.amazon.com/elasticloadbalancing/) are known to work. 4. Should be run on reputable Cloud Provider or Self Managed solutions. More information can be found in the [Configure the object storage](#configure-the-object-storage) section. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. +6. Gitaly has been designed and tested with repositories of varying sizes that follow best practices. However, large repositories or monorepos that don't follow these practices can significantly impact Gitaly requirements. Refer to the [Large Repositories](#large-repositories) for more info. <!-- markdownlint-enable MD029 --> NOTE: @@ -161,10 +162,45 @@ Any "burstable" instance types are not recommended due to inconsistent performan ### Supported infrastructure -As a general guidance, GitLab should run on most infrastructure such as reputable Cloud Providers (AWS, GCP) and their services, or self managed (ESXi) that meet both the specs detailed above, as well as any requirements in this section. However, this does not constitute a guarantee for every potential permutation. +As a general guidance, GitLab should run on most infrastructure such as reputable Cloud Providers (AWS, GCP) and their services, +or self managed (ESXi) that meet both the specs detailed above, as well as any requirements in this section. +However, this does not constitute a guarantee for every potential permutation. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. +### Additional workloads + +The Reference Architectures have been [designed and tested](index.md#validation-and-test-results) for standard GitLab setups with +good headroom in mind to cover most scenarios. However, if any additional workloads are being added on the nodes, +such as security software, you may still need to adjust the specs accordingly to compensate. + +This also applies for some GitLab features where it's possible to run custom scripts, for example [server hooks](../server_hooks.md). + +As a general rule, it's recommended to have robust monitoring in place to measure the impact of +any additional workloads to inform any changes needed to be made. + +### Large repositories + +The Reference Architectures were tested with repositories of varying sizes that follow best practices. + +However, large repositories or monorepos (Several gigabytes or more) can **significantly** impact the performance +of Git and in turn the environment itself if best practices aren't being followed such as not storing +binary or blob files in LFS. Repositories are at the core of any environment the consequences can be wide-ranging +when they are not optimized. Some examples of this impact include [Git packing operations](https://git-scm.com/book/en/v2/Git-Internals-Packfiles) +taking longer and consuming high CPU / Memory resources or Git checkouts taking longer that affect both users and +CI pipelines alike. + +As such, large repositories come with notable cost and typically will require more resources to handle, +significantly so in some cases. It's therefore **strongly** recommended then to review large repositories +to ensure they maintain good repo health and reduce their size wherever possible. + +NOTE: +If best practices aren't followed and large repositories are present on the environment, +increased Gitaly specs may be required to ensure stable performance. + +Refer the [Managing large repositories documentation](../../user/project/repository/managing_large_repositories.md) +for more information and guidance. + ### Praefect PostgreSQL It's worth noting that at this time [Praefect requires its own database server](../gitaly/praefect.md#postgresql) and @@ -237,12 +273,11 @@ The following list includes descriptions of each server and its assigned IP: ## Configure the external load balancer -In a multi-node GitLab configuration, you need a load balancer to route +In a multi-node GitLab configuration, you'll need a load balancer to route traffic to the application servers. The specifics on which load balancer to use or its exact configuration is beyond the scope of GitLab documentation. We assume that if you're managing multi-node systems like GitLab, you already have a load -balancer of choice and that the routing methods used are distributing calls evenly -between all nodes. Some load balancer examples include HAProxy (open-source), +balancer of choice. Some load balancer examples include HAProxy (open-source), F5 Big-IP LTM, and Citrix Net Scaler. This documentation outline the ports and protocols needed for use with GitLab. @@ -259,45 +294,20 @@ There are several different options: - [The load balancer terminates SSL with backend SSL](#load-balancer-terminates-ssl-with-backend-ssl) and communication is *secure* between the load balancer and the application node. -### Application node terminates SSL - -Configure your load balancer to pass connections on port 443 as `TCP` rather -than `HTTP(S)` protocol. This passes the connection to the application node's -NGINX service untouched. NGINX has the SSL certificate and listen on port 443. - -See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) -for details on managing SSL certificates and configuring NGINX. - -### Load balancer terminates SSL without backend SSL - -Configure your load balancer to use the `HTTP(S)` protocol rather than `TCP`. -The load balancer is then responsible for managing SSL certificates and -terminating SSL. - -Since communication between the load balancer and GitLab is not secure, -there is some additional configuration needed. See the -[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) -for details. - -### Load balancer terminates SSL with backend SSL +### Balancing algorithm -Configure your load balancers to use the 'HTTP(S)' protocol rather than 'TCP'. -The load balancers are responsible for managing SSL certificates that -end users see. +We recommend that a least-connection load balancing algorithm or equivalent +is used wherever possible to ensure equal spread of calls to the nodes and good performance. -Traffic is also secure between the load balancers and NGINX in this -scenario. There is no need to add configuration for proxied SSL since the -connection is secure all the way. However, configuration needs to be -added to GitLab to configure SSL certificates. See the -[HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) -for details on managing SSL certificates and configuring NGINX. +We don't recommend the use of round-robin algorithms as they are known to not +spread connections equally in practice. ### Readiness checks Ensure the external load balancer only routes to working services with built in monitoring endpoints. The [readiness checks](../../user/admin_area/monitoring/health_check.md) all require [additional configuration](../monitoring/ip_allowlist.md) -on the nodes being checked, otherwise, the external load balancer is not able to +on the nodes being checked, otherwise, the external load balancer will not be able to connect. ### Ports @@ -316,11 +326,11 @@ The basic ports to be used are shown in the table below. to pass through the `Connection` and `Upgrade` hop-by-hop headers. See the [web terminal](../integration/terminal.md) integration guide for more details. -- (*2*): When using HTTPS protocol for port 443, you need to add an SSL +- (*2*): When using HTTPS protocol for port 443, you will need to add an SSL certificate to the load balancers. If you wish to terminate SSL at the GitLab application server instead, use TCP protocol. -If you're using GitLab Pages with custom domain support you need some +If you're using GitLab Pages with custom domain support you will need some additional port configurations. GitLab Pages requires a separate virtual IP address. Configure DNS to point the `pages_external_url` from `/etc/gitlab/gitlab.rb` at the new virtual IP address. See the @@ -342,7 +352,7 @@ GitLab Pages requires a separate virtual IP address. Configure DNS to point the Some organizations have policies against opening SSH port 22. In this case, it may be helpful to configure an alternate SSH hostname that allows users -to use SSH on port 443. An alternate SSH hostname requires a new virtual IP address +to use SSH on port 443. An alternate SSH hostname will require a new virtual IP address compared to the other GitLab HTTP configuration above. Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. @@ -351,6 +361,50 @@ Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. | ------- | ------------ | -------- | | 443 | 22 | TCP | +### SSL + +The next question is how you will handle SSL in your environment. +There are several different options: + +- [The application node terminates SSL](#application-node-terminates-ssl). +- [The load balancer terminates SSL without backend SSL](#load-balancer-terminates-ssl-without-backend-ssl) + and communication is not secure between the load balancer and the application node. +- [The load balancer terminates SSL with backend SSL](#load-balancer-terminates-ssl-with-backend-ssl) + and communication is *secure* between the load balancer and the application node. + +#### Application node terminates SSL + +Configure your load balancer to pass connections on port 443 as `TCP` rather +than `HTTP(S)` protocol. This will pass the connection to the application node's +NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. + +See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +for details on managing SSL certificates and configuring NGINX. + +#### Load balancer terminates SSL without backend SSL + +Configure your load balancer to use the `HTTP(S)` protocol rather than `TCP`. +The load balancer will then be responsible for managing SSL certificates and +terminating SSL. + +Since communication between the load balancer and GitLab will not be secure, +there is some additional configuration needed. See the +[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) +for details. + +#### Load balancer terminates SSL with backend SSL + +Configure your load balancers to use the 'HTTP(S)' protocol rather than 'TCP'. +The load balancers will be responsible for managing SSL certificates that +end users will see. + +Traffic will also be secure between the load balancers and NGINX in this +scenario. There is no need to add configuration for proxied SSL since the +connection will be secure all the way. However, configuration will need to be +added to GitLab to configure SSL certificates. See +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +for details on managing SSL certificates and configuring NGINX. + <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> @@ -415,8 +469,14 @@ backend praefect ``` Refer to your preferred Load Balancer's documentation for further guidance. -Also ensure that the routing methods used are distributing calls evenly across -all nodes. + +### Balancing algorithm + +We recommend that a least-connection load balancing algorithm or equivalent +is used wherever possible to ensure equal spread of calls to the nodes and good performance. + +We don't recommend the use of round-robin algorithms as they are known to not +spread connections equally in practice. <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -1110,6 +1170,12 @@ NOTE: Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). For implementations with sharded Gitaly, use the same Gitaly specs. Follow the [separate Gitaly documentation](../gitaly/configure_gitaly.md) instead of this section. +NOTE: +Gitaly has been designed and tested with repositories of varying sizes that follow best practices. +However, large repositories or monorepos not following these practices can significantly +impact Gitaly performance and requirements. +Refer to the [Large Repositories](#large-repositories) for more info. + The recommended cluster setup includes the following components: - 3 Gitaly nodes: Replicated storage of Git repositories. @@ -1415,9 +1481,15 @@ The [Gitaly](../gitaly/index.md) server nodes that make up the cluster have requirements that are dependent on data and load. NOTE: -The Reference Architecture specs have been designed with good headroom in mind -but for Gitaly, increased specs or additional -Gitaly Cluster arrays may be required for notably large data sets or load. +Increased specs for Gitaly nodes may be required in some circumstances such as +significantly large repositories or if any [additional workloads](#additional-workloads), +such as [server hooks](../server_hooks.md), have been added. + +NOTE: +Gitaly has been designed and tested with repositories of varying sizes that follow best practices. +However, large repositories or monorepos not following these practices can significantly +impact Gitaly performance and requirements. +Refer to the [Large Repositories](#large-repositories) for more info. Due to Gitaly having notable input and output requirements, we strongly recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs @@ -2234,7 +2306,7 @@ services where applicable): | PostgreSQL<sup>1</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | | PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Internal load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Gitaly<sup>5</sup> | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | `m5.2xlarge` | +| Gitaly<sup>5</sup><sup>6</sup> | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | `m5.2xlarge` | | Praefect<sup>5</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Object storage<sup>4</sup> | - | - | - | - | @@ -2252,6 +2324,7 @@ services where applicable): - [Google Cloud Load Balancing](https://cloud.google.com/load-balancing) and [Amazon Elastic Load Balancing](https://aws.amazon.com/elasticloadbalancing/) are known to work. 4. Should be run on reputable Cloud Provider or Self Managed solutions. More information can be found in the [Configure the object storage](#configure-the-object-storage) section. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. +6. Gitaly has been designed and tested with repositories of varying sizes that follow best practices. However, large repositories or monorepos that don't follow these practices can significantly impact Gitaly requirements. Refer to the [Large Repositories](#large-repositories) for more info. <!-- markdownlint-enable MD029 --> NOTE: diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index b7a3a5c6b3f..e077b28c0ac 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -673,6 +673,7 @@ Input type: `AdminSidekiqQueuesDeleteJobsInput` | <a id="mutationadminsidekiqqueuesdeletejobsrootnamespace"></a>`rootNamespace` | [`String`](#string) | Delete jobs matching root_namespace in the context metadata. | | <a id="mutationadminsidekiqqueuesdeletejobssubscriptionplan"></a>`subscriptionPlan` | [`String`](#string) | Delete jobs matching subscription_plan in the context metadata. | | <a id="mutationadminsidekiqqueuesdeletejobsuser"></a>`user` | [`String`](#string) | Delete jobs matching user in the context metadata. | +| <a id="mutationadminsidekiqqueuesdeletejobsuserid"></a>`userId` | [`String`](#string) | Delete jobs matching user_id in the context metadata. | | <a id="mutationadminsidekiqqueuesdeletejobsworkerclass"></a>`workerClass` | [`String`](#string) | Delete jobs with the given worker class. | #### Fields diff --git a/doc/api/protected_branches.md b/doc/api/protected_branches.md index f7047f145cf..96bd5c15f13 100644 --- a/doc/api/protected_branches.md +++ b/doc/api/protected_branches.md @@ -435,9 +435,93 @@ PATCH /projects/:id/protected_branches/:name curl --request PATCH --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/protected_branches/feature-branch?allow_force_push=true&code_owner_approval_required=true" ``` -| Attribute | Type | Required | Description | +| Attribute | Type | Required | Description | | -------------------------------------------- | ---- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | -| `name` | string | yes | The name of the branch | -| `allow_force_push` | boolean | no | When enabled, members who can push to this branch can also force push. | -| `code_owner_approval_required` **(PREMIUM)** | boolean | no | Prevent pushes to this branch if it matches an item in the [`CODEOWNERS` file](../user/project/code_owners.md). Defaults to `false`. | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | +| `name` | string | yes | The name of the branch | +| `allow_force_push` | boolean | no | When enabled, members who can push to this branch can also force push. | +| `code_owner_approval_required` **(PREMIUM)** | boolean | no | Prevent pushes to this branch if it matches an item in the [`CODEOWNERS` file](../user/project/code_owners.md). Defaults to `false`. | +| `allowed_to_push` **(PREMIUM)** | array | no | Array of push access levels, with each described by a hash. | +| `allowed_to_merge` **(PREMIUM)** | array | no | Array of merge access levels, with each described by a hash. | +| `allowed_to_unprotect` **(PREMIUM)** | array | no | Array of unprotect access levels, with each described by a hash. | + +Elements in the `allowed_to_push`, `allowed_to_merge` and `allowed_to_unprotect` arrays should be one of `user_id`, `group_id` or +`access_level`, and take the form `{user_id: integer}`, `{group_id: integer}` or +`{access_level: integer}`. + +To update: + +- `user_id`: Ensure the updated user has access to the project. You must also pass the + `id` of the `access_level` in the respective hash. +- `group_id`: Ensure the updated group [has this project shared](../user/project/members/share_project_with_groups.md). + You must also pass the `id` of the `access_level` in the respective hash. + +To delete: + +- You must pass `_destroy` set to `true`. See the following examples. + +### Example: create a `push_access_level` record + +```shell +curl --header 'Content-Type: application/json' --request PATCH \ + --data '{"push_access_levels": [{"group_id": 9899829, access_level: 40}]}' \ + --header "PRIVATE-TOKEN: <your_access_token>" \ + "https://gitlab.example.com/api/v4/projects/22034114/protected_branches/master" +``` + +Example response: + +```json +{ + "name": "master", + "allowed_to_push": [ + { + "id": 12, + "access_level": 40, + "access_level_description": "Administrator", + "user_id": null, + "group_id": 9899829 + } + ] +} +``` + +### Example: update a `push_access_level` record + +```shell +curl --header 'Content-Type: application/json' --request PUT \ + --data '{"push_access_levels": [{"id": 12, "group_id": 22034120}]' \ + --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/22034114/protected_branches/master" +``` + +```json +{ + "name": "master", + "deploy_access_levels": [ + { + "id": 12, + "access_level": 40, + "access_level_description": "Administrator", + "user_id": null, + "group_id": 22034120 + } + ] +} +``` + +### Example: delete a `push_access_level` record + +```shell +curl --header 'Content-Type: application/json' --request PUT \ + --data '{"push_access_levels": [{"id": 12, "_destroy": true}]' \ + --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/22034114/protected_branches/master" +``` + +Example response: + +```json +{ + "name": "master", + "push_access_levels": [] +} +``` diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index e03f029a6ef..56db6ee4c5c 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -124,7 +124,12 @@ module API repository: repository.gitaly_repository.to_h, address: Gitlab::GitalyClient.address(repository.shard), token: Gitlab::GitalyClient.token(repository.shard), - features: Feature::Gitaly.server_feature_flags(repository.project) + features: Feature::Gitaly.server_feature_flags( + user: ::Feature::Gitaly.user_actor(actor.user), + repository: repository, + project: ::Feature::Gitaly.project_actor(repository.container), + group: ::Feature::Gitaly.group_actor(repository.container) + ) } end end diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb index 0c6b9dfde7a..fd798862fa8 100644 --- a/lib/feature/gitaly.rb +++ b/lib/feature/gitaly.rb @@ -4,26 +4,60 @@ module Feature class Gitaly PREFIX = "gitaly_" + # Wrapper for feature flag actor to avoid unnecessarily SQL queries + class ActorWrapper + def initialize(klass, id) + @klass = klass + @id = id + end + + def flipper_id + "#{@klass.name}:#{@id}" + end + end + class << self - def enabled?(feature_flag, project = nil) + def enabled_for_any?(feature_flag, *actors) return false unless Feature::FlipperFeature.table_exists? - Feature.enabled?("#{PREFIX}#{feature_flag}", project, type: :undefined, default_enabled_if_undefined: false) + actors = actors.compact + return Feature.enabled?(feature_flag, type: :undefined, default_enabled_if_undefined: false) if actors.empty? + + actors.any? do |actor| + Feature.enabled?(feature_flag, actor, type: :undefined, default_enabled_if_undefined: false) + end rescue ActiveRecord::NoDatabaseError, PG::ConnectionBad false end - def server_feature_flags(project = nil) + def server_feature_flags(repository: nil, user: nil, project: nil, group: nil) # We need to check that both the DB connection and table exists return {} unless FlipperFeature.database.cached_table_exists? + # The order of actors here is significant. Percentage-based actor selection may not work as expected if this + # order changes. We want repository actor to take highest precedence. + actors = [repository, user, project, group].compact + Feature.persisted_names .select { |f| f.start_with?(PREFIX) } .to_h do |f| - flag = f.delete_prefix(PREFIX) + ["gitaly-feature-#{f.delete_prefix(PREFIX).tr('_', '-')}", enabled_for_any?(f, *actors).to_s] + end + end - ["gitaly-feature-#{flag.tr('_', '-')}", enabled?(flag, project).to_s] - end + def user_actor(user = nil) + return ::Feature::Gitaly::ActorWrapper.new(::User, user.id) if user.is_a?(::User) + + user_id = Gitlab::ApplicationContext.current_context_attribute(:user_id) + ::Feature::Gitaly::ActorWrapper.new(::User, user_id) if user_id + end + + def project_actor(container) + ::Feature::Gitaly::ActorWrapper.new(::Project, container.id) if container.is_a?(::Project) + end + + def group_actor(container) + ::Feature::Gitaly::ActorWrapper.new(::Group, container.namespace_id) if container.is_a?(::Project) end end end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index 1920e1443da..b6ad25e700b 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -11,6 +11,7 @@ module Gitlab LOG_KEY = Labkit::Context::LOG_KEY KNOWN_KEYS = [ :user, + :user_id, :project, :root_namespace, :client_id, @@ -98,6 +99,7 @@ module Gitlab assign_hash_if_value(hash, :artifacts_dependencies_count) hash[:user] = -> { username } if include_user? + hash[:user_id] = -> { user_id } if include_user? hash[:project] = -> { project_path } if include_project? hash[:root_namespace] = -> { root_namespace_path } if include_namespace? hash[:client_id] = -> { client } if include_client? @@ -147,6 +149,11 @@ module Gitlab associated_user&.username end + def user_id + associated_user = user || job_user + associated_user&.id + end + def root_namespace_path associated_routable = namespace || project || runner_project || runner_group || job_project associated_routable&.full_path_components&.first diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb index 1244c7f7475..21a57640aee 100644 --- a/lib/gitlab/ci/config/external/file/artifact.rb +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -42,29 +42,20 @@ module Gitlab context&.parent_pipeline&.project end - def validate_content! - return unless ensure_preconditions_satisfied! - - errors.push("File `#{masked_location}` is empty!") unless content.present? - end - - def ensure_preconditions_satisfied! - unless creating_child_pipeline? - errors.push('Including configs from artifacts is only allowed when triggering child pipelines') - return false - end - - unless job_name.present? - errors.push("Job must be provided when including configs from artifacts") - return false - end - - unless artifact_job.present? - errors.push("Job `#{masked_job_name}` not found in parent pipeline or does not have artifacts!") - return false + def validate_context! + context.logger.instrument(:config_file_artifact_validate_context) do + if !creating_child_pipeline? + errors.push('Including configs from artifacts is only allowed when triggering child pipelines') + elsif !job_name.present? + errors.push("Job must be provided when including configs from artifacts") + elsif !artifact_job.present? + errors.push("Job `#{masked_job_name}` not found in parent pipeline or does not have artifacts!") + end end + end - true + def validate_content! + errors.push("File `#{masked_location}` is empty!") unless content.present? end def artifact_job diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 89da0796906..57ff606c9ee 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -47,12 +47,11 @@ module Gitlab end def validate! - context.logger.instrument(:config_file_validation) do - validate_execution_time! - validate_location! - validate_content! if errors.none? - validate_hash! if errors.none? - end + validate_execution_time! + validate_location! + validate_context! if valid? + fetch_and_validate_content! if valid? + load_and_validate_expanded_hash! if valid? end def metadata @@ -100,6 +99,34 @@ module Gitlab end end + def validate_context! + raise NotImplementedError, 'subclass must implement validate_context' + end + + def fetch_and_validate_content! + context.logger.instrument(:config_file_fetch_content) do + content # calling the method fetches then memoizes the result + end + + return if errors.any? + + context.logger.instrument(:config_file_validate_content) do + validate_content! + end + end + + def load_and_validate_expanded_hash! + context.logger.instrument(:config_file_fetch_content_hash) do + content_hash # calling the method loads then memoizes the result + end + + context.logger.instrument(:config_file_expand_content_includes) do + expanded_content_hash # calling the method expands then memoizes the result + end + + validate_hash! + end + def validate_content! if content.blank? errors.push("Included file `#{masked_location}` is empty or does not exist!") diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index 36fc5c656fc..0912a732158 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -31,10 +31,14 @@ module Gitlab private + def validate_context! + return if context.project&.repository + + errors.push("Local file `#{masked_location}` does not have project!") + end + def validate_content! - if context.project&.repository.nil? - errors.push("Local file `#{masked_location}` does not have project!") - elsif content.nil? + if content.nil? errors.push("Local file `#{masked_location}` does not exist!") elsif content.blank? errors.push("Local file `#{masked_location}` is empty!") diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index 89418bd6a21..553cbd819ad 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -39,12 +39,16 @@ module Gitlab private - def validate_content! + def validate_context! if !can_access_local_content? errors.push("Project `#{masked_project_name}` not found or access denied! Make sure any includes in the pipeline configuration are correctly defined.") elsif sha.nil? errors.push("Project `#{masked_project_name}` reference `#{masked_ref_name}` does not exist!") - elsif content.nil? + end + end + + def validate_content! + if content.nil? errors.push("Project `#{masked_project_name}` file `#{masked_location}` does not exist!") elsif content.blank? errors.push("Project `#{masked_project_name}` file `#{masked_location}` is empty!") @@ -58,7 +62,11 @@ module Gitlab end def can_access_local_content? - Ability.allowed?(context.user, :download_code, project) + strong_memoize(:can_access_local_content) do + context.logger.instrument(:config_file_project_validate_access) do + Ability.allowed?(context.user, :download_code, project) + end + end end def fetch_local_content diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index 3984bf9e4f8..b0c540685d4 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -30,6 +30,10 @@ module Gitlab private + def validate_context! + # no-op + end + def validate_location! super diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb index 5fcf7c71bdf..53236cb317b 100644 --- a/lib/gitlab/ci/config/external/file/template.rb +++ b/lib/gitlab/ci/config/external/file/template.rb @@ -33,6 +33,10 @@ module Gitlab private + def validate_context! + # no-op + end + def validate_location! super diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index dc7ecb7ec45..b8f4ff0e9c4 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -45,7 +45,7 @@ module Gitlab # Relative path of repo attr_reader :relative_path - attr_reader :storage, :gl_repository, :gl_project_path + attr_reader :storage, :gl_repository, :gl_project_path, :container # This remote name has to be stable for all types of repositories that # can join an object pool. If it's structure ever changes, a migration @@ -56,11 +56,12 @@ module Gitlab # This initializer method is only used on the client side (gitlab-ce). # Gitaly-ruby uses a different initializer. - def initialize(storage, relative_path, gl_repository, gl_project_path) + def initialize(storage, relative_path, gl_repository, gl_project_path, container: nil) @storage = storage @relative_path = relative_path @gl_repository = gl_repository @gl_project_path = gl_project_path + @container = container @name = @relative_path.split("/").last end @@ -69,6 +70,11 @@ module Gitlab "<#{self.class.name}: #{self.gl_project_path}>" end + # Support Feature Flag Repository actor + def flipper_id + "Repository:#{@relative_path}" + end + def ==(other) other.is_a?(self.class) && [storage, relative_path] == [other.storage, other.relative_path] end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 870d93a951f..1344e084323 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -205,7 +205,7 @@ module Gitlab metadata['gitaly-session-id'] = session_id metadata['username'] = context_data['meta.user'] if context_data&.fetch('meta.user', nil) metadata['remote_ip'] = context_data['meta.remote_ip'] if context_data&.fetch('meta.remote_ip', nil) - metadata.merge!(Feature::Gitaly.server_feature_flags) + metadata.merge!(Feature::Gitaly.server_feature_flags(**feature_flag_actors)) metadata.merge!(route_to_primary) deadline_info = request_deadline(timeout) @@ -293,7 +293,7 @@ module Gitlab # check if the limit is being exceeded while testing in those environments # In that case we can use a feature flag to indicate that we do want to # enforce request limits. - return true if Feature::Gitaly.enabled?('enforce_requests_limits') + return true if Feature::Gitaly.enabled_for_any?(:gitaly_enforce_requests_limits) !Rails.env.production? end @@ -504,5 +504,24 @@ module Gitlab end private_class_method :max_stacks + + def self.with_feature_flag_actors(repository: nil, user: nil, project: nil, group: nil, &block) + feature_flag_actors[:repository] = repository + feature_flag_actors[:user] = user + feature_flag_actors[:project] = project + feature_flag_actors[:group] = group + + yield + ensure + feature_flag_actors.clear + end + + def self.feature_flag_actors + if Gitlab::SafeRequestStore.active? + Gitlab::SafeRequestStore[:gitaly_feature_flag_actors] ||= {} + else + Thread.current[:gitaly_feature_flag_actors] ||= {} + end + end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 312d1dddff1..6bcf4802fbe 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -4,12 +4,15 @@ module Gitlab module GitalyClient class CommitService include Gitlab::EncodingHelper + include WithFeatureFlagActors TREE_ENTRIES_DEFAULT_LIMIT = 100_000 def initialize(repository) @gitaly_repo = repository.gitaly_repository @repository = repository + + self.repository_actor = repository end def ls_files(revision) @@ -18,7 +21,7 @@ module Gitlab revision: encode_binary(revision) ) - response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :list_files, request, timeout: GitalyClient.medium_timeout) response.flat_map do |msg| msg.paths.map { |d| EncodingHelper.encode!(d.dup) } end @@ -31,7 +34,7 @@ module Gitlab child_id: child_id ) - GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request, timeout: GitalyClient.fast_timeout).value + gitaly_client_call(@repository.storage, :commit_service, :commit_is_ancestor, request, timeout: GitalyClient.fast_timeout).value end def diff(from, to, options = {}) @@ -74,7 +77,7 @@ module Gitlab def commit_deltas(commit) request = Gitaly::CommitDeltaRequest.new(diff_from_parent_request_params(commit)) - response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@repository.storage, :diff_service, :commit_delta, request, timeout: GitalyClient.fast_timeout) response.flat_map { |msg| msg.deltas } end @@ -93,7 +96,7 @@ module Gitlab limit: limit.to_i ) - response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :tree_entry, request, timeout: GitalyClient.medium_timeout) entry = nil data = [] @@ -127,7 +130,7 @@ module Gitlab ) request.sort = Gitaly::GetTreeEntriesRequest::SortBy::TREES_FIRST if pagination_params - response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout) cursor = nil @@ -163,7 +166,7 @@ module Gitlab request.path = encode_binary(options[:path]) if options[:path].present? request.max_count = options[:max_count] if options[:max_count].present? - GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count + gitaly_client_call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count end def diverging_commit_count(from, to, max_count:) @@ -173,7 +176,7 @@ module Gitlab to: encode_binary(to), max_count: max_count ) - response = GitalyClient.call(@repository.storage, :commit_service, :count_diverging_commits, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :count_diverging_commits, request, timeout: GitalyClient.medium_timeout) [response.left_count, response.right_count] end @@ -187,7 +190,7 @@ module Gitlab global_options: parse_global_options!(literal_pathspec: literal_pathspec) ) - response = GitalyClient.call(@repository.storage, :commit_service, :list_last_commits_for_tree, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :list_last_commits_for_tree, request, timeout: GitalyClient.medium_timeout) response.each_with_object({}) do |gitaly_response, hsh| gitaly_response.commits.each do |commit_for_tree| @@ -204,7 +207,7 @@ module Gitlab global_options: parse_global_options!(literal_pathspec: literal_pathspec) ) - gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request, timeout: GitalyClient.fast_timeout).commit + gitaly_commit = gitaly_client_call(@repository.storage, :commit_service, :last_commit_for_path, request, timeout: GitalyClient.fast_timeout).commit return unless gitaly_commit Gitlab::Git::Commit.new(@repository, gitaly_commit) @@ -217,7 +220,7 @@ module Gitlab right_commit_id: right_commit_sha ) - response = GitalyClient.call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout) response.flat_map { |rsp| rsp.stats.to_a } end @@ -227,7 +230,7 @@ module Gitlab commits: commits ) - response = GitalyClient.call(@repository.storage, :diff_service, :find_changed_paths, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :diff_service, :find_changed_paths, request, timeout: GitalyClient.medium_timeout) response.flat_map do |msg| msg.paths.map do |path| Gitlab::Git::ChangedPath.new( @@ -247,7 +250,7 @@ module Gitlab ) request.order = opts[:order].upcase if opts[:order].present? - response = GitalyClient.call(@repository.storage, :commit_service, :find_all_commits, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :find_all_commits, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -268,7 +271,7 @@ module Gitlab request.before = GitalyClient.timestamp(params[:before]) if params[:before] request.after = GitalyClient.timestamp(params[:after]) if params[:after] - response = GitalyClient.call(@repository.storage, :commit_service, :list_commits, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :list_commits, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -290,7 +293,7 @@ module Gitlab repository: quarantined_repo ) - response = GitalyClient.call(@repository.storage, :commit_service, :list_all_commits, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :list_all_commits, request, timeout: GitalyClient.medium_timeout) quarantined_commits = consume_commits_response(response) quarantined_commit_ids = quarantined_commits.map(&:id) @@ -328,7 +331,7 @@ module Gitlab request = Gitaly::ListCommitsByOidRequest.new(repository: @gitaly_repo, oid: oids) - response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) rescue GRPC::NotFound # If no repository is found, happens mainly during testing [] @@ -345,13 +348,13 @@ module Gitlab global_options: parse_global_options!(literal_pathspec: literal_pathspec) ) - response = GitalyClient.call(@repository.storage, :commit_service, :commits_by_message, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :commits_by_message, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end def languages(ref = nil) request = Gitaly::CommitLanguagesRequest.new(repository: @gitaly_repo, revision: ref || '') - response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request, timeout: GitalyClient.long_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :commit_languages, request, timeout: GitalyClient.long_timeout) response.languages.map { |l| { value: l.share.round(2), label: l.name, color: l.color, highlight: l.color } } end @@ -364,7 +367,7 @@ module Gitlab range: (encode_binary(range) if range) ) - response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) response.reduce([]) { |memo, msg| memo << msg.data }.join end @@ -400,7 +403,7 @@ module Gitlab repository: @gitaly_repo, revision: encode_binary(revision) ) - GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request, timeout: GitalyClient.medium_timeout) + gitaly_client_call(@repository.storage, :commit_service, :commit_stats, request, timeout: GitalyClient.medium_timeout) end def find_commits(options) @@ -424,7 +427,7 @@ module Gitlab request.paths = encode_repeated(Array(options[:path])) if options[:path].present? - response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -443,7 +446,7 @@ module Gitlab end end - response = GitalyClient.call( + response = gitaly_client_call( @repository.storage, :commit_service, :check_objects_exist, enum, timeout: GitalyClient.medium_timeout ) @@ -470,7 +473,7 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :commit_service, :filter_shas_with_signatures, enum, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :filter_shas_with_signatures, enum, timeout: GitalyClient.fast_timeout) response.flat_map do |msg| msg.shas.map { |sha| EncodingHelper.encode!(sha) } end @@ -478,7 +481,7 @@ module Gitlab def get_commit_signatures(commit_ids) request = Gitaly::GetCommitSignaturesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids) - response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_signatures, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :get_commit_signatures, request, timeout: GitalyClient.fast_timeout) signatures = Hash.new { |h, k| h[k] = [+''.b, +''.b] } current_commit_id = nil @@ -497,7 +500,7 @@ module Gitlab def get_commit_messages(commit_ids) request = Gitaly::GetCommitMessagesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids) - response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_messages, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :get_commit_messages, request, timeout: GitalyClient.fast_timeout) messages = Hash.new { |h, k| h[k] = +''.b } current_commit_id = nil @@ -515,7 +518,7 @@ module Gitlab request = Gitaly::ListCommitsByRefNameRequest .new(repository: @gitaly_repo, ref_names: refs.map { |ref| encode_binary(ref) }) - response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_ref_name, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :list_commits_by_ref_name, request, timeout: GitalyClient.medium_timeout) commit_refs = response.flat_map do |message| message.commit_refs.map do |commit_ref| @@ -540,7 +543,7 @@ module Gitlab request_params.merge!(Gitlab::Git::DiffCollection.limits(options)) request = Gitaly::CommitDiffRequest.new(request_params) - response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :diff_service, :commit_diff, request, timeout: GitalyClient.medium_timeout) GitalyClient::DiffStitcher.new(response) end @@ -577,7 +580,7 @@ module Gitlab revision: encode_binary(revision) ) - response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) response.commit end diff --git a/lib/gitlab/gitaly_client/with_feature_flag_actors.rb b/lib/gitlab/gitaly_client/with_feature_flag_actors.rb new file mode 100644 index 00000000000..f89de276c50 --- /dev/null +++ b/lib/gitlab/gitaly_client/with_feature_flag_actors.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module GitalyClient + # This module is responsible for collecting feature flag actors in Gitaly Client. Unlike normal feature flags used + # in Gitlab development, feature flags passed to Gitaly are pre-evaluated at Rails side before being passed to + # Gitaly. As a result, we need to collect all possible actors for the evaluation before issue any RPC. At this + # layer, the only parameter we have is raw repository. We need to infer other actors from the repository. Adding + # extra SQL queries before any RPC are not good for the performance. We applied some quirky optimizations here to + # avoid issuing SQL queries. However, in some less common code paths, a couple of queries are expected. + module WithFeatureFlagActors + include Gitlab::Utils::StrongMemoize + + attr_accessor :repository_actor + + # Use actor here means the user who originally perform the action. It is collected from ApplicationContext. As + # this information is widely propagated in all entry points, User actor should be available everywhere, even in + # background jobs. + def user_actor + strong_memoize(:user_actor) do + Feature::Gitaly.user_actor + end + end + + # TODO: replace this project actor by Repo actor + def project_actor + strong_memoize(:project_actor) do + Feature::Gitaly.project_actor(repository_container) + end + end + + def group_actor + strong_memoize(:group_actor) do + Feature::Gitaly.group_actor(repository_container) + end + end + + def gitaly_client_call(*args, **kargs) + if Feature.enabled?(:actors_aware_gitaly_calls) + # The order of actors here is significant. Percentage-based actor selection may not work as expected if this + # order changes. + GitalyClient.with_feature_flag_actors( + repository: repository_actor, + user: user_actor, + project: project_actor, + group: group_actor + ) do + GitalyClient.call(*args, **kargs) + end + else + GitalyClient.call(*args, **kargs) + end + end + + def repository_container + strong_memoize(:repository_container) do + next if repository_actor&.gl_repository.blank? + + if repository_actor.container.nil? + identifier = Gitlab::GlRepository::Identifier.parse(repository_actor.gl_repository) + identifier.container + else + repository_actor.container + end + end + end + end + end +end + +Gitlab::GitalyClient::WithFeatureFlagActors.prepend_mod_with('Gitlab::GitalyClient::WithFeatureFlagActors') diff --git a/lib/gitlab/slash_commands/application_help.rb b/lib/gitlab/slash_commands/application_help.rb index 1a92346be15..bfdb65a816d 100644 --- a/lib/gitlab/slash_commands/application_help.rb +++ b/lib/gitlab/slash_commands/application_help.rb @@ -3,14 +3,9 @@ module Gitlab module SlashCommands class ApplicationHelp < BaseCommand - def initialize(project, params) - @project = project - @params = params - end - def execute Gitlab::SlashCommands::Presenters::Help - .new(project, commands) + .new(project, commands, params) .present(trigger, params[:text]) end @@ -21,7 +16,11 @@ module Gitlab end def commands - Gitlab::SlashCommands::Command.commands + Gitlab::SlashCommands::Command.new( + project, + chat_name, + params + ).commands end end end diff --git a/lib/gitlab/slash_commands/command.rb b/lib/gitlab/slash_commands/command.rb index 239479f99d2..265eda46489 100644 --- a/lib/gitlab/slash_commands/command.rb +++ b/lib/gitlab/slash_commands/command.rb @@ -3,8 +3,8 @@ module Gitlab module SlashCommands class Command < BaseCommand - def self.commands - [ + def commands + commands = [ Gitlab::SlashCommands::IssueShow, Gitlab::SlashCommands::IssueNew, Gitlab::SlashCommands::IssueSearch, @@ -14,6 +14,12 @@ module Gitlab Gitlab::SlashCommands::Deploy, Gitlab::SlashCommands::Run ] + + if Feature.enabled?(:incident_declare_slash_command, current_user) + commands << Gitlab::SlashCommands::IncidentManagement::IncidentNew + end + + commands end def execute @@ -44,7 +50,7 @@ module Gitlab private def available_commands - self.class.commands.keep_if do |klass| + commands.keep_if do |klass| klass.available?(project) end end diff --git a/lib/gitlab/slash_commands/incident_management/incident_command.rb b/lib/gitlab/slash_commands/incident_management/incident_command.rb new file mode 100644 index 00000000000..13d371151f9 --- /dev/null +++ b/lib/gitlab/slash_commands/incident_management/incident_command.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + module IncidentManagement + class IncidentCommand < BaseCommand + def self.available?(project) + true + end + + def collection + IssuesFinder.new(current_user, project_id: project.id, issue_types: :incident).execute + end + end + end + end +end diff --git a/lib/gitlab/slash_commands/incident_management/incident_new.rb b/lib/gitlab/slash_commands/incident_management/incident_new.rb new file mode 100644 index 00000000000..ce148f888b8 --- /dev/null +++ b/lib/gitlab/slash_commands/incident_management/incident_new.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + module IncidentManagement + class IncidentNew < IncidentCommand + def self.help_message + 'incident declare' + end + + def self.allowed?(project, user) + Feature.enabled?(:incident_declare_slash_command, user) && can?(user, :create_incident, project) + end + + def self.match(text) + text == 'incident declare' + end + + def execute(_match) + response = ServiceResponse.success(message: 'It works!') + + presenter.present(response.message) + end + + private + + def presenter + Gitlab::SlashCommands::Presenters::IncidentManagement::IncidentNew.new + end + end + end + end +end diff --git a/lib/gitlab/slash_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb index 71bc0dc0123..61b36308d20 100644 --- a/lib/gitlab/slash_commands/presenters/help.rb +++ b/lib/gitlab/slash_commands/presenters/help.rb @@ -4,9 +4,10 @@ module Gitlab module SlashCommands module Presenters class Help < Presenters::Base - def initialize(project, commands) + def initialize(project, commands, params = {}) @project = project @commands = commands + @params = params end def present(trigger, text) @@ -66,7 +67,13 @@ module Gitlab def full_commands_message(trigger) list = @commands - .map { |command| "#{trigger} #{command.help_message}" } + .map do |command| + if command < Gitlab::SlashCommands::IncidentManagement::IncidentCommand + "#{@params[:command]} #{command.help_message}" + else + "#{trigger} #{command.help_message}" + end + end .join("\n") <<~MESSAGE diff --git a/lib/gitlab/slash_commands/presenters/incident_management/incident_new.rb b/lib/gitlab/slash_commands/presenters/incident_management/incident_new.rb new file mode 100644 index 00000000000..5030c8282db --- /dev/null +++ b/lib/gitlab/slash_commands/presenters/incident_management/incident_new.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + module Presenters + module IncidentManagement + class IncidentNew < Presenters::Base + def present(message) + ephemeral_response(text: message) + end + end + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 906439d5e71..0d5daeefe90 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -33,7 +33,12 @@ module Gitlab GitalyServer: { address: Gitlab::GitalyClient.address(repository.storage), token: Gitlab::GitalyClient.token(repository.storage), - features: Feature::Gitaly.server_feature_flags(repository.project) + features: Feature::Gitaly.server_feature_flags( + user: ::Feature::Gitaly.user_actor(user), + repository: repository, + project: ::Feature::Gitaly.project_actor(repository.container), + group: ::Feature::Gitaly.group_actor(repository.container) + ) } } @@ -252,7 +257,12 @@ module Gitlab { address: Gitlab::GitalyClient.address(repository.shard), token: Gitlab::GitalyClient.token(repository.shard), - features: Feature::Gitaly.server_feature_flags(repository.project) + features: Feature::Gitaly.server_feature_flags( + user: ::Feature::Gitaly.user_actor, + repository: repository, + project: ::Feature::Gitaly.project_actor(repository.container), + group: ::Feature::Gitaly.group_actor(repository.container) + ) } end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 3a26183da38..c5160f4c751 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -41891,6 +41891,9 @@ msgstr "" msgid "This variable can not be masked." msgstr "" +msgid "This vulnerability type has been deprecated from GitLab's default ruleset and automatically resolved." +msgstr "" + msgid "This will invalidate your registered applications and U2F / WebAuthn devices." msgstr "" @@ -48695,6 +48698,9 @@ msgstr "" msgid "mrWidget|Merge blocked: all required approvals must be given." msgstr "" +msgid "mrWidget|Merge blocked: all status checks must pass." +msgstr "" + msgid "mrWidget|Merge blocked: all threads must be resolved." msgstr "" diff --git a/spec/frontend/boards/components/board_filtered_search_spec.js b/spec/frontend/boards/components/board_filtered_search_spec.js index 1a07b9f0b78..f1113ee3073 100644 --- a/spec/frontend/boards/components/board_filtered_search_spec.js +++ b/spec/frontend/boards/components/board_filtered_search_spec.js @@ -3,7 +3,10 @@ import Vue from 'vue'; import Vuex from 'vuex'; import BoardFilteredSearch from '~/boards/components/board_filtered_search.vue'; import * as urlUtility from '~/lib/utils/url_utility'; -import { __ } from '~/locale'; +import { + TOKEN_TITLE_AUTHOR, + TOKEN_TITLE_LABEL, +} from '~/vue_shared/components/filtered_search_bar/constants'; import FilteredSearchBarRoot from '~/vue_shared/components/filtered_search_bar/filtered_search_bar_root.vue'; import AuthorToken from '~/vue_shared/components/filtered_search_bar/tokens/author_token.vue'; import LabelToken from '~/vue_shared/components/filtered_search_bar/tokens/label_token.vue'; @@ -17,7 +20,7 @@ describe('BoardFilteredSearch', () => { const tokens = [ { icon: 'labels', - title: __('Label'), + title: TOKEN_TITLE_LABEL, type: 'label', operators: [ { value: '=', description: 'is' }, @@ -30,7 +33,7 @@ describe('BoardFilteredSearch', () => { }, { icon: 'pencil', - title: __('Author'), + title: TOKEN_TITLE_AUTHOR, type: 'author', operators: [ { value: '=', description: 'is' }, diff --git a/spec/frontend/boards/mock_data.js b/spec/frontend/boards/mock_data.js index 17dcf5868a2..c99ae118db7 100644 --- a/spec/frontend/boards/mock_data.js +++ b/spec/frontend/boards/mock_data.js @@ -1,7 +1,14 @@ import { GlFilteredSearchToken } from '@gitlab/ui'; import { keyBy } from 'lodash'; import { ListType } from '~/boards/constants'; -import { __ } from '~/locale'; +import { + TOKEN_TITLE_ASSIGNEE, + TOKEN_TITLE_AUTHOR, + TOKEN_TITLE_LABEL, + TOKEN_TITLE_MILESTONE, + TOKEN_TITLE_RELEASE, + TOKEN_TITLE_TYPE, +} from '~/vue_shared/components/filtered_search_bar/constants'; import AuthorToken from '~/vue_shared/components/filtered_search_bar/tokens/author_token.vue'; import EmojiToken from '~/vue_shared/components/filtered_search_bar/tokens/emoji_token.vue'; import LabelToken from '~/vue_shared/components/filtered_search_bar/tokens/label_token.vue'; @@ -742,7 +749,7 @@ export const mockConfidentialToken = { export const mockTokens = (fetchLabels, fetchAuthors, fetchMilestones, isSignedIn) => [ { icon: 'user', - title: __('Assignee'), + title: TOKEN_TITLE_ASSIGNEE, type: 'assignee', operators: [ { value: '=', description: 'is' }, @@ -755,7 +762,7 @@ export const mockTokens = (fetchLabels, fetchAuthors, fetchMilestones, isSignedI }, { icon: 'pencil', - title: __('Author'), + title: TOKEN_TITLE_AUTHOR, type: 'author', operators: [ { value: '=', description: 'is' }, @@ -769,7 +776,7 @@ export const mockTokens = (fetchLabels, fetchAuthors, fetchMilestones, isSignedI }, { icon: 'labels', - title: __('Label'), + title: TOKEN_TITLE_LABEL, type: 'label', operators: [ { value: '=', description: 'is' }, @@ -783,7 +790,7 @@ export const mockTokens = (fetchLabels, fetchAuthors, fetchMilestones, isSignedI ...(isSignedIn ? [mockEmojiToken, mockConfidentialToken] : []), { icon: 'clock', - title: __('Milestone'), + title: TOKEN_TITLE_MILESTONE, symbol: '%', type: 'milestone', shouldSkipSort: true, @@ -793,7 +800,7 @@ export const mockTokens = (fetchLabels, fetchAuthors, fetchMilestones, isSignedI }, { icon: 'issues', - title: __('Type'), + title: TOKEN_TITLE_TYPE, type: 'type', token: GlFilteredSearchToken, unique: true, @@ -804,7 +811,7 @@ export const mockTokens = (fetchLabels, fetchAuthors, fetchMilestones, isSignedI }, { type: 'release', - title: __('Release'), + title: TOKEN_TITLE_RELEASE, icon: 'rocket', token: ReleaseToken, fetchReleases: expect.any(Function), diff --git a/spec/frontend/vue_merge_request_widget/components/states/merge_checks_failed_spec.js b/spec/frontend/vue_merge_request_widget/components/states/merge_checks_failed_spec.js index d85574262fe..8eeba4d6274 100644 --- a/spec/frontend/vue_merge_request_widget/components/states/merge_checks_failed_spec.js +++ b/spec/frontend/vue_merge_request_widget/components/states/merge_checks_failed_spec.js @@ -1,5 +1,6 @@ import { shallowMount } from '@vue/test-utils'; import MergeChecksFailed from '~/vue_merge_request_widget/components/states/merge_checks_failed.vue'; +import { DETAILED_MERGE_STATUS } from '~/vue_merge_request_widget/constants'; let wrapper; @@ -15,9 +16,10 @@ describe('Merge request widget merge checks failed state component', () => { }); it.each` - mrState | displayText - ${{ approvals: true, isApproved: false }} | ${'approvalNeeded'} - ${{ detailedMergeStatus: 'BLOCKED_STATUS' }} | ${'blockingMergeRequests'} + mrState | displayText + ${{ approvals: true, isApproved: false }} | ${'approvalNeeded'} + ${{ detailedMergeStatus: DETAILED_MERGE_STATUS.BLOCKED_STATUS }} | ${'blockingMergeRequests'} + ${{ detailedMergeStatus: DETAILED_MERGE_STATUS.EXTERNAL_STATUS_CHECKS }} | ${'externalStatusChecksFailed'} `('display $displayText text for $mrState', ({ mrState, displayText }) => { factory({ mr: mrState }); diff --git a/spec/helpers/users_helper_spec.rb b/spec/helpers/users_helper_spec.rb index 6971ec83ce2..c2c78be6a0f 100644 --- a/spec/helpers/users_helper_spec.rb +++ b/spec/helpers/users_helper_spec.rb @@ -391,7 +391,7 @@ RSpec.describe UsersHelper do expect_next_instance_of(Admin::UserSerializer) do |instance| expect(instance).to receive(:represent).with([user], { current_user: user }).and_return(entity) end - expect(entity).to receive(:as_json).and_return({ "username" => "admin" }) + expect(entity).to receive(:to_json).and_return("{\"username\":\"admin\"}") expect(data[:users]).to eq "{\"username\":\"admin\"}" end diff --git a/spec/lib/feature/gitaly_spec.rb b/spec/lib/feature/gitaly_spec.rb index ed80e31e3cd..33696290483 100644 --- a/spec/lib/feature/gitaly_spec.rb +++ b/spec/lib/feature/gitaly_spec.rb @@ -6,73 +6,207 @@ RSpec.describe Feature::Gitaly do let_it_be(:project) { create(:project) } let_it_be(:project_2) { create(:project) } + let_it_be(:repository) { project.repository.raw } + let_it_be(:repository_2) { project_2.repository.raw } + before do skip_feature_flags_yaml_validation + allow(Feature::Definition).to receive(:get).with(any_args).and_return( + Feature::Definition.new('flag.yml', name: :flag, type: :development) + ) end - describe ".enabled?" do + describe ".enabled_for_any?" do context 'when the flag is set globally' do - let(:feature_flag) { 'global_flag' } - context 'when the gate is closed' do before do stub_feature_flags(gitaly_global_flag: false) end it 'returns false' do - expect(described_class.enabled?(feature_flag)).to be(false) + expect(described_class.enabled_for_any?(:gitaly_global_flag)).to be(false) end end context 'when the flag defaults to on' do it 'returns true' do - expect(described_class.enabled?(feature_flag)).to be(true) + expect(described_class.enabled_for_any?(:gitaly_global_flag)).to be(true) end end end context 'when the flag is enabled for a particular project' do - let(:feature_flag) { 'project_flag' } - before do stub_feature_flags(gitaly_project_flag: project) end it 'returns true for that project' do - expect(described_class.enabled?(feature_flag, project)).to be(true) + expect(described_class.enabled_for_any?(:gitaly_project_flag, project)).to be(true) end it 'returns false for any other project' do - expect(described_class.enabled?(feature_flag, project_2)).to be(false) + expect(described_class.enabled_for_any?(:gitaly_project_flag, project_2)).to be(false) end it 'returns false when no project is passed' do - expect(described_class.enabled?(feature_flag)).to be(false) + expect(described_class.enabled_for_any?(:gitaly_project_flag)).to be(false) + end + end + + context 'when the flag is enabled for a particular repository' do + before do + stub_feature_flags(gitaly_repository_flag: repository) + end + + it 'returns true for that repository' do + expect(described_class.enabled_for_any?(:gitaly_repository_flag, repository)).to be(true) + end + + it 'returns false for any other repository' do + expect(described_class.enabled_for_any?(:gitaly_repository_flag, repository_2)).to be(false) + end + + it 'returns false when no repository is passed' do + expect(described_class.enabled_for_any?(:gitaly_repository_flag)).to be(false) + end + end + + context 'when the flag is checked with multiple input actors' do + before do + stub_feature_flags(gitaly_flag: repository) + end + + it 'returns true if any of the flag is enabled for any of the input actors' do + expect(described_class.enabled_for_any?(:gitaly_flag, project, repository)).to be(true) + end + + it 'returns false if any of the flag is not enabled for any of the input actors' do + expect( + described_class.enabled_for_any?(:gitaly_flag, project, project_2, repository_2) + ).to be(false) end end end describe ".server_feature_flags" do + let(:group) { create(:group) } + let(:user) { create(:user) } + before do - stub_feature_flags(gitaly_global_flag: true, gitaly_project_flag: project, non_gitaly_flag: false) + stub_feature_flags( + gitaly_global_flag: true, + gitaly_project_flag: project, + gitaly_repository_flag: repository, + gitaly_user_flag: user, + gitaly_group_flag: group, + non_gitaly_flag: false + ) end subject { described_class.server_feature_flags } it 'returns a hash of flags starting with the prefix, with dashes instead of underscores' do expect(subject).to eq('gitaly-feature-global-flag' => 'true', - 'gitaly-feature-project-flag' => 'false') + 'gitaly-feature-project-flag' => 'false', + 'gitaly-feature-repository-flag' => 'false', + 'gitaly-feature-user-flag' => 'false', + 'gitaly-feature-group-flag' => 'false') end context 'when a project is passed' do it 'returns the value for the flag on the given project' do - expect(described_class.server_feature_flags(project)) + expect(described_class.server_feature_flags(project: project)) + .to eq('gitaly-feature-global-flag' => 'true', + 'gitaly-feature-project-flag' => 'true', + 'gitaly-feature-repository-flag' => 'false', + 'gitaly-feature-user-flag' => 'false', + 'gitaly-feature-group-flag' => 'false') + + expect(described_class.server_feature_flags(project: project_2)) + .to eq('gitaly-feature-global-flag' => 'true', + 'gitaly-feature-project-flag' => 'false', + 'gitaly-feature-repository-flag' => 'false', + 'gitaly-feature-user-flag' => 'false', + 'gitaly-feature-group-flag' => 'false') + end + end + + context 'when a repository is passed' do + it 'returns the value for the flag on the given repository' do + expect(described_class.server_feature_flags(repository: repository)) + .to eq('gitaly-feature-global-flag' => 'true', + 'gitaly-feature-project-flag' => 'false', + 'gitaly-feature-repository-flag' => 'true', + 'gitaly-feature-user-flag' => 'false', + 'gitaly-feature-group-flag' => 'false') + + expect(described_class.server_feature_flags(repository: repository_2)) + .to eq('gitaly-feature-global-flag' => 'true', + 'gitaly-feature-project-flag' => 'false', + 'gitaly-feature-repository-flag' => 'false', + 'gitaly-feature-user-flag' => 'false', + 'gitaly-feature-group-flag' => 'false') + end + end + + context 'when a user is passed' do + it 'returns the value for the flag on the given user' do + expect(described_class.server_feature_flags(user: user)) + .to eq('gitaly-feature-global-flag' => 'true', + 'gitaly-feature-project-flag' => 'false', + 'gitaly-feature-repository-flag' => 'false', + 'gitaly-feature-user-flag' => 'true', + 'gitaly-feature-group-flag' => 'false') + + expect(described_class.server_feature_flags(user: create(:user))) + .to eq('gitaly-feature-global-flag' => 'true', + 'gitaly-feature-project-flag' => 'false', + 'gitaly-feature-repository-flag' => 'false', + 'gitaly-feature-user-flag' => 'false', + 'gitaly-feature-group-flag' => 'false') + end + end + + context 'when a group is passed' do + it 'returns the value for the flag on the given group' do + expect(described_class.server_feature_flags(group: group)) .to eq('gitaly-feature-global-flag' => 'true', - 'gitaly-feature-project-flag' => 'true') + 'gitaly-feature-project-flag' => 'false', + 'gitaly-feature-repository-flag' => 'false', + 'gitaly-feature-user-flag' => 'false', + 'gitaly-feature-group-flag' => 'true') - expect(described_class.server_feature_flags(project_2)) + expect(described_class.server_feature_flags(group: create(:group))) .to eq('gitaly-feature-global-flag' => 'true', - 'gitaly-feature-project-flag' => 'false') + 'gitaly-feature-project-flag' => 'false', + 'gitaly-feature-repository-flag' => 'false', + 'gitaly-feature-user-flag' => 'false', + 'gitaly-feature-group-flag' => 'false') + end + end + + context 'when multiple actors are passed' do + it 'returns the corresponding enablement status for actors' do + expect(described_class.server_feature_flags(project: project_2, repository: repository)) + .to eq('gitaly-feature-global-flag' => 'true', + 'gitaly-feature-project-flag' => 'false', + 'gitaly-feature-repository-flag' => 'true', + 'gitaly-feature-user-flag' => 'false', + 'gitaly-feature-group-flag' => 'false') + + expect(described_class.server_feature_flags(project: project, repository: repository_2)) + .to eq('gitaly-feature-global-flag' => 'true', + 'gitaly-feature-project-flag' => 'true', + 'gitaly-feature-repository-flag' => 'false', + 'gitaly-feature-user-flag' => 'false', + 'gitaly-feature-group-flag' => 'false') + + expect(described_class.server_feature_flags(user: user, project: project, repository: repository, group: group)) + .to eq('gitaly-feature-global-flag' => 'true', + 'gitaly-feature-project-flag' => 'true', + 'gitaly-feature-repository-flag' => 'true', + 'gitaly-feature-user-flag' => 'true', + 'gitaly-feature-group-flag' => 'true') end end @@ -88,4 +222,67 @@ RSpec.describe Feature::Gitaly do end end end + + describe ".user_actor" do + let(:user) { create(:user) } + + context 'when user is passed in' do + it 'returns a actor wrapper from user' do + expect(described_class.user_actor(user).flipper_id).to eql(user.flipper_id) + end + end + + context 'when called without user and user_id is available in application context' do + it 'returns a actor wrapper from user_id' do + ::Gitlab::ApplicationContext.with_context(user: user) do + expect(described_class.user_actor.flipper_id).to eql(user.flipper_id) + end + end + end + + context 'when called without user and user_id is absent from application context' do + it 'returns nil' do + expect(described_class.user_actor).to be(nil) + end + end + + context 'when something else is passed' do + it 'returns nil' do + expect(described_class.user_actor(1234)).to be(nil) + end + end + end + + describe ".project_actor" do + let_it_be(:project) { create(:project) } + + context 'when project is passed in' do + it 'returns a actor wrapper from project' do + expect(described_class.project_actor(project).flipper_id).to eql(project.flipper_id) + end + end + + context 'when something else is passed in' do + it 'returns nil' do + expect(described_class.project_actor(1234)).to be(nil) + end + end + end + + describe ".group_actor" do + let_it_be(:group) { create(:group ) } + let_it_be(:project) { create(:project, group: group) } + + context 'when project is passed in' do + it "returns a actor wrapper from project's group" do + expect(described_class.group_actor(project).flipper_id).to eql(group.flipper_id) + end + end + + context 'when something else is passed in' do + it 'returns nil' do + expect(described_class.group_actor(1234)).to be(nil) + end + end + end end diff --git a/spec/lib/gitlab/application_context_spec.rb b/spec/lib/gitlab/application_context_spec.rb index 8b2a228b935..58d462aa27f 100644 --- a/spec/lib/gitlab/application_context_spec.rb +++ b/spec/lib/gitlab/application_context_spec.rb @@ -44,7 +44,7 @@ RSpec.describe Gitlab::ApplicationContext do describe '.push' do it 'passes the expected context on to labkit' do fake_proc = duck_type(:call) - expected_context = { user: fake_proc, client_id: fake_proc } + expected_context = { user: fake_proc, user_id: fake_proc, client_id: fake_proc } expect(Labkit::Context).to receive(:push).with(expected_context) @@ -108,14 +108,16 @@ RSpec.describe Gitlab::ApplicationContext do context = described_class.new(user: -> { user }, project: -> { project }, namespace: -> { subgroup }) expect(result(context)) - .to include(user: user.username, project: project.full_path, root_namespace: namespace.full_path) + .to include(user: user.username, user_id: user.id, project: project.full_path, + root_namespace: namespace.full_path) end it 'correctly loads the expected values when passed directly' do context = described_class.new(user: user, project: project, namespace: subgroup) expect(result(context)) - .to include(user: user.username, project: project.full_path, root_namespace: namespace.full_path) + .to include(user: user.username, user_id: user.id, project: project.full_path, + root_namespace: namespace.full_path) end it 'falls back to a projects namespace when a project is passed but no namespace' do diff --git a/spec/lib/gitlab/ci/config/external/file/base_spec.rb b/spec/lib/gitlab/ci/config/external/file/base_spec.rb index 1306d61d99c..8475c3a8b19 100644 --- a/spec/lib/gitlab/ci/config/external/file/base_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/base_spec.rb @@ -14,6 +14,10 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base do super end + + def validate_context! + # no-op + end end end @@ -95,6 +99,24 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base do expect(file.error_message).to eq('Included file `some/file/xxxxxxxxxxxxxxxx.yml` does not have valid YAML syntax!') end end + + context 'when the class has no validate_context!' do + let(:test_class) do + Class.new(described_class) do + def initialize(params, context) + @location = params + + super + end + end + end + + let(:location) { 'some/file/config.yaml' } + + it 'raises an error' do + expect { valid? }.to raise_error(NotImplementedError) + end + end end describe '#to_hash' do diff --git a/spec/lib/gitlab/git/object_pool_spec.rb b/spec/lib/gitlab/git/object_pool_spec.rb index 3b1eb0319f8..b158c7227d4 100644 --- a/spec/lib/gitlab/git/object_pool_spec.rb +++ b/spec/lib/gitlab/git/object_pool_spec.rb @@ -78,44 +78,40 @@ RSpec.describe Gitlab::Git::ObjectPool do end describe '#fetch' do - let(:commit_count) { source_repository.commit_count } + context 'when the object pool repository exists' do + let!(:pool_repository) { create(:pool_repository, :ready) } - context "when the object's pool repository exists" do - it 'does not raise an error' do - expect { subject.fetch }.not_to raise_error + context 'without changes' do + it 'does not raise an error' do + expect { subject.fetch }.not_to raise_error + end end - end - - context "when the object's pool repository does not exist" do - before do - subject.delete - end - - it "re-creates the object pool's repository" do - subject.fetch - - expect(subject.repository.exists?).to be true - end - - it 'does not raise an error' do - expect { subject.fetch }.not_to raise_error - end - - it 'fetches objects from the source repository' do - new_commit_id = source_repository.create_file( - pool_repository.source_project.owner, - 'a.file', - 'This is a file', - branch_name: source_repository.root_ref, - message: 'Add a file' - ) - - expect(subject.repository.exists?).to be false - - subject.fetch - expect(subject.repository.commit_count('refs/remotes/origin/heads/master')).to eq(commit_count) - expect(subject.repository.commit(new_commit_id).id).to eq(new_commit_id) + context 'with new commit in source repository' do + let(:branch_name) { Gitlab::Git::Ref.extract_branch_name(source_repository.root_ref) } + let(:source_ref_name) { "refs/heads/#{branch_name}" } + let(:pool_ref_name) { "refs/remotes/origin/heads/#{branch_name}" } + + let(:new_commit_id) do + source_repository.create_file( + pool_repository.source_project.owner, + 'a.file', + 'This is a file', + branch_name: branch_name, + message: 'Add a file' + ) + end + + it 'fetches objects from the source repository' do + # Sanity-check that the commit does not yet exist in the pool repository. + expect(subject.repository.commit(new_commit_id)).to be_nil + + subject.fetch + + expect(subject.repository.commit(pool_ref_name).id).to eq(new_commit_id) + expect(subject.repository.commit_count(pool_ref_name)) + .to eq(source_repository.raw_repository.commit_count(source_ref_name)) + end end end end diff --git a/spec/lib/gitlab/gitaly_client/with_feature_flag_actors_spec.rb b/spec/lib/gitlab/gitaly_client/with_feature_flag_actors_spec.rb new file mode 100644 index 00000000000..dd54ac0c6ac --- /dev/null +++ b/spec/lib/gitlab/gitaly_client/with_feature_flag_actors_spec.rb @@ -0,0 +1,210 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GitalyClient::WithFeatureFlagActors do + let(:user) { create(:user) } + let(:service) do + Class.new do + include Gitlab::GitalyClient::WithFeatureFlagActors + end.new + end + + describe '#user_actor' do + context 'when user is not available in ApplicationContext' do + it 'returns nil' do + expect(service.user_actor).to be(nil) + end + end + + context 'when user is available in ApplicationContext' do + around do |example| + ::Gitlab::ApplicationContext.with_context(user: user) { example.run } + end + + it 'returns corresponding user record' do + expect(service.user_actor.flipper_id).to eql(user.flipper_id) + end + end + + context 'when user does not exist' do + around do |example| + ::Gitlab::ApplicationContext.with_context(user: SecureRandom.uuid) { example.run } + end + + it 'returns corresponding user record' do + expect(service.user_actor).to be(nil) + end + end + end + + describe '#repository, #project_actor, #group_actor' do + context 'when normal project repository' do + let_it_be(:project) { create(:project, group: create(:group)) } + let(:expected_project) { project } + let(:expected_group) { Feature::Gitaly::ActorWrapper.new(::Group, project.group.id) } + + it_behaves_like 'Gitaly feature flag actors are inferred from repository' do + let(:repository) { project.repository } + end + + it_behaves_like 'Gitaly feature flag actors are inferred from repository' do + let(:repository) { project.repository.raw } + end + + it_behaves_like 'Gitaly feature flag actors are inferred from repository' do + let(:repository) { raw_repo_without_container(project.repository) } + end + end + + context 'when project wiki repository' do + let_it_be(:project) { create(:project, :wiki_repo, group: create(:group)) } + let(:expected_project) { nil } + let(:expected_group) { nil } + + it_behaves_like 'Gitaly feature flag actors are inferred from repository' do + let(:repository) { project.wiki.repository } + end + + it_behaves_like 'Gitaly feature flag actors are inferred from repository' do + let(:repository) { project.wiki.repository.raw } + end + + it_behaves_like 'Gitaly feature flag actors are inferred from repository' do + let(:repository) { raw_repo_without_container(project.wiki.repository) } + end + end + + context 'when repository of project in user namespace' do + let_it_be(:project) { create(:project, namespace: create(:user).namespace) } + let(:expected_project) { project } + let(:expected_group) { Feature::Gitaly::ActorWrapper.new(::Group, project.namespace_id) } + + it_behaves_like 'Gitaly feature flag actors are inferred from repository' do + let(:repository) { project.repository } + end + + it_behaves_like 'Gitaly feature flag actors are inferred from repository' do + let(:repository) { project.repository.raw } + end + + it_behaves_like 'Gitaly feature flag actors are inferred from repository' do + let(:repository) { raw_repo_without_container(project.repository) } + end + end + + context 'when personal snippet' do + let(:snippet) { create(:personal_snippet) } + let(:expected_project) { nil } + let(:expected_group) { nil } + + it_behaves_like 'Gitaly feature flag actors are inferred from repository' do + let(:repository) { snippet.repository } + end + + it_behaves_like 'Gitaly feature flag actors are inferred from repository' do + let(:repository) { snippet.repository.raw } + end + + it_behaves_like 'Gitaly feature flag actors are inferred from repository' do + let(:repository) { raw_repo_without_container(snippet.repository) } + end + end + + context 'when project snippet' do + let_it_be(:project) { create(:project, group: create(:group)) } + let(:snippet) { create(:project_snippet, project: project) } + let(:expected_project) { nil } + let(:expected_group) { nil } + + it_behaves_like 'Gitaly feature flag actors are inferred from repository' do + let(:repository) { snippet.repository } + end + + it_behaves_like 'Gitaly feature flag actors are inferred from repository' do + let(:repository) { snippet.repository.raw } + end + + it_behaves_like 'Gitaly feature flag actors are inferred from repository' do + let(:repository) { raw_repo_without_container(snippet.repository) } + end + end + + context 'when project design' do + let_it_be(:project) { create(:project, group: create(:group)) } + let(:issue) { create(:issue, project: project) } + let(:design) { create(:design, issue: issue) } + + let(:expected_project) { project } + let(:expected_group) { project.group } + + it_behaves_like 'Gitaly feature flag actors are inferred from repository' do + let(:repository) { design.repository } + end + + it_behaves_like 'Gitaly feature flag actors are inferred from repository' do + let(:repository) { design.repository.raw } + end + + it_behaves_like 'Gitaly feature flag actors are inferred from repository' do + let(:repository) { raw_repo_without_container(design.repository) } + end + end + end + + describe '#gitaly_client_call' do + let(:call_arg_1) { double } + let(:call_arg_2) { double } + let(:call_arg_3) { double } + let(:call_result) { double } + + before do + allow(Gitlab::GitalyClient).to receive(:call).and_return(call_result) + end + + context 'when actors_aware_gitaly_calls flag is enabled' do + let(:repository_actor) { instance_double(::Repository) } + let(:user_actor) { instance_double(::User) } + let(:project_actor) { instance_double(Project) } + let(:group_actor) { instance_double(Group) } + + before do + stub_feature_flags(actors_aware_gitaly_calls: true) + + allow(service).to receive(:user_actor).and_return(user_actor) + allow(service).to receive(:repository_actor).and_return(repository_actor) + allow(service).to receive(:project_actor).and_return(project_actor) + allow(service).to receive(:group_actor).and_return(group_actor) + allow(Gitlab::GitalyClient).to receive(:with_feature_flag_actors).and_call_original + end + + it 'triggers client call with feature flag actors' do + result = service.gitaly_client_call(call_arg_1, call_arg_2, karg: call_arg_3) + + expect(Gitlab::GitalyClient).to have_received(:call).with(call_arg_1, call_arg_2, karg: call_arg_3) + expect(Gitlab::GitalyClient).to have_received(:with_feature_flag_actors).with( + repository: repository_actor, + user: user_actor, + project: project_actor, + group: group_actor + ) + expect(result).to be(call_result) + end + end + + context 'when actors_aware_gitaly_calls not enabled' do + before do + stub_feature_flags(actors_aware_gitaly_calls: false) + end + + it 'triggers client call without feature flag actors' do + expect(Gitlab::GitalyClient).not_to receive(:with_feature_flag_actors) + + result = service.gitaly_client_call(call_arg_1, call_arg_2, karg: call_arg_3) + + expect(Gitlab::GitalyClient).to have_received(:call).with(call_arg_1, call_arg_2, karg: call_arg_3) + expect(result).to be(call_result) + end + end + end +end diff --git a/spec/lib/gitlab/gitaly_client_spec.rb b/spec/lib/gitlab/gitaly_client_spec.rb index a3840ca843f..a3b2fb2d9d0 100644 --- a/spec/lib/gitlab/gitaly_client_spec.rb +++ b/spec/lib/gitlab/gitaly_client_spec.rb @@ -259,6 +259,58 @@ RSpec.describe Gitlab::GitalyClient do end end + shared_examples 'gitaly feature flags in metadata' do + before do + allow(Feature::Gitaly).to receive(:server_feature_flags).and_return( + 'gitaly-feature-a' => 'true', + 'gitaly-feature-b' => 'false' + ) + end + + it 'evaluates Gitaly server feature flags' do + metadata = described_class.request_kwargs('default', timeout: 1)[:metadata] + + expect(Feature::Gitaly).to have_received(:server_feature_flags).with(no_args) + expect(metadata['gitaly-feature-a']).to be('true') + expect(metadata['gitaly-feature-b']).to be('false') + end + + context 'when there are actors' do + let(:repository_actor) { double(:actor) } + let(:project_actor) { double(:actor) } + let(:user_actor) { double(:actor) } + let(:group_actor) { double(:actor) } + + it 'evaluates Gitaly server feature flags with actors' do + metadata = described_class.with_feature_flag_actors( + repository: repository_actor, + project: project_actor, + user: user_actor, + group: group_actor + ) do + described_class.request_kwargs('default', timeout: 1)[:metadata] + end + + expect(Feature::Gitaly).to have_received(:server_feature_flags).with( + repository: repository_actor, + project: project_actor, + user: user_actor, + group: group_actor + ) + expect(metadata['gitaly-feature-a']).to be('true') + expect(metadata['gitaly-feature-b']).to be('false') + end + end + end + + context 'server_feature_flags when RequestStore is activated', :request_store do + it_behaves_like 'gitaly feature flags in metadata' + end + + context 'server_feature_flags when RequestStore is not activated' do + it_behaves_like 'gitaly feature flags in metadata' + end + context 'gitlab_git_env' do let(:policy) { 'gitaly-route-repository-accessor-policy' } @@ -585,4 +637,42 @@ RSpec.describe Gitlab::GitalyClient do end end end + + describe '.with_feature_flag_actor', :request_store do + shared_examples 'with_feature_flag_actor' do + let(:repository_actor) { double(:actor) } + let(:project_actor) { double(:actor) } + let(:user_actor) { double(:actor) } + let(:group_actor) { double(:actor) } + + it 'allows access to feature flag actors inside the block' do + expect(described_class.feature_flag_actors).to eql({}) + + described_class.with_feature_flag_actors( + repository: repository_actor, + project: project_actor, + user: user_actor, + group: group_actor + ) do + expect( + described_class.feature_flag_actors + ).to eql( + repository: repository_actor, + project: project_actor, + user: user_actor, + group: group_actor) + end + + expect(described_class.feature_flag_actors).to eql({}) + end + end + + context 'when RequestStore is activated', :request_store do + it_behaves_like 'with_feature_flag_actor' + end + + context 'when RequestStore is not activated' do + it_behaves_like 'with_feature_flag_actor' + end + end end diff --git a/spec/lib/gitlab/slash_commands/application_help_spec.rb b/spec/lib/gitlab/slash_commands/application_help_spec.rb index b82121bf3a8..b182c0e5cc6 100644 --- a/spec/lib/gitlab/slash_commands/application_help_spec.rb +++ b/spec/lib/gitlab/slash_commands/application_help_spec.rb @@ -4,11 +4,13 @@ require 'spec_helper' RSpec.describe Gitlab::SlashCommands::ApplicationHelp do let(:params) { { command: '/gitlab', text: 'help' } } + let_it_be(:user) { create(:user) } + let_it_be(:chat_user) { create(:chat_name, user: user) } let(:project) { build(:project) } describe '#execute' do subject do - described_class.new(project, params).execute + described_class.new(project, chat_user, params).execute end it 'displays the help section' do @@ -16,5 +18,23 @@ RSpec.describe Gitlab::SlashCommands::ApplicationHelp do expect(subject[:text]).to include('Available commands') expect(subject[:text]).to include('/gitlab [project name or alias] issue show') end + + context 'with incident declare command' do + context 'when feature flag is enabled' do + it 'displays the declare command' do + expect(subject[:text]).to include('/gitlab incident declare') + end + end + + context 'when feature flag is disabled' do + before do + stub_feature_flags(incident_declare_slash_command: false) + end + + it 'does not displays the declare command' do + expect(subject[:text]).not_to include('/gitlab incident declare') + end + end + end end end diff --git a/spec/lib/gitlab/slash_commands/command_spec.rb b/spec/lib/gitlab/slash_commands/command_spec.rb index 069577b3846..f4664bcfef9 100644 --- a/spec/lib/gitlab/slash_commands/command_spec.rb +++ b/spec/lib/gitlab/slash_commands/command_spec.rb @@ -122,5 +122,25 @@ RSpec.describe Gitlab::SlashCommands::Command do it { is_expected.to eq(Gitlab::SlashCommands::IssueComment) } end + + context 'when incident declare is triggered' do + context 'IncidentNew is triggered' do + let(:params) { { text: 'incident declare' } } + + it { is_expected.to eq(Gitlab::SlashCommands::IncidentManagement::IncidentNew) } + end + + context 'when feature flag is disabled' do + before do + stub_feature_flags(incident_declare_slash_command: false) + end + + context 'IncidentNew is triggered' do + let(:params) { { text: 'incident declare' } } + + it { is_expected.not_to eq(Gitlab::SlashCommands::IncidentManagement::IncidentNew) } + end + end + end end end diff --git a/spec/lib/gitlab/slash_commands/incident_management/incident_new_spec.rb b/spec/lib/gitlab/slash_commands/incident_management/incident_new_spec.rb new file mode 100644 index 00000000000..ee54f29aec7 --- /dev/null +++ b/spec/lib/gitlab/slash_commands/incident_management/incident_new_spec.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::SlashCommands::IncidentManagement::IncidentNew do + let_it_be(:project) { create(:project) } + let_it_be(:user) { create(:user) } + let_it_be(:chat_name) { create(:chat_name, user: user) } + let_it_be(:regex_match) { described_class.match('declare') } + + subject do + described_class.new(project, chat_name) + end + + describe '#execute' do + context 'when invoked' do + it 'sends ephemeral response' do + response = subject.execute(regex_match) + + expect(response[:response_type]).to be(:ephemeral) + expect(response[:text]).to eq('It works!') + end + end + end + + describe '#allowed?' do + context 'when user has permissions' do + before do + project.add_developer(user) + end + + it 'returns true' do + expect(described_class).to be_allowed(project, user) + end + end + + context 'when feature flag is disabled' do + before do + project.add_developer(user) + stub_feature_flags(incident_declare_slash_command: false) + end + + it 'returns false in allowed?' do + expect(described_class).not_to be_allowed(project, user) + end + end + end + + describe '#collection' do + context 'when collection method id called' do + it 'calls IssuesFinder' do + expect_next_instance_of(IssuesFinder) do |finder| + expect(finder).to receive(:execute) + end + + subject.collection + end + end + end +end diff --git a/spec/lib/gitlab/slash_commands/presenters/incident_management/incident_new_spec.rb b/spec/lib/gitlab/slash_commands/presenters/incident_management/incident_new_spec.rb new file mode 100644 index 00000000000..cbc584b931f --- /dev/null +++ b/spec/lib/gitlab/slash_commands/presenters/incident_management/incident_new_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::SlashCommands::Presenters::IncidentManagement::IncidentNew do + subject { described_class.new } + + it 'returns the ephemeral message' do + message = subject.present('It works!') + + expect(message).to be_a(Hash) + expect(message[:text]).to eq('It works!') + expect(message[:response_type]).to be(:ephemeral) + end +end diff --git a/spec/migrations/sync_new_amount_used_for_ci_namespace_monthly_usages_spec.rb b/spec/migrations/sync_new_amount_used_for_ci_namespace_monthly_usages_spec.rb new file mode 100644 index 00000000000..9a17f375f82 --- /dev/null +++ b/spec/migrations/sync_new_amount_used_for_ci_namespace_monthly_usages_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require_migration! + +RSpec.describe SyncNewAmountUsedForCiNamespaceMonthlyUsages, migration: :gitlab_ci do + let(:namespace_usages) { table(:ci_namespace_monthly_usages) } + + before do + # Disabling the trigger temporarily to allow records being created with out-of-sync + # `new_amount_used` and `amount_used`. This will simulate existing records before + # we add the trigger. + ActiveRecord::Base.connection + .execute("ALTER TABLE ci_namespace_monthly_usages DISABLE TRIGGER sync_namespaces_amount_used_columns") + + this_month = Time.now.utc.beginning_of_month + last_month = 1.month.ago.utc.beginning_of_month + last_year = 1.year.ago.utc.beginning_of_month + + namespace_usages.create!(namespace_id: 1, date: last_year) + namespace_usages.create!(namespace_id: 1, date: this_month, amount_used: 10, new_amount_used: 0) + namespace_usages.create!(namespace_id: 1, date: last_month, amount_used: 20, new_amount_used: 0) + + namespace_usages.create!(namespace_id: 2, date: last_year) + namespace_usages.create!(namespace_id: 2, date: this_month, amount_used: 30, new_amount_used: 0) + namespace_usages.create!(namespace_id: 2, date: last_month, amount_used: 40, new_amount_used: 0) + + ActiveRecord::Base.connection + .execute("ALTER TABLE ci_namespace_monthly_usages ENABLE TRIGGER sync_namespaces_amount_used_columns") + end + + it 'updates `new_amount_used` with values from `amount_used`' do + expect(namespace_usages.where(new_amount_used: 0).count).to eq(6) + + migrate! + + expect(namespace_usages.where(new_amount_used: 0).count).to eq(2) + expect(namespace_usages.order(:id).pluck(:new_amount_used)) + .to contain_exactly(0, 0, 10, 20, 30, 40) + end +end diff --git a/spec/migrations/sync_new_amount_used_for_ci_project_monthly_usages_spec.rb b/spec/migrations/sync_new_amount_used_for_ci_project_monthly_usages_spec.rb new file mode 100644 index 00000000000..8d45f1107ea --- /dev/null +++ b/spec/migrations/sync_new_amount_used_for_ci_project_monthly_usages_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require_migration! + +RSpec.describe SyncNewAmountUsedForCiProjectMonthlyUsages, migration: :gitlab_ci do + let(:project_usages) { table(:ci_project_monthly_usages) } + + before do + # Disabling the trigger temporarily to allow records being created with out-of-sync + # `new_amount_used` and `amount_used`. This will simulate existing records before + # we add the trigger. + ActiveRecord::Base.connection + .execute("ALTER TABLE ci_project_monthly_usages DISABLE TRIGGER sync_projects_amount_used_columns") + + this_month = Time.now.utc.beginning_of_month + last_month = 1.month.ago.utc.beginning_of_month + last_year = 1.year.ago.utc.beginning_of_month + + project_usages.create!(project_id: 1, date: last_year) + project_usages.create!(project_id: 1, date: this_month, amount_used: 10, new_amount_used: 0) + project_usages.create!(project_id: 1, date: last_month, amount_used: 20, new_amount_used: 0) + + project_usages.create!(project_id: 2, date: last_year) + project_usages.create!(project_id: 2, date: this_month, amount_used: 30, new_amount_used: 0) + project_usages.create!(project_id: 2, date: last_month, amount_used: 40, new_amount_used: 0) + + ActiveRecord::Base.connection + .execute("ALTER TABLE ci_project_monthly_usages ENABLE TRIGGER sync_projects_amount_used_columns") + end + + it 'updates `new_amount_used` with values from `amount_used`' do + expect(project_usages.where(new_amount_used: 0).count).to eq(6) + + migrate! + + expect(project_usages.where(new_amount_used: 0).count).to eq(2) + expect(project_usages.order(:id).pluck(:new_amount_used)) + .to contain_exactly(0, 0, 10, 20, 30, 40) + end +end diff --git a/spec/models/clusters/applications/cert_manager_spec.rb b/spec/models/clusters/applications/cert_manager_spec.rb index 05ab8c4108e..427a99efadd 100644 --- a/spec/models/clusters/applications/cert_manager_spec.rb +++ b/spec/models/clusters/applications/cert_manager_spec.rb @@ -10,6 +10,11 @@ RSpec.describe Clusters::Applications::CertManager do include_examples 'cluster application version specs', :clusters_applications_cert_manager include_examples 'cluster application initial status specs' + describe 'default values' do + it { expect(cert_manager.version).to eq(described_class::VERSION) } + it { expect(cert_manager.email).to eq("admin@example.com") } + end + describe '#can_uninstall?' do subject { cert_manager.can_uninstall? } diff --git a/spec/models/clusters/applications/crossplane_spec.rb b/spec/models/clusters/applications/crossplane_spec.rb index 7082576028b..d1abaa52c7f 100644 --- a/spec/models/clusters/applications/crossplane_spec.rb +++ b/spec/models/clusters/applications/crossplane_spec.rb @@ -14,6 +14,11 @@ RSpec.describe Clusters::Applications::Crossplane do it { is_expected.to validate_presence_of(:stack) } end + describe 'default values' do + it { expect(subject.version).to eq(described_class::VERSION) } + it { expect(subject.stack).to be_empty } + end + describe '#can_uninstall?' do subject { crossplane.can_uninstall? } diff --git a/spec/models/clusters/applications/helm_spec.rb b/spec/models/clusters/applications/helm_spec.rb index 5212e321a55..1b8be92475a 100644 --- a/spec/models/clusters/applications/helm_spec.rb +++ b/spec/models/clusters/applications/helm_spec.rb @@ -5,6 +5,10 @@ require 'spec_helper' RSpec.describe Clusters::Applications::Helm do include_examples 'cluster application core specs', :clusters_applications_helm + describe 'default values' do + it { expect(subject.version).to eq(Gitlab::Kubernetes::Helm::V2::BaseCommand::HELM_VERSION) } + end + describe '.available' do subject { described_class.available } diff --git a/spec/models/clusters/applications/ingress_spec.rb b/spec/models/clusters/applications/ingress_spec.rb index e16d97c42d9..e5caa11452e 100644 --- a/spec/models/clusters/applications/ingress_spec.rb +++ b/spec/models/clusters/applications/ingress_spec.rb @@ -18,6 +18,11 @@ RSpec.describe Clusters::Applications::Ingress do allow(ClusterWaitForIngressIpAddressWorker).to receive(:perform_async) end + describe 'default values' do + it { expect(subject.ingress_type).to eq("nginx") } + it { expect(subject.version).to eq(described_class::VERSION) } + end + describe '#can_uninstall?' do subject { ingress.can_uninstall? } diff --git a/spec/models/clusters/applications/jupyter_spec.rb b/spec/models/clusters/applications/jupyter_spec.rb index e7de2d24334..9336d2352f8 100644 --- a/spec/models/clusters/applications/jupyter_spec.rb +++ b/spec/models/clusters/applications/jupyter_spec.rb @@ -10,6 +10,10 @@ RSpec.describe Clusters::Applications::Jupyter do it { is_expected.to belong_to(:oauth_application) } + describe 'default values' do + it { expect(subject.version).to eq(described_class::VERSION) } + end + describe '#can_uninstall?' do let(:ingress) { create(:clusters_applications_ingress, :installed, external_hostname: 'localhost.localdomain') } let(:jupyter) { create(:clusters_applications_jupyter, cluster: ingress.cluster) } diff --git a/spec/models/clusters/applications/knative_spec.rb b/spec/models/clusters/applications/knative_spec.rb index d0e470bfa42..3914450339a 100644 --- a/spec/models/clusters/applications/knative_spec.rb +++ b/spec/models/clusters/applications/knative_spec.rb @@ -21,6 +21,10 @@ RSpec.describe Clusters::Applications::Knative do it { is_expected.to have_one(:serverless_domain_cluster).class_name('::Serverless::DomainCluster').with_foreign_key('clusters_applications_knative_id').inverse_of(:knative) } end + describe 'default values' do + it { expect(subject.version).to eq(described_class::VERSION) } + end + describe 'when cloud run is enabled' do let(:cluster) { create(:cluster, :provided_by_gcp, :cloud_run_enabled) } let(:knative_cloud_run) { create(:clusters_applications_knative, cluster: cluster) } diff --git a/spec/models/clusters/applications/prometheus_spec.rb b/spec/models/clusters/applications/prometheus_spec.rb index b60c5ec114d..15c3162270e 100644 --- a/spec/models/clusters/applications/prometheus_spec.rb +++ b/spec/models/clusters/applications/prometheus_spec.rb @@ -12,6 +12,13 @@ RSpec.describe Clusters::Applications::Prometheus do include_examples 'cluster application helm specs', :clusters_applications_prometheus include_examples 'cluster application initial status specs' + describe 'default values' do + subject(:prometheus) { build(:clusters_applications_prometheus) } + + it { expect(prometheus.alert_manager_token).to be_an_instance_of(String) } + it { expect(prometheus.version).to eq(described_class::VERSION) } + end + describe 'after_destroy' do let(:cluster) { create(:cluster, :with_installed_helm) } let(:application) { create(:clusters_applications_prometheus, :installed, cluster: cluster) } diff --git a/spec/models/clusters/applications/runner_spec.rb b/spec/models/clusters/applications/runner_spec.rb index 8f02161843b..04b5ae9641d 100644 --- a/spec/models/clusters/applications/runner_spec.rb +++ b/spec/models/clusters/applications/runner_spec.rb @@ -13,6 +13,10 @@ RSpec.describe Clusters::Applications::Runner do it { is_expected.to belong_to(:runner) } + describe 'default values' do + it { expect(subject.version).to eq(described_class::VERSION) } + end + describe '#can_uninstall?' do let(:gitlab_runner) { create(:clusters_applications_runner, runner: ci_runner) } diff --git a/spec/models/clusters/cluster_spec.rb b/spec/models/clusters/cluster_spec.rb index 73cd7bb9075..be64d72e031 100644 --- a/spec/models/clusters/cluster_spec.rb +++ b/spec/models/clusters/cluster_spec.rb @@ -49,6 +49,10 @@ RSpec.describe Clusters::Cluster, :use_clean_rails_memory_store_caching do it { is_expected.to respond_to :project } it { is_expected.to be_namespace_per_environment } + describe 'default values' do + it { expect(subject.helm_major_version).to eq(3) } + end + it_behaves_like 'it has loose foreign keys' do let(:factory_name) { :cluster } end diff --git a/spec/models/clusters/integrations/prometheus_spec.rb b/spec/models/clusters/integrations/prometheus_spec.rb index 90e99aefdce..d6d1105cdb1 100644 --- a/spec/models/clusters/integrations/prometheus_spec.rb +++ b/spec/models/clusters/integrations/prometheus_spec.rb @@ -15,6 +15,16 @@ RSpec.describe Clusters::Integrations::Prometheus do it { is_expected.not_to allow_value(nil).for(:enabled) } end + describe 'default values' do + subject(:integration) { build(:clusters_integrations_prometheus) } + + before do + allow(SecureRandom).to receive(:hex).and_return('randomtoken') + end + + it { expect(integration.alert_manager_token).to eq('randomtoken') } + end + describe 'after_destroy' do subject(:integration) { create(:clusters_integrations_prometheus, cluster: cluster, enabled: true) } diff --git a/spec/models/clusters/platforms/kubernetes_spec.rb b/spec/models/clusters/platforms/kubernetes_spec.rb index 35ed6f4e2c6..99dbb654b7f 100644 --- a/spec/models/clusters/platforms/kubernetes_spec.rb +++ b/spec/models/clusters/platforms/kubernetes_spec.rb @@ -21,6 +21,12 @@ RSpec.describe Clusters::Platforms::Kubernetes do it_behaves_like 'having unique enum values' + describe 'default values' do + let(:kubernetes) { create(:cluster_platform_kubernetes) } + + it { expect(kubernetes.authorization_type).to eq("rbac") } + end + describe 'before_validation' do let(:kubernetes) { create(:cluster_platform_kubernetes, :configured, namespace: namespace) } diff --git a/spec/models/clusters/providers/aws_spec.rb b/spec/models/clusters/providers/aws_spec.rb index 3b4a48cc5be..2afed663edf 100644 --- a/spec/models/clusters/providers/aws_spec.rb +++ b/spec/models/clusters/providers/aws_spec.rb @@ -13,7 +13,7 @@ RSpec.describe Clusters::Providers::Aws do include_examples 'provider status', :cluster_provider_aws - describe 'default_value_for' do + describe 'default values' do let(:provider) { build(:cluster_provider_aws) } it "sets default values" do diff --git a/spec/models/clusters/providers/gcp_spec.rb b/spec/models/clusters/providers/gcp_spec.rb index ad9ada04875..a1f00069937 100644 --- a/spec/models/clusters/providers/gcp_spec.rb +++ b/spec/models/clusters/providers/gcp_spec.rb @@ -8,13 +8,14 @@ RSpec.describe Clusters::Providers::Gcp do include_examples 'provider status', :cluster_provider_gcp - describe 'default_value_for' do + describe 'default values' do let(:gcp) { build(:cluster_provider_gcp) } it "has default value" do expect(gcp.zone).to eq('us-central1-a') expect(gcp.num_nodes).to eq(3) expect(gcp.machine_type).to eq('n1-standard-2') + expect(gcp.cloud_run).to eq(false) end end diff --git a/spec/support/shared_examples/lib/cache_helpers_shared_examples.rb b/spec/support/shared_examples/lib/cache_helpers_shared_examples.rb index 2e00abe2f8e..6cdd7954b5f 100644 --- a/spec/support/shared_examples/lib/cache_helpers_shared_examples.rb +++ b/spec/support/shared_examples/lib/cache_helpers_shared_examples.rb @@ -129,6 +129,7 @@ RSpec.shared_examples_for 'collection cache helper' do before do allow(::Gitlab::Metrics::WebTransaction).to receive(:current).and_return(transaction) allow(transaction).to receive(:increment) + allow(Gitlab::ApplicationContext).to receive(:current_context_attribute).with(any_args).and_call_original allow(Gitlab::ApplicationContext).to receive(:current_context_attribute).with(:caller_id).and_return(caller_id) end diff --git a/spec/support/shared_examples/lib/gitlab/gitaly_client_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/gitaly_client_shared_examples.rb new file mode 100644 index 00000000000..f26b9a4a7bd --- /dev/null +++ b/spec/support/shared_examples/lib/gitlab/gitaly_client_shared_examples.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +def raw_repo_without_container(repository) + Gitlab::Git::Repository.new(repository.shard, + "#{repository.disk_path}.git", + repository.repo_type.identifier_for_container(repository.container), + repository.container.full_path) +end + +RSpec.shared_examples 'Gitaly feature flag actors are inferred from repository' do + it 'captures correct actors' do + service.repository_actor = repository + + expect(service.repository_actor.flipper_id).to eql(repository.flipper_id) + + if expected_project.nil? + expect(service.project_actor).to be(nil) + else + expect(service.project_actor.flipper_id).to eql(expected_project.flipper_id) + end + + if expected_group.nil? + expect(service.group_actor).to be(nil) + else + expect(service.group_actor.flipper_id).to eql(expected_group.flipper_id) + end + end + + it 'does not issues SQL queries after the first invocation' do + service.repository_actor = repository + + service.repository_actor + service.project_actor + service.group_actor + + recorder = ActiveRecord::QueryRecorder.new do + 3.times do + service.repository_actor + service.project_actor + service.group_actor + end + end + + expect(recorder.count).to be(0) + end +end diff --git a/spec/workers/repository_check/single_repository_worker_spec.rb b/spec/workers/repository_check/single_repository_worker_spec.rb index a0cf3b1e978..b8db262598b 100644 --- a/spec/workers/repository_check/single_repository_worker_spec.rb +++ b/spec/workers/repository_check/single_repository_worker_spec.rb @@ -16,7 +16,7 @@ RSpec.describe RepositoryCheck::SingleRepositoryWorker do repository = instance_double(::Gitlab::Git::Repository) allow(::Gitlab::Git::Repository).to receive(:new) - .with(project.repository_storage, "#{project.disk_path}.git", anything, anything) + .with(project.repository_storage, "#{project.disk_path}.git", anything, anything, container: project) .and_return(repository) worker.perform(project.id) @@ -31,7 +31,7 @@ RSpec.describe RepositoryCheck::SingleRepositoryWorker do repository = project.repository.raw expect(repository).to receive(:fsck).and_raise(::Gitlab::Git::Repository::GitError) expect(::Gitlab::Git::Repository).to receive(:new) - .with(project.repository_storage, "#{project.disk_path}.git", anything, anything) + .with(project.repository_storage, "#{project.disk_path}.git", anything, anything, container: project) .and_return(repository) worker.perform(project.id) @@ -46,7 +46,7 @@ RSpec.describe RepositoryCheck::SingleRepositoryWorker do repository = project.repository.raw expect(repository).to receive(:fsck).and_call_original expect(::Gitlab::Git::Repository).to receive(:new) - .with(project.repository_storage, "#{project.disk_path}.git", anything, anything) + .with(project.repository_storage, "#{project.disk_path}.git", anything, anything, container: project) .and_return(repository) expect do @@ -68,7 +68,7 @@ RSpec.describe RepositoryCheck::SingleRepositoryWorker do repository = project.wiki.repository.raw expect(repository).to receive(:fsck).and_raise(::Gitlab::Git::Repository::GitError) expect(::Gitlab::Git::Repository).to receive(:new) - .with(project.repository_storage, "#{project.disk_path}.wiki.git", anything, anything) + .with(project.repository_storage, "#{project.disk_path}.wiki.git", anything, anything, container: project.wiki) .and_return(repository) worker.perform(project.id) @@ -81,7 +81,7 @@ RSpec.describe RepositoryCheck::SingleRepositoryWorker do # Make sure the test would fail if the wiki repo was checked repository = instance_double(::Gitlab::Git::Repository) allow(::Gitlab::Git::Repository).to receive(:new) - .with(project.repository_storage, "#{project.disk_path}.wiki.git", anything, anything) + .with(project.repository_storage, "#{project.disk_path}.wiki.git", anything, anything, container: project) .and_return(repository) subject.perform(project.id) diff --git a/workhorse/go.mod b/workhorse/go.mod index 0cfbd4c0092..c3b4aa0a20b 100644 --- a/workhorse/go.mod +++ b/workhorse/go.mod @@ -20,7 +20,7 @@ require ( github.com/johannesboyne/gofakes3 v0.0.0-20220627085814-c3ac35da23b2 github.com/jpillora/backoff v1.0.0 github.com/mitchellh/copystructure v1.2.0 - github.com/prometheus/client_golang v1.13.1 + github.com/prometheus/client_golang v1.14.0 github.com/rafaeljusto/redigomock/v3 v3.1.1 github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a github.com/sirupsen/logrus v1.9.0 @@ -92,7 +92,7 @@ require ( github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/prometheus/prometheus v0.37.0 // indirect diff --git a/workhorse/go.sum b/workhorse/go.sum index 7884b69dbe1..c17b6e049b9 100644 --- a/workhorse/go.sum +++ b/workhorse/go.sum @@ -1259,16 +1259,17 @@ github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.1 h1:3gMjIY2+/hzmqhtUC/aQNYldJA6DtH3CgQvwS+02K1c= -github.com/prometheus/client_golang v1.13.1/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= |