diff options
82 files changed, 1318 insertions, 1006 deletions
diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index 6a4a58fe5db..64955d67e34 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -604,6 +604,7 @@ lib/gitlab/checks/** @proglottis @toon @zj-gitlab /doc/gitlab-basics/ @aqualls /doc/install/ @axil /doc/integration/ @kpaizee +/doc/integration/advanced_search/ @sselhorn /doc/integration/elasticsearch.md @sselhorn /doc/integration/gitpod.md @aqualls /doc/integration/kerberos.md @eread diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index b18cda2278a..7459b428021 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -1076,10 +1076,8 @@ rules: - <<: *if-merge-request-labels-run-all-rspec - <<: *if-merge-request - changes: *core-backend-patterns - - <<: *if-merge-request - changes: *ci-patterns - - changes: ["config/**/*"] + changes: *backend-patterns + - changes: *core-backend-patterns .rails:rules:code-backstage-qa: rules: @@ -170,7 +170,7 @@ gem 'asciidoctor-kroki', '~> 0.5.0', require: false gem 'rouge', '~> 3.27.0' gem 'truncato', '~> 0.7.11' gem 'bootstrap_form', '~> 4.2.0' -gem 'nokogiri', '~> 1.12' +gem 'nokogiri', '~> 1.13.6' gem 'escape_utils', '~> 1.1' # Calendar rendering diff --git a/Gemfile.lock b/Gemfile.lock index 43f7109a728..d84ff5769f4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -807,7 +807,7 @@ GEM netrc (0.11.0) nio4r (2.5.8) no_proxy_fix (0.1.2) - nokogiri (1.13.3) + nokogiri (1.13.6) mini_portile2 (~> 2.8.0) racc (~> 1.4) notiffany (0.1.3) @@ -1603,7 +1603,7 @@ DEPENDENCIES multi_json (~> 1.14.1) net-ldap (~> 0.16.3) net-ntp - nokogiri (~> 1.12) + nokogiri (~> 1.13.6) oauth2 (~> 1.4) octokit (~> 4.15) ohai (~> 16.10) diff --git a/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_form.vue b/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_form.vue index 71e40fde77d..c44ce8b0057 100644 --- a/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_form.vue +++ b/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_form.vue @@ -8,7 +8,7 @@ import { confidentialityQueries } from '~/sidebar/constants'; export default { i18n: { confidentialityOnWarning: __( - 'You are going to turn on confidentiality. Only %{context} members with %{strongStart}at least Reporter role%{strongEnd} can view or be notified about this %{issuableType}.', + 'You are going to turn on confidentiality. Only %{context} members with %{strongStart}%{permissions}%{strongEnd} can view or be notified about this %{issuableType}.', ), confidentialityOffWarning: __( 'You are going to turn off the confidentiality. This means %{strongStart}everyone%{strongEnd} will be able to see and leave a comment on this %{issuableType}.', @@ -65,6 +65,11 @@ export default { groupPath: this.fullPath, }; }, + permissions() { + return this.issuableType === IssuableType.Issue + ? __('at least the Reporter role, the author, and assignees') + : __('at least the Reporter role'); + }, }, methods: { submitForm() { @@ -120,7 +125,11 @@ export default { <p data-testid="warning-message"> <gl-sprintf :message="warningMessage"> <template #strong="{ content }"> - <strong>{{ content }}</strong> + <strong> + <gl-sprintf :message="content"> + <template #permissions>{{ permissions }}</template> + </gl-sprintf> + </strong> </template> <template #context>{{ context }}</template> <template #issuableType>{{ issuableType }}</template> diff --git a/app/assets/javascripts/vue_shared/constants.js b/app/assets/javascripts/vue_shared/constants.js index 3ebeec4a50b..14328b1f25f 100644 --- a/app/assets/javascripts/vue_shared/constants.js +++ b/app/assets/javascripts/vue_shared/constants.js @@ -71,10 +71,14 @@ export const AVATAR_SHAPE_OPTION_RECT = 'rect'; export const confidentialityInfoText = (workspaceType, issuableType) => sprintf( __( - 'Only %{workspaceType} members with at least Reporter role can view or be notified about this %{issuableType}.', + 'Only %{workspaceType} members with %{permissions} can view or be notified about this %{issuableType}.', ), { workspaceType: workspaceType === WorkspaceType.project ? __('project') : __('group'), issuableType: issuableType === IssuableType.Issue ? __('issue') : __('epic'), + permissions: + issuableType === IssuableType.Issue + ? __('at least the Reporter role, the author, and assignees') + : __('at least the Reporter role'), }, ); diff --git a/app/assets/stylesheets/highlight/common.scss b/app/assets/stylesheets/highlight/common.scss index 5c58d220075..fcbd05141b9 100644 --- a/app/assets/stylesheets/highlight/common.scss +++ b/app/assets/stylesheets/highlight/common.scss @@ -126,3 +126,11 @@ @include gl-display-inline-block; } } + +@mixin hljs-override($suffix, $color) { + &.blob-viewer { + .hljs-#{$suffix} { + color: $color; + } + } +} diff --git a/app/assets/stylesheets/highlight/themes/monokai.scss b/app/assets/stylesheets/highlight/themes/monokai.scss index 4a6d06a6444..dfa32d4b773 100644 --- a/app/assets/stylesheets/highlight/themes/monokai.scss +++ b/app/assets/stylesheets/highlight/themes/monokai.scss @@ -96,6 +96,25 @@ $monokai-gh: #75715e; } .code.monokai { + // Highlight.js theme overrides (https://gitlab.com/gitlab-org/gitlab/-/issues/365167) + // We should be able to remove the overrides once the upstream issue is fixed (https://github.com/sourcegraph/sourcegraph/issues/23251) + @include hljs-override('string', $monokai-s); + @include hljs-override('attr', $monokai-na); + @include hljs-override('keyword', $monokai-k); + @include hljs-override('variable', $monokai-nv); + @include hljs-override('variable.language_', $monokai-k); + @include hljs-override('title', $monokai-nf); + @include hljs-override('name', $monokai-k); + @include hljs-override('tag', $monokai-nt); + @include hljs-override('type', $monokai-nc); + @include hljs-override('number', $monokai-mf); + @include hljs-override('literal', $monokai-kc); + @include hljs-override('built_in', $monokai-n); + @include hljs-override('section', $monokai-gh); + @include hljs-override('bullet', $monokai-n); + @include hljs-override('subst', $monokai-p); + @include hljs-override('symbol', $monokai-ni); + // Line numbers .file-line-num { @include line-number-link($monokai-line-num-color); diff --git a/app/assets/stylesheets/highlight/themes/none.scss b/app/assets/stylesheets/highlight/themes/none.scss index 2f35126dd3b..f70c53c9eaa 100644 --- a/app/assets/stylesheets/highlight/themes/none.scss +++ b/app/assets/stylesheets/highlight/themes/none.scss @@ -15,6 +15,14 @@ } .code.none { + // Highlight.js theme overrides (https://gitlab.com/gitlab-org/gitlab/-/issues/365167) + // We should be able to remove the overrides once the upstream issue is fixed (https://github.com/sourcegraph/sourcegraph/issues/23251) + &.blob-viewer { + [class^="hljs-"] { + color: $gl-text-color; + } + } + // Line numbers .file-line-num { @include line-number-link($black-transparent); diff --git a/app/assets/stylesheets/highlight/themes/solarized-dark.scss b/app/assets/stylesheets/highlight/themes/solarized-dark.scss index acfc98c13ab..73aa6275d17 100644 --- a/app/assets/stylesheets/highlight/themes/solarized-dark.scss +++ b/app/assets/stylesheets/highlight/themes/solarized-dark.scss @@ -99,6 +99,25 @@ $solarized-dark-il: #2aa198; } .code.solarized-dark { + // Highlight.js theme overrides (https://gitlab.com/gitlab-org/gitlab/-/issues/365167) + // We should be able to remove the overrides once the upstream issue is fixed (https://github.com/sourcegraph/sourcegraph/issues/23251) + @include hljs-override('string', $solarized-dark-s); + @include hljs-override('attr', $solarized-dark-na); + @include hljs-override('keyword', $solarized-dark-k); + @include hljs-override('variable', $solarized-dark-nv); + @include hljs-override('variable.language_', $solarized-dark-k); + @include hljs-override('title', $solarized-dark-nf); + @include hljs-override('name', $solarized-dark-k); + @include hljs-override('tag', $solarized-dark-nt); + @include hljs-override('type', $solarized-dark-nc); + @include hljs-override('number', $solarized-dark-mf); + @include hljs-override('literal', $solarized-dark-kc); + @include hljs-override('built_in', $solarized-dark-n); + @include hljs-override('section', $solarized-dark-gh); + @include hljs-override('bullet', $solarized-dark-n); + @include hljs-override('subst', $solarized-dark-p); + @include hljs-override('symbol', $solarized-dark-ni); + // Line numbers .file-line-num { @include line-number-link($solarized-dark-line-color); diff --git a/app/views/groups/edit.html.haml b/app/views/groups/edit.html.haml index f3494149087..bca1c874cc6 100644 --- a/app/views/groups/edit.html.haml +++ b/app/views/groups/edit.html.haml @@ -9,7 +9,7 @@ .settings-header %h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only{ role: 'button' } = _('Naming, visibility') - %button.btn.gl-button.js-settings-toggle{ type: 'button' } + = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = _('Collapse') %p = _('Update your group name, description, avatar, and visibility.') @@ -21,7 +21,7 @@ .settings-header %h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only{ role: 'button' } = _('Permissions and group features') - %button.btn.gl-button.js-settings-toggle{ type: 'button' } + = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded ? _('Collapse') : _('Expand') %p = _('Configure advanced permissions, Large File Storage, two-factor authentication, and customer relations settings.') @@ -35,7 +35,7 @@ .settings-header %h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only{ role: 'button' } = s_('GroupSettings|Badges') - %button.btn.gl-button.js-settings-toggle{ type: 'button' } + = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded ? _('Collapse') : _('Expand') %p = s_('GroupSettings|Customize this group\'s badges.') @@ -51,7 +51,7 @@ .settings-header %h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only{ role: 'button' } = _('Advanced') - %button.btn.gl-button.js-settings-toggle{ type: 'button' } + = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded ? _('Collapse') : _('Expand') %p = _('Perform advanced options such as changing path, transferring, exporting, or removing the group.') diff --git a/app/views/groups/merge_requests.html.haml b/app/views/groups/merge_requests.html.haml index 011c76e5ae7..b33d1443706 100644 --- a/app/views/groups/merge_requests.html.haml +++ b/app/views/groups/merge_requests.html.haml @@ -10,7 +10,7 @@ - if current_user .nav-controls - if @can_bulk_update - = render_if_exists 'shared/issuable/bulk_update_button', type: :merge_requests + = render_if_exists 'projects/merge_requests/bulk_update_button' = render 'shared/new_project_item_select', path: 'merge_requests/new', label: _("merge request"), type: :merge_requests, with_feature_enabled: 'merge_requests', with_shared: false, include_projects_in_subgroups: true diff --git a/app/views/groups/settings/_advanced.html.haml b/app/views/groups/settings/_advanced.html.haml index d72f52f39cd..3624ff2bcb3 100644 --- a/app/views/groups/settings/_advanced.html.haml +++ b/app/views/groups/settings/_advanced.html.haml @@ -4,7 +4,7 @@ .sub-section %h4.warning-title= s_('GroupSettings|Change group URL') = form_for @group, html: { multipart: true, class: 'gl-show-field-errors' }, authenticity_token: true do |f| - = form_errors(@group) + = form_errors(@group, pajamas_alert: true) .form-group %p = s_("GroupSettings|Changing a group's URL can have unintended side effects.") diff --git a/app/views/groups/settings/ci_cd/show.html.haml b/app/views/groups/settings/ci_cd/show.html.haml index 3e113b1f35f..51e9e344d26 100644 --- a/app/views/groups/settings/ci_cd/show.html.haml +++ b/app/views/groups/settings/ci_cd/show.html.haml @@ -11,7 +11,7 @@ .settings-header %h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only = _("General pipelines") - %button.btn.gl-button.js-settings-toggle{ type: 'button' } + = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded ? _('Collapse') : _('Expand') %p = _("Customize your pipeline configuration.") @@ -28,7 +28,7 @@ .settings-header %h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only = _('Runners') - %button.btn.gl-button.btn-default.js-settings-toggle{ type: "button" } + = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded ? _('Collapse') : _('Expand') %p = _("Runners are processes that pick up and execute CI/CD jobs for GitLab.") @@ -40,7 +40,7 @@ .settings-header %h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only = _('Auto DevOps') - %button.btn.gl-button.btn-default.js-settings-toggle{ type: "button" } + = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded ? _('Collapse') : _('Expand') %p - auto_devops_url = help_page_path('topics/autodevops/index') diff --git a/app/views/groups/settings/repository/_default_branch.html.haml b/app/views/groups/settings/repository/_default_branch.html.haml index f2644465a49..844a5f890a4 100644 --- a/app/views/groups/settings/repository/_default_branch.html.haml +++ b/app/views/groups/settings/repository/_default_branch.html.haml @@ -2,7 +2,7 @@ .settings-header %h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only = _('Default branch') - %button.gl-button.js-settings-toggle{ type: 'button' } + = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded_by_default? ? _('Collapse') : _('Expand') %p = s_('GroupSettings|Set the initial name and protections for the default branch of new repositories created in the group.') diff --git a/app/views/groups/settings/repository/show.html.haml b/app/views/groups/settings/repository/show.html.haml index 072c8c4d821..d3b9117c05b 100644 --- a/app/views/groups/settings/repository/show.html.haml +++ b/app/views/groups/settings/repository/show.html.haml @@ -1,5 +1,6 @@ - breadcrumb_title _('Repository Settings') - page_title _('Repository') +- @content_class = "limit-container-width" unless fluid_layout - deploy_token_description = s_('DeployTokens|Group deploy tokens allow access to the packages, repositories, and registry images within the group.') diff --git a/app/views/projects/branches/new.html.haml b/app/views/projects/branches/new.html.haml index 5b6834d3d82..f8bee5a69e9 100644 --- a/app/views/projects/branches/new.html.haml +++ b/app/views/projects/branches/new.html.haml @@ -26,7 +26,8 @@ = render 'shared/ref_dropdown', dropdown_class: 'wide' .form-text.text-muted Existing branch name, tag, or commit SHA .form-actions - = button_tag 'Create branch', class: 'gl-button btn btn-confirm' + = render Pajamas::ButtonComponent.new(variant: :confirm, button_options: { type: 'submit', class: 'gl-mr-3' }) do + = _('Create branch') = link_to _('Cancel'), project_branches_path(@project), class: 'gl-button btn btn-default btn-cancel' -# haml-lint:disable InlineJavaScript %script#availableRefs{ type: "application/json" }= @project.repository.ref_names.to_json.html_safe diff --git a/app/views/projects/issuable/_show.html.haml b/app/views/projects/issuable/_show.html.haml index f311ed2d8ae..0898e0ae52d 100644 --- a/app/views/projects/issuable/_show.html.haml +++ b/app/views/projects/issuable/_show.html.haml @@ -4,7 +4,7 @@ - if issuable.relocation_target - page_canonical_link issuable.relocation_target.present(current_user: current_user).web_url -= render "projects/issues/alert_moved_from_service_desk", issue: issuable += render "projects/issues/service_desk/alert_moved_from_service_desk", issue: issuable = render 'shared/issue_type/details_header', issuable: issuable = render 'shared/issue_type/details_content', issuable: issuable, api_awards_path: api_awards_path diff --git a/app/views/projects/issues/_by_email_description.html.haml b/app/views/projects/issues/_by_email_description.html.haml deleted file mode 100644 index aeed5fb69c9..00000000000 --- a/app/views/projects/issues/_by_email_description.html.haml +++ /dev/null @@ -1,6 +0,0 @@ -The subject will be used as the title of the new issue, and the message will be the description. - -= link_to 'Quick actions', help_page_path('user/project/quick_actions'), target: '_blank', rel: 'noopener noreferrer' -and styling with -= link_to 'Markdown', help_page_path('user/markdown'), target: '_blank', rel: 'noopener noreferrer' -are supported. diff --git a/app/views/projects/issues/service_desk.html.haml b/app/views/projects/issues/service_desk.html.haml index fb5880f633a..93cb5ddd7e2 100644 --- a/app/views/projects/issues/service_desk.html.haml +++ b/app/views/projects/issues/service_desk.html.haml @@ -3,7 +3,7 @@ - page_title _("Service Desk") - add_page_specific_style 'page_bundles/issues_list' - content_for :breadcrumbs_extra do - = render "projects/issues/nav_btns", show_export_button: false, show_rss_button: false + = render "projects/issues/service_desk/nav_btns", show_export_button: false, show_rss_button: false - support_bot_attrs = { service_desk_enabled: @project.service_desk_enabled?, **UserSerializer.new.represent(User.support_bot) }.to_json @@ -11,12 +11,12 @@ .top-area = render 'shared/issuable/nav', type: :issues .nav-controls.d-block.d-sm-none - = render "projects/issues/nav_btns", show_feed_buttons: false, show_import_button: false, show_export_button: false + = render "projects/issues/service_desk/nav_btns", show_feed_buttons: false, show_import_button: false, show_export_button: false - if @issues.present? = render 'shared/issuable/search_bar', type: :issues - if Gitlab::ServiceDesk.supported? - = render 'service_desk_info_content' + = render 'projects/issues/service_desk/service_desk_info_content' .issues-holder - = render 'projects/issues/issues', empty_state_path: 'service_desk_empty_state' + = render 'projects/issues/issues', empty_state_path: 'projects/issues/service_desk/service_desk_empty_state' diff --git a/app/views/projects/issues/_alert_moved_from_service_desk.html.haml b/app/views/projects/issues/service_desk/_alert_moved_from_service_desk.html.haml index cc8d5bdaeec..cc8d5bdaeec 100644 --- a/app/views/projects/issues/_alert_moved_from_service_desk.html.haml +++ b/app/views/projects/issues/service_desk/_alert_moved_from_service_desk.html.haml diff --git a/app/views/projects/issues/_nav_btns.html.haml b/app/views/projects/issues/service_desk/_nav_btns.html.haml index 8d16c3d978f..8d16c3d978f 100644 --- a/app/views/projects/issues/_nav_btns.html.haml +++ b/app/views/projects/issues/service_desk/_nav_btns.html.haml diff --git a/app/views/projects/issues/_service_desk_empty_state.html.haml b/app/views/projects/issues/service_desk/_service_desk_empty_state.html.haml index 1c9143c633d..1c9143c633d 100644 --- a/app/views/projects/issues/_service_desk_empty_state.html.haml +++ b/app/views/projects/issues/service_desk/_service_desk_empty_state.html.haml diff --git a/app/views/projects/issues/_service_desk_info_content.html.haml b/app/views/projects/issues/service_desk/_service_desk_info_content.html.haml index bad75ac2cd9..bad75ac2cd9 100644 --- a/app/views/projects/issues/_service_desk_info_content.html.haml +++ b/app/views/projects/issues/service_desk/_service_desk_info_content.html.haml diff --git a/app/views/shared/deploy_tokens/_index.html.haml b/app/views/shared/deploy_tokens/_index.html.haml index 860fb5614af..aa4a3deaac4 100644 --- a/app/views/shared/deploy_tokens/_index.html.haml +++ b/app/views/shared/deploy_tokens/_index.html.haml @@ -3,8 +3,8 @@ %section.settings.no-animate#js-deploy-tokens{ class: ('expanded' if expanded), data: { qa_selector: 'deploy_tokens_settings_content' } } .settings-header %h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only= s_('DeployTokens|Deploy tokens') - %button.btn.gl-button.btn-default.js-settings-toggle - = expanded ? 'Collapse' : 'Expand' + = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do + = expanded ? _('Collapse') : _('Expand') %p = description .settings-content diff --git a/config/feature_flags/development/vsa_consistency_worker.yml b/config/feature_flags/development/vsa_consistency_worker.yml deleted file mode 100644 index c038efeb9cd..00000000000 --- a/config/feature_flags/development/vsa_consistency_worker.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: vsa_consistency_worker -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82591 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/355709 -milestone: '14.9' -type: development -group: group::optimize -default_enabled: true diff --git a/config/initializers/truncato.rb b/config/initializers/truncato.rb new file mode 100644 index 00000000000..dd0a8a313c3 --- /dev/null +++ b/config/initializers/truncato.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +# Source: https://github.com/jorgemanrubia/truncato/issues/20#issuecomment-1135105823 +silence_warnings do + Truncato::ARTIFICIAL_ROOT_NAME = 'truncato-artificial-root' +end diff --git a/db/post_migrate/20220602070502_queue_backfill_project_feature_package_registry_access_level.rb b/db/post_migrate/20220602070502_queue_backfill_project_feature_package_registry_access_level.rb new file mode 100644 index 00000000000..2c772a79327 --- /dev/null +++ b/db/post_migrate/20220602070502_queue_backfill_project_feature_package_registry_access_level.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class QueueBackfillProjectFeaturePackageRegistryAccessLevel < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + MIGRATION = 'BackfillProjectFeaturePackageRegistryAccessLevel' + DELAY_INTERVAL = 2.minutes + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :projects, + :id, + job_interval: DELAY_INTERVAL + ) + end + + def down + delete_batched_background_migration(MIGRATION, :projects, :id, []) + end +end diff --git a/db/schema_migrations/20220602070502 b/db/schema_migrations/20220602070502 new file mode 100644 index 00000000000..68cfe781ef8 --- /dev/null +++ b/db/schema_migrations/20220602070502 @@ -0,0 +1 @@ +9c66d020895c534280862136a08477fe3715465bdeb9d3e7dd632005c19de474
\ No newline at end of file diff --git a/doc/administration/geo/replication/datatypes.md b/doc/administration/geo/replication/datatypes.md index aa9958b2fb2..8a5aa935d11 100644 --- a/doc/administration/geo/replication/datatypes.md +++ b/doc/administration/geo/replication/datatypes.md @@ -206,7 +206,7 @@ successfully, you must replicate their data using some other means. |[Incident Metric Images](../../../operations/incident_management/incidents.md#metrics) | [Planned](https://gitlab.com/gitlab-org/gitlab/-/issues/352326) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/362561) | No | | |[Alert Metric Images](../../../operations/incident_management/alerts.md#metrics-tab) | [Planned](https://gitlab.com/gitlab-org/gitlab/-/issues/352326) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/362561) | No | | |[Server-side Git hooks](../../server_hooks.md) | [Not planned](https://gitlab.com/groups/gitlab-org/-/epics/1867) | No | No | Not planned because of current implementation complexity, low customer interest, and availability of alternatives to hooks. | -|[Elasticsearch integration](../../../integration/elasticsearch.md) | [Not planned](https://gitlab.com/gitlab-org/gitlab/-/issues/1186) | No | No | Not planned because further product discovery is required and Elasticsearch (ES) clusters can be rebuilt. Secondaries use the same ES cluster as the primary. | +|[Elasticsearch integration](../../../integration/advanced_search/elasticsearch.md) | [Not planned](https://gitlab.com/gitlab-org/gitlab/-/issues/1186) | No | No | Not planned because further product discovery is required and Elasticsearch (ES) clusters can be rebuilt. Secondaries use the same ES cluster as the primary. | |[Dependency proxy images](../../../user/packages/dependency_proxy/index.md) | [Not planned](https://gitlab.com/gitlab-org/gitlab/-/issues/259694) | No | No | Blocked by [Geo: Secondary Mimicry](https://gitlab.com/groups/gitlab-org/-/epics/1528). Replication of this cache is not needed for disaster recovery purposes because it can be recreated from external sources. | |[Vulnerability Export](../../../user/application_security/vulnerability_report/#export-vulnerability-details) | [Not planned](https://gitlab.com/groups/gitlab-org/-/epics/3111) | No | No | Not planned because they are ephemeral and sensitive information. They can be regenerated on demand. | diff --git a/doc/administration/gitaly/configure_gitaly.md b/doc/administration/gitaly/configure_gitaly.md index 5a9da925c47..26aa750900b 100644 --- a/doc/administration/gitaly/configure_gitaly.md +++ b/doc/administration/gitaly/configure_gitaly.md @@ -67,7 +67,7 @@ When running Gitaly on its own server, note the following regarding GitLab versi servers. - From GitLab 11.8 to 12.2, it is possible to use Elasticsearch in a Gitaly setup that doesn't use NFS. To use Elasticsearch in these versions, the - [repository indexer](../../integration/elasticsearch.md#elasticsearch-repository-indexer) + [repository indexer](../../integration/advanced_search/elasticsearch.md#elasticsearch-repository-indexer) must be enabled in your GitLab configuration. - [In GitLab 12.3 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/6481), the new indexer is the default and no configuration is required. diff --git a/doc/administration/index.md b/doc/administration/index.md index 6b9691bea2b..29fb65e2deb 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -65,7 +65,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. requests diffs external storage. - [Broadcast Messages](../user/admin_area/broadcast_messages.md): Send messages to GitLab users through the UI. -- [Elasticsearch](../integration/elasticsearch.md): Enable Elasticsearch to +- [Elasticsearch](../integration/advanced_search/elasticsearch.md): Enable Elasticsearch to empower Advanced Search. Use when you deal with a huge amount of data. - [External Classification Policy Authorization](../user/admin_area/settings/external_authorization.md) - [Add a license](../user/admin_area/license.md): Add a license to unlock diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index b3c8586db62..7ff22549a84 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -924,7 +924,7 @@ indexed, which have a separate limit. For more information, read - For self-managed installations, the field length is unlimited by default. You can configure this limit for self-managed installations when you -[enable Elasticsearch](../integration/elasticsearch.md#enable-advanced-search). +[enable Elasticsearch](../integration/advanced_search/elasticsearch.md#enable-advanced-search). Set the limit to `0` to disable it. ## Wiki limits diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index 61b23527f8f..99272cdd226 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -2233,13 +2233,13 @@ While sharing the job logs through NFS is supported, it's recommended to avoid t ## Configure Advanced Search -You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific data. For recommended best practices about how to set up your Elasticsearch cluster alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). +[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> diff --git a/doc/administration/reference_architectures/1k_users.md b/doc/administration/reference_architectures/1k_users.md index bc2504af4c1..c356c7224cb 100644 --- a/doc/administration/reference_architectures/1k_users.md +++ b/doc/administration/reference_architectures/1k_users.md @@ -106,13 +106,13 @@ performance and reliability at an increased complexity cost. ## Configure Advanced Search **(PREMIUM SELF)** -You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific data. For recommended best practices about how to set up your Elasticsearch cluster alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). +[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). ## Cloud Native Hybrid reference architecture with Helm Charts diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index d5e13940f01..367654b8e59 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -2237,13 +2237,13 @@ While sharing the job logs through NFS is supported, it's recommended to avoid t ## Configure Advanced Search -You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific data. For recommended best practices about how to set up your Elasticsearch cluster alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). +[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index 65c0d417a44..3b9a8f966c8 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -941,13 +941,13 @@ in the future. ## Configure Advanced Search **(PREMIUM SELF)** -You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific data. For recommended best practices about how to set up your Elasticsearch cluster alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). +[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index de5f7c7455c..a43d33a4d4a 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -2172,13 +2172,13 @@ While sharing the job logs through NFS is supported, it's recommended to avoid t ## Configure Advanced Search -You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific data. For recommended best practices about how to set up your Elasticsearch cluster alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). +[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index 53e39a576c8..316969dfacc 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -2253,13 +2253,13 @@ While sharing the job logs through NFS is supported, it's recommended to avoid t ## Configure Advanced Search -You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific data. For recommended best practices about how to set up your Elasticsearch cluster alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). +[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 170496d84e9..ef44d688f31 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -2172,13 +2172,13 @@ While sharing the job logs through NFS is supported, it's recommended to avoid t ## Configure Advanced Search -You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific data. For recommended best practices about how to set up your Elasticsearch cluster alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). +[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> diff --git a/doc/administration/troubleshooting/elasticsearch.md b/doc/administration/troubleshooting/elasticsearch.md index bfea8d2587a..7ce09252680 100644 --- a/doc/administration/troubleshooting/elasticsearch.md +++ b/doc/administration/troubleshooting/elasticsearch.md @@ -6,8 +6,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Troubleshooting Elasticsearch **(PREMIUM SELF)** -To install and configure Elasticsearch, and for common and known issues, -visit the [administrator documentation](../../integration/elasticsearch.md). +To install and configure Elasticsearch, +visit the [administrator documentation](../../integration/advanced_search/elasticsearch.md). + +For troubleshooting, visit the +[administrator troubleshooting documentation](../../integration/advanced_search/elasticsearch_troubleshooting.md). Troubleshooting Elasticsearch requires: @@ -242,7 +245,7 @@ The best place to start is to determine if the issue is with creating an empty i If it is, check on the Elasticsearch side to determine if the `gitlab-production` (the name for the GitLab index) exists. If it exists, manually delete it on the Elasticsearch side and attempt to recreate it from the -[`recreate_index`](../../integration/elasticsearch.md#gitlab-advanced-search-rake-tasks) +[`recreate_index`](../../integration/advanced_search/elasticsearch.md#gitlab-advanced-search-rake-tasks) Rake task. If you still encounter issues, try creating an index manually on the Elasticsearch @@ -261,8 +264,8 @@ during the indexing of projects. If errors do occur, they stem from either the i If the indexing process does not present errors, check the status of the indexed projects. You can do this via the following Rake tasks: -- [`sudo gitlab-rake gitlab:elastic:index_projects_status`](../../integration/elasticsearch.md#gitlab-advanced-search-rake-tasks) (shows the overall status) -- [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](../../integration/elasticsearch.md#gitlab-advanced-search-rake-tasks) (shows specific projects that are not indexed) +- [`sudo gitlab-rake gitlab:elastic:index_projects_status`](../../integration/advanced_search/elasticsearch.md#gitlab-advanced-search-rake-tasks) (shows the overall status) +- [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](../../integration/advanced_search/elasticsearch.md#gitlab-advanced-search-rake-tasks) (shows specific projects that are not indexed) If: diff --git a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md index 64c51ead46d..b57bc0a1119 100644 --- a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md +++ b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md @@ -1446,7 +1446,7 @@ Open the rails console (`gitlab rails c`) and run the following command to see a ApplicationSetting.last.attributes ``` -Among other attributes, the output contains all the settings available in the [Elasticsearch Integration page](../../integration/elasticsearch.md), such as `elasticsearch_indexing`, `elasticsearch_url`, `elasticsearch_replicas`, and `elasticsearch_pause_indexing`. +Among other attributes, the output contains all the settings available in the [Elasticsearch Integration page](../../integration/advanced_search/elasticsearch.md), such as `elasticsearch_indexing`, `elasticsearch_url`, `elasticsearch_replicas`, and `elasticsearch_pause_indexing`. #### Setting attributes @@ -1462,7 +1462,7 @@ ApplicationSetting.last.update(elasticsearch_indexing: false) #### Getting attributes -You can then check if the settings have been set in the [Elasticsearch Integration page](../../integration/elasticsearch.md) or in the rails console by issuing: +You can then check if the settings have been set in the [Elasticsearch Integration page](../../integration/advanced_search/elasticsearch.md) or in the rails console by issuing: ```ruby Gitlab::CurrentSettings.elasticsearch_url diff --git a/doc/api/search.md b/doc/api/search.md index 688dc8195cf..4e644a6c087 100644 --- a/doc/api/search.md +++ b/doc/api/search.md @@ -29,7 +29,7 @@ GET /search Search the expression in the specified scope. These scopes are supported: `projects`, `issues`, `merge_requests`, `milestones`, `snippet_titles`, `users`. -If Elasticsearch is enabled additional scopes available are `blobs`, `wiki_blobs`, `notes`, and `commits`. Find more about [the feature](../integration/elasticsearch.md). **(PREMIUM)** +If Elasticsearch is enabled additional scopes available are `blobs`, `wiki_blobs`, `notes`, and `commits`. Find more about [the feature](../integration/advanced_search/elasticsearch.md). **(PREMIUM)** The response depends on the requested scope. @@ -270,7 +270,7 @@ Example response: > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=wiki_blobs&search=bye" @@ -301,7 +301,7 @@ NOTE: > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=commits&search=bye" @@ -336,7 +336,7 @@ Example response: > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. Filters are available for this scope: @@ -377,7 +377,7 @@ NOTE: > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=notes&search=maxime" @@ -452,7 +452,7 @@ GET /groups/:id/search Search the expression in the specified scope. These scopes are supported: `projects`, `issues`, `merge_requests`, `milestones`, `users`. -If Elasticsearch is enabled additional scopes available are `blobs`, `wiki_blobs`, `notes`, and `commits`. Find more about [the feature](../integration/elasticsearch.md). **(PREMIUM)** +If Elasticsearch is enabled additional scopes available are `blobs`, `wiki_blobs`, `notes`, and `commits`. Find more about [the feature](../integration/advanced_search/elasticsearch.md). **(PREMIUM)** The response depends on the requested scope. @@ -662,7 +662,7 @@ Example response: > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/6/search?scope=wiki_blobs&search=bye" @@ -693,7 +693,7 @@ NOTE: > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/6/search?scope=commits&search=bye" @@ -728,7 +728,7 @@ Example response: > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. Filters are available for this scope: @@ -769,7 +769,7 @@ NOTE: > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=notes&search=maxime" @@ -1021,7 +1021,7 @@ Example response: > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=notes&search=maxime" @@ -1057,7 +1057,7 @@ Example response: > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. Filters are available for this scope: @@ -1105,7 +1105,7 @@ NOTE: > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=commits&search=bye" @@ -1140,7 +1140,7 @@ Example response: > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. Filters are available for this scope: diff --git a/doc/development/adding_service_component.md b/doc/development/adding_service_component.md index 2894250ec1b..51c6e86bb49 100644 --- a/doc/development/adding_service_component.md +++ b/doc/development/adding_service_component.md @@ -47,7 +47,7 @@ Adding a new service follows the same [merge request workflow](contributing/merg The first iteration should be to add the ability to connect and use the service as an externally installed component. Often this involves providing settings in GitLab to connect to the service, or allow connections from it. And then shipping documentation on how to install and configure the service with GitLab. -[Elasticsearch](../integration/elasticsearch.md#install-elasticsearch) is an example of a service that has been integrated this way. Many of the other services, including internal projects like Gitaly, started off as separately installed alternatives. +[Elasticsearch](../integration/advanced_search/elasticsearch.md#install-elasticsearch) is an example of a service that has been integrated this way. Many of the other services, including internal projects like Gitaly, started off as separately installed alternatives. **For services that depend on the existing GitLab codebase:** diff --git a/doc/development/architecture.md b/doc/development/architecture.md index 486ef6d27fc..a61a891b096 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -442,9 +442,9 @@ Consul is a tool for service discovery and configuration. Consul is distributed, - [Project page](https://github.com/elastic/elasticsearch/) - Configuration: - - [Omnibus](../integration/elasticsearch.md) - - [Charts](../integration/elasticsearch.md) - - [Source](../integration/elasticsearch.md) + - [Omnibus](../integration/advanced_search/elasticsearch.md) + - [Charts](../integration/advanced_search/elasticsearch.md) + - [Source](../integration/advanced_search/elasticsearch.md) - [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/howto/elasticsearch.md) - Layer: Core Service (Data) - GitLab.com: [Get Advanced Search working on GitLab.com (Closed)](https://gitlab.com/groups/gitlab-org/-/epics/153) epic. diff --git a/doc/development/database/batched_background_migrations.md b/doc/development/database/batched_background_migrations.md index c05ffe2cbf5..d77cbd23935 100644 --- a/doc/development/database/batched_background_migrations.md +++ b/doc/development/database/batched_background_migrations.md @@ -261,7 +261,7 @@ background migration. end def down - delete_batched_background_migration(MIGRATION_NAME, :routes, :id, []) + delete_batched_background_migration(MIGRATION, :routes, :id, []) end end ``` diff --git a/doc/development/elasticsearch.md b/doc/development/elasticsearch.md index 1b655cca4f8..d32ceb43ce9 100644 --- a/doc/development/elasticsearch.md +++ b/doc/development/elasticsearch.md @@ -9,17 +9,17 @@ info: To determine the technical writer assigned to the Stage/Group associated w This area is to maintain a compendium of useful information when working with Elasticsearch. Information on how to enable Elasticsearch and perform the initial indexing is in -the [Elasticsearch integration documentation](../integration/elasticsearch.md#enable-advanced-search). +the [Elasticsearch integration documentation](../integration/advanced_search/elasticsearch.md#enable-advanced-search). ## Deep Dive -In June 2019, Mario de la Ossa hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/issues/1`) on the GitLab [Elasticsearch integration](../integration/elasticsearch.md) to share his domain specific knowledge with anyone who may work in this part of the codebase in the future. You can find the <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [recording on YouTube](https://www.youtube.com/watch?v=vrvl-tN2EaA), and the slides on [Google Slides](https://docs.google.com/presentation/d/1H-pCzI_LNrgrL5pJAIQgvLX8Ji0-jIKOg1QeJQzChug/edit) and in [PDF](https://gitlab.com/gitlab-org/create-stage/uploads/c5aa32b6b07476fa8b597004899ec538/Elasticsearch_Deep_Dive.pdf). Everything covered in this deep dive was accurate as of GitLab 12.0, and while specific details may have changed since then, it should still serve as a good introduction. +In June 2019, Mario de la Ossa hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/issues/1`) on the GitLab [Elasticsearch integration](../integration/advanced_search/elasticsearch.md) to share his domain specific knowledge with anyone who may work in this part of the codebase in the future. You can find the <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [recording on YouTube](https://www.youtube.com/watch?v=vrvl-tN2EaA), and the slides on [Google Slides](https://docs.google.com/presentation/d/1H-pCzI_LNrgrL5pJAIQgvLX8Ji0-jIKOg1QeJQzChug/edit) and in [PDF](https://gitlab.com/gitlab-org/create-stage/uploads/c5aa32b6b07476fa8b597004899ec538/Elasticsearch_Deep_Dive.pdf). Everything covered in this deep dive was accurate as of GitLab 12.0, and while specific details may have changed since then, it should still serve as a good introduction. In August 2020, a second Deep Dive was hosted, focusing on [GitLab-specific architecture for multi-indices support](#zero-downtime-reindexing-with-multiple-indices). The <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [recording on YouTube](https://www.youtube.com/watch?v=0WdPR9oB2fg) and the [slides](https://lulalala.gitlab.io/gitlab-elasticsearch-deepdive/) are available. Everything covered in this deep dive was accurate as of GitLab 13.3. ## Supported Versions -See [Version Requirements](../integration/elasticsearch.md#version-requirements). +See [Version Requirements](../integration/advanced_search/elasticsearch.md#version-requirements). Developers making significant changes to Elasticsearch queries should test their features against all our supported versions. diff --git a/doc/index.md b/doc/index.md index 2f3bfefe819..1ac8cdb19ce 100644 --- a/doc/index.md +++ b/doc/index.md @@ -50,7 +50,7 @@ Have a look at some of our most popular topics: | [Activate GitLab EE with a license](user/admin_area/license.md) | Activate GitLab Enterprise Edition functionality with a license. | | [Back up and restore GitLab](raketasks/backup_restore.md) | Rake tasks for backing up and restoring GitLab self-managed instances. | | [GitLab release and maintenance policy](policy/maintenance.md) | Policies for version naming and cadence, and also upgrade recommendations. | -| [Elasticsearch integration](integration/elasticsearch.md) | Integrate Elasticsearch with GitLab to enable advanced searching. | +| [Elasticsearch integration](integration/advanced_search/elasticsearch.md) | Integrate Elasticsearch with GitLab to enable advanced searching. | | [Omnibus GitLab database settings](https://docs.gitlab.com/omnibus/settings/database.html) | Database settings for Omnibus GitLab self-managed instances. | | [Omnibus GitLab NGINX settings](https://docs.gitlab.com/omnibus/settings/nginx.html) | NGINX settings for Omnibus GitLab self-managed instances. | | [Omnibus GitLab SSL configuration](https://docs.gitlab.com/omnibus/settings/ssl.html) | SSL settings for Omnibus GitLab self-managed instances. | diff --git a/doc/install/next_steps.md b/doc/install/next_steps.md index b1b563fb57f..4defb62d254 100644 --- a/doc/install/next_steps.md +++ b/doc/install/next_steps.md @@ -56,7 +56,7 @@ installation. ## Cross-repository Code Search -- [Advanced Search](../integration/elasticsearch.md): Leverage [Elasticsearch](https://www.elastic.co/) or [OpenSearch](https://opensearch.org/) for +- [Advanced Search](../integration/advanced_search/elasticsearch.md): Leverage [Elasticsearch](https://www.elastic.co/) or [OpenSearch](https://opensearch.org/) for faster, more advanced code search across your entire GitLab instance. ## Scaling and replication diff --git a/doc/integration/advanced_search/elasticsearch.md b/doc/integration/advanced_search/elasticsearch.md new file mode 100644 index 00000000000..c123611fa44 --- /dev/null +++ b/doc/integration/advanced_search/elasticsearch.md @@ -0,0 +1,857 @@ +--- +type: reference +stage: Data Stores +group: Global Search +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Elasticsearch integration **(PREMIUM SELF)** + +This page describes how to enable Advanced Search. When enabled, +Advanced Search provides faster search response times and [improved search features](../../user/search/advanced_search.md). + +## Version requirements + +### Elasticsearch version requirements + +> Support for Elasticsearch 6.8 was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/350275) in GitLab 15.0. + +Advanced Search works with the following versions of Elasticsearch. + +| GitLab version | Elasticsearch version | +|-----------------------|--------------------------| +| GitLab 15.0 or later | Elasticsearch 7.x - 8.x | +| GitLab 13.9 - 14.10 | Elasticsearch 6.8 - 7.x | +| GitLab 13.3 - 13.8 | Elasticsearch 6.4 - 7.x | +| GitLab 12.7 - 13.2 | Elasticsearch 6.x - 7.x | + +Advanced Search follows Elasticsearch's [End of Life Policy](https://www.elastic.co/support/eol). +When we change Elasticsearch supported versions in GitLab, we announce them in [deprecation notes](https://about.gitlab.com/handbook/marketing/blog/release-posts/#deprecations) in monthly release posts +before we remove them. + +### OpenSearch version requirements + +| GitLab version | Elasticsearch version | +|-----------------------|--------------------------| +| GitLab 15.0 or later | OpenSearch 1.x or later | + +If you are using a compatible version and after connecting to OpenSearch, you get the message `Elasticsearch version not compatible`, [unpause indexing](#unpause-indexing). + +## System requirements + +Elasticsearch requires additional resources to those documented in the +[GitLab system requirements](../../install/requirements.md). + +Memory, CPU, and storage resource amounts vary depending on the amount of data you index into the Elasticsearch cluster. Heavily used Elasticsearch clusters may require more resources. According to +[Elasticsearch official guidelines](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_memory), +each node should have: + +- [Memory](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_memory): 8 GiB (minimum). +- [CPU](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_cpus): Modern processor with multiple cores. GitLab.com has minimal CPU requirements for Elasticsearch. Multiple cores provide extra concurrency, which is more beneficial than faster CPUs. +- [Storage](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_disks): Use SSD storage. The total storage size of all Elasticsearch nodes is about 50% of the total size of your Git repositories. It includes one primary and one replica. The [`estimate_cluster_size`](#gitlab-advanced-search-rake-tasks) Rake task ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221177) in GitLab 13.10) uses total repository size to estimate the Advanced Search storage requirements. + +## Install Elasticsearch + +Elasticsearch is *not* included in the Omnibus packages or when you install from +source. You must [install it separately](https://www.elastic.co/guide/en/elasticsearch/reference/7.x/install-elasticsearch.html "Elasticsearch 7.x installation documentation") and ensure you select your version. Detailed information on how to install Elasticsearch is out of the scope of this page. + +You can install Elasticsearch yourself, or use a cloud hosted offering such as [Elasticsearch Service](https://www.elastic.co/elasticsearch/service) (available on AWS, GCP, or Azure) or the [Amazon OpenSearch](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/gsg.html) +service. + +You should install Elasticsearch on a separate server. Running Elasticsearch on the same server as GitLab is not recommended and can cause a degradation in GitLab instance performance. + +For a single node Elasticsearch cluster, the functional cluster health status is always yellow due to the allocation of the primary shard. Elasticsearch cannot assign replica shards to the same node as primary shards. + +The search index updates after you: + +- Add data to the database or repository. +- [Enable Elasticsearch](#enable-advanced-search) in the Admin Area. + +## Upgrade to a new Elasticsearch major version + +> - Elasticsearch 6.8 support is removed with GitLab 15.0. +> - Upgrading from GitLab 14.10 to 15.0 requires that you are using any version of Elasticsearch 7.x. + +You are not required to change the GitLab configuration when you upgrade Elasticsearch. + +## Elasticsearch repository indexer + +To index Git repository data, GitLab uses an [indexer written in Go](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). + +Depending on your GitLab version, there are different installation procedures for the Go indexer: + +- For Omnibus GitLab 11.8 or greater, see [Omnibus GitLab](#omnibus-gitlab). +- For installations from source or older versions of Omnibus GitLab, + [install the indexer from source](#from-source). +- If you are using GitLab Development Kit, see [GDK Elasticsearch how-to](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/elasticsearch.md). + +### Omnibus GitLab + +Starting with GitLab 11.8, the Go indexer is included in Omnibus GitLab. +The former Ruby-based indexer was removed in [GitLab 12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/6481). + +### From source + +First, we need to install some dependencies, then we build and install +the indexer itself. + +This project relies on [International Components for Unicode](https://icu.unicode.org/) (ICU) for text encoding, +therefore we must ensure the development packages for your platform are +installed before running `make`. + +#### Debian / Ubuntu + +To install on Debian or Ubuntu, run: + +```shell +sudo apt install libicu-dev +``` + +#### CentOS / RHEL + +To install on CentOS or RHEL, run: + +```shell +sudo yum install libicu-devel +``` + +#### macOS + +NOTE: +You must first [install Homebrew](https://brew.sh/). + +To install on macOS, run: + +```shell +brew install icu4c +export PKG_CONFIG_PATH="/usr/local/opt/icu4c/lib/pkgconfig:$PKG_CONFIG_PATH" +``` + +### Build and install + +To build and install the indexer, run: + +```shell +indexer_path=/home/git/gitlab-elasticsearch-indexer + +# Run the installation task for gitlab-elasticsearch-indexer: +sudo -u git -H bundle exec rake gitlab:indexer:install[$indexer_path] RAILS_ENV=production +cd $indexer_path && sudo make install +``` + +The `gitlab-elasticsearch-indexer` is installed to `/usr/local/bin`. + +You can change the installation path with the `PREFIX` environment variable. +Please remember to pass the `-E` flag to `sudo` if you do so. + +Example: + +```shell +PREFIX=/usr sudo -E make install +``` + +After installation, be sure to [enable Elasticsearch](#enable-advanced-search). + +NOTE: +If you see an error such as `Permission denied - /home/git/gitlab-elasticsearch-indexer/` while indexing, you +may need to set the `production -> elasticsearch -> indexer_path` setting in your `gitlab.yml` file to +`/usr/local/bin/gitlab-elasticsearch-indexer`, which is where the binary is installed. + +## Enable Advanced Search + +For GitLab instances with more than 50GB repository data you can follow the instructions for [how to index large instances efficiently](#how-to-index-large-instances-efficiently) below. + +To enable Advanced Search, you must have administrator access to GitLab: + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Advanced Search**. + + NOTE: + To see the Advanced Search section, you need an active GitLab Premium + [license](../../user/admin_area/license.md). + +1. Configure the [Advanced Search settings](#advanced-search-configuration) for + your Elasticsearch cluster. Do not enable **Search with Elasticsearch enabled** + yet. +1. Enable **Elasticsearch indexing** and select **Save changes**. This creates + an empty index if one does not already exist. +1. Select **Index all projects**. +1. Select **Check progress** in the confirmation message to see the status of + the background jobs. +1. Personal snippets must be indexed using another Rake task: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:index_snippets + + # Installations from source + bundle exec rake gitlab:elastic:index_snippets RAILS_ENV=production + ``` + +1. After indexing completes, enable **Search with Elasticsearch enabled** and select **Save changes**. + +NOTE: +When your Elasticsearch cluster is down while Elasticsearch is enabled, +you might have problems updating documents such as issues because your +instance queues a job to index the change, but cannot find a valid +Elasticsearch cluster. + +### Advanced Search configuration + +The following Elasticsearch settings are available: + +| Parameter | Description | +|-------------------------------------------------------|-------------| +| `Elasticsearch indexing` | Enables or disables Elasticsearch indexing and creates an empty index if one does not already exist. You may want to enable indexing but disable search to give the index time to be fully completed, for example. Also, keep in mind that this option doesn't have any impact on existing data, this only enables/disables the background indexer which tracks data changes and ensures new data is indexed. | +| `Pause Elasticsearch indexing` | Enables or disables temporary indexing pause. This is useful for cluster migration/reindexing. All changes are still tracked, but they are not committed to the Elasticsearch index until resumed. | +| `Search with Elasticsearch enabled` | Enables or disables using Elasticsearch in search. | +| `URL` | The URL of your Elasticsearch instance. Use a comma-separated list to support clustering (for example, `http://host1, https://host2:9200`). If your Elasticsearch instance is password-protected, use the `Username` and `Password` fields described below. Alternatively, use inline credentials such as `http://<username>:<password>@<elastic_host>:9200/`. | +| `Username` | The `username` of your Elasticsearch instance. | +| `Password` | The password of your Elasticsearch instance. | +| `Number of Elasticsearch shards` | Elasticsearch indices are split into multiple shards for performance reasons. In general, you should use at least 5 shards, and indices with tens of millions of documents need to have more shards ([see below](#guidance-on-choosing-optimal-cluster-configuration)). Changes to this value do not take effect until the index is recreated. You can read more about tradeoffs in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/scalability.html). | +| `Number of Elasticsearch replicas` | Each Elasticsearch shard can have a number of replicas. These are a complete copy of the shard, and can provide increased query performance or resilience against hardware failure. Increasing this value increases total disk space required by the index. | +| `Limit the number of namespaces and projects that can be indexed` | Enabling this allows you to select namespaces and projects to index. All other namespaces and projects use database search instead. If you enable this option but do not select any namespaces or projects, none are indexed. [Read more below](#limit-the-number-of-namespaces-and-projects-that-can-be-indexed). +| `Using AWS hosted Elasticsearch with IAM credentials` | Sign your Elasticsearch requests using [AWS IAM authorization](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html), [AWS EC2 Instance Profile Credentials](https://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html#getting-started-create-iam-instance-profile-cli), or [AWS ECS Tasks Credentials](https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-iam-roles.html). Please refer to [Identity and Access Management in Amazon OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html) for details of AWS hosted OpenSearch domain access policy configuration. | +| `AWS Region` | The AWS region in which your OpenSearch Service is located. | +| `AWS Access Key` | The AWS access key. | +| `AWS Secret Access Key` | The AWS secret access key. | +| `Maximum file size indexed` | See [the explanation in instance limits.](../../administration/instance_limits.md#maximum-file-size-indexed). | +| `Maximum field length` | See [the explanation in instance limits.](../../administration/instance_limits.md#maximum-field-length). | +| `Maximum bulk request size (MiB)` | The Maximum Bulk Request size is used by the GitLab Golang-based indexer processes and indicates how much data it ought to collect (and store in memory) in a given indexing process before submitting the payload to Elasticsearch's Bulk API. This setting should be used with the Bulk request concurrency setting (see below) and needs to accommodate the resource constraints of both the Elasticsearch hosts and the hosts running the GitLab Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. | +| `Bulk request concurrency` | The Bulk request concurrency indicates how many of the GitLab Golang-based indexer processes (or threads) can run in parallel to collect data to subsequently submit to Elasticsearch's Bulk API. This increases indexing performance, but fills the Elasticsearch bulk requests queue faster. This setting should be used together with the Maximum bulk request size setting (see above) and needs to accommodate the resource constraints of both the Elasticsearch hosts and the hosts running the GitLab Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. | +| `Client request timeout` | Elasticsearch HTTP client request timeout value in seconds. `0` means using the system default timeout value, which depends on the libraries that GitLab application is built upon. | + +WARNING: +Increasing the values of `Maximum bulk request size (MiB)` and `Bulk request concurrency` can negatively impact +Sidekiq performance. Return them to their default values if you see increased `scheduling_latency_s` durations +in your Sidekiq logs. For more information, see +[issue 322147](https://gitlab.com/gitlab-org/gitlab/-/issues/322147). + +### Limit the number of namespaces and projects that can be indexed + +If you check checkbox `Limit the number of namespaces and projects that can be indexed` +under **Elasticsearch indexing restrictions** more options become available. + + + +You can select namespaces and projects to index exclusively. Note that if the namespace is a group, it includes +any subgroups and projects belonging to those subgroups to be indexed as well. + +Advanced Search only provides cross-group code/commit search (global) if all name-spaces are indexed. In this particular scenario where only a subset of namespaces are indexed, a global search does not provide a code or commit scope. This is possible only in the scope of an indexed namespace. There is no way to code/commit search in multiple indexed namespaces (when only a subset of namespaces has been indexed). For example if two groups are indexed, there is no way to run a single code search on both. You can only run a code search on the first group and then on the second. + +You can filter the selection dropdown by writing part of the namespace or project name you're interested in. + + + +NOTE: +If no namespaces or projects are selected, no Advanced Search indexing takes place. + +WARNING: +If you have already indexed your instance, you must regenerate the index to delete all existing data +for filtering to work correctly. To do this, run the Rake tasks `gitlab:elastic:recreate_index` and +`gitlab:elastic:clear_index_status`. Afterwards, removing a namespace or a project from the list deletes the data +from the Elasticsearch index as expected. + +## Enable custom language analyzers + +You can improve the language support for Chinese and Japanese languages by utilizing [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) and/or [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) analysis plugins from Elastic. + +To enable languages support: + +1. Install the desired plugins, please refer to [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/plugins/7.9/installation.html) for plugins installation instructions. The plugins must be installed on every node in the cluster, and each node must be restarted after installation. For a list of plugins, see the table later in this section. +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Advanced Search**. +1. Locate **Custom analyzers: language support**. +1. Enable plugins support for **Indexing**. +1. Select **Save changes** for the changes to take effect. +1. Trigger [Zero downtime reindexing](#zero-downtime-reindexing) or reindex everything from scratch to create a new index with updated mappings. +1. Enable plugins support for **Searching** after the previous step is completed. + +For guidance on what to install, see the following Elasticsearch language plugin options: + +| Parameter | Description | +|-------------------------------------------------------|-------------| +| `Enable Chinese (smartcn) custom analyzer: Indexing` | Enables or disables Chinese language support using [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) custom analyzer for newly created indices.| +| `Enable Chinese (smartcn) custom analyzer: Search` | Enables or disables using [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) fields for Advanced Search. Please only enable this after [installing the plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html), enabling custom analyzer indexing and recreating the index.| +| `Enable Japanese (kuromoji) custom analyzer: Indexing` | Enables or disables Japanese language support using [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) custom analyzer for newly created indices.| +| `Enable Japanese (kuromoji) custom analyzer: Search` | Enables or disables using [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) fields for Advanced Search. Please only enable this after [installing the plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html), enabling custom analyzer indexing and recreating the index.| + +## Disable Advanced Search + +To disable the Elasticsearch integration: + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Advanced Search**. +1. Uncheck **Elasticsearch indexing** and **Search with Elasticsearch enabled**. +1. Select **Save changes**. +1. Optional. Delete the existing indices: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:delete_index + + # Installations from source + bundle exec rake gitlab:elastic:delete_index RAILS_ENV=production + ``` + +## Unpause Indexing + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Advanced Search**. +1. Expand **Advanced Search**. +1. Clear the **Pause Elasticsearch indexing** checkbox. + +## Zero downtime reindexing + +The idea behind this reindexing method is to leverage the [Elasticsearch reindex API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html) +and Elasticsearch index alias feature to perform the operation. We set up an index alias which connects to a +`primary` index which is used by GitLab for reads/writes. When reindexing process starts, we temporarily pause +the writes to the `primary` index. Then, we create another index and invoke the Reindex API which migrates the +index data onto the new index. After the reindexing job is complete, we switch to the new index by connecting the +index alias to it which becomes the new `primary` index. At the end, we resume the writes and normal operation resumes. + +### Trigger the reindex via the Advanced Search administration + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34069) in GitLab 13.2. +> - A scheduled index deletion and the ability to cancel it was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38914) in GitLab 13.3. +> - Support for retries during reindexing was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55681) in GitLab 13.12. + +To trigger the reindexing process: + +1. Sign in to your GitLab instance as an administrator. +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Advanced Search**. +1. Expand **Elasticsearch zero-downtime reindexing**. +1. Select **Trigger cluster reindexing**. + +Reindexing can be a lengthy process depending on the size of your Elasticsearch cluster. + +After this process is completed, the original index is scheduled to be deleted after +14 days. You can cancel this action by pressing the **Cancel** button on the same +page you triggered the reindexing process. + +While the reindexing is running, you can follow its progress under that same section. + +#### Elasticsearch zero-downtime reindexing + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55681) in GitLab 13.12. + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Advanced Search**. +1. Expand **Elasticsearch zero-downtime reindexing**, and you'll + find the following options: + +- [Slice multiplier](#slice-multiplier) +- [Maximum running slices](#maximum-running-slices) + +##### Slice multiplier + +The slice multiplier calculates the [number of slices during reindexing](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html#docs-reindex-slice). + +GitLab uses [manual slicing](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html#docs-reindex-manual-slice) +to control the reindex efficiently and safely, which enables users to retry only +failed slices. + +The multiplier defaults to `2` and applies to the number of shards per index. +For example, if this value is `2` and your index has 20 shards, then the +reindex task is split into 40 slices. + +##### Maximum running slices + +The maximum running slices parameter defaults to `60` and corresponds to the +maximum number of slices allowed to run concurrently during Elasticsearch +reindexing. + +Setting this value too high can have adverse performance impacts as your cluster +may become heavily saturated with searches and writes. Setting this value too +low may lead the reindexing process to take a very long time to complete. + +The best value for this depends on your cluster size, whether you're willing +to accept some degraded search performance during reindexing, and how important +it is for the reindex to finish quickly and resume indexing. + +### Mark the most recent reindex job as failed and resume the indexing + +Sometimes, you might want to abandon the unfinished reindex job and resume the indexing. You can achieve this via the following steps: + +1. Mark the most recent reindex job as failed: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:mark_reindex_failed + + # Installations from source + bundle exec rake gitlab:elastic:mark_reindex_failed RAILS_ENV=production + ``` + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Advanced Search**. +1. Expand **Advanced Search**. +1. Clear the **Pause Elasticsearch indexing** checkbox. + +## Advanced Search migrations + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/234046) in GitLab 13.6. + +With reindex migrations running in the background, there's no need for a manual +intervention. This usually happens in situations where new features are added to +Advanced Search, which means adding or changing the way content is indexed. + +To confirm that the Advanced Search migrations ran, you can check with: + +```shell +curl "$CLUSTER_URL/gitlab-production-migrations/_search?q=*" | jq . +``` + +This should return something similar to: + +```json +{ + "took": 14, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 1, + "hits": [ + { + "_index": "gitlab-production-migrations", + "_type": "_doc", + "_id": "20201105181100", + "_score": 1, + "_source": { + "completed": true + } + } + ] + } +} +``` + +In order to debug issues with the migrations you can check the [`elasticsearch.log` file](../../administration/logs.md#elasticsearchlog). + +### Retry a halted migration + +Some migrations are built with a retry limit. If the migration cannot finish within the retry limit, +it is halted and a notification is displayed in the Advanced Search integration settings. +It is recommended to check the [`elasticsearch.log` file](../../administration/logs.md#elasticsearchlog) to +debug why the migration was halted and make any changes before retrying the migration. Once you believe you've +fixed the cause of the failure, select "Retry migration", and the migration is scheduled to be retried +in the background. + +If you cannot get the migration to succeed, you may +consider the +[last resort to recreate the index from scratch](elasticsearch_troubleshooting.md#last-resort-to-recreate-an-index). +This may allow you to skip over +the problem because a newly created index skips all migrations as the index +is recreated with the correct up-to-date schema. + +### All migrations must be finished before doing a major upgrade + +Before doing a major version GitLab upgrade, you should have completed all +migrations that exist up until the latest minor version before that major +version. If you have halted migrations, these need to be resolved and +[retried](#retry-a-halted-migration) before proceeding with a major version +upgrade. Read more about [upgrading to a new major +version](../../update/index.md#upgrading-to-a-new-major-version). + +## GitLab Advanced Search Rake tasks + +Rake tasks are available to: + +- [Build and install](#build-and-install) the indexer. +- Delete indices when [disabling Elasticsearch](#disable-advanced-search). +- Add GitLab data to an index. + +The following are some available Rake tasks: + +| Task | Description | +|:--------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [`sudo gitlab-rake gitlab:elastic:index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enables Elasticsearch indexing and run `gitlab:elastic:create_empty_index`, `gitlab:elastic:clear_index_status`, `gitlab:elastic:index_projects`, and `gitlab:elastic:index_snippets`. | +| [`sudo gitlab-rake gitlab:elastic:pause_indexing`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Pauses Elasticsearch indexing. Changes are still tracked. Useful for cluster/index migrations. | +| [`sudo gitlab-rake gitlab:elastic:resume_indexing`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Resumes Elasticsearch indexing. | +| [`sudo gitlab-rake gitlab:elastic:index_projects`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Iterates over all projects, and queues Sidekiq jobs to index them in the background. It can only be used after the index is created. | +| [`sudo gitlab-rake gitlab:elastic:index_projects_status`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Determines the overall status of the indexing. It is done by counting the total number of indexed projects, dividing by a count of the total number of projects, then multiplying by 100. | +| [`sudo gitlab-rake gitlab:elastic:clear_index_status`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Deletes all instances of IndexStatus for all projects. Note that this command results in a complete wipe of the index, and it should be used with caution. | +| [`sudo gitlab-rake gitlab:elastic:create_empty_index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Generates empty indices (the default index and a separate issues index) and assigns an alias for each on the Elasticsearch side only if it doesn't already exist. | +| [`sudo gitlab-rake gitlab:elastic:delete_index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Removes the GitLab indices and aliases (if they exist) on the Elasticsearch instance. | +| [`sudo gitlab-rake gitlab:elastic:recreate_index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Wrapper task for `gitlab:elastic:delete_index` and `gitlab:elastic:create_empty_index`. | +| [`sudo gitlab-rake gitlab:elastic:index_snippets`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Performs an Elasticsearch import that indexes the snippets data. | +| [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Displays which projects are not indexed. | +| [`sudo gitlab-rake gitlab:elastic:reindex_cluster`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Schedules a zero-downtime cluster reindexing task. This feature should be used with an index that was created after GitLab 13.0. | +| [`sudo gitlab-rake gitlab:elastic:mark_reindex_failed`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Mark the most recent re-index job as failed. | +| [`sudo gitlab-rake gitlab:elastic:list_pending_migrations`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | List pending migrations. Pending migrations include those that have not yet started, have started but not finished, and those that are halted. | +| [`sudo gitlab-rake gitlab:elastic:estimate_cluster_size`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Get an estimate of cluster size based on the total repository size. | +| [`sudo gitlab-rake gitlab:elastic:enable_search_with_elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enable advanced search with Elasticsearch. | +| [`sudo gitlab-rake gitlab:elastic:disable_search_with_elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Disables advanced search with Elasticsearch. | + +### Environment variables + +In addition to the Rake tasks, there are some environment variables that can be used to modify the process: + +| Environment Variable | Data Type | What it does | +| -------------------- |:---------:| ---------------------------------------------------------------------------- | +| `UPDATE_INDEX` | Boolean | Tells the indexer to overwrite any existing index data (true/false). | +| `ID_TO` | Integer | Tells the indexer to only index projects less than or equal to the value. | +| `ID_FROM` | Integer | Tells the indexer to only index projects greater than or equal to the value. | + +### Indexing a range of projects or a specific project + +Using the `ID_FROM` and `ID_TO` environment variables, you can index a limited number of projects. This can be useful for staging indexing. + +```shell +root@git:~# sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=1 ID_TO=100 +``` + +Because `ID_FROM` and `ID_TO` use the `or equal to` comparison, you can use them to index only one project +by setting both to the same project ID: + +```shell +root@git:~# sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=5 ID_TO=5 +Indexing project repositories...I, [2019-03-04T21:27:03.083410 #3384] INFO -- : Indexing GitLab User / test (ID=33)... +I, [2019-03-04T21:27:05.215266 #3384] INFO -- : Indexing GitLab User / test (ID=33) is done! +``` + +## Advanced Search index scopes + +When performing a search, the GitLab index uses the following scopes: + +| Scope Name | What it searches | +| ---------------- | ---------------------- | +| `commits` | Commit data | +| `projects` | Project data (default) | +| `blobs` | Code | +| `issues` | Issue data | +| `merge_requests` | Merge request data | +| `milestones` | Milestone data | +| `notes` | Note data | +| `snippets` | Snippet data | +| `wiki_blobs` | Wiki contents | + +## Tuning + +### Guidance on choosing optimal cluster configuration + +For basic guidance on choosing a cluster configuration you may refer to [Elastic Cloud Calculator](https://cloud.elastic.co/pricing). You can find more information below. + +- Generally, you want to use at least a 2-node cluster configuration with one replica, which allows you to have resilience. If your storage usage is growing quickly, you may want to plan horizontal scaling (adding more nodes) beforehand. +- It's not recommended to use HDD storage with the search cluster, because it takes a hit on performance. It's better to use SSD storage (NVMe or SATA SSD drives for example). +- You can use the [GitLab Performance Tool](https://gitlab.com/gitlab-org/quality/performance) to benchmark search performance with different search cluster sizes and configurations. +- `Heap size` should be set to no more than 50% of your physical RAM. Additionally, it shouldn't be set to more than the threshold for zero-based compressed oops. The exact threshold varies, but 26 GB is safe on most systems, but can also be as large as 30 GB on some systems. See [Heap size settings](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#heap-size-settings) and [Setting JVM options](https://www.elastic.co/guide/en/elasticsearch/reference/current/jvm-options.html) for more details. +- Number of CPUs (CPU cores) per node usually corresponds to the `Number of Elasticsearch shards` setting described below. +- A good guideline is to ensure you keep the number of shards per node below 20 per GB heap it has configured. A node with a 30GB heap should therefore have a maximum of 600 shards, but the further below this limit you can keep it the better. This generally helps the cluster stay in good health. +- Number of Elasticsearch shards: + - Small shards result in small segments, which increases overhead. Aim to keep the average shard size between at least a few GB and a few tens of GB. + - Another consideration is the number of documents. To determine the number of shards to use, sum the numbers in the **Menu > Admin > Dashboard > Statistics** pane (the number of documents to be indexed), divide by 5 million, and add 5. For example: + - If you have fewer than about 2,000,000 documents, use the default of 5 shards + - 10,000,000 documents: `10000000/5000000 + 5` = 7 shards + - 100,000,000 documents: `100000000/5000000 + 5` = 25 shards +- `refresh_interval` is a per index setting. You may want to adjust that from default `1s` to a bigger value if you don't need data in real-time. This changes how soon you see fresh results. If that's important for you, you should leave it as close as possible to the default value. +- You might want to raise [`indices.memory.index_buffer_size`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indexing-buffer.html) to 30% or 40% if you have a lot of heavy indexing operations. + +### Advanced Search integration settings guidance + +- The `Number of Elasticsearch shards` setting usually corresponds with the number of CPUs available in your cluster. For example, if you have a 3-node cluster with 4 cores each, this means you benefit from having at least 3*4=12 shards in the cluster. It's only possible to change the shards number by using [Split index API](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-split-index.html) or by reindexing to a different index with a changed number of shards. +- The `Number of Elasticsearch replicas` setting should most of the time be equal to `1` (each shard has 1 replica). Using `0` is not recommended, because losing one node corrupts the index. + +### How to index large instances efficiently + +This section may be helpful in the event that the other +[basic instructions](#enable-advanced-search) cause problems +due to large volumes of data being indexed. + +WARNING: +Indexing a large instance generates a lot of Sidekiq jobs. +Make sure to prepare for this task by having a [Scalable and Highly Available +Setup](../../administration/reference_architectures/index.md) or creating [extra +Sidekiq processes](../../administration/operations/extra_sidekiq_processes.md). + +1. [Configure your Elasticsearch host and port](#enable-advanced-search). +1. Create empty indices: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:create_empty_index + + # Installations from source + bundle exec rake gitlab:elastic:create_empty_index RAILS_ENV=production + ``` + +1. If this is a re-index of your GitLab instance, clear the index status: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:clear_index_status + + # Installations from source + bundle exec rake gitlab:elastic:clear_index_status RAILS_ENV=production + ``` + +1. [Enable **Elasticsearch indexing**](#enable-advanced-search). +1. Indexing large Git repositories can take a while. To speed up the process, you can [tune for indexing speed](https://www.elastic.co/guide/en/elasticsearch/reference/current/tune-for-indexing-speed.html#tune-for-indexing-speed): + + - You can temporarily disable [`refresh`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html), the operation responsible for making changes to an index available to search. + + - You can set the number of replicas to 0. This setting controls the number of copies each primary shard of an index will have. Thus, having 0 replicas effectively disables the replication of shards across nodes, which should increase the indexing performance. This is an important trade-off in terms of reliability and query performance. It is important to remember to set the replicas to a considered value after the initial indexing is complete. + + In our experience, you can expect a 20% decrease in indexing time. After completing indexing in a later step, you can return `refresh` and `number_of_replicas` to their desired settings. + + NOTE: + This step is optional but may help significantly speed up large indexing operations. + + ```shell + curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ + --data '{ + "index" : { + "refresh_interval" : "-1", + "number_of_replicas" : 0 + } }' + ``` + +1. Index projects and their associated data: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:index_projects + + # Installations from source + bundle exec rake gitlab:elastic:index_projects RAILS_ENV=production + ``` + + This enqueues a Sidekiq job for each project that needs to be indexed. + You can view the jobs in **Menu > Admin > Monitoring > Background Jobs > Queues Tab** + and select `elastic_commit_indexer`, or you can query indexing status using a Rake task: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:index_projects_status + + # Installations from source + bundle exec rake gitlab:elastic:index_projects_status RAILS_ENV=production + + Indexing is 65.55% complete (6555/10000 projects) + ``` + + If you want to limit the index to a range of projects you can provide the + `ID_FROM` and `ID_TO` parameters: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=1001 ID_TO=2000 + + # Installations from source + bundle exec rake gitlab:elastic:index_projects ID_FROM=1001 ID_TO=2000 RAILS_ENV=production + ``` + + Where `ID_FROM` and `ID_TO` are project IDs. Both parameters are optional. + The above example will index all projects from ID `1001` up to (and including) ID `2000`. + + NOTE: + Sometimes the project indexing jobs queued by `gitlab:elastic:index_projects` + can get interrupted. This may happen for many reasons, but it's always safe + to run the indexing task again. It will skip repositories that have + already been indexed. + + As the indexer stores the last commit SHA of every indexed repository in the + database, you can run the indexer with the special parameter `UPDATE_INDEX` and + it will check every project repository again to make sure that every commit in + a repository is indexed, which can be useful in case if your index is outdated: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:index_projects UPDATE_INDEX=true ID_TO=1000 + + # Installations from source + bundle exec rake gitlab:elastic:index_projects UPDATE_INDEX=true ID_TO=1000 RAILS_ENV=production + ``` + + You can also use the `gitlab:elastic:clear_index_status` Rake task to force the + indexer to "forget" all progress, so it retries the indexing process from the + start. + +1. Personal snippets are not associated with a project and need to be indexed separately: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:index_snippets + + # Installations from source + bundle exec rake gitlab:elastic:index_snippets RAILS_ENV=production + ``` + +1. Enable replication and refreshing again after indexing (only if you previously disabled it): + + ```shell + curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ + --data '{ + "index" : { + "number_of_replicas" : 1, + "refresh_interval" : "1s" + } }' + ``` + + A force merge should be called after enabling the refreshing above. + + For Elasticsearch 6.x, the index should be in read-only mode before proceeding with the force merge: + + ```shell + curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ + --data '{ + "settings": { + "index.blocks.write": true + } }' + ``` + + Then, initiate the force merge: + + ```shell + curl --request POST 'localhost:9200/gitlab-production/_forcemerge?max_num_segments=5' + ``` + + After this, if your index is in read-only mode, switch back to read-write: + + ```shell + curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ + --data '{ + "settings": { + "index.blocks.write": false + } }' + ``` + +1. After the indexing has completed, enable [**Search with Elasticsearch enabled**](#enable-advanced-search). + +### Deleted documents + +Whenever a change or deletion is made to an indexed GitLab object (a merge request description is changed, a file is deleted from the default branch in a repository, a project is deleted, etc), a document in the index is deleted. However, since these are "soft" deletes, the overall number of "deleted documents", and therefore wasted space, increases. Elasticsearch does intelligent merging of segments in order to remove these deleted documents. However, depending on the amount and type of activity in your GitLab installation, it's possible to see as much as 50% wasted space in the index. + +In general, we recommend letting Elasticsearch merge and reclaim space automatically, with the default settings. From [Lucene's Handling of Deleted Documents](https://www.elastic.co/blog/lucenes-handling-of-deleted-documents "Lucene's Handling of Deleted Documents"), _"Overall, besides perhaps decreasing the maximum segment size, it is best to leave Lucene's defaults as-is and not fret too much about when deletes are reclaimed."_ + +However, some larger installations may wish to tune the merge policy settings: + +- Consider reducing the `index.merge.policy.max_merged_segment` size from the default 5 GB to maybe 2 GB or 3 GB. Merging only happens when a segment has at least 50% deletions. Smaller segment sizes will allow merging to happen more frequently. + + ```shell + curl --request PUT localhost:9200/gitlab-production/_settings ---header 'Content-Type: application/json' \ + --data '{ + "index" : { + "merge.policy.max_merged_segment": "2gb" + } + }' + ``` + +- You can also adjust `index.merge.policy.reclaim_deletes_weight`, which controls how aggressively deletions are targeted. But this can lead to costly merge decisions, so we recommend not changing this unless you understand the tradeoffs. + + ```shell + curl --request PUT localhost:9200/gitlab-production/_settings ---header 'Content-Type: application/json' \ + --data '{ + "index" : { + "merge.policy.reclaim_deletes_weight": "3.0" + } + }' + ``` + +- Do not do a [force merge](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html "Force Merge") to remove deleted documents. A warning in the [documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html "Force Merge") states that this can lead to very large segments that may never get reclaimed, and can also cause significant performance or availability issues. + +## Index large instances with dedicated Sidekiq nodes or processes + +Indexing a large instance can be a lengthy and resource-intensive process that has the potential +of overwhelming Sidekiq nodes and processes. This negatively affects the GitLab performance and +availability. + +As GitLab allows you to start multiple Sidekiq processes, you can create an +additional process dedicated to indexing a set of queues (or queue group). This way, you can +ensure that indexing queues always have a dedicated worker, while the rest of the queues have +another dedicated worker to avoid contention. + +For this purpose, use the [queue selector](../../administration/operations/extra_sidekiq_processes.md#queue-selector) +option that allows a more general selection of queue groups using a [worker matching query](../../administration/operations/extra_sidekiq_routing.md#worker-matching-query). + +To handle these two queue groups, we generally recommend one of the following two options. You can either: + +- [Use two queue groups on one single node](#single-node-two-processes). +- [Use two queue groups, one on each node](#two-nodes-one-process-for-each). + +For the steps below, consider: + +- `feature_category=global_search` as an indexing queue group with its own Sidekiq process. +- `feature_category!=global_search` as a non-indexing queue group that has its own Sidekiq process. + +### Single node, two processes + +To create both an indexing and a non-indexing Sidekiq process in one node: + +1. On your Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to: + + ```ruby + sidekiq['enable'] = true + sidekiq['queue_selector'] = true + sidekiq['queue_groups'] = [ + "feature_category=global_search", + "feature_category!=global_search" + ] + ``` + +1. Save the file and [reconfigure GitLab](../../administration/restart_gitlab.md) +for the changes to take effect. + +WARNING: +When starting multiple processes, the number of processes cannot exceed the number of CPU +cores you want to dedicate to Sidekiq. Each Sidekiq process can use only one CPU core, subject +to the available workload and concurrency settings. For more details, see how to +[run multiple Sidekiq processes](../../administration/operations/extra_sidekiq_processes.md). + +### Two nodes, one process for each + +To handle these queue groups on two nodes: + +1. To set up the indexing Sidekiq process, on your indexing Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to: + + ```ruby + sidekiq['enable'] = true + sidekiq['queue_selector'] = true + sidekiq['queue_groups'] = [ + "feature_category=global_search" + ] + ``` + +1. Save the file and [reconfigure GitLab](../../administration/restart_gitlab.md) +for the changes to take effect. + +1. To set up the non-indexing Sidekiq process, on your non-indexing Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to: + + ```ruby + sidekiq['enable'] = true + sidekiq['queue_selector'] = true + sidekiq['queue_groups'] = [ + "feature_category!=global_search" + ] + ``` + + to set up a non-indexing Sidekiq process. + +1. Save the file and [reconfigure GitLab](../../administration/restart_gitlab.md) +for the changes to take effect. + +## Reverting to Basic Search + +Sometimes there may be issues with your Elasticsearch index data and as such +GitLab allows you to revert to "basic search" when there are no search +results and assuming that basic search is supported in that scope. This "basic +search" behaves as though you don't have Advanced Search enabled at all for +your instance and search using other data sources (such as PostgreSQL data and Git +data). + +## Data recovery: Elasticsearch is a secondary data store only + +The use of Elasticsearch in GitLab is only ever as a secondary data store. +This means that all of the data stored in Elasticsearch can always be derived +again from other data sources, specifically PostgreSQL and Gitaly. Therefore, if +the Elasticsearch data store is ever corrupted for whatever reason, you can reindex everything from scratch. diff --git a/doc/integration/advanced_search/elasticsearch_troubleshooting.md b/doc/integration/advanced_search/elasticsearch_troubleshooting.md index cd99b857399..97abf456baa 100644 --- a/doc/integration/advanced_search/elasticsearch_troubleshooting.md +++ b/doc/integration/advanced_search/elasticsearch_troubleshooting.md @@ -57,11 +57,11 @@ There are a couple of ways to achieve that: We continuously make updates to our indexing strategies and aim to support newer versions of Elasticsearch. When indexing changes are made, it may -be necessary for you to [reindex](../elasticsearch.md#zero-downtime-reindexing) after updating GitLab. +be necessary for you to [reindex](elasticsearch.md#zero-downtime-reindexing) after updating GitLab. ## I indexed all the repositories but I can't get any hits for my search term in the UI -Make sure you [indexed all the database data](../elasticsearch.md#enable-advanced-search). +Make sure you [indexed all the database data](elasticsearch.md#enable-advanced-search). If there aren't any results (hits) in the UI search, check if you are seeing the same results via the rails console (`sudo gitlab-rails console`): @@ -82,9 +82,9 @@ More [complex Elasticsearch API calls](https://www.elastic.co/guide/en/elasticse It is important to understand at which level the problem is manifesting (UI, Rails code, Elasticsearch side) to be able to [troubleshoot further](../../administration/troubleshooting/elasticsearch.md#search-results-workflow). NOTE: -The above instructions are not to be used for scenarios that only index a [subset of namespaces](../elasticsearch.md#limit-the-number-of-namespaces-and-projects-that-can-be-indexed). +The above instructions are not to be used for scenarios that only index a [subset of namespaces](elasticsearch.md#limit-the-number-of-namespaces-and-projects-that-can-be-indexed). -See [Elasticsearch Index Scopes](../elasticsearch.md#advanced-search-index-scopes) for more information on searching for specific types of data. +See [Elasticsearch Index Scopes](elasticsearch.md#advanced-search-index-scopes) for more information on searching for specific types of data. ## I indexed all the repositories but then switched Elasticsearch servers and now I can't find anything @@ -133,7 +133,7 @@ see details in the [update guide](../../update/upgrading_from_source.md). ## `Elasticsearch::Transport::Transport::Errors::BadRequest` -If you have this exception (just like in the case above but the actual message is different) please check if you have the correct Elasticsearch version and you met the other [requirements](../elasticsearch.md#system-requirements). +If you have this exception (just like in the case above but the actual message is different) please check if you have the correct Elasticsearch version and you met the other [requirements](elasticsearch.md#system-requirements). There is also an easy way to check it automatically with `sudo gitlab-rake gitlab:check` command. ## `Elasticsearch::Transport::Transport::Errors::RequestEntityTooLarge` @@ -173,7 +173,7 @@ Gitlab::Elastic::Indexer::Error: time="2020-01-23T09:13:00Z" level=fatal msg="he ``` You probably have not used either `http://` or `https://` as part of your value in the **"URL"** field of the Elasticsearch Integration Menu. Please make sure you are using either `http://` or `https://` in this field as the [Elasticsearch client for Go](https://github.com/olivere/elastic) that we are using [needs the prefix for the URL to be accepted as valid](https://github.com/olivere/elastic/commit/a80af35aa41856dc2c986204e2b64eab81ccac3a). -After you have corrected the formatting of the URL, delete the index (via the [dedicated Rake task](../elasticsearch.md#gitlab-advanced-search-rake-tasks)) and [reindex the content of your instance](../elasticsearch.md#enable-advanced-search). +After you have corrected the formatting of the URL, delete the index (via the [dedicated Rake task](elasticsearch.md#gitlab-advanced-search-rake-tasks)) and [reindex the content of your instance](elasticsearch.md#enable-advanced-search). ## My Elasticsearch cluster has a plugin and the integration is not working @@ -239,13 +239,13 @@ filtering out projects that a user does not have access to at search time. If `ElasticCommitIndexerWorker` Sidekiq workers are failing with this error during indexing, it usually means that Elasticsearch is unable to keep up with the concurrency of indexing request. To address change the following settings: -- To decrease the indexing throughput you can decrease `Bulk request concurrency` (see [Advanced Search settings](../elasticsearch.md#advanced-search-configuration)). This is set to `10` by default, but you change it to as low as 1 to reduce the number of concurrent indexing operations. -- If changing `Bulk request concurrency` didn't help, you can use the [queue selector](../../administration/operations/extra_sidekiq_processes.md#queue-selector) option to [limit indexing jobs only to specific Sidekiq nodes](../elasticsearch.md#index-large-instances-with-dedicated-sidekiq-nodes-or-processes), which should reduce the number of indexing requests. +- To decrease the indexing throughput you can decrease `Bulk request concurrency` (see [Advanced Search settings](elasticsearch.md#advanced-search-configuration)). This is set to `10` by default, but you change it to as low as 1 to reduce the number of concurrent indexing operations. +- If changing `Bulk request concurrency` didn't help, you can use the [queue selector](../../administration/operations/extra_sidekiq_processes.md#queue-selector) option to [limit indexing jobs only to specific Sidekiq nodes](elasticsearch.md#index-large-instances-with-dedicated-sidekiq-nodes-or-processes), which should reduce the number of indexing requests. ## Indexing is very slow or fails with `rejected execution of coordinating operation` messages Bulk requests getting rejected by the Elasticsearch nodes are likely due to load and lack of available memory. -Ensure that your Elasticsearch cluster meets the [system requirements](../elasticsearch.md#system-requirements) and has enough resources +Ensure that your Elasticsearch cluster meets the [system requirements](elasticsearch.md#system-requirements) and has enough resources to perform bulk operations. See also the error ["429 (Too Many Requests)"](#indexing-fails-with-error-elastic-error-429-too-many-requests). ## Access requirements for the self-managed AWS OpenSearch Service diff --git a/doc/integration/img/limit_namespace_filter.png b/doc/integration/advanced_search/img/limit_namespace_filter.png Binary files differindex 437aecad467..437aecad467 100644 --- a/doc/integration/img/limit_namespace_filter.png +++ b/doc/integration/advanced_search/img/limit_namespace_filter.png diff --git a/doc/integration/img/limit_namespaces_projects_options.png b/doc/integration/advanced_search/img/limit_namespaces_projects_options.png Binary files differindex a3cf9933b7e..a3cf9933b7e 100644 --- a/doc/integration/img/limit_namespaces_projects_options.png +++ b/doc/integration/advanced_search/img/limit_namespaces_projects_options.png diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index a014a73d9ac..0b34f7018da 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -1,857 +1,11 @@ --- -type: reference -stage: Data Stores -group: Global Search -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'advanced_search/elasticsearch.md' +remove_date: '2022-09-13' --- -# Elasticsearch integration **(PREMIUM SELF)** +This document was moved to [another location](advanced_search/elasticsearch.md). -This page describes how to enable Advanced Search. When enabled, -Advanced Search provides faster search response times and [improved search features](../user/search/advanced_search.md). - -## Version requirements - -### Elasticsearch version requirements - -> Support for Elasticsearch 6.8 was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/350275) in GitLab 15.0. - -Advanced Search works with the following versions of Elasticsearch. - -| GitLab version | Elasticsearch version | -|-----------------------|--------------------------| -| GitLab 15.0 or later | Elasticsearch 7.x - 8.x | -| GitLab 13.9 - 14.10 | Elasticsearch 6.8 - 7.x | -| GitLab 13.3 - 13.8 | Elasticsearch 6.4 - 7.x | -| GitLab 12.7 - 13.2 | Elasticsearch 6.x - 7.x | - -Advanced Search follows Elasticsearch's [End of Life Policy](https://www.elastic.co/support/eol). -When we change Elasticsearch supported versions in GitLab, we announce them in [deprecation notes](https://about.gitlab.com/handbook/marketing/blog/release-posts/#deprecations) in monthly release posts -before we remove them. - -### OpenSearch version requirements - -| GitLab version | Elasticsearch version | -|-----------------------|--------------------------| -| GitLab 15.0 or later | OpenSearch 1.x or later | - -If you are using a compatible version and after connecting to OpenSearch, you get the message `Elasticsearch version not compatible`, [unpause indexing](#unpause-indexing). - -## System requirements - -Elasticsearch requires additional resources to those documented in the -[GitLab system requirements](../install/requirements.md). - -Memory, CPU, and storage resource amounts vary depending on the amount of data you index into the Elasticsearch cluster. Heavily used Elasticsearch clusters may require more resources. According to -[Elasticsearch official guidelines](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_memory), -each node should have: - -- [Memory](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_memory): 8 GiB (minimum). -- [CPU](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_cpus): Modern processor with multiple cores. GitLab.com has minimal CPU requirements for Elasticsearch. Multiple cores provide extra concurrency, which is more beneficial than faster CPUs. -- [Storage](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_disks): Use SSD storage. The total storage size of all Elasticsearch nodes is about 50% of the total size of your Git repositories. It includes one primary and one replica. The [`estimate_cluster_size`](#gitlab-advanced-search-rake-tasks) Rake task ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221177) in GitLab 13.10) uses total repository size to estimate the Advanced Search storage requirements. - -## Install Elasticsearch - -Elasticsearch is *not* included in the Omnibus packages or when you install from -source. You must [install it separately](https://www.elastic.co/guide/en/elasticsearch/reference/7.x/install-elasticsearch.html "Elasticsearch 7.x installation documentation") and ensure you select your version. Detailed information on how to install Elasticsearch is out of the scope of this page. - -You can install Elasticsearch yourself, or use a cloud hosted offering such as [Elasticsearch Service](https://www.elastic.co/elasticsearch/service) (available on AWS, GCP, or Azure) or the [Amazon OpenSearch](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/gsg.html) -service. - -You should install Elasticsearch on a separate server. Running Elasticsearch on the same server as GitLab is not recommended and can cause a degradation in GitLab instance performance. - -For a single node Elasticsearch cluster, the functional cluster health status is always yellow due to the allocation of the primary shard. Elasticsearch cannot assign replica shards to the same node as primary shards. - -The search index updates after you: - -- Add data to the database or repository. -- [Enable Elasticsearch](#enable-advanced-search) in the Admin Area. - -## Upgrade to a new Elasticsearch major version - -> - Elasticsearch 6.8 support is removed with GitLab 15.0. -> - Upgrading from GitLab 14.10 to 15.0 requires that you are using any version of Elasticsearch 7.x. - -You are not required to change the GitLab configuration when you upgrade Elasticsearch. - -## Elasticsearch repository indexer - -To index Git repository data, GitLab uses an [indexer written in Go](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). - -Depending on your GitLab version, there are different installation procedures for the Go indexer: - -- For Omnibus GitLab 11.8 or greater, see [Omnibus GitLab](#omnibus-gitlab). -- For installations from source or older versions of Omnibus GitLab, - [install the indexer from source](#from-source). -- If you are using GitLab Development Kit, see [GDK Elasticsearch how-to](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/elasticsearch.md). - -### Omnibus GitLab - -Starting with GitLab 11.8, the Go indexer is included in Omnibus GitLab. -The former Ruby-based indexer was removed in [GitLab 12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/6481). - -### From source - -First, we need to install some dependencies, then we build and install -the indexer itself. - -This project relies on [International Components for Unicode](https://icu.unicode.org/) (ICU) for text encoding, -therefore we must ensure the development packages for your platform are -installed before running `make`. - -#### Debian / Ubuntu - -To install on Debian or Ubuntu, run: - -```shell -sudo apt install libicu-dev -``` - -#### CentOS / RHEL - -To install on CentOS or RHEL, run: - -```shell -sudo yum install libicu-devel -``` - -#### macOS - -NOTE: -You must first [install Homebrew](https://brew.sh/). - -To install on macOS, run: - -```shell -brew install icu4c -export PKG_CONFIG_PATH="/usr/local/opt/icu4c/lib/pkgconfig:$PKG_CONFIG_PATH" -``` - -### Build and install - -To build and install the indexer, run: - -```shell -indexer_path=/home/git/gitlab-elasticsearch-indexer - -# Run the installation task for gitlab-elasticsearch-indexer: -sudo -u git -H bundle exec rake gitlab:indexer:install[$indexer_path] RAILS_ENV=production -cd $indexer_path && sudo make install -``` - -The `gitlab-elasticsearch-indexer` is installed to `/usr/local/bin`. - -You can change the installation path with the `PREFIX` environment variable. -Please remember to pass the `-E` flag to `sudo` if you do so. - -Example: - -```shell -PREFIX=/usr sudo -E make install -``` - -After installation, be sure to [enable Elasticsearch](#enable-advanced-search). - -NOTE: -If you see an error such as `Permission denied - /home/git/gitlab-elasticsearch-indexer/` while indexing, you -may need to set the `production -> elasticsearch -> indexer_path` setting in your `gitlab.yml` file to -`/usr/local/bin/gitlab-elasticsearch-indexer`, which is where the binary is installed. - -## Enable Advanced Search - -For GitLab instances with more than 50GB repository data you can follow the instructions for [how to index large instances efficiently](#how-to-index-large-instances-efficiently) below. - -To enable Advanced Search, you must have administrator access to GitLab: - -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Advanced Search**. - - NOTE: - To see the Advanced Search section, you need an active GitLab Premium - [license](../user/admin_area/license.md). - -1. Configure the [Advanced Search settings](#advanced-search-configuration) for - your Elasticsearch cluster. Do not enable **Search with Elasticsearch enabled** - yet. -1. Enable **Elasticsearch indexing** and select **Save changes**. This creates - an empty index if one does not already exist. -1. Select **Index all projects**. -1. Select **Check progress** in the confirmation message to see the status of - the background jobs. -1. Personal snippets must be indexed using another Rake task: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:index_snippets - - # Installations from source - bundle exec rake gitlab:elastic:index_snippets RAILS_ENV=production - ``` - -1. After indexing completes, enable **Search with Elasticsearch enabled** and select **Save changes**. - -NOTE: -When your Elasticsearch cluster is down while Elasticsearch is enabled, -you might have problems updating documents such as issues because your -instance queues a job to index the change, but cannot find a valid -Elasticsearch cluster. - -### Advanced Search configuration - -The following Elasticsearch settings are available: - -| Parameter | Description | -|-------------------------------------------------------|-------------| -| `Elasticsearch indexing` | Enables or disables Elasticsearch indexing and creates an empty index if one does not already exist. You may want to enable indexing but disable search to give the index time to be fully completed, for example. Also, keep in mind that this option doesn't have any impact on existing data, this only enables/disables the background indexer which tracks data changes and ensures new data is indexed. | -| `Pause Elasticsearch indexing` | Enables or disables temporary indexing pause. This is useful for cluster migration/reindexing. All changes are still tracked, but they are not committed to the Elasticsearch index until resumed. | -| `Search with Elasticsearch enabled` | Enables or disables using Elasticsearch in search. | -| `URL` | The URL of your Elasticsearch instance. Use a comma-separated list to support clustering (for example, `http://host1, https://host2:9200`). If your Elasticsearch instance is password-protected, use the `Username` and `Password` fields described below. Alternatively, use inline credentials such as `http://<username>:<password>@<elastic_host>:9200/`. | -| `Username` | The `username` of your Elasticsearch instance. | -| `Password` | The password of your Elasticsearch instance. | -| `Number of Elasticsearch shards` | Elasticsearch indices are split into multiple shards for performance reasons. In general, you should use at least 5 shards, and indices with tens of millions of documents need to have more shards ([see below](#guidance-on-choosing-optimal-cluster-configuration)). Changes to this value do not take effect until the index is recreated. You can read more about tradeoffs in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/scalability.html). | -| `Number of Elasticsearch replicas` | Each Elasticsearch shard can have a number of replicas. These are a complete copy of the shard, and can provide increased query performance or resilience against hardware failure. Increasing this value increases total disk space required by the index. | -| `Limit the number of namespaces and projects that can be indexed` | Enabling this allows you to select namespaces and projects to index. All other namespaces and projects use database search instead. If you enable this option but do not select any namespaces or projects, none are indexed. [Read more below](#limit-the-number-of-namespaces-and-projects-that-can-be-indexed). -| `Using AWS hosted Elasticsearch with IAM credentials` | Sign your Elasticsearch requests using [AWS IAM authorization](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html), [AWS EC2 Instance Profile Credentials](https://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html#getting-started-create-iam-instance-profile-cli), or [AWS ECS Tasks Credentials](https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-iam-roles.html). Please refer to [Identity and Access Management in Amazon OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html) for details of AWS hosted OpenSearch domain access policy configuration. | -| `AWS Region` | The AWS region in which your OpenSearch Service is located. | -| `AWS Access Key` | The AWS access key. | -| `AWS Secret Access Key` | The AWS secret access key. | -| `Maximum file size indexed` | See [the explanation in instance limits.](../administration/instance_limits.md#maximum-file-size-indexed). | -| `Maximum field length` | See [the explanation in instance limits.](../administration/instance_limits.md#maximum-field-length). | -| `Maximum bulk request size (MiB)` | The Maximum Bulk Request size is used by the GitLab Golang-based indexer processes and indicates how much data it ought to collect (and store in memory) in a given indexing process before submitting the payload to Elasticsearch's Bulk API. This setting should be used with the Bulk request concurrency setting (see below) and needs to accommodate the resource constraints of both the Elasticsearch hosts and the hosts running the GitLab Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. | -| `Bulk request concurrency` | The Bulk request concurrency indicates how many of the GitLab Golang-based indexer processes (or threads) can run in parallel to collect data to subsequently submit to Elasticsearch's Bulk API. This increases indexing performance, but fills the Elasticsearch bulk requests queue faster. This setting should be used together with the Maximum bulk request size setting (see above) and needs to accommodate the resource constraints of both the Elasticsearch hosts and the hosts running the GitLab Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. | -| `Client request timeout` | Elasticsearch HTTP client request timeout value in seconds. `0` means using the system default timeout value, which depends on the libraries that GitLab application is built upon. | - -WARNING: -Increasing the values of `Maximum bulk request size (MiB)` and `Bulk request concurrency` can negatively impact -Sidekiq performance. Return them to their default values if you see increased `scheduling_latency_s` durations -in your Sidekiq logs. For more information, see -[issue 322147](https://gitlab.com/gitlab-org/gitlab/-/issues/322147). - -### Limit the number of namespaces and projects that can be indexed - -If you check checkbox `Limit the number of namespaces and projects that can be indexed` -under **Elasticsearch indexing restrictions** more options become available. - - - -You can select namespaces and projects to index exclusively. Note that if the namespace is a group, it includes -any subgroups and projects belonging to those subgroups to be indexed as well. - -Advanced Search only provides cross-group code/commit search (global) if all name-spaces are indexed. In this particular scenario where only a subset of namespaces are indexed, a global search does not provide a code or commit scope. This is possible only in the scope of an indexed namespace. There is no way to code/commit search in multiple indexed namespaces (when only a subset of namespaces has been indexed). For example if two groups are indexed, there is no way to run a single code search on both. You can only run a code search on the first group and then on the second. - -You can filter the selection dropdown by writing part of the namespace or project name you're interested in. - - - -NOTE: -If no namespaces or projects are selected, no Advanced Search indexing takes place. - -WARNING: -If you have already indexed your instance, you must regenerate the index to delete all existing data -for filtering to work correctly. To do this, run the Rake tasks `gitlab:elastic:recreate_index` and -`gitlab:elastic:clear_index_status`. Afterwards, removing a namespace or a project from the list deletes the data -from the Elasticsearch index as expected. - -## Enable custom language analyzers - -You can improve the language support for Chinese and Japanese languages by utilizing [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) and/or [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) analysis plugins from Elastic. - -To enable languages support: - -1. Install the desired plugins, please refer to [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/plugins/7.9/installation.html) for plugins installation instructions. The plugins must be installed on every node in the cluster, and each node must be restarted after installation. For a list of plugins, see the table later in this section. -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Advanced Search**. -1. Locate **Custom analyzers: language support**. -1. Enable plugins support for **Indexing**. -1. Select **Save changes** for the changes to take effect. -1. Trigger [Zero downtime reindexing](#zero-downtime-reindexing) or reindex everything from scratch to create a new index with updated mappings. -1. Enable plugins support for **Searching** after the previous step is completed. - -For guidance on what to install, see the following Elasticsearch language plugin options: - -| Parameter | Description | -|-------------------------------------------------------|-------------| -| `Enable Chinese (smartcn) custom analyzer: Indexing` | Enables or disables Chinese language support using [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) custom analyzer for newly created indices.| -| `Enable Chinese (smartcn) custom analyzer: Search` | Enables or disables using [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) fields for Advanced Search. Please only enable this after [installing the plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html), enabling custom analyzer indexing and recreating the index.| -| `Enable Japanese (kuromoji) custom analyzer: Indexing` | Enables or disables Japanese language support using [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) custom analyzer for newly created indices.| -| `Enable Japanese (kuromoji) custom analyzer: Search` | Enables or disables using [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) fields for Advanced Search. Please only enable this after [installing the plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html), enabling custom analyzer indexing and recreating the index.| - -## Disable Advanced Search - -To disable the Elasticsearch integration: - -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Advanced Search**. -1. Uncheck **Elasticsearch indexing** and **Search with Elasticsearch enabled**. -1. Select **Save changes**. -1. Optional. Delete the existing indices: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:delete_index - - # Installations from source - bundle exec rake gitlab:elastic:delete_index RAILS_ENV=production - ``` - -## Unpause Indexing - -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Advanced Search**. -1. Expand **Advanced Search**. -1. Clear the **Pause Elasticsearch indexing** checkbox. - -## Zero downtime reindexing - -The idea behind this reindexing method is to leverage the [Elasticsearch reindex API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html) -and Elasticsearch index alias feature to perform the operation. We set up an index alias which connects to a -`primary` index which is used by GitLab for reads/writes. When reindexing process starts, we temporarily pause -the writes to the `primary` index. Then, we create another index and invoke the Reindex API which migrates the -index data onto the new index. After the reindexing job is complete, we switch to the new index by connecting the -index alias to it which becomes the new `primary` index. At the end, we resume the writes and normal operation resumes. - -### Trigger the reindex via the Advanced Search administration - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34069) in GitLab 13.2. -> - A scheduled index deletion and the ability to cancel it was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38914) in GitLab 13.3. -> - Support for retries during reindexing was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55681) in GitLab 13.12. - -To trigger the reindexing process: - -1. Sign in to your GitLab instance as an administrator. -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Advanced Search**. -1. Expand **Elasticsearch zero-downtime reindexing**. -1. Select **Trigger cluster reindexing**. - -Reindexing can be a lengthy process depending on the size of your Elasticsearch cluster. - -After this process is completed, the original index is scheduled to be deleted after -14 days. You can cancel this action by pressing the **Cancel** button on the same -page you triggered the reindexing process. - -While the reindexing is running, you can follow its progress under that same section. - -#### Elasticsearch zero-downtime reindexing - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55681) in GitLab 13.12. - -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Advanced Search**. -1. Expand **Elasticsearch zero-downtime reindexing**, and you'll - find the following options: - -- [Slice multiplier](#slice-multiplier) -- [Maximum running slices](#maximum-running-slices) - -##### Slice multiplier - -The slice multiplier calculates the [number of slices during reindexing](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html#docs-reindex-slice). - -GitLab uses [manual slicing](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html#docs-reindex-manual-slice) -to control the reindex efficiently and safely, which enables users to retry only -failed slices. - -The multiplier defaults to `2` and applies to the number of shards per index. -For example, if this value is `2` and your index has 20 shards, then the -reindex task is split into 40 slices. - -##### Maximum running slices - -The maximum running slices parameter defaults to `60` and corresponds to the -maximum number of slices allowed to run concurrently during Elasticsearch -reindexing. - -Setting this value too high can have adverse performance impacts as your cluster -may become heavily saturated with searches and writes. Setting this value too -low may lead the reindexing process to take a very long time to complete. - -The best value for this depends on your cluster size, whether you're willing -to accept some degraded search performance during reindexing, and how important -it is for the reindex to finish quickly and resume indexing. - -### Mark the most recent reindex job as failed and resume the indexing - -Sometimes, you might want to abandon the unfinished reindex job and resume the indexing. You can achieve this via the following steps: - -1. Mark the most recent reindex job as failed: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:mark_reindex_failed - - # Installations from source - bundle exec rake gitlab:elastic:mark_reindex_failed RAILS_ENV=production - ``` - -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Advanced Search**. -1. Expand **Advanced Search**. -1. Clear the **Pause Elasticsearch indexing** checkbox. - -## Advanced Search migrations - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/234046) in GitLab 13.6. - -With reindex migrations running in the background, there's no need for a manual -intervention. This usually happens in situations where new features are added to -Advanced Search, which means adding or changing the way content is indexed. - -To confirm that the Advanced Search migrations ran, you can check with: - -```shell -curl "$CLUSTER_URL/gitlab-production-migrations/_search?q=*" | jq . -``` - -This should return something similar to: - -```json -{ - "took": 14, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": { - "value": 1, - "relation": "eq" - }, - "max_score": 1, - "hits": [ - { - "_index": "gitlab-production-migrations", - "_type": "_doc", - "_id": "20201105181100", - "_score": 1, - "_source": { - "completed": true - } - } - ] - } -} -``` - -In order to debug issues with the migrations you can check the [`elasticsearch.log` file](../administration/logs.md#elasticsearchlog). - -### Retry a halted migration - -Some migrations are built with a retry limit. If the migration cannot finish within the retry limit, -it is halted and a notification is displayed in the Advanced Search integration settings. -It is recommended to check the [`elasticsearch.log` file](../administration/logs.md#elasticsearchlog) to -debug why the migration was halted and make any changes before retrying the migration. Once you believe you've -fixed the cause of the failure, select "Retry migration", and the migration is scheduled to be retried -in the background. - -If you cannot get the migration to succeed, you may -consider the -[last resort to recreate the index from scratch](advanced_search/elasticsearch_troubleshooting.md#last-resort-to-recreate-an-index). -This may allow you to skip over -the problem because a newly created index skips all migrations as the index -is recreated with the correct up-to-date schema. - -### All migrations must be finished before doing a major upgrade - -Before doing a major version GitLab upgrade, you should have completed all -migrations that exist up until the latest minor version before that major -version. If you have halted migrations, these need to be resolved and -[retried](#retry-a-halted-migration) before proceeding with a major version -upgrade. Read more about [upgrading to a new major -version](../update/index.md#upgrading-to-a-new-major-version). - -## GitLab Advanced Search Rake tasks - -Rake tasks are available to: - -- [Build and install](#build-and-install) the indexer. -- Delete indices when [disabling Elasticsearch](#disable-advanced-search). -- Add GitLab data to an index. - -The following are some available Rake tasks: - -| Task | Description | -|:--------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [`sudo gitlab-rake gitlab:elastic:index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enables Elasticsearch indexing and run `gitlab:elastic:create_empty_index`, `gitlab:elastic:clear_index_status`, `gitlab:elastic:index_projects`, and `gitlab:elastic:index_snippets`. | -| [`sudo gitlab-rake gitlab:elastic:pause_indexing`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Pauses Elasticsearch indexing. Changes are still tracked. Useful for cluster/index migrations. | -| [`sudo gitlab-rake gitlab:elastic:resume_indexing`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Resumes Elasticsearch indexing. | -| [`sudo gitlab-rake gitlab:elastic:index_projects`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Iterates over all projects, and queues Sidekiq jobs to index them in the background. It can only be used after the index is created. | -| [`sudo gitlab-rake gitlab:elastic:index_projects_status`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Determines the overall status of the indexing. It is done by counting the total number of indexed projects, dividing by a count of the total number of projects, then multiplying by 100. | -| [`sudo gitlab-rake gitlab:elastic:clear_index_status`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Deletes all instances of IndexStatus for all projects. Note that this command results in a complete wipe of the index, and it should be used with caution. | -| [`sudo gitlab-rake gitlab:elastic:create_empty_index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Generates empty indices (the default index and a separate issues index) and assigns an alias for each on the Elasticsearch side only if it doesn't already exist. | -| [`sudo gitlab-rake gitlab:elastic:delete_index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Removes the GitLab indices and aliases (if they exist) on the Elasticsearch instance. | -| [`sudo gitlab-rake gitlab:elastic:recreate_index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Wrapper task for `gitlab:elastic:delete_index` and `gitlab:elastic:create_empty_index`. | -| [`sudo gitlab-rake gitlab:elastic:index_snippets`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Performs an Elasticsearch import that indexes the snippets data. | -| [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Displays which projects are not indexed. | -| [`sudo gitlab-rake gitlab:elastic:reindex_cluster`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Schedules a zero-downtime cluster reindexing task. This feature should be used with an index that was created after GitLab 13.0. | -| [`sudo gitlab-rake gitlab:elastic:mark_reindex_failed`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Mark the most recent re-index job as failed. | -| [`sudo gitlab-rake gitlab:elastic:list_pending_migrations`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | List pending migrations. Pending migrations include those that have not yet started, have started but not finished, and those that are halted. | -| [`sudo gitlab-rake gitlab:elastic:estimate_cluster_size`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Get an estimate of cluster size based on the total repository size. | -| [`sudo gitlab-rake gitlab:elastic:enable_search_with_elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enable advanced search with Elasticsearch. | -| [`sudo gitlab-rake gitlab:elastic:disable_search_with_elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Disables advanced search with Elasticsearch. | - -### Environment variables - -In addition to the Rake tasks, there are some environment variables that can be used to modify the process: - -| Environment Variable | Data Type | What it does | -| -------------------- |:---------:| ---------------------------------------------------------------------------- | -| `UPDATE_INDEX` | Boolean | Tells the indexer to overwrite any existing index data (true/false). | -| `ID_TO` | Integer | Tells the indexer to only index projects less than or equal to the value. | -| `ID_FROM` | Integer | Tells the indexer to only index projects greater than or equal to the value. | - -### Indexing a range of projects or a specific project - -Using the `ID_FROM` and `ID_TO` environment variables, you can index a limited number of projects. This can be useful for staging indexing. - -```shell -root@git:~# sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=1 ID_TO=100 -``` - -Because `ID_FROM` and `ID_TO` use the `or equal to` comparison, you can use them to index only one project -by setting both to the same project ID: - -```shell -root@git:~# sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=5 ID_TO=5 -Indexing project repositories...I, [2019-03-04T21:27:03.083410 #3384] INFO -- : Indexing GitLab User / test (ID=33)... -I, [2019-03-04T21:27:05.215266 #3384] INFO -- : Indexing GitLab User / test (ID=33) is done! -``` - -## Advanced Search index scopes - -When performing a search, the GitLab index uses the following scopes: - -| Scope Name | What it searches | -| ---------------- | ---------------------- | -| `commits` | Commit data | -| `projects` | Project data (default) | -| `blobs` | Code | -| `issues` | Issue data | -| `merge_requests` | Merge request data | -| `milestones` | Milestone data | -| `notes` | Note data | -| `snippets` | Snippet data | -| `wiki_blobs` | Wiki contents | - -## Tuning - -### Guidance on choosing optimal cluster configuration - -For basic guidance on choosing a cluster configuration you may refer to [Elastic Cloud Calculator](https://cloud.elastic.co/pricing). You can find more information below. - -- Generally, you want to use at least a 2-node cluster configuration with one replica, which allows you to have resilience. If your storage usage is growing quickly, you may want to plan horizontal scaling (adding more nodes) beforehand. -- It's not recommended to use HDD storage with the search cluster, because it takes a hit on performance. It's better to use SSD storage (NVMe or SATA SSD drives for example). -- You can use the [GitLab Performance Tool](https://gitlab.com/gitlab-org/quality/performance) to benchmark search performance with different search cluster sizes and configurations. -- `Heap size` should be set to no more than 50% of your physical RAM. Additionally, it shouldn't be set to more than the threshold for zero-based compressed oops. The exact threshold varies, but 26 GB is safe on most systems, but can also be as large as 30 GB on some systems. See [Heap size settings](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#heap-size-settings) and [Setting JVM options](https://www.elastic.co/guide/en/elasticsearch/reference/current/jvm-options.html) for more details. -- Number of CPUs (CPU cores) per node usually corresponds to the `Number of Elasticsearch shards` setting described below. -- A good guideline is to ensure you keep the number of shards per node below 20 per GB heap it has configured. A node with a 30GB heap should therefore have a maximum of 600 shards, but the further below this limit you can keep it the better. This generally helps the cluster stay in good health. -- Number of Elasticsearch shards: - - Small shards result in small segments, which increases overhead. Aim to keep the average shard size between at least a few GB and a few tens of GB. - - Another consideration is the number of documents. To determine the number of shards to use, sum the numbers in the **Menu > Admin > Dashboard > Statistics** pane (the number of documents to be indexed), divide by 5 million, and add 5. For example: - - If you have fewer than about 2,000,000 documents, use the default of 5 shards - - 10,000,000 documents: `10000000/5000000 + 5` = 7 shards - - 100,000,000 documents: `100000000/5000000 + 5` = 25 shards -- `refresh_interval` is a per index setting. You may want to adjust that from default `1s` to a bigger value if you don't need data in real-time. This changes how soon you see fresh results. If that's important for you, you should leave it as close as possible to the default value. -- You might want to raise [`indices.memory.index_buffer_size`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indexing-buffer.html) to 30% or 40% if you have a lot of heavy indexing operations. - -### Advanced Search integration settings guidance - -- The `Number of Elasticsearch shards` setting usually corresponds with the number of CPUs available in your cluster. For example, if you have a 3-node cluster with 4 cores each, this means you benefit from having at least 3*4=12 shards in the cluster. It's only possible to change the shards number by using [Split index API](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-split-index.html) or by reindexing to a different index with a changed number of shards. -- The `Number of Elasticsearch replicas` setting should most of the time be equal to `1` (each shard has 1 replica). Using `0` is not recommended, because losing one node corrupts the index. - -### How to index large instances efficiently - -This section may be helpful in the event that the other -[basic instructions](#enable-advanced-search) cause problems -due to large volumes of data being indexed. - -WARNING: -Indexing a large instance generates a lot of Sidekiq jobs. -Make sure to prepare for this task by having a [Scalable and Highly Available -Setup](../administration/reference_architectures/index.md) or creating [extra -Sidekiq processes](../administration/operations/extra_sidekiq_processes.md). - -1. [Configure your Elasticsearch host and port](#enable-advanced-search). -1. Create empty indices: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:create_empty_index - - # Installations from source - bundle exec rake gitlab:elastic:create_empty_index RAILS_ENV=production - ``` - -1. If this is a re-index of your GitLab instance, clear the index status: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:clear_index_status - - # Installations from source - bundle exec rake gitlab:elastic:clear_index_status RAILS_ENV=production - ``` - -1. [Enable **Elasticsearch indexing**](#enable-advanced-search). -1. Indexing large Git repositories can take a while. To speed up the process, you can [tune for indexing speed](https://www.elastic.co/guide/en/elasticsearch/reference/current/tune-for-indexing-speed.html#tune-for-indexing-speed): - - - You can temporarily disable [`refresh`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html), the operation responsible for making changes to an index available to search. - - - You can set the number of replicas to 0. This setting controls the number of copies each primary shard of an index will have. Thus, having 0 replicas effectively disables the replication of shards across nodes, which should increase the indexing performance. This is an important trade-off in terms of reliability and query performance. It is important to remember to set the replicas to a considered value after the initial indexing is complete. - - In our experience, you can expect a 20% decrease in indexing time. After completing indexing in a later step, you can return `refresh` and `number_of_replicas` to their desired settings. - - NOTE: - This step is optional but may help significantly speed up large indexing operations. - - ```shell - curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ - --data '{ - "index" : { - "refresh_interval" : "-1", - "number_of_replicas" : 0 - } }' - ``` - -1. Index projects and their associated data: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:index_projects - - # Installations from source - bundle exec rake gitlab:elastic:index_projects RAILS_ENV=production - ``` - - This enqueues a Sidekiq job for each project that needs to be indexed. - You can view the jobs in **Menu > Admin > Monitoring > Background Jobs > Queues Tab** - and select `elastic_commit_indexer`, or you can query indexing status using a Rake task: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:index_projects_status - - # Installations from source - bundle exec rake gitlab:elastic:index_projects_status RAILS_ENV=production - - Indexing is 65.55% complete (6555/10000 projects) - ``` - - If you want to limit the index to a range of projects you can provide the - `ID_FROM` and `ID_TO` parameters: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=1001 ID_TO=2000 - - # Installations from source - bundle exec rake gitlab:elastic:index_projects ID_FROM=1001 ID_TO=2000 RAILS_ENV=production - ``` - - Where `ID_FROM` and `ID_TO` are project IDs. Both parameters are optional. - The above example will index all projects from ID `1001` up to (and including) ID `2000`. - - NOTE: - Sometimes the project indexing jobs queued by `gitlab:elastic:index_projects` - can get interrupted. This may happen for many reasons, but it's always safe - to run the indexing task again. It will skip repositories that have - already been indexed. - - As the indexer stores the last commit SHA of every indexed repository in the - database, you can run the indexer with the special parameter `UPDATE_INDEX` and - it will check every project repository again to make sure that every commit in - a repository is indexed, which can be useful in case if your index is outdated: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:index_projects UPDATE_INDEX=true ID_TO=1000 - - # Installations from source - bundle exec rake gitlab:elastic:index_projects UPDATE_INDEX=true ID_TO=1000 RAILS_ENV=production - ``` - - You can also use the `gitlab:elastic:clear_index_status` Rake task to force the - indexer to "forget" all progress, so it retries the indexing process from the - start. - -1. Personal snippets are not associated with a project and need to be indexed separately: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:index_snippets - - # Installations from source - bundle exec rake gitlab:elastic:index_snippets RAILS_ENV=production - ``` - -1. Enable replication and refreshing again after indexing (only if you previously disabled it): - - ```shell - curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ - --data '{ - "index" : { - "number_of_replicas" : 1, - "refresh_interval" : "1s" - } }' - ``` - - A force merge should be called after enabling the refreshing above. - - For Elasticsearch 6.x, the index should be in read-only mode before proceeding with the force merge: - - ```shell - curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ - --data '{ - "settings": { - "index.blocks.write": true - } }' - ``` - - Then, initiate the force merge: - - ```shell - curl --request POST 'localhost:9200/gitlab-production/_forcemerge?max_num_segments=5' - ``` - - After this, if your index is in read-only mode, switch back to read-write: - - ```shell - curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ - --data '{ - "settings": { - "index.blocks.write": false - } }' - ``` - -1. After the indexing has completed, enable [**Search with Elasticsearch enabled**](#enable-advanced-search). - -### Deleted documents - -Whenever a change or deletion is made to an indexed GitLab object (a merge request description is changed, a file is deleted from the default branch in a repository, a project is deleted, etc), a document in the index is deleted. However, since these are "soft" deletes, the overall number of "deleted documents", and therefore wasted space, increases. Elasticsearch does intelligent merging of segments in order to remove these deleted documents. However, depending on the amount and type of activity in your GitLab installation, it's possible to see as much as 50% wasted space in the index. - -In general, we recommend letting Elasticsearch merge and reclaim space automatically, with the default settings. From [Lucene's Handling of Deleted Documents](https://www.elastic.co/blog/lucenes-handling-of-deleted-documents "Lucene's Handling of Deleted Documents"), _"Overall, besides perhaps decreasing the maximum segment size, it is best to leave Lucene's defaults as-is and not fret too much about when deletes are reclaimed."_ - -However, some larger installations may wish to tune the merge policy settings: - -- Consider reducing the `index.merge.policy.max_merged_segment` size from the default 5 GB to maybe 2 GB or 3 GB. Merging only happens when a segment has at least 50% deletions. Smaller segment sizes will allow merging to happen more frequently. - - ```shell - curl --request PUT localhost:9200/gitlab-production/_settings ---header 'Content-Type: application/json' \ - --data '{ - "index" : { - "merge.policy.max_merged_segment": "2gb" - } - }' - ``` - -- You can also adjust `index.merge.policy.reclaim_deletes_weight`, which controls how aggressively deletions are targeted. But this can lead to costly merge decisions, so we recommend not changing this unless you understand the tradeoffs. - - ```shell - curl --request PUT localhost:9200/gitlab-production/_settings ---header 'Content-Type: application/json' \ - --data '{ - "index" : { - "merge.policy.reclaim_deletes_weight": "3.0" - } - }' - ``` - -- Do not do a [force merge](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html "Force Merge") to remove deleted documents. A warning in the [documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html "Force Merge") states that this can lead to very large segments that may never get reclaimed, and can also cause significant performance or availability issues. - -## Index large instances with dedicated Sidekiq nodes or processes - -Indexing a large instance can be a lengthy and resource-intensive process that has the potential -of overwhelming Sidekiq nodes and processes. This negatively affects the GitLab performance and -availability. - -As GitLab allows you to start multiple Sidekiq processes, you can create an -additional process dedicated to indexing a set of queues (or queue group). This way, you can -ensure that indexing queues always have a dedicated worker, while the rest of the queues have -another dedicated worker to avoid contention. - -For this purpose, use the [queue selector](../administration/operations/extra_sidekiq_processes.md#queue-selector) -option that allows a more general selection of queue groups using a [worker matching query](../administration/operations/extra_sidekiq_routing.md#worker-matching-query). - -To handle these two queue groups, we generally recommend one of the following two options. You can either: - -- [Use two queue groups on one single node](#single-node-two-processes). -- [Use two queue groups, one on each node](#two-nodes-one-process-for-each). - -For the steps below, consider: - -- `feature_category=global_search` as an indexing queue group with its own Sidekiq process. -- `feature_category!=global_search` as a non-indexing queue group that has its own Sidekiq process. - -### Single node, two processes - -To create both an indexing and a non-indexing Sidekiq process in one node: - -1. On your Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to: - - ```ruby - sidekiq['enable'] = true - sidekiq['queue_selector'] = true - sidekiq['queue_groups'] = [ - "feature_category=global_search", - "feature_category!=global_search" - ] - ``` - -1. Save the file and [reconfigure GitLab](../administration/restart_gitlab.md) -for the changes to take effect. - -WARNING: -When starting multiple processes, the number of processes cannot exceed the number of CPU -cores you want to dedicate to Sidekiq. Each Sidekiq process can use only one CPU core, subject -to the available workload and concurrency settings. For more details, see how to -[run multiple Sidekiq processes](../administration/operations/extra_sidekiq_processes.md). - -### Two nodes, one process for each - -To handle these queue groups on two nodes: - -1. To set up the indexing Sidekiq process, on your indexing Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to: - - ```ruby - sidekiq['enable'] = true - sidekiq['queue_selector'] = true - sidekiq['queue_groups'] = [ - "feature_category=global_search" - ] - ``` - -1. Save the file and [reconfigure GitLab](../administration/restart_gitlab.md) -for the changes to take effect. - -1. To set up the non-indexing Sidekiq process, on your non-indexing Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to: - - ```ruby - sidekiq['enable'] = true - sidekiq['queue_selector'] = true - sidekiq['queue_groups'] = [ - "feature_category!=global_search" - ] - ``` - - to set up a non-indexing Sidekiq process. - -1. Save the file and [reconfigure GitLab](../administration/restart_gitlab.md) -for the changes to take effect. - -## Reverting to Basic Search - -Sometimes there may be issues with your Elasticsearch index data and as such -GitLab allows you to revert to "basic search" when there are no search -results and assuming that basic search is supported in that scope. This "basic -search" behaves as though you don't have Advanced Search enabled at all for -your instance and search using other data sources (such as PostgreSQL data and Git -data). - -## Data recovery: Elasticsearch is a secondary data store only - -The use of Elasticsearch in GitLab is only ever as a secondary data store. -This means that all of the data stored in Elasticsearch can always be derived -again from other data sources, specifically PostgreSQL and Gitaly. Therefore, if -the Elasticsearch data store is ever corrupted for whatever reason, you can reindex everything from scratch. +<!-- This redirect file can be deleted after <2022-09-13>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->
\ No newline at end of file diff --git a/doc/integration/index.md b/doc/integration/index.md index bcc3eb6c030..85ebac5b40c 100644 --- a/doc/integration/index.md +++ b/doc/integration/index.md @@ -60,7 +60,7 @@ GitLab can be integrated with the following enhancements: or [Kroki](../administration/integration/kroki.md) to use diagrams in AsciiDoc and Markdown documents. - Attach merge requests to [Trello](trello_power_up.md) cards. - Enable integrated code intelligence powered by [Sourcegraph](sourcegraph.md). -- Add [Elasticsearch](elasticsearch.md) for [Advanced Search](../user/search/advanced_search.md). +- Add [Elasticsearch](advanced_search/elasticsearch.md) for [Advanced Search](../user/search/advanced_search.md). ## Integrations diff --git a/doc/integration/jira/index.md b/doc/integration/jira/index.md index 8ff9b68facd..4e8b5ca8927 100644 --- a/doc/integration/jira/index.md +++ b/doc/integration/jira/index.md @@ -67,10 +67,13 @@ The authentication method in Jira depends on whether you host Jira on your own s ## Privacy considerations -If you integrate a private GitLab project with Jira using the [**Jira integration**](#jira-integration), -actions in GitLab issues and merge requests linked to a Jira issue leak information -about the private project to non-administrator Jira users. If your installation uses Jira Cloud, -you can use the [GitLab.com for Jira Cloud app](connect-app.md) to avoid this risk. +All Jira integrations share data with Jira to make it visible outside of GitLab. +If you integrate a private GitLab project with Jira, the private data is +shared with users who have access to your Jira project. + +The [**Jira project integration**](#jira-integration) posts GitLab data in the form of comments in Jira issues. +The GitLab.com for Jira Cloud app and Jira DVCS connector share this data through the [**Jira Development Panel**](development_panel.md). +This method provides more fine-grained access control because access can be restricted to certain user groups or roles. ## Third-party Jira integrations diff --git a/doc/raketasks/index.md b/doc/raketasks/index.md index d8acdf8fbfb..38b57fab8ca 100644 --- a/doc/raketasks/index.md +++ b/doc/raketasks/index.md @@ -26,7 +26,7 @@ The following Rake tasks are available for use with GitLab: | [Back up and restore](backup_restore.md) | Back up, restore, and migrate GitLab instances between servers. | | [Clean up](cleanup.md) | Clean up unneeded items from GitLab instances. | | [Development](../development/rake_tasks.md) | Tasks for GitLab contributors. | -| [Elasticsearch](../integration/elasticsearch.md#gitlab-advanced-search-rake-tasks) | Maintain Elasticsearch in a GitLab instance. | +| [Elasticsearch](../integration/advanced_search/elasticsearch.md#gitlab-advanced-search-rake-tasks) | Maintain Elasticsearch in a GitLab instance. | | [Enable namespaces](features.md) | Enable usernames and namespaces for user projects. | | [General maintenance](../administration/raketasks/maintenance.md) | General maintenance and self-check tasks. | | [Geo maintenance](../administration/raketasks/geo.md) | [Geo](../administration/geo/index.md)-related maintenance. | diff --git a/doc/subscriptions/bronze_starter.md b/doc/subscriptions/bronze_starter.md index 5e2d5c47bb6..6d88b61dd05 100644 --- a/doc/subscriptions/bronze_starter.md +++ b/doc/subscriptions/bronze_starter.md @@ -129,7 +129,7 @@ Bronze-level subscribers: - [Group iterations API](../api/group_iterations.md) - Project milestones API: [Get all burndown chart events for a single milestone](../api/milestones.md#get-all-burndown-chart-events-for-a-single-milestone) - [Project iterations API](../api/iterations.md) - - Fields in the [Search API](../api/search.md) available only to [Advanced Search (Elasticsearch)](../integration/elasticsearch.md) users + - Fields in the [Search API](../api/search.md) available only to [Advanced Search (Elasticsearch)](../integration/advanced_search/elasticsearch.md) users - Fields in the [Merge requests API](../api/merge_requests.md) for [merge request approvals](../user/project/merge_requests/approvals/index.md) - Fields in the [Protected branches API](../api/protected_branches.md) that specify users or groups allowed to merge - [Merge request approvals API](../api/merge_request_approvals.md) diff --git a/doc/update/index.md b/doc/update/index.md index 27be6ff842e..416adb621d0 100644 --- a/doc/update/index.md +++ b/doc/update/index.md @@ -311,9 +311,9 @@ To address the above two scenarios, it is advised to do the following prior to u ## Checking for pending Advanced Search migrations **(PREMIUM SELF)** -This section is only applicable if you have enabled the [Elasticsearch integration](../integration/elasticsearch.md) **(PREMIUM SELF)**. +This section is only applicable if you have enabled the [Elasticsearch integration](../integration/advanced_search/elasticsearch.md) **(PREMIUM SELF)**. -Major releases require all [Advanced Search migrations](../integration/elasticsearch.md#advanced-search-migrations) +Major releases require all [Advanced Search migrations](../integration/advanced_search/elasticsearch.md#advanced-search-migrations) to be finished from the most recent minor release in your current version before the major version upgrade. You can find pending migrations by running the following command: @@ -333,11 +333,11 @@ sudo -u git -H bundle exec rake gitlab:elastic:list_pending_migrations ### What do I do if my Advanced Search migrations are stuck? -See [how to retry a halted migration](../integration/elasticsearch.md#retry-a-halted-migration). +See [how to retry a halted migration](../integration/advanced_search/elasticsearch.md#retry-a-halted-migration). ### What do I do for the error `Elasticsearch version not compatible` -Confirm that your version of Elasticsearch or OpenSearch is [compatible with your version of GitLab](../integration/elasticsearch.md#version-requirements). +Confirm that your version of Elasticsearch or OpenSearch is [compatible with your version of GitLab](../integration/advanced_search/elasticsearch.md#version-requirements). ## Upgrading without downtime @@ -360,7 +360,7 @@ A *major* upgrade requires the following steps: It's also important to ensure that any [background migrations have been fully completed](#checking-for-background-migrations-before-upgrading) before upgrading to a new major version. -If you have enabled the [Elasticsearch integration](../integration/elasticsearch.md) **(PREMIUM SELF)**, then +If you have enabled the [Elasticsearch integration](../integration/advanced_search/elasticsearch.md) **(PREMIUM SELF)**, then [ensure all Advanced Search migrations are completed](#checking-for-pending-advanced-search-migrations) in the last minor version within your current version before proceeding with the major version upgrade. @@ -466,7 +466,7 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap ### 15.0.0 -- Elasticsearch 6.8 [is no longer supported](../integration/elasticsearch.md#version-requirements). Before you upgrade to GitLab 15.0, [update Elasticsearch to any 7.x version](../integration/elasticsearch.md#upgrade-to-a-new-elasticsearch-major-version). +- Elasticsearch 6.8 [is no longer supported](../integration/advanced_search/elasticsearch.md#version-requirements). Before you upgrade to GitLab 15.0, [update Elasticsearch to any 7.x version](../integration/advanced_search/elasticsearch.md#upgrade-to-a-new-elasticsearch-major-version). - If you run external PostgreSQL, particularly AWS RDS, [check you have a PostgreSQL bug fix](#postgresql-segmentation-fault-issue) to avoid the database crashing. diff --git a/doc/update/patch_versions.md b/doc/update/patch_versions.md index dd0bd5b240c..dc09f6063c8 100644 --- a/doc/update/patch_versions.md +++ b/doc/update/patch_versions.md @@ -107,7 +107,7 @@ sudo -u git -H make ### 8. Install/Update `gitlab-elasticsearch-indexer` **(PREMIUM SELF)** -Please follow the [install instruction](../integration/elasticsearch.md#install-elasticsearch). +Please follow the [install instruction](../integration/advanced_search/elasticsearch.md#install-elasticsearch). ### 9. Start application diff --git a/doc/update/plan_your_upgrade.md b/doc/update/plan_your_upgrade.md index 80b1b3e5cb1..cdc3ec39f9a 100644 --- a/doc/update/plan_your_upgrade.md +++ b/doc/update/plan_your_upgrade.md @@ -170,7 +170,7 @@ After updating GitLab, upgrade your runners to match #### Elasticsearch After updating GitLab, you may have to upgrade -[Elasticsearch if the new version breaks compatibility](../integration/elasticsearch.md#version-requirements). +[Elasticsearch if the new version breaks compatibility](../integration/advanced_search/elasticsearch.md#version-requirements). Updating Elasticsearch is **out of scope for GitLab Support**. ## Troubleshooting diff --git a/doc/update/upgrading_from_ce_to_ee.md b/doc/update/upgrading_from_ce_to_ee.md index dada3dd070a..d6548620356 100644 --- a/doc/update/upgrading_from_ce_to_ee.md +++ b/doc/update/upgrading_from_ce_to_ee.md @@ -88,7 +88,7 @@ sudo -u git -H bundle exec rake cache:clear RAILS_ENV=production ### 4. Install `gitlab-elasticsearch-indexer` **(PREMIUM SELF)** -Please follow the [install instruction](../integration/elasticsearch.md#install-elasticsearch). +Please follow the [install instruction](../integration/advanced_search/elasticsearch.md#install-elasticsearch). ### 5. Start application diff --git a/doc/user/admin_area/settings/index.md b/doc/user/admin_area/settings/index.md index c877ca9837d..034a432c570 100644 --- a/doc/user/admin_area/settings/index.md +++ b/doc/user/admin_area/settings/index.md @@ -79,7 +79,7 @@ The **Geo** setting contains: The **Integrations** settings contain: -- [Elasticsearch](../../../integration/elasticsearch.md#enable-advanced-search) - +- [Elasticsearch](../../../integration/advanced_search/elasticsearch.md#enable-advanced-search) - Elasticsearch integration. Elasticsearch AWS IAM. - [Kroki](../../../administration/integration/kroki.md#enable-kroki-in-gitlab) - Allow rendering of diagrams in AsciiDoc and Markdown documents using [kroki.io](https://kroki.io). diff --git a/doc/user/application_security/vulnerability_report/index.md b/doc/user/application_security/vulnerability_report/index.md index a20281dde46..15a287356f8 100644 --- a/doc/user/application_security/vulnerability_report/index.md +++ b/doc/user/application_security/vulnerability_report/index.md @@ -259,7 +259,7 @@ Group, Project, and Security Center Vulnerability Reports. To filter them, use t > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/6345) in GitLab 14.6. -The **Operational vulnerabilities** tab lists vulnerabilities found by the `cluster_image_scanner`. +The **Operational vulnerabilities** tab lists vulnerabilities found by [Operational container scanning](../../clusters/agent/vulnerabilities.md). This tab appears on the project, group, and Security Center vulnerability reports.  diff --git a/doc/user/clusters/agent/vulnerabilities.md b/doc/user/clusters/agent/vulnerabilities.md index e56167de4ae..706ed122f7b 100644 --- a/doc/user/clusters/agent/vulnerabilities.md +++ b/doc/user/clusters/agent/vulnerabilities.md @@ -26,6 +26,8 @@ To view vulnerability information in GitLab:  +This information can also be found under [operational vulnerabilities](../../../user/application_security/vulnerability_report/index.md#operational-vulnerabilities). + ## Enable cluster vulnerability scanning **(ULTIMATE)** You can use [cluster image scanning](../../application_security/cluster_image_scanning/index.md) diff --git a/doc/user/crm/index.md b/doc/user/crm/index.md index a46ba317cd4..796f714874f 100644 --- a/doc/user/crm/index.md +++ b/doc/user/crm/index.md @@ -177,7 +177,6 @@ API. FLAG: On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](../../administration/feature_flags.md) named `contacts_autocomplete`. On GitLab.com, this feature is available. -This feature is not ready for production use. When you use the `/add_contacts` or `/remove_contacts` quick actions, follow them with `[contact:` and an autocomplete list appears: diff --git a/doc/user/permissions.md b/doc/user/permissions.md index c025967579a..801c107e371 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -213,7 +213,7 @@ The following table lists project permissions available for each role: public and internal projects (not on private projects). [External users](#external-users) must be given explicit access even if the project is internal. For GitLab.com, see the [GitLab.com visibility settings](gitlab_com/index.md#visibility-settings). -2. Guest users can only view the [confidential issues](project/issues/confidential_issues.md) they created themselves. +2. Guest users can only view the [confidential issues](project/issues/confidential_issues.md) they created themselves or are assigned to. 3. Not allowed for Guest, Reporter, Developer, Maintainer, or Owner. See [protected branches](project/protected_branches.md). 4. If the [branch is protected](project/protected_branches.md), this depends on the access Developers and Maintainers are given. 5. Guest users can access GitLab [**Releases**](project/releases/index.md) for downloading assets but are not allowed to download the source code nor see [repository information like commits and release evidence](project/releases/index.md#view-a-release-and-download-assets). @@ -358,7 +358,7 @@ Read through the documentation on [permissions for File Locking](project/file_lo ### Confidential Issues permissions [Confidential issues](project/issues/confidential_issues.md) can be accessed by users with reporter and higher permission levels, -as well as by guest users that create a confidential issue. To learn more, +as well as by guest users that create a confidential issue or are assigned to it. To learn more, read through the documentation on [permissions and access to confidential issues](project/issues/confidential_issues.md#permissions-and-access-to-confidential-issues). ### Container Registry visibility permissions diff --git a/doc/user/project/issues/confidential_issues.md b/doc/user/project/issues/confidential_issues.md index 2e10e15ba5a..402ce4bebec 100644 --- a/doc/user/project/issues/confidential_issues.md +++ b/doc/user/project/issues/confidential_issues.md @@ -36,7 +36,7 @@ The second way is to locate the **Confidentiality** section in the sidebar and s | Turn off confidentiality | Turn on confidentiality | | :-----------: | :----------: | -|  |  | +|  |  | Every change from regular to confidential and vice versa, is indicated by a system note in the issue's comments. @@ -84,6 +84,9 @@ There are two kinds of level access for confidential issues. The general rule is that confidential issues are visible only to members of a project with at least the Reporter role. However, a guest user can also create confidential issues, but can only view the ones that they created themselves. +Users with the Guest role or non-members can also read the confidential issue if they are assigned to the issue. +When a Guest user or non-member is unassigned from a confidential issue, +they can no longer view it. Confidential issues are also hidden in search results for unprivileged users. For example, here's what a user with the Maintainer role and the Guest role diff --git a/doc/user/project/issues/img/turn_off_confidentiality_v15_1.png b/doc/user/project/issues/img/turn_off_confidentiality_v15_1.png Binary files differnew file mode 100644 index 00000000000..e6050aec0de --- /dev/null +++ b/doc/user/project/issues/img/turn_off_confidentiality_v15_1.png diff --git a/doc/user/project/issues/img/turn_on_confidentiality_v15_1.png b/doc/user/project/issues/img/turn_on_confidentiality_v15_1.png Binary files differnew file mode 100644 index 00000000000..24a7ad554f8 --- /dev/null +++ b/doc/user/project/issues/img/turn_on_confidentiality_v15_1.png diff --git a/doc/user/project/repository/push_rules.md b/doc/user/project/repository/push_rules.md index 994cf78d98a..592aff85434 100644 --- a/doc/user/project/repository/push_rules.md +++ b/doc/user/project/repository/push_rules.md @@ -26,6 +26,9 @@ For custom push rules use [server hooks](../../../administration/server_hooks.md You can create push rules for all new projects to inherit, but they can be overridden at the project level or the [group level](../../group/index.md#group-push-rules). +All projects created after you configure global push rules inherit this +configuration. However, each existing project must be updated manually, using the +process described in [Override global push rules per project](#override-global-push-rules-per-project). Prerequisite: @@ -42,7 +45,8 @@ To create global push rules: ## Override global push rules per project The push rule of an individual project overrides the global push rule. -To override global push rules for a specific project: +To override global push rules for a specific project, or to update the rules +for an existing project to match new global push rules: 1. On the top bar, select **Menu > Projects** and find your project. 1. On the left sidebar, select **Settings > Repository**. @@ -77,10 +81,12 @@ Use these rules for your commit messages. ## Validate branch names To validate your branch names, enter a regular expression for **Branch name**. -To allow any branch name, leave empty. Your -[default branch](branches/default.md) is always allowed. +To allow any branch name, leave empty. Your [default branch](branches/default.md) +is always allowed. Certain formats of branch names are restricted by default for +security purposes. Names with 40 hexadecimal characters, similar to Git commit hashes, +are prohibited. -Examples: +Some validation examples: - Branches must start with `JIRA-`. @@ -101,11 +107,6 @@ Examples: `^[a-z0-9\\-]{4,15}$` ``` -NOTE: -In GitLab 12.10 and later, certain formats of branch names are restricted by -default for security purposes. 40-character hexadecimal names, similar to Git -commit hashes, are prohibited. - ## Prevent unintended consequences Use these rules to prevent unintended consequences. @@ -134,6 +135,8 @@ Never commit secrets, such as credential files and SSH private keys, to a versio system. In GitLab, you can use a predefined list of files to block those files from a repository. Merge requests that contain a file that matches the list are blocked. This push rule does not restrict files already committed to the repository. +You must update the configuration of existing projects to use the rule, using the +process described in [Override global push rules per project](#override-global-push-rules-per-project). Files blocked by this rule are listed below. For a complete list of criteria, refer to [`files_denylist.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/gitlab/checks/files_denylist.yml). diff --git a/doc/user/search/advanced_search.md b/doc/user/search/advanced_search.md index d050bbce19f..075c9b6154b 100644 --- a/doc/user/search/advanced_search.md +++ b/doc/user/search/advanced_search.md @@ -45,7 +45,7 @@ Advanced Search can be useful in various scenarios: ## Configuring Advanced Search For self-managed GitLab instances, an administrator must -[configure Advanced Search](../../integration/elasticsearch.md). +[configure Advanced Search](../../integration/advanced_search/elasticsearch.md). On GitLab.com, Advanced Search is enabled. diff --git a/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb new file mode 100644 index 00000000000..814f5a897a9 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill projectfeatures.package_registry_access_level depending on projects.packages_enabled + class BackfillProjectFeaturePackageRegistryAccessLevel < ::Gitlab::BackgroundMigration::BatchedMigrationJob + FEATURE_DISABLED = 0 # ProjectFeature::DISABLED + FEATURE_PRIVATE = 10 # ProjectFeature::PRIVATE + FEATURE_ENABLED = 20 # ProjectFeature::ENABLED + FEATURE_PUBLIC = 30 # ProjectFeature::PUBLIC + PROJECT_PRIVATE = 0 # Gitlab::VisibilityLevel::PRIVATE + PROJECT_INTERNAL = 10 # Gitlab::VisibilityLevel::INTERNAL + PROJECT_PUBLIC = 20 # Gitlab::VisibilityLevel::PUBLIC + + # Migration only version of ProjectFeature table + class ProjectFeature < ::ApplicationRecord + self.table_name = 'project_features' + end + + def perform + each_sub_batch(operation_name: :update_all) do |sub_batch| + ProjectFeature.connection.execute( + <<~SQL + UPDATE project_features pf + SET package_registry_access_level = (CASE p.packages_enabled + WHEN true THEN (CASE p.visibility_level + WHEN #{PROJECT_PUBLIC} THEN #{FEATURE_PUBLIC} + WHEN #{PROJECT_INTERNAL} THEN #{FEATURE_ENABLED} + WHEN #{PROJECT_PRIVATE} THEN #{FEATURE_PRIVATE} + END) + WHEN false THEN #{FEATURE_DISABLED} + ELSE #{FEATURE_DISABLED} + END) + FROM projects p + WHERE pf.project_id = p.id AND + pf.project_id BETWEEN #{start_id} AND #{end_id} + SQL + ) + end + end + end + end +end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 64589f872c2..6104f371c22 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -13640,9 +13640,6 @@ msgstr "" msgid "Edit" msgstr "" -msgid "Edit %{issuable}" -msgstr "" - msgid "Edit %{name}" msgstr "" @@ -18318,16 +18315,16 @@ msgstr "" msgid "GroupSettings|Projects will be permanently deleted after a %{waiting_period}-day delay. This delay can be %{link_start}customized by an admin%{link_end} in instance settings. Inherited by subgroups." msgstr "" -msgid "GroupSettings|Select a project with the %{code_start}.gitlab/insights.yml%{code_end} file" -msgstr "" - msgid "GroupSettings|Select a subgroup to use as the source for custom project templates for this group." msgstr "" msgid "GroupSettings|Select parent group" msgstr "" -msgid "GroupSettings|Select the project that contains your custom Insights file." +msgid "GroupSettings|Select the project containing the %{code_start}.gitlab/insights.yml%{code_end} file" +msgstr "" + +msgid "GroupSettings|Select the project containing your custom Insights file. %{help_link_start}What is Insights?%{help_link_end}" msgstr "" msgid "GroupSettings|Set a size limit for all content in each Pages site in this group. %{link_start}Learn more.%{link_end}" @@ -26643,7 +26640,7 @@ msgstr "" msgid "One or more of your personal access tokens will expire in %{days_to_expire} days or less:" msgstr "" -msgid "Only %{workspaceType} members with at least Reporter role can view or be notified about this %{issuableType}." +msgid "Only %{workspaceType} members with %{permissions} can view or be notified about this %{issuableType}." msgstr "" msgid "Only 'Reporter' roles and above on tiers Premium and above can see Value Stream Analytics." @@ -26673,7 +26670,7 @@ msgstr "" msgid "Only effective when remote storage is enabled. Set to 0 for no size limit." msgstr "" -msgid "Only group members with at least Reporter role can view or be notified about this epic." +msgid "Only group members with at least the Reporter role can view or be notified about this epic" msgstr "" msgid "Only include features new to your current subscription tier." @@ -43375,7 +43372,7 @@ msgstr "" msgid "You are going to turn off the confidentiality. This means %{strongStart}everyone%{strongEnd} will be able to see and leave a comment on this %{issuableType}." msgstr "" -msgid "You are going to turn on confidentiality. Only %{context} members with %{strongStart}at least Reporter role%{strongEnd} can view or be notified about this %{issuableType}." +msgid "You are going to turn on confidentiality. Only %{context} members with %{strongStart}%{permissions}%{strongEnd} can view or be notified about this %{issuableType}." msgstr "" msgid "You are not allowed to %{action} a user" @@ -44444,6 +44441,12 @@ msgstr "" msgid "at" msgstr "" +msgid "at least the Reporter role" +msgstr "" + +msgid "at least the Reporter role, the author, and assignees" +msgstr "" + msgid "at risk" msgstr "" diff --git a/spec/fixtures/glfm/example_snapshots/html.yml b/spec/fixtures/glfm/example_snapshots/html.yml index fcbc7c0572a..96c01340f8e 100644 --- a/spec/fixtures/glfm/example_snapshots/html.yml +++ b/spec/fixtures/glfm/example_snapshots/html.yml @@ -1909,7 +1909,18 @@ ]]> <p>okay</p> static: |2- + <![CDATA[ + function matchwo(a,b) + { + if (a < b && a < 0) then { + return 1; + + } else { + return 0; + } + } + ]]> <p data-sourcepos="13:1-13:4" dir="auto">okay</p> wysiwyg: |- Error - check implementation: @@ -7233,7 +7244,7 @@ canonical: | <p>foo <!ELEMENT br EMPTY></p> static: |- - <p data-sourcepos="1:1-1:23" dir="auto">foo </p> + <p data-sourcepos="1:1-1:23" dir="auto">foo <!ELEMENT br EMPTY></p> wysiwyg: |- Error - check implementation: Cannot destructure property 'className' of 'hastNode.properties' as it is undefined. @@ -7241,7 +7252,7 @@ canonical: | <p>foo <![CDATA[>&<]]></p> static: |- - <p data-sourcepos="1:1-1:19" dir="auto">foo &</p> + <p data-sourcepos="1:1-1:19" dir="auto">foo <![CDATA[>&<]]></p> wysiwyg: |- Error - check implementation: Cannot destructure property 'className' of 'hastNode.properties' as it is undefined. diff --git a/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_content_spec.js b/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_content_spec.js index ab45fdf03bc..81354d64a90 100644 --- a/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_content_spec.js +++ b/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_content_spec.js @@ -69,14 +69,14 @@ describe('Sidebar Confidentiality Content', () => { variant: 'warning', }); expect(alertEl.text()).toBe( - 'Only project members with at least Reporter role can view or be notified about this issue.', + 'Only project members with at least the Reporter role, the author, and assignees can view or be notified about this issue.', ); }); it('displays a correct confidential text for epic', () => { createComponent({ confidential: true, issuableType: 'epic' }); expect(findText().findComponent(GlAlert).text()).toBe( - 'Only group members with at least Reporter role can view or be notified about this epic.', + 'Only group members with at least the Reporter role can view or be notified about this epic.', ); }); }); diff --git a/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_form_spec.js b/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_form_spec.js index 85d6bc7b782..1ea035c7184 100644 --- a/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_form_spec.js +++ b/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_form_spec.js @@ -89,7 +89,7 @@ describe('Sidebar Confidentiality Form', () => { it('renders a message about making an issue confidential', () => { expect(findWarningMessage().text()).toBe( - 'You are going to turn on confidentiality. Only project members with at least Reporter role can view or be notified about this issue.', + 'You are going to turn on confidentiality. Only project members with at least the Reporter role, the author, and assignees can view or be notified about this issue.', ); }); diff --git a/spec/frontend/vue_shared/components/confidentiality_badge_spec.js b/spec/frontend/vue_shared/components/confidentiality_badge_spec.js index 9d11fbbaf55..e1860d3399b 100644 --- a/spec/frontend/vue_shared/components/confidentiality_badge_spec.js +++ b/spec/frontend/vue_shared/components/confidentiality_badge_spec.js @@ -29,8 +29,8 @@ describe('ConfidentialityBadge', () => { it.each` workspaceType | issuableType | expectedTooltip - ${WorkspaceType.project} | ${IssuableType.Issue} | ${'Only project members with at least Reporter role can view or be notified about this issue.'} - ${WorkspaceType.group} | ${IssuableType.Epic} | ${'Only group members with at least Reporter role can view or be notified about this epic.'} + ${WorkspaceType.project} | ${IssuableType.Issue} | ${'Only project members with at least the Reporter role, the author, and assignees can view or be notified about this issue.'} + ${WorkspaceType.group} | ${IssuableType.Epic} | ${'Only group members with at least the Reporter role can view or be notified about this epic.'} `( 'should render gl-badge with correct tooltip when workspaceType is $workspaceType and issuableType is $issuableType', ({ workspaceType, issuableType, expectedTooltip }) => { diff --git a/spec/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level_spec.rb new file mode 100644 index 00000000000..fd6c055b9f6 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level_spec.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillProjectFeaturePackageRegistryAccessLevel do + let(:non_null_project_features) { { pages_access_level: 20 } } + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:project_features) { table(:project_features) } + + let(:namespace1) { namespaces.create!(name: 'namespace 1', path: 'namespace1') } + let(:namespace2) { namespaces.create!(name: 'namespace 2', path: 'namespace2') } + let(:namespace3) { namespaces.create!(name: 'namespace 3', path: 'namespace3') } + let(:namespace4) { namespaces.create!(name: 'namespace 4', path: 'namespace4') } + let(:namespace5) { namespaces.create!(name: 'namespace 5', path: 'namespace5') } + let(:namespace6) { namespaces.create!(name: 'namespace 6', path: 'namespace6') } + + let(:project1) do + projects.create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id, packages_enabled: false) + end + + let(:project2) do + projects.create!(namespace_id: namespace2.id, project_namespace_id: namespace2.id, packages_enabled: nil) + end + + let(:project3) do + projects.create!( + namespace_id: namespace3.id, + project_namespace_id: namespace3.id, + packages_enabled: true, + visibility_level: Gitlab::VisibilityLevel::PRIVATE + ) + end + + let(:project4) do + projects.create!( + namespace_id: namespace4.id, + project_namespace_id: namespace4.id, + packages_enabled: true, visibility_level: Gitlab::VisibilityLevel::INTERNAL) + end + + let(:project5) do + projects.create!( + namespace_id: namespace5.id, + project_namespace_id: namespace5.id, + packages_enabled: true, + visibility_level: Gitlab::VisibilityLevel::PUBLIC + ) + end + + let(:project6) do + projects.create!(namespace_id: namespace6.id, project_namespace_id: namespace6.id, packages_enabled: false) + end + + let!(:project_feature1) do + project_features.create!( + project_id: project1.id, + package_registry_access_level: ProjectFeature::ENABLED, + **non_null_project_features + ) + end + + let!(:project_feature2) do + project_features.create!( + project_id: project2.id, + package_registry_access_level: ProjectFeature::ENABLED, + **non_null_project_features + ) + end + + let!(:project_feature3) do + project_features.create!( + project_id: project3.id, + package_registry_access_level: ProjectFeature::DISABLED, + **non_null_project_features + ) + end + + let!(:project_feature4) do + project_features.create!( + project_id: project4.id, + package_registry_access_level: ProjectFeature::DISABLED, + **non_null_project_features + ) + end + + let!(:project_feature5) do + project_features.create!( + project_id: project5.id, + package_registry_access_level: ProjectFeature::DISABLED, + **non_null_project_features + ) + end + + let!(:project_feature6) do + project_features.create!( + project_id: project6.id, + package_registry_access_level: ProjectFeature::ENABLED, + **non_null_project_features + ) + end + + subject(:perform_migration) do + described_class.new(start_id: project1.id, + end_id: project5.id, + batch_table: :projects, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection) + .perform + end + + it 'backfills project_features.package_registry_access_level', :aggregate_failures do + perform_migration + + expect(project_feature1.reload.package_registry_access_level).to eq(ProjectFeature::DISABLED) + expect(project_feature2.reload.package_registry_access_level).to eq(ProjectFeature::DISABLED) + expect(project_feature3.reload.package_registry_access_level).to eq(ProjectFeature::PRIVATE) + expect(project_feature4.reload.package_registry_access_level).to eq(ProjectFeature::ENABLED) + expect(project_feature5.reload.package_registry_access_level).to eq(ProjectFeature::PUBLIC) + expect(project_feature6.reload.package_registry_access_level).to eq(ProjectFeature::ENABLED) + end +end diff --git a/spec/migrations/queue_backfill_project_feature_package_registry_access_level_spec.rb b/spec/migrations/queue_backfill_project_feature_package_registry_access_level_spec.rb new file mode 100644 index 00000000000..487d94b82a1 --- /dev/null +++ b/spec/migrations/queue_backfill_project_feature_package_registry_access_level_spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillProjectFeaturePackageRegistryAccessLevel do + let_it_be(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :projects, + column_name: :id, + interval: described_class::DELAY_INTERVAL + ) + } + end + end +end diff --git a/spec/support/shared_examples/features/sidebar_shared_examples.rb b/spec/support/shared_examples/features/sidebar_shared_examples.rb index af3ea0600a2..77334db6a36 100644 --- a/spec/support/shared_examples/features/sidebar_shared_examples.rb +++ b/spec/support/shared_examples/features/sidebar_shared_examples.rb @@ -109,9 +109,8 @@ RSpec.shared_examples 'issue boards sidebar' do wait_for_requests expect(page).to have_content( - _('Only project members with at least' \ - ' Reporter role can view or be' \ - ' notified about this issue.') + _('Only project members with at least the Reporter role, the author, and assignees' \ + ' can view or be notified about this issue.') ) end end diff --git a/spec/views/projects/issues/_service_desk_info_content.html.haml_spec.rb b/spec/views/projects/issues/_service_desk_info_content.html.haml_spec.rb index 1c6d729ddce..92ca0bb52d6 100644 --- a/spec/views/projects/issues/_service_desk_info_content.html.haml_spec.rb +++ b/spec/views/projects/issues/_service_desk_info_content.html.haml_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe 'projects/issues/_service_desk_info_content' do +RSpec.describe 'projects/issues/service_desk/_service_desk_info_content' do let_it_be(:project) { create(:project) } let_it_be(:user) { create(:user) } let_it_be(:service_desk_address) { 'address@example.com' } |
