diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2024-01-21 18:08:07 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2024-01-21 18:08:07 +0300 |
commit | 61d7d07541aafcf49a97159a6048cf6847adbd51 (patch) | |
tree | be1a381102a73d97ec3cd071d138cc4338623d83 | |
parent | 0d932b7e02b1d3618bf71012e20a33d7f2d282a6 (diff) |
Add latest changes from gitlab-org/gitlab@master
8 files changed, 93 insertions, 0 deletions
diff --git a/db/migrate/20240116205943_add_elasticsearch_max_code_indexing_concurrency_to_application_settings.rb b/db/migrate/20240116205943_add_elasticsearch_max_code_indexing_concurrency_to_application_settings.rb new file mode 100644 index 00000000000..c12edcedc84 --- /dev/null +++ b/db/migrate/20240116205943_add_elasticsearch_max_code_indexing_concurrency_to_application_settings.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddElasticsearchMaxCodeIndexingConcurrencyToApplicationSettings < Gitlab::Database::Migration[2.2] + enable_lock_retries! + milestone '16.9' + + def change + add_column :application_settings, + :elasticsearch_max_code_indexing_concurrency, + :integer, + default: 30, + null: false, + if_not_exists: true + end +end diff --git a/db/migrate/20240116211138_update_max_code_indexing_concurrency_in_application_settings_for_gitlab_com.rb b/db/migrate/20240116211138_update_max_code_indexing_concurrency_in_application_settings_for_gitlab_com.rb new file mode 100644 index 00000000000..1850da16341 --- /dev/null +++ b/db/migrate/20240116211138_update_max_code_indexing_concurrency_in_application_settings_for_gitlab_com.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class UpdateMaxCodeIndexingConcurrencyInApplicationSettingsForGitlabCom < Gitlab::Database::Migration[2.2] + restrict_gitlab_migration gitlab_schema: :gitlab_main + enable_lock_retries! + + milestone '16.9' + + def up + return unless Gitlab.com? + + execute 'UPDATE application_settings SET elasticsearch_max_code_indexing_concurrency = 60' + end + + def down + return unless Gitlab.com? + + execute 'UPDATE application_settings SET elasticsearch_max_code_indexing_concurrency = 30' + end +end diff --git a/db/schema_migrations/20240116205943 b/db/schema_migrations/20240116205943 new file mode 100644 index 00000000000..1b9bd9e0a39 --- /dev/null +++ b/db/schema_migrations/20240116205943 @@ -0,0 +1 @@ +f86c644bc630c52bf8647576f391b7a77d4b0d9a13a49c7552d0ec68f59c21c8
\ No newline at end of file diff --git a/db/schema_migrations/20240116211138 b/db/schema_migrations/20240116211138 new file mode 100644 index 00000000000..e5245276818 --- /dev/null +++ b/db/schema_migrations/20240116211138 @@ -0,0 +1 @@ +72a612645aea55077e46a0c99860312f677e03fb9956cfd13cee61a4bfa53faa
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 14b6be62159..7ce29e40688 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -12632,6 +12632,7 @@ CREATE TABLE application_settings ( toggle_security_policies_policy_scope boolean DEFAULT false NOT NULL, lock_toggle_security_policies_policy_scope boolean DEFAULT false NOT NULL, rate_limits jsonb DEFAULT '{}'::jsonb NOT NULL, + elasticsearch_max_code_indexing_concurrency integer DEFAULT 30 NOT NULL, CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)), CONSTRAINT app_settings_container_registry_pre_import_tags_rate_positive CHECK ((container_registry_pre_import_tags_rate >= (0)::numeric)), CONSTRAINT app_settings_dep_proxy_ttl_policies_worker_capacity_positive CHECK ((dependency_proxy_ttl_group_policy_worker_capacity >= 0)), diff --git a/doc/api/settings.md b/doc/api/settings.md index bde4c769b92..a2854a14320 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -402,6 +402,7 @@ listed in the descriptions of the relevant settings. | `elasticsearch_requeue_workers` **(PREMIUM ALL)** | boolean | no | Enable automatic requeuing of indexing workers. This improves non-code indexing throughput by enqueuing Sidekiq jobs until all documents are processed. | | `elasticsearch_limit_indexing` **(PREMIUM ALL)** | boolean | no | Limit Elasticsearch to index certain namespaces and projects. | | `elasticsearch_max_bulk_concurrency` **(PREMIUM ALL)** | integer | no | Maximum concurrency of Elasticsearch bulk requests per indexing operation. This only applies to repository indexing operations. | +| `elasticsearch_max_code_indexing_concurrency` **(PREMIUM ALL)** | integer | no | Maximum concurrency of Elasticsearch code indexing background jobs. This only applies to repository indexing operations. | | `elasticsearch_worker_number_of_shards` **(PREMIUM ALL)** | integer | no | Number of indexing worker shards. This improves non-code indexing throughput by enqueuing more parallel Sidekiq jobs. Default is `2`. | | `elasticsearch_max_bulk_size_mb` **(PREMIUM ALL)** | integer | no | Maximum size of Elasticsearch bulk indexing requests in MB. This only applies to repository indexing operations. | | `elasticsearch_namespace_ids` **(PREMIUM ALL)** | array of integers | no | The namespaces to index via Elasticsearch if `elasticsearch_limit_indexing` is enabled. | diff --git a/locale/gitlab.pot b/locale/gitlab.pot index e478817381d..b025513cce9 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -12038,6 +12038,9 @@ msgstr "" msgid "Code coverage statistics for %{ref} %{start_date} - %{end_date}" msgstr "" +msgid "Code indexing concurrency" +msgstr "" + msgid "Code owner approval is required" msgstr "" @@ -24561,6 +24564,9 @@ msgstr "" msgid "How is progress calculated?" msgstr "" +msgid "How many code indexing jobs are allowed to run concurrently. A higher value improves repository indexing performance by increasing Sidekiq and Elasticsearch load. Setting it to 0 allows an unlimited number of indexing jobs." +msgstr "" + msgid "How many seconds an IP counts toward the IP address limit." msgstr "" diff --git a/spec/migrations/20240116211138_update_max_code_indexing_concurrency_in_application_settings_for_gitlab_com_spec.rb b/spec/migrations/20240116211138_update_max_code_indexing_concurrency_in_application_settings_for_gitlab_com_spec.rb new file mode 100644 index 00000000000..8bed7e1b8a1 --- /dev/null +++ b/spec/migrations/20240116211138_update_max_code_indexing_concurrency_in_application_settings_for_gitlab_com_spec.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe UpdateMaxCodeIndexingConcurrencyInApplicationSettingsForGitlabCom, feature_category: :global_search do + let(:settings) { table(:application_settings) } + + describe "#up" do + subject(:up) { migrate! } + + it 'does nothing when not in gitlab.com' do + record = settings.create! + + expect { up }.not_to change { record.reload.elasticsearch_max_code_indexing_concurrency } + end + + it 'updates elasticsearch_worker_number_of_shards when gitlab.com' do + allow(Gitlab).to receive(:com?).and_return(true) + + record = settings.create! + + expect { up }.to change { record.reload.elasticsearch_max_code_indexing_concurrency }.from(30).to(60) + end + end + + describe "#down" do + subject(:down) { schema_migrate_down! } + + it 'does nothing when not in gitlab.com' do + record = settings.create!(elasticsearch_max_code_indexing_concurrency: 60) + + migrate! + + expect { down }.not_to change { record.reload.elasticsearch_max_code_indexing_concurrency } + end + + it 'updates elasticsearch_worker_number_of_shards when gitlab.com' do + allow(Gitlab).to receive(:com?).and_return(true) + + record = settings.create!(elasticsearch_max_code_indexing_concurrency: 60) + + migrate! + + expect { down }.to change { record.reload.elasticsearch_max_code_indexing_concurrency }.from(60).to(30) + end + end +end |