Add latest changes from gitlab-org/gitlab@master

3 files changed, 334 insertions, 351 deletions

diff --git a/doc/update/index.md b/doc/update/index.md
index 6147911a14a..d89c35fd024 100644
--- a/doc/update/index.md
+++ b/doc/update/index.md
@@ -299,350 +299,6 @@ Before upgrading to GitLab 15, see [GitLab 15 changes](versions/gitlab_15_change
 
 Before upgrading to GitLab 14, see [GitLab 14 changes](versions/gitlab_14_changes.md).
 
-### User profile data loss bug in 15.9.x
-
-There is a database migration bug in 15.9.0, 15.9.1, and 15.9.2 that can cause data loss from the user profile fields `linkedin`, `twitter`, `skype`, `website_url`, `location`, and `organization`.
-
-This bug is fixed in patch releases 15.9.3 and later.
-
-The following upgrade path also works around the bug:
-
-1. Upgrade to GitLab 15.6.x, 15.7.x, or 15.8.x.
-1. [Ensure batched background migrations](background_migrations.md#batched-background-migrations) are complete.
-1. Upgrade to an earlier GitLab 15.9 patch release that doesn't have the bug fix.
-
-It is not then required to upgrade to 15.9.3 or later for this issue.
-
-[Read the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/393216) for more information.
-
-### Gitaly: Omnibus GitLab configuration structure change
-
-Gitaly configuration structure in Omnibus GitLab [changes](https://gitlab.com/gitlab-org/gitaly/-/issues/4467) in GitLab 16.0 to be consistent with the Gitaly configuration
-structure used in self-compiled installations.
-
-As a result of this change, a single hash under `gitaly['configuration']` holds most Gitaly
-configuration. Some `gitaly['..']` configuration options will continue to be used by Omnibus GitLab 16.0 and later:
-
-- `enable`
-- `dir`
-- `bin_path`
-- `env_directory`
-- `env`
-- `open_files_ulimit`
-- `consul_service_name`
-- `consul_service_meta`
-
-Migrate by moving your existing configuration under the new structure. The new structure is supported from Omnibus GitLab 15.10.
-
-The new structure is documented below with the old keys described in a comment above the new keys. When applying the new structure to your configuration:
-
-1. Replace the `...` with the value from the old key.
-1. Skip any keys you haven't configured a value for previously.
-1. Remove the old keys from the configuration once migrated.
-1. Optional but recommended. Include a trailing comma for all hash keys so the hash remains valid when keys are re-ordered or additional keys are added.
-1. When configuring `storage` to replace `git_data_dirs`, you must append `repositories` to the path as documented below. If you omit this step, your Git repositories are
-   inaccessible until the configuration is fixed.
-
-  ```ruby
-gitaly['configuration'] = {
-  # gitaly['socket_path']
-  socket_path: ...,
-  # gitaly['runtime_dir']
-  runtime_dir: ...,
-  # gitaly['listen_addr']
-  listen_addr: ...,
-  # gitaly['prometheus_listen_addr']
-  prometheus_listen_addr: ...,
-  # gitaly['tls_listen_addr']
-  tls_listen_addr: ...,
-  tls: {
-    # gitaly['certificate_path']
-    certificate_path: ...,
-    # gitaly['key_path']
-    key_path: ...,
-  },
-  # gitaly['graceful_restart_timeout']
-  graceful_restart_timeout: ...,
-  logging: {
-    # gitaly['logging_level']
-    level: ...,
-    # gitaly['logging_format']
-    format: ...,
-    # gitaly['logging_sentry_dsn']
-    sentry_dsn: ...,
-    # gitaly['logging_ruby_sentry_dsn']
-    ruby_sentry_dsn: ...,
-    # gitaly['logging_sentry_environment']
-    sentry_environment: ...,
-    # gitaly['log_directory']
-    dir: ...,
-  },
-  prometheus: {
-    # gitaly['prometheus_grpc_latency_buckets']. The old value was configured as a string
-    # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2].
-    grpc_latency_buckets: ...,
-  },
-  auth: {
-    # gitaly['auth_token']
-    token: ...,
-    # gitaly['auth_transitioning']
-    transitioning: ...,
-  },
-  git: {
-    # gitaly['git_catfile_cache_size']
-    catfile_cache_size: ...,
-    # gitaly['git_bin_path']
-    bin_path: ...,
-    # gitaly['use_bundled_git']
-    use_bundled_binaries: ...,
-    # gitaly['gpg_signing_key_path']
-    signing_key: ...,
-    # gitaly['gitconfig']. This is still an array but the type of the elements have changed.
-    config: [
-      {
-        # Previously the elements contained 'section', and 'subsection' in addition to 'key'. Now
-        # these all should be concatenated into just 'key', separated by dots. For example,
-        # {section: 'first', subsection: 'middle', key: 'last', value: 'value'}, should become
-        # {key: 'first.middle.last', value: 'value'}.
-        key: ...,
-        value: ...,
-      },
-    ],
-  },
-  # Storage could previously be configured through either gitaly['storage'] or 'git_data_dirs'. Migrate
-  # the relevant configuration according to the instructions below.
-  # For 'git_data_dirs', migrate only the 'path' to the gitaly['configuration'] and leave the rest of it untouched.
-  storage: [
-    {
-      # gitaly['storage'][<index>]['name']
-      #
-      # git_data_dirs[<name>]. The storage name was configured as a key in the map.
-      name: ...,
-      # gitaly['storage'][<index>]['path']
-      #
-      # git_data_dirs[<name>]['path']. Use the value from git_data_dirs[<name>]['path'] and append '/repositories' to it.
-      #
-      # For example, if the path in 'git_data_dirs' was '/var/opt/gitlab/git-data', use
-      # '/var/opt/gitlab/git-data/repositories'. The '/repositories' extension was automatically
-      # appended to the path configured in `git_data_dirs`.
-      path: ...,
-    },
-  ],
-  hooks: {
-    # gitaly['custom_hooks_dir']
-    custom_hooks_dir: ...,
-  },
-  daily_maintenance: {
-    # gitaly['daily_maintenance_disabled']
-    disabled: ...,
-    # gitaly['daily_maintenance_start_hour']
-    start_hour: ...,
-    # gitaly['daily_maintenance_start_minute']
-    start_minute: ...,
-    # gitaly['daily_maintenance_duration']
-    duration: ...,
-    # gitaly['daily_maintenance_storages']
-    storages: ...,
-  },
-  cgroups: {
-    # gitaly['cgroups_mountpoint']
-    mountpoint: ...,
-    # gitaly['cgroups_hierarchy_root']
-    hierarchy_root: ...,
-    # gitaly['cgroups_memory_bytes']
-    memory_bytes: ...,
-    # gitaly['cgroups_cpu_shares']
-    cpu_shares: ...,
-    repositories: {
-      # gitaly['cgroups_repositories_count']
-      count: ...,
-      # gitaly['cgroups_repositories_memory_bytes']
-      memory_bytes: ...,
-      # gitaly['cgroups_repositories_cpu_shares']
-      cpu_shares: ...,
-    }
-  },
-  # gitaly['concurrency']. While the structure is the same, the string keys in the array elements
-  # should be replaced by symbols as elsewhere. {'key' => 'value'}, should become {key: 'value'}.
-  concurrency: ...,
-  # gitaly['rate_limiting']. While the structure is the same, the string keys in the array elements
-  # should be replaced by symbols as elsewhere. {'key' => 'value'}, should become {key: 'value'}.
-  rate_limiting: ...,
-  pack_objects_cache: {
-    # gitaly['pack_objects_cache_enabled']
-    enabled: ...,
-    # gitaly['pack_objects_cache_dir']
-    dir: ...,
-    # gitaly['pack_objects_cache_max_age']
-    max_age: ...,
-  }
-}
-```
-
-### Praefect: Omnibus GitLab configuration structure change
-
-Praefect configuration structure in Omnibus GitLab [changes](https://gitlab.com/gitlab-org/gitaly/-/issues/4467) in GitLab 16.0 to be consistent with the Praefect configuration
-structure used in self-compiled installations.
-
-As a result of this change, a single hash under `praefect['configuration']` holds most Praefect
-configuration. Some `praefect['..']` configuration options will continue to be used by Omnibus GitLab 16.0 and later:
-
-- `enable`
-- `dir`
-- `log_directory`
-- `env_directory`
-- `env`
-- `wrapper_path`
-- `auto_migrate`
-- `consul_service_name`
-
-Migrate by moving your existing configuration under the new structure. The new structure is supported from Omnibus GitLab 15.9.
-
-The new structure is documented below with the old keys described in a comment above the new keys. When applying the new structure to your configuration:
-
-1. Replace the `...` with the value from the old key.
-1. Skip any keys you haven't configured a value for previously.
-1. Remove the old keys from the configuration once migrated.
-1. Optional but recommended. Include a trailing comma for all hash keys so the hash remains valid when keys are re-ordered or additional keys are added.
-
-```ruby
-praefect['configuration'] = {
-  # praefect['listen_addr']
-  listen_addr: ...,
-  # praefect['socket_path']
-  socket_path: ...,
-  # praefect['prometheus_listen_addr']
-  prometheus_listen_addr: ...,
-  # praefect['tls_listen_addr']
-  tls_listen_addr: ...,
-  # praefect['separate_database_metrics']
-  prometheus_exclude_database_from_default_metrics: ...,
-  auth: {
-    # praefect['auth_token']
-    token: ...,
-    # praefect['auth_transitioning']
-    transitioning: ...,
-  },
-  logging: {
-    # praefect['logging_format']
-    format: ...,
-    # praefect['logging_level']
-    level: ...,
-  },
-  failover: {
-    # praefect['failover_enabled']
-    enabled: ...,
-  },
-  background_verification: {
-    # praefect['background_verification_delete_invalid_records']
-    delete_invalid_records: ...,
-    # praefect['background_verification_verification_interval']
-    verification_interval: ...,
-  },
-  reconciliation: {
-    # praefect['reconciliation_scheduling_interval']
-    scheduling_interval: ...,
-    # praefect['reconciliation_histogram_buckets']. The old value was configured as a string
-    # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2].
-    histogram_buckets: ...,
-  },
-  tls: {
-    # praefect['certificate_path']
-    certificate_path: ...,
-    # praefect['key_path']
-    key_path: ...,
-  },
-  database: {
-    # praefect['database_host']
-    host: ...,
-    # praefect['database_port']
-    port: ...,
-    # praefect['database_user']
-    user: ...,
-    # praefect['database_password']
-    password: ...,
-    # praefect['database_dbname']
-    dbname: ...,
-    # praefect['database_sslmode']
-    sslmode: ...,
-    # praefect['database_sslcert']
-    sslcert: ...,
-    # praefect['database_sslkey']
-    sslkey: ...,
-    # praefect['database_sslrootcert']
-    sslrootcert: ...,
-    session_pooled: {
-      # praefect['database_direct_host']
-      host: ...,
-      # praefect['database_direct_port']
-      port: ...,
-      # praefect['database_direct_user']
-      user: ...,
-      # praefect['database_direct_password']
-      password: ...,
-      # praefect['database_direct_dbname']
-      dbname: ...,
-      # praefect['database_direct_sslmode']
-      sslmode: ...,
-      # praefect['database_direct_sslcert']
-      sslcert: ...,
-      # praefect['database_direct_sslkey']
-      sslkey: ...,
-      # praefect['database_direct_sslrootcert']
-      sslrootcert: ...,
-    }
-  },
-  sentry: {
-    # praefect['sentry_dsn']
-    sentry_dsn: ...,
-    # praefect['sentry_environment']
-    sentry_environment: ...,
-  },
-  prometheus: {
-    # praefect['prometheus_grpc_latency_buckets']. The old value was configured as a string
-    # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2].
-    grpc_latency_buckets: ...,
-  },
-  # praefect['graceful_stop_timeout']
-  graceful_stop_timeout: ...,
-
-  # praefect['virtual_storages']. The old value was a hash map but the new value is an array.
-  virtual_storage: [
-    {
-      # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]. The name was previously the key in
-      # the 'virtual_storages' hash.
-      name: ...,
-      # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]. The old value was a hash map
-      # but the new value is an array.
-      node: [
-        {
-          # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]. Use NODE_NAME key as the
-          # storage.
-          storage: ...,
-          # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]['address'].
-          address: ...,
-          # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]['token'].
-          token: ...,
-        },
-      ],
-    }
-  ]
-}
-```
-
-### Change to Praefect-generated replica paths in GitLab 15.3
-
-New Git repositories created in Gitaly cluster no longer use the `@hashed` storage path.
-
-Praefect now generates replica paths for use by Gitaly cluster.
-This change is a pre-requisite for Gitaly cluster atomically creating, deleting, and
-renaming Git repositories.
-
-To identify the replica path, [query the Praefect repository metadata](../administration/gitaly/troubleshooting.md#view-repository-metadata)
-and pass the `@hashed` storage path to `-relative-path`.
-
-With this information, you can correctly install [server hooks](../administration/server_hooks.md).
-
 ### PostgreSQL segmentation fault issue
 
 If you run GitLab with external PostgreSQL, particularly AWS RDS, ensure you upgrade PostgreSQL
diff --git a/doc/update/versions/gitlab_15_changes.md b/doc/update/versions/gitlab_15_changes.md
index 6ebbeeb1d32..5e937a0ec98 100644
--- a/doc/update/versions/gitlab_15_changes.md
+++ b/doc/update/versions/gitlab_15_changes.md
@@ -64,7 +64,7 @@ see [Packaged PostgreSQL deployed in an HA/Geo Cluster](https://docs.gitlab.com/
   - To resolve this issue, upgrade to 15.11 or use the workaround in the issue.
 - A [bug with zero-downtime reindexing](https://gitlab.com/gitlab-org/gitlab/-/issues/422938) can cause a `Couldn't load task status` error when you reindex. You might also get a `sliceId must be greater than 0 but was [-1]` error on the Elasticsearch host. As a workaround, consider [reindexing from scratch](../../integration/advanced_search/elasticsearch_troubleshooting.md#last-resort-to-recreate-an-index) or upgrading to GitLab 16.3.
 - Gitaly configuration changes significantly in Omnibus GitLab 16.0. You can begin migrating to the new structure in Omnibus GitLab 15.10 while backwards compatibility is
-  maintained in the lead up to Omnibus GitLab 16.0. [Read more about this change](../index.md#gitaly-omnibus-gitlab-configuration-structure-change).
+  maintained in the lead up to Omnibus GitLab 16.0. [Read more about this change](gitlab_16_changes.md#1600).
 - You might encounter the following error while upgrading to GitLab 15.10 or later:
 
   ```shell
@@ -124,7 +124,10 @@ see [Packaged PostgreSQL deployed in an HA/Geo Cluster](https://docs.gitlab.com/
   - Elasticsearch does not need to be enabled for this to occur.
   - To resolve this issue, upgrade to 15.11 or use the workaround in the issue.
 - **Upgrade to patch release 15.9.3 or later**. This provides fixes for two database migration bugs:
-  - Patch releases 15.9.0, 15.9.1, 15.9.2 have [a bug that can cause data loss](../index.md#user-profile-data-loss-bug-in-159x) from the user profile fields.
+  - Patch releases 15.9.0, 15.9.1, 15.9.2 have a bug that can cause data loss
+    from the user profile fields `linkedin`, `twitter`, `skype`, `website_url`,
+    `location`, and `organization`. For more information, see
+    [issue 393216](https://gitlab.com/gitlab-org/gitlab/-/issues/393216).
   - The second [bug fix](https://gitlab.com/gitlab-org/gitlab/-/issues/394760) ensures it is possible to upgrade directly from 15.4.x.
 - As part of the [CI Partitioning effort](../../architecture/blueprints/ci_data_decay/pipeline_partitioning.md), a [new Foreign Key](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/107547) was added to `ci_builds_needs`. On GitLab instances with large CI tables, adding this constraint can take longer than usual.
 - Praefect's metadata verifier's [invalid metadata deletion behavior](../../administration/gitaly/praefect.md#enable-deletions) is now enabled by default.
@@ -140,7 +143,7 @@ see [Packaged PostgreSQL deployed in an HA/Geo Cluster](https://docs.gitlab.com/
 
   You can find repositories with invalid metadata records prior in GitLab 15.0 and later by searching for the log records outputted by the verifier. [Read more about repository verification, and to see an example log entry](../../administration/gitaly/praefect.md#repository-verification).
 - Praefect configuration changes significantly in Omnibus GitLab 16.0. You can begin migrating to the new structure in Omnibus GitLab 15.9 while backwards compatibility is
-  maintained in the lead up to Omnibus GitLab 16.0. [Read more about this change](../index.md#praefect-omnibus-gitlab-configuration-structure-change).
+  maintained in the lead up to Omnibus GitLab 16.0. [Read more about this change](gitlab_16_changes.md#1600).
 
 ### Self-compiled installations
 
@@ -526,8 +529,6 @@ potentially cause downtime.
     sidekiq['routing_rules'] = [['*', 'default']]
     ```
 
-- New Git repositories created in Gitaly cluster [no longer use the `@hashed` storage path](../index.md#change-to-praefect-generated-replica-paths-in-gitlab-153). Server
-  hooks for new repositories must be copied into a different location.
 - The structure of `/etc/gitlab/gitlab-secrets.json` was modified in [GitLab 15.4](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/6310),
   and new configuration was added to `gitlab_pages`, `grafana`, and `mattermost` sections.
   In a highly available or GitLab Geo environment, secrets need to be the same on all nodes.
@@ -594,8 +595,19 @@ A [license caching issue](https://gitlab.com/gitlab-org/gitlab/-/issues/376706)
 
 ## 15.3.0
 
-- New Git repositories created in Gitaly cluster [no longer use the `@hashed` storage path](../index.md#change-to-praefect-generated-replica-paths-in-gitlab-153). Server
-  hooks for new repositories must be copied into a different location.
+- New Git repositories created in Gitaly cluster no longer use the `@hashed`
+  storage path. Server hooks for new repositories must be copied into a
+  different location. Praefect now generates replica paths for use by Gitaly
+  cluster. This change is a pre-requisite for Gitaly cluster atomically
+  creating, deleting, and renaming Git repositories.
+
+  To identify the replica path,
+  [query the Praefect repository metadata](../../administration/gitaly/troubleshooting.md#view-repository-metadata)
+  and pass the `@hashed` storage path to `-relative-path`.
+
+  With this information, you can correctly install
+  [server hooks](../../administration/server_hooks.md).
+
 - A [license caching issue](https://gitlab.com/gitlab-org/gitlab/-/issues/376706) prevents some premium features of GitLab from working correctly if you add a new license. Workarounds for this issue:
 
   - Restart all Rails, Sidekiq and Gitaly nodes after applying a new license. This clears the relevant license caches and allows all premium features to operate correctly.
diff --git a/doc/update/versions/gitlab_16_changes.md b/doc/update/versions/gitlab_16_changes.md
index 10cc673da8c..45aee9e56ca 100644
--- a/doc/update/versions/gitlab_16_changes.md
+++ b/doc/update/versions/gitlab_16_changes.md
@@ -197,6 +197,321 @@ Specific information applies to Linux package installations:
 
   Workaround is to make use of a different key type, or upgrade the client OpenSSH to a version >= 8.7.
 
+- The Gitaly configuration structure in the Linux package
+  [changes](https://gitlab.com/gitlab-org/gitaly/-/issues/4467) in GitLab 16.0
+  to be consistent with the Gitaly configuration structure used in
+  self-compiled installations.
+
+  As a result of this change, a single hash under `gitaly['configuration']` holds most Gitaly
+  configuration. Some `gitaly['..']` configuration options will continue to be used by GitLab 16.0 and later:
+
+  - `enable`
+  - `dir`
+  - `bin_path`
+  - `env_directory`
+  - `env`
+  - `open_files_ulimit`
+  - `consul_service_name`
+  - `consul_service_meta`
+
+  Migrate by moving your existing configuration under the new structure. The new structure is supported from GitLab 15.10.
+
+  The new structure is documented below with the old keys described in a comment above the new keys. When applying the new structure to your configuration:
+
+  1. Replace the `...` with the value from the old key.
+  1. Skip any keys you haven't configured a value for previously.
+  1. Remove the old keys from the configuration once migrated.
+  1. Optional but recommended. Include a trailing comma for all hash keys so the hash remains valid when keys are re-ordered or additional keys are added.
+  1. When configuring `storage` to replace `git_data_dirs`, you must append `repositories` to the path as documented below. If you omit this step, your Git repositories are
+     inaccessible until the configuration is fixed.
+
+     ```ruby
+     gitaly['configuration'] = {
+       # gitaly['socket_path']
+       socket_path: ...,
+       # gitaly['runtime_dir']
+       runtime_dir: ...,
+       # gitaly['listen_addr']
+       listen_addr: ...,
+       # gitaly['prometheus_listen_addr']
+       prometheus_listen_addr: ...,
+       # gitaly['tls_listen_addr']
+       tls_listen_addr: ...,
+       tls: {
+         # gitaly['certificate_path']
+         certificate_path: ...,
+         # gitaly['key_path']
+         key_path: ...,
+       },
+       # gitaly['graceful_restart_timeout']
+       graceful_restart_timeout: ...,
+       logging: {
+         # gitaly['logging_level']
+         level: ...,
+         # gitaly['logging_format']
+         format: ...,
+         # gitaly['logging_sentry_dsn']
+         sentry_dsn: ...,
+         # gitaly['logging_ruby_sentry_dsn']
+         ruby_sentry_dsn: ...,
+         # gitaly['logging_sentry_environment']
+         sentry_environment: ...,
+         # gitaly['log_directory']
+         dir: ...,
+       },
+       prometheus: {
+         # gitaly['prometheus_grpc_latency_buckets']. The old value was configured as a string
+         # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2].
+         grpc_latency_buckets: ...,
+       },
+       auth: {
+         # gitaly['auth_token']
+         token: ...,
+         # gitaly['auth_transitioning']
+         transitioning: ...,
+       },
+       git: {
+         # gitaly['git_catfile_cache_size']
+         catfile_cache_size: ...,
+         # gitaly['git_bin_path']
+         bin_path: ...,
+         # gitaly['use_bundled_git']
+         use_bundled_binaries: ...,
+         # gitaly['gpg_signing_key_path']
+         signing_key: ...,
+         # gitaly['gitconfig']. This is still an array but the type of the elements have changed.
+         config: [
+           {
+             # Previously the elements contained 'section', and 'subsection' in addition to 'key'. Now
+             # these all should be concatenated into just 'key', separated by dots. For example,
+             # {section: 'first', subsection: 'middle', key: 'last', value: 'value'}, should become
+             # {key: 'first.middle.last', value: 'value'}.
+             key: ...,
+             value: ...,
+           },
+         ],
+       },
+       # Storage could previously be configured through either gitaly['storage'] or 'git_data_dirs'. Migrate
+       # the relevant configuration according to the instructions below.
+       # For 'git_data_dirs', migrate only the 'path' to the gitaly['configuration'] and leave the rest of it untouched.
+       storage: [
+         {
+           # gitaly['storage'][<index>]['name']
+           #
+           # git_data_dirs[<name>]. The storage name was configured as a key in the map.
+           name: ...,
+           # gitaly['storage'][<index>]['path']
+           #
+           # git_data_dirs[<name>]['path']. Use the value from git_data_dirs[<name>]['path'] and append '/repositories' to it.
+           #
+           # For example, if the path in 'git_data_dirs' was '/var/opt/gitlab/git-data', use
+           # '/var/opt/gitlab/git-data/repositories'. The '/repositories' extension was automatically
+           # appended to the path configured in `git_data_dirs`.
+           path: ...,
+         },
+       ],
+       hooks: {
+         # gitaly['custom_hooks_dir']
+         custom_hooks_dir: ...,
+       },
+       daily_maintenance: {
+         # gitaly['daily_maintenance_disabled']
+         disabled: ...,
+         # gitaly['daily_maintenance_start_hour']
+         start_hour: ...,
+         # gitaly['daily_maintenance_start_minute']
+         start_minute: ...,
+         # gitaly['daily_maintenance_duration']
+         duration: ...,
+         # gitaly['daily_maintenance_storages']
+         storages: ...,
+       },
+       cgroups: {
+         # gitaly['cgroups_mountpoint']
+         mountpoint: ...,
+         # gitaly['cgroups_hierarchy_root']
+         hierarchy_root: ...,
+         # gitaly['cgroups_memory_bytes']
+         memory_bytes: ...,
+         # gitaly['cgroups_cpu_shares']
+         cpu_shares: ...,
+         repositories: {
+           # gitaly['cgroups_repositories_count']
+           count: ...,
+           # gitaly['cgroups_repositories_memory_bytes']
+           memory_bytes: ...,
+           # gitaly['cgroups_repositories_cpu_shares']
+           cpu_shares: ...,
+         }
+       },
+       # gitaly['concurrency']. While the structure is the same, the string keys in the array elements
+       # should be replaced by symbols as elsewhere. {'key' => 'value'}, should become {key: 'value'}.
+       concurrency: ...,
+       # gitaly['rate_limiting']. While the structure is the same, the string keys in the array elements
+       # should be replaced by symbols as elsewhere. {'key' => 'value'}, should become {key: 'value'}.
+       rate_limiting: ...,
+       pack_objects_cache: {
+         # gitaly['pack_objects_cache_enabled']
+         enabled: ...,
+         # gitaly['pack_objects_cache_dir']
+         dir: ...,
+         # gitaly['pack_objects_cache_max_age']
+         max_age: ...,
+       }
+     }
+     ```
+
+- The Praefect configuration structure in the Linux package
+  [changes](https://gitlab.com/gitlab-org/gitaly/-/issues/4467) in GitLab 16.0
+  to be consistent with the Praefect configuration structure used in
+  self-compiled installations.
+
+  As a result of this change, a single hash under `praefect['configuration']` holds most Praefect
+  configuration. Some `praefect['..']` configuration options will continue to be used by GitLab 16.0 and later:
+
+  - `enable`
+  - `dir`
+  - `log_directory`
+  - `env_directory`
+  - `env`
+  - `wrapper_path`
+  - `auto_migrate`
+  - `consul_service_name`
+
+  Migrate by moving your existing configuration under the new structure. The new structure is supported from GitLab 15.9.
+
+  The new structure is documented below with the old keys described in a comment above the new keys. When applying the new structure to your configuration:
+
+  1. Replace the `...` with the value from the old key.
+  1. Skip any keys you haven't configured a value for previously.
+  1. Remove the old keys from the configuration once migrated.
+  1. Optional but recommended. Include a trailing comma for all hash keys so the hash remains valid when keys are re-ordered or additional keys are added.
+
+     ```ruby
+     praefect['configuration'] = {
+       # praefect['listen_addr']
+       listen_addr: ...,
+       # praefect['socket_path']
+       socket_path: ...,
+       # praefect['prometheus_listen_addr']
+       prometheus_listen_addr: ...,
+       # praefect['tls_listen_addr']
+       tls_listen_addr: ...,
+       # praefect['separate_database_metrics']
+       prometheus_exclude_database_from_default_metrics: ...,
+       auth: {
+         # praefect['auth_token']
+         token: ...,
+         # praefect['auth_transitioning']
+         transitioning: ...,
+       },
+       logging: {
+         # praefect['logging_format']
+         format: ...,
+         # praefect['logging_level']
+         level: ...,
+       },
+       failover: {
+         # praefect['failover_enabled']
+         enabled: ...,
+       },
+       background_verification: {
+         # praefect['background_verification_delete_invalid_records']
+         delete_invalid_records: ...,
+         # praefect['background_verification_verification_interval']
+         verification_interval: ...,
+       },
+       reconciliation: {
+         # praefect['reconciliation_scheduling_interval']
+         scheduling_interval: ...,
+         # praefect['reconciliation_histogram_buckets']. The old value was configured as a string
+         # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2].
+         histogram_buckets: ...,
+       },
+       tls: {
+         # praefect['certificate_path']
+         certificate_path: ...,
+        # praefect['key_path']
+         key_path: ...,
+       },
+       database: {
+         # praefect['database_host']
+         host: ...,
+         # praefect['database_port']
+         port: ...,
+         # praefect['database_user']
+         user: ...,
+         # praefect['database_password']
+         password: ...,
+         # praefect['database_dbname']
+         dbname: ...,
+         # praefect['database_sslmode']
+         sslmode: ...,
+         # praefect['database_sslcert']
+         sslcert: ...,
+         # praefect['database_sslkey']
+         sslkey: ...,
+         # praefect['database_sslrootcert']
+         sslrootcert: ...,
+         session_pooled: {
+           # praefect['database_direct_host']
+           host: ...,
+           # praefect['database_direct_port']
+           port: ...,
+           # praefect['database_direct_user']
+           user: ...,
+           # praefect['database_direct_password']
+           password: ...,
+           # praefect['database_direct_dbname']
+           dbname: ...,
+           # praefect['database_direct_sslmode']
+           sslmode: ...,
+           # praefect['database_direct_sslcert']
+           sslcert: ...,
+           # praefect['database_direct_sslkey']
+           sslkey: ...,
+           # praefect['database_direct_sslrootcert']
+           sslrootcert: ...,
+         }
+       },
+       sentry: {
+         # praefect['sentry_dsn']
+         sentry_dsn: ...,
+         # praefect['sentry_environment']
+         sentry_environment: ...,
+       },
+       prometheus: {
+         # praefect['prometheus_grpc_latency_buckets']. The old value was configured as a string
+         # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2].
+         grpc_latency_buckets: ...,
+       },
+       # praefect['graceful_stop_timeout']
+       graceful_stop_timeout: ...,
+       
+       # praefect['virtual_storages']. The old value was a hash map but the new value is an array.
+       virtual_storage: [
+         {
+           # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]. The name was previously the key in
+           # the 'virtual_storages' hash.
+           name: ...,
+           # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]. The old value was a hash map
+           # but the new value is an array.
+           node: [
+             {
+               # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]. Use NODE_NAME key as the
+               # storage.
+               storage: ...,
+               # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]['address'].
+               address: ...,
+               # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]['token'].
+               token: ...,
+             },
+           ],
+         }
+       ]
+     }
+     ```
+
 ### Geo installations **(PREMIUM SELF)**
 
 Specific information applies to installations using Geo: