Add latest changes from gitlab-org/gitlab@master

16 files changed, 305 insertions, 62 deletions

diff --git a/doc/administration/geo/setup/database.md b/doc/administration/geo/setup/database.md
index a3b48476941..03908e6fc45 100644
--- a/doc/administration/geo/setup/database.md
+++ b/doc/administration/geo/setup/database.md
@@ -487,7 +487,7 @@ The replication process is now complete.
 PostgreSQL connections, which can improve performance even when using in a
 single instance installation.
 
-We recommend using PgBouncer if you use GitLab in a highly available  
+We recommend using PgBouncer if you use GitLab in a highly available
 configuration with a cluster of nodes supporting a Geo **primary** site and
 two other clusters of nodes supporting a Geo **secondary** site. One for the
 main database and the other for the tracking database. For more information,
@@ -550,12 +550,12 @@ Leader instance**:
 
    ```ruby
    roles(['patroni_role'])
-   
+
    consul['services'] = %w(postgresql)
    consul['configuration'] = {
      retry_join: %w[CONSUL_PRIMARY1_IP CONSUL_PRIMARY2_IP CONSUL_PRIMARY3_IP]
    }
-   
+
    # You need one entry for each secondary, with a unique name following PostgreSQL slot_name constraints:
    #
    # Configuration syntax is: 'unique_slotname' => { 'type' => 'physical' },
@@ -567,6 +567,8 @@ Leader instance**:
    patroni['use_pg_rewind'] = true
    patroni['postgresql']['max_wal_senders'] = 8 # Use double of the amount of patroni/reserved slots (3 patronis + 1 reserved slot for a Geo secondary).
    patroni['postgresql']['max_replication_slots'] = 8 # Use double of the amount of patroni/reserved slots (3 patronis + 1 reserved slot for a Geo secondary).
+   patroni['username'] = 'PATRONI_API_USERNAME'
+   patroni['password'] = 'PATRONI_API_PASSWORD'
    patroni['replication_password'] = 'PLAIN_TEXT_POSTGRESQL_REPLICATION_PASSWORD'
 
    # We list all secondary instances as they can all become a Standby Leader
@@ -727,16 +729,18 @@ For each Patroni instance on the secondary site:
    patroni['standby_cluster']['host'] = 'INTERNAL_LOAD_BALANCER_PRIMARY_IP'
    patroni['standby_cluster']['port'] = INTERNAL_LOAD_BALANCER_PRIMARY_PORT
    patroni['standby_cluster']['primary_slot_name'] = 'geo_secondary' # Or the unique replication slot name you setup before
+   patroni['username'] = 'PATRONI_API_USERNAME'
+   patroni['password'] = 'PATRONI_API_PASSWORD'
    patroni['replication_password'] = 'PLAIN_TEXT_POSTGRESQL_REPLICATION_PASSWORD'
    patroni['use_pg_rewind'] = true
    patroni['postgresql']['max_wal_senders'] = 5 # A minimum of three for one replica, plus two for each additional replica
    patroni['postgresql']['max_replication_slots'] = 5 # A minimum of three for one replica, plus two for each additional replica
-   
+
    postgresql['pgbouncer_user_password'] = 'PGBOUNCER_PASSWORD_HASH'
    postgresql['sql_replication_password'] = 'POSTGRESQL_REPLICATION_PASSWORD_HASH'
    postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH'
    postgresql['listen_address'] = '0.0.0.0' # You can use a public or VPC address here instead
-   
+
    gitlab_rails['dbpassword'] = 'POSTGRESQL_PASSWORD'
    gitlab_rails['enable'] = true
    gitlab_rails['auto_migrate'] = false
@@ -754,7 +758,7 @@ For each Patroni instance on the secondary site:
    - If you are configuring a Patroni standby cluster on a site that previously had a working Patroni cluster:
 
      ```shell
-     gitlab-ctl stop patroni 
+     gitlab-ctl stop patroni
      rm -rf /var/opt/gitlab/postgresql/data
      /opt/gitlab/embedded/bin/patronictl -c /var/opt/gitlab/patroni/patroni.yaml remove postgresql-ha
      gitlab-ctl reconfigure
@@ -900,6 +904,8 @@ For each Patroni instance on the secondary site for the tracking database:
    ]
 
    # Patroni configuration
+   patroni['username'] = 'PATRONI_API_USERNAME'
+   patroni['password'] = 'PATRONI_API_PASSWORD'
    patroni['replication_password'] = 'PLAIN_TEXT_POSTGRESQL_REPLICATION_PASSWORD'
    patroni['postgresql']['max_wal_senders'] = 5 # A minimum of three for one replica, plus two for each additional replica
 
diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md
index 4a9e525ead5..8b9ba55bd3c 100644
--- a/doc/administration/monitoring/prometheus/gitlab_metrics.md
+++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md
@@ -121,6 +121,7 @@ The following metrics are available:
 | `action_cable_single_client_transmissions_total`                 | Counter     | 13.10   | The number of ActionCable messages transmitted to any client in any channel                                           | `server_mode`                                             |
 | `action_cable_subscription_confirmations_total`                  | Counter     | 13.10   | The number of ActionCable subscriptions from clients confirmed                                                        | `server_mode`                                             |
 | `action_cable_subscription_rejections_total`                     | Counter     | 13.10   | The number of ActionCable subscriptions from clients rejected                                                         | `server_mode`                                             |
+| `action_cable_transmitted_bytes`                | Histogram     | 14.1   | Message size, in bytes, transmitted over action cable                                           | `operation`, `channel` |
 | `gitlab_issuable_fast_count_by_state_total`                      | Counter     | 13.5    | Total number of row count operations on issue/merge request list pages                                                |                                                           |
 | `gitlab_issuable_fast_count_by_state_failures_total`             | Counter     | 13.5    | Number of soft-failed row count operations on issue/merge request list pages                                          |                                                           |
 | `gitlab_external_http_total`                                     | Counter     | 13.8    | Total number of HTTP calls to external systems                                                                        | `controller`, `action`                                    |
diff --git a/doc/administration/postgresql/replication_and_failover.md b/doc/administration/postgresql/replication_and_failover.md
index 3c10a245c3f..870b1f6aea2 100644
--- a/doc/administration/postgresql/replication_and_failover.md
+++ b/doc/administration/postgresql/replication_and_failover.md
@@ -157,6 +157,13 @@ We will need the following password information for the application's database u
   sudo gitlab-ctl pg-password-md5 POSTGRESQL_USERNAME
   ```
 
+#### Patroni information
+
+We will need the following password information for the Patroni API:
+
+- `PATRONI_API_USERNAME`. A username for basic auth to the API
+- `PATRONI_API_PASSWORD`. A password for basic auth to the API
+
 #### PgBouncer information
 
 When using default setup, minimum configuration requires:
@@ -236,6 +243,11 @@ postgresql['sql_replication_password'] = 'POSTGRESQL_REPLICATION_PASSWORD_HASH'
 # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value
 postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH'
 
+# Replace PATRONI_API_USERNAME with a username for Patroni Rest API calls (use the same username in all nodes)
+patroni['username'] = 'PATRONI_API_USERNAME'
+# Replace PATRONI_API_PASSWORD with a password for Patroni Rest API calls (use the same password in all nodes)
+patroni['password'] = 'PATRONI_API_PASSWORD'
+
 # Sets `max_replication_slots` to double the number of database nodes.
 # Patroni uses one extra slot per node when initiating the replication.
 patroni['postgresql']['max_replication_slots'] = X
@@ -246,7 +258,7 @@ patroni['postgresql']['max_replication_slots'] = X
 patroni['postgresql']['max_wal_senders'] = X+1
 
 # Replace XXX.XXX.XXX.XXX/YY with Network Address
-postgresql['trust_auth_cidr_addresses'] = %w(XXX.XXX.XXX.XXX/YY)
+postgresql['trust_auth_cidr_addresses'] = %w(XXX.XXX.XXX.XXX/YY 127.0.0.1/32)
 
 # Replace placeholders:
 #
@@ -259,8 +271,8 @@ consul['configuration'] = {
 # END user configuration
 ```
 
-You do not need an additional or different configuration for replica nodes. As a matter of fact, you don't have to have
-a predetermined primary node. Therefore all database nodes use the same configuration.
+All database nodes use the same configuration. The leader node is not determined in configuration,
+and there is no additional or different configuration for either leader or replica nodes.
 
 Once the configuration of a node is done, you must [reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure)
 on each node for the changes to take effect.
@@ -555,10 +567,12 @@ gitlab_rails['auto_migrate'] = false
 
 postgresql['pgbouncer_user_password'] = '771a8625958a529132abe6f1a4acb19c'
 postgresql['sql_user_password'] = '450409b85a0223a214b5fb1484f34d0f'
+patroni['username'] = 'PATRONI_API_USERNAME'
+patroni['password'] = 'PATRONI_API_PASSWORD'
 patroni['postgresql']['max_replication_slots'] = 6
 patroni['postgresql']['max_wal_senders'] = 7
 
-postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/16)
+postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/16 127.0.0.1/32)
 
 # Configure the Consul agent
 consul['services'] = %w(postgresql)
@@ -642,12 +656,15 @@ postgresql['sql_user_password'] = '450409b85a0223a214b5fb1484f34d0f'
 # Patroni uses one extra slot per node when initiating the replication.
 patroni['postgresql']['max_replication_slots'] = 6
 
+patroni['username'] = 'PATRONI_API_USERNAME'
+patroni['password'] = 'PATRONI_API_PASSWORD'
+
 # Set `max_wal_senders` to one more than the number of replication slots in the cluster.
 # This is used to prevent replication from using up all of the
 # available database connections.
 patroni['postgresql']['max_wal_senders'] = 7
 
-postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/16)
+postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/16 127.0.0.1/32)
 
 consul['configuration'] = {
   server: true,
@@ -721,6 +738,97 @@ functional or does not have a leader, Patroni and by extension PostgreSQL will n
 API which can be accessed via its [default port](https://docs.gitlab.com/omnibus/package-information/defaults.html#patroni)
 on each node.
 
+### Check replication status
+
+Run `gitlab-ctl patroni members` to query Patroni for a summary of the cluster status:
+
+```plaintext
++ Cluster: postgresql-ha (6970678148837286213) ------+---------+---------+----+-----------+
+| Member                              | Host         | Role    | State   | TL | Lag in MB |
++-------------------------------------+--------------+---------+---------+----+-----------+
+| gitlab-database-1.example.com       | 172.18.0.111 | Replica | running |  5 |         0 |
+| gitlab-database-2.example.com       | 172.18.0.112 | Replica | running |  5 |       100 |
+| gitlab-database-3.example.com       | 172.18.0.113 | Leader  | running |  5 |           |
++-------------------------------------+--------------+---------+---------+----+-----------+
+```
+
+To verify the status of replication:
+
+```shell
+echo 'select * from pg_stat_wal_receiver\x\g\x \n select * from pg_stat_replication\x\g\x' | gitlab-psql
+```
+
+The same command can be run on all three database servers, and will return any information
+about replication available depending on the role the server is performing.
+
+The leader should return one record per replica:
+
+```sql
+-[ RECORD 1 ]----+------------------------------
+pid              | 371
+usesysid         | 16384
+usename          | gitlab_replicator
+application_name | gitlab-database-1.example.com
+client_addr      | 172.18.0.111
+client_hostname  |
+client_port      | 42900
+backend_start    | 2021-06-14 08:01:59.580341+00
+backend_xmin     |
+state            | streaming
+sent_lsn         | 0/EA13220
+write_lsn        | 0/EA13220
+flush_lsn        | 0/EA13220
+replay_lsn       | 0/EA13220
+write_lag        |
+flush_lag        |
+replay_lag       |
+sync_priority    | 0
+sync_state       | async
+reply_time       | 2021-06-18 19:17:14.915419+00
+```
+
+Investigate further if:
+
+- There are missing or extra records.
+- `reply_time` is not current.
+
+The `lsn` fields relate to which write-ahead-log segments have been replicated.
+Run the following on the leader to find out the current LSN:
+
+```shell
+echo 'SELECT pg_current_wal_lsn();' | gitlab-psql
+```
+
+If a replica is not in sync, `gitlab-ctl patroni members` indicates the volume
+of missing data, and the `lag` fields indicate the elapsed time.
+
+Read more about the data returned by the leader
+[in the PostgreSQL documentation](https://www.postgresql.org/docs/12/monitoring-stats.html#PG-STAT-REPLICATION-VIEW),
+including other values for the `state` field.
+
+The replicas should return:
+
+```sql
+-[ RECORD 1 ]---------+-------------------------------------------------------------------------------------------------
+pid                   | 391
+status                | streaming
+receive_start_lsn     | 0/D000000
+receive_start_tli     | 5
+received_lsn          | 0/EA13220
+received_tli          | 5
+last_msg_send_time    | 2021-06-18 19:16:54.807375+00
+last_msg_receipt_time | 2021-06-18 19:16:54.807512+00
+latest_end_lsn        | 0/EA13220
+latest_end_time       | 2021-06-18 19:07:23.844879+00
+slot_name             | gitlab-database-1.example.com
+sender_host           | 172.18.0.113
+sender_port           | 5432
+conninfo              | user=gitlab_replicator host=172.18.0.113 port=5432 application_name=gitlab-database-1.example.com
+```
+
+Read more about the data returned by the replica
+[in the PostgreSQL documentation](https://www.postgresql.org/docs/12/monitoring-stats.html#PG-STAT-WAL-RECEIVER-VIEW).
+
 ### Selecting the appropriate Patroni replication method
 
 [Review the Patroni documentation carefully](https://patroni.readthedocs.io/en/latest/SETTINGS.html#postgresql)
@@ -1017,6 +1125,29 @@ postgresql['trust_auth_cidr_addresses'] = %w(123.123.123.123/32 <other_cidrs>)
 
 [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect.
 
+### Reinitialize a replica
+
+If replication is not occurring, it may be necessary to reinitialize a replica.
+
+1. On any server in the cluster, determine the Cluster and Member names,
+   and check the replication lag by running `gitlab-ctl patroni members`. Here is an example:
+
+   ```plaintext
+   + Cluster: postgresql-ha (6970678148837286213) ------+---------+---------+----+-----------+
+   | Member                              | Host         | Role    | State   | TL | Lag in MB |
+   +-------------------------------------+--------------+---------+---------+----+-----------+
+   | gitlab-database-1.example.com       | 172.18.0.111 | Replica | running |  5 |         0 |
+   | gitlab-database-2.example.com       | 172.18.0.112 | Replica | running |  5 |       100 |
+   | gitlab-database-3.example.com       | 172.18.0.113 | Leader  | running |  5 |           |
+   +-------------------------------------+--------------+---------+---------+----+-----------+
+   ```
+
+1. Reinitialize the affected replica server:
+
+   ```plaintext
+   gitlab-ctl patroni reinitialize-replica postgresql-ha gitlab-database-2.example.com
+   ```
+
 ### Reset the Patroni state in Consul
 
 WARNING:
@@ -1058,6 +1189,70 @@ To reset the Patroni state in Consul:
 
 If you are still seeing issues, the next step is restoring the last healthy backup.
 
+### Errors in the Patroni log about a `pg_hba.conf` entry for `127.0.0.1`
+
+The following log entry in the Patroni log indicates the replication is not working
+and a configuration change is needed:
+
+```plaintext
+FATAL:  no pg_hba.conf entry for replication connection from host "127.0.0.1", user "gitlab_replicator"
+```
+
+To fix the problem, ensure the loopback interface is included in the CIDR addresses list:
+
+1. Edit `/etc/gitlab/gitlab.rb`:
+
+   ```ruby
+   postgresql['trust_auth_cidr_addresses'] = %w(<other_cidrs> 127.0.0.1/32)
+   ```
+
+1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect.
+1. Check that [all the replicas are synchronized](#check-replication-status)
+
+### Errors in Patroni logs: the requested start point is ahead of the WAL flush position
+
+This error indicates that the database is not replicating:
+
+```plaintext
+FATAL:  could not receive data from WAL stream: ERROR:  requested starting point 0/5000000 is ahead of the WAL flush position of this server 0/4000388
+```
+
+This example error is from a replica that was initially misconfigured, and had never replicated.
+
+Fix it [by reinitializing the replica](#reinitialize-a-replica).
+
+### Patroni fails to start with `MemoryError`
+
+Patroni may fail to start, logging an error and stack trace:
+
+```plaintext
+MemoryError
+Traceback (most recent call last):
+  File "/opt/gitlab/embedded/bin/patroni", line 8, in <module>
+    sys.exit(main())
+[..]
+  File "/opt/gitlab/embedded/lib/python3.7/ctypes/__init__.py", line 273, in _reset_cache
+    CFUNCTYPE(c_int)(lambda: None)
+```
+
+If the stack trace ends with `CFUNCTYPE(c_int)(lambda: None)`, this code triggers `MemoryError`
+if the Linux server has been hardened for security.
+
+The code causes Python to write temporary executable files, and if it cannot find a filesystem
+in which to do this, for example if `noexec` is set on the `/tmp` filesystem, it fails with
+`MemoryError` ([read more in the issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6184)).
+
+Workarounds:
+
+- Remove `noexec` from the mount options for filesystems like `/tmp` and `/var/tmp`.
+- If set to enforcing, SELinux may also prevent these operations. Verify the issue is fixed by setting
+  SELinux to permissive.
+
+Omnibus GitLab has shipped with Patroni since 13.1 along with a build of Python 3.7.
+Workarounds should stop being required when GitLab 14.x starts shipping with
+[a later version of Python](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6164) as
+the code which causes this was removed from Python 3.8.
+
 ### Issues with other components
 
 If you're running into an issue with a component not outlined here, be sure to check the troubleshooting section of their specific documentation page:
diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md
index e554f5c3693..862f35f4b5c 100644
--- a/doc/administration/reference_architectures/10k_users.md
+++ b/doc/administration/reference_architectures/10k_users.md
@@ -598,8 +598,12 @@ in the second step, do not supply the `EXTERNAL_URL` value.
    # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value
    postgresql['sql_user_password'] = '<postgresql_password_hash>'
 
+   # Set up basic authentication for the Patroni API (use the same username/password in all nodes).
+   patroni['username'] = '<patroni_api_username>'
+   patroni['password'] = '<patroni_api_password>'
+
    # Replace XXX.XXX.XXX.XXX/YY with Network Address
-   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24)
+   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32)
 
    # Set the network addresses that the exporters will listen on for monitoring
    node_exporter['listen_address'] = '0.0.0.0:9100'
@@ -1403,7 +1407,7 @@ in the second step, do not supply the `EXTERNAL_URL` value.
    postgresql['sql_user_password'] = "<praefect_postgresql_password_hash>"
 
    # Replace XXX.XXX.XXX.XXX/YY with Network Address
-   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24)
+   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32)
 
    # Set the network addresses that the exporters will listen on for monitoring
    node_exporter['listen_address'] = '0.0.0.0:9100'
@@ -1681,7 +1685,7 @@ On each node:
    # balancer.
    gitlab_rails['internal_api_url'] = 'https://gitlab.example.com'
 
-   # Gitaly 
+   # Gitaly
    gitaly['enable'] = true
 
    # Make Gitaly accept connections on all network interfaces. You must use
diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md
index 920ed5a7652..76477a6ffb2 100644
--- a/doc/administration/reference_architectures/25k_users.md
+++ b/doc/administration/reference_architectures/25k_users.md
@@ -600,8 +600,12 @@ in the second step, do not supply the `EXTERNAL_URL` value.
    # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value
    postgresql['sql_user_password'] = '<postgresql_password_hash>'
 
+   # Set up basic authentication for the Patroni API (use the same username/password in all nodes).
+   patroni['username'] = '<patroni_api_username>'
+   patroni['password'] = '<patroni_api_password>'
+
    # Replace XXX.XXX.XXX.XXX/YY with Network Address
-   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24)
+   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32)
 
    # Set the network addresses that the exporters will listen on for monitoring
    node_exporter['listen_address'] = '0.0.0.0:9100'
@@ -863,7 +867,7 @@ a node and change its status from primary to replica (and vice versa).
    redis_exporter['flags'] = {
         'redis.addr' => 'redis://10.6.0.51:6379',
         'redis.password' => 'redis-password-goes-here',
-   }   
+   }
 
    # Prevent database migrations from running on upgrade automatically
    gitlab_rails['auto_migrate'] = false
@@ -1421,7 +1425,7 @@ in the second step, do not supply the `EXTERNAL_URL` value.
    postgresql['sql_user_password'] = "<praefect_postgresql_password_hash>"
 
    # Replace XXX.XXX.XXX.XXX/YY with Network Address
-   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24)
+   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32)
 
    # Set the network addresses that the exporters will listen on for monitoring
    node_exporter['listen_address'] = '0.0.0.0:9100'
@@ -1699,7 +1703,7 @@ On each node:
    # balancer.
    gitlab_rails['internal_api_url'] = 'https://gitlab.example.com'
 
-   # Gitaly 
+   # Gitaly
    gitaly['enable'] = true
 
    # Make Gitaly accept connections on all network interfaces. You must use
diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md
index 71ca67075d3..655afc58343 100644
--- a/doc/administration/reference_architectures/3k_users.md
+++ b/doc/administration/reference_architectures/3k_users.md
@@ -848,7 +848,7 @@ in the second step, do not supply the `EXTERNAL_URL` value.
    ```ruby
    # Disable all components except Patroni and Consul
    roles(['patroni_role'])
-   
+
    # PostgreSQL configuration
    postgresql['listen_address'] = '0.0.0.0'
 
@@ -866,7 +866,7 @@ in the second step, do not supply the `EXTERNAL_URL` value.
 
    # Prevent database migrations from running on upgrade automatically
    gitlab_rails['auto_migrate'] = false
-   
+
    # Configure the Consul agent
    consul['services'] = %w(postgresql)
    ## Enable service discovery for Prometheus
@@ -882,8 +882,12 @@ in the second step, do not supply the `EXTERNAL_URL` value.
    # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value
    postgresql['sql_user_password'] = '<postgresql_password_hash>'
 
+   # Set up basic authentication for the Patroni API (use the same username/password in all nodes).
+   patroni['username'] = '<patroni_api_username>'
+   patroni['password'] = '<patroni_api_password>'
+
    # Replace XXX.XXX.XXX.XXX/YY with Network Address
-   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24)
+   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32)
 
    # Set the network addresses that the exporters will listen on for monitoring
    node_exporter['listen_address'] = '0.0.0.0:9100'
@@ -1127,7 +1131,7 @@ in the second step, do not supply the `EXTERNAL_URL` value.
    postgresql['sql_user_password'] = "<praefect_postgresql_password_hash>"
 
    # Replace XXX.XXX.XXX.XXX/YY with Network Address
-   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24)
+   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32)
 
    # Set the network addresses that the exporters will listen on for monitoring
    node_exporter['listen_address'] = '0.0.0.0:9100'
diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md
index 3b3b1cf8ced..3b362f91a61 100644
--- a/doc/administration/reference_architectures/50k_users.md
+++ b/doc/administration/reference_architectures/50k_users.md
@@ -608,8 +608,12 @@ in the second step, do not supply the `EXTERNAL_URL` value.
    # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value
    postgresql['sql_user_password'] = '<postgresql_password_hash>'
 
+   # Set up basic authentication for the Patroni API (use the same username/password in all nodes).
+   patroni['username'] = '<patroni_api_username>'
+   patroni['password'] = '<patroni_api_password>'
+
    # Replace XXX.XXX.XXX.XXX/YY with Network Address
-   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24)
+   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32)
 
    # Set the network addresses that the exporters will listen on for monitoring
    node_exporter['listen_address'] = '0.0.0.0:9100'
@@ -872,7 +876,7 @@ a node and change its status from primary to replica (and vice versa).
         'redis.addr' => 'redis://10.6.0.51:6379',
         'redis.password' => 'redis-password-goes-here',
    }
-   
+
    # Prevent database migrations from running on upgrade automatically
    gitlab_rails['auto_migrate'] = false
    ```
@@ -1425,7 +1429,7 @@ in the second step, do not supply the `EXTERNAL_URL` value.
    postgresql['sql_user_password'] = "<praefect_postgresql_password_hash>"
 
    # Replace XXX.XXX.XXX.XXX/YY with Network Address
-   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24)
+   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32)
 
    # Set the network addresses that the exporters will listen on for monitoring
    node_exporter['listen_address'] = '0.0.0.0:9100'
@@ -1703,7 +1707,7 @@ On each node:
    # balancer.
    gitlab_rails['internal_api_url'] = 'https://gitlab.example.com'
 
-   # Gitaly 
+   # Gitaly
    gitaly['enable'] = true
 
    # Make Gitaly accept connections on all network interfaces. You must use
@@ -1929,7 +1933,7 @@ To configure the Sidekiq nodes, on each one:
    ## Set number of Sidekiq threads per queue process to the recommend number of 10
    sidekiq['max_concurrency'] = 10
 
-   # Monitoring 
+   # Monitoring
    consul['enable'] = true
    consul['monitoring_service_discovery'] =  true
 
diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md
index b5001068945..960f7148eda 100644
--- a/doc/administration/reference_architectures/5k_users.md
+++ b/doc/administration/reference_architectures/5k_users.md
@@ -844,7 +844,7 @@ in the second step, do not supply the `EXTERNAL_URL` value.
    # Sets `max_replication_slots` to double the number of database nodes.
    # Patroni uses one extra slot per node when initiating the replication.
    patroni['postgresql']['max_replication_slots'] = 8
-   
+
    # Set `max_wal_senders` to one more than the number of replication slots in the cluster.
    # This is used to prevent replication from using up all of the
    # available database connections.
@@ -871,8 +871,12 @@ in the second step, do not supply the `EXTERNAL_URL` value.
    # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value
    postgresql['sql_user_password'] = '<postgresql_password_hash>'
 
+   # Set up basic authentication for the Patroni API (use the same username/password in all nodes).
+   patroni['username'] = '<patroni_api_username>'
+   patroni['password'] = '<patroni_api_password>'
+
    # Replace XXX.XXX.XXX.XXX/YY with Network Address
-   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24)
+   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32)
 
    # Set the network addresses that the exporters will listen on for monitoring
    node_exporter['listen_address'] = '0.0.0.0:9100'
@@ -1116,7 +1120,7 @@ in the second step, do not supply the `EXTERNAL_URL` value.
    postgresql['sql_user_password'] = "<praefect_postgresql_password_hash>"
 
    # Replace XXX.XXX.XXX.XXX/YY with Network Address
-   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24)
+   postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32)
 
    # Set the network addresses that the exporters will listen on for monitoring
    node_exporter['listen_address'] = '0.0.0.0:9100'
diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md
index 4a0572569a3..de07d601c4f 100644
--- a/doc/api/graphql/reference/index.md
+++ b/doc/api/graphql/reference/index.md
@@ -15290,6 +15290,7 @@ Name of the feature that the callout is for.
 | <a id="usercalloutfeaturenameenumpipeline_needs_banner"></a>`PIPELINE_NEEDS_BANNER` | Callout feature name for pipeline_needs_banner. |
 | <a id="usercalloutfeaturenameenumpipeline_needs_hover_tip"></a>`PIPELINE_NEEDS_HOVER_TIP` | Callout feature name for pipeline_needs_hover_tip. |
 | <a id="usercalloutfeaturenameenumregistration_enabled_callout"></a>`REGISTRATION_ENABLED_CALLOUT` | Callout feature name for registration_enabled_callout. |
+| <a id="usercalloutfeaturenameenumsecurity_configuration_devops_alert"></a>`SECURITY_CONFIGURATION_DEVOPS_ALERT` | Callout feature name for security_configuration_devops_alert. |
 | <a id="usercalloutfeaturenameenumsecurity_configuration_upgrade_banner"></a>`SECURITY_CONFIGURATION_UPGRADE_BANNER` | Callout feature name for security_configuration_upgrade_banner. |
 | <a id="usercalloutfeaturenameenumservice_templates_deprecated_callout"></a>`SERVICE_TEMPLATES_DEPRECATED_CALLOUT` | Callout feature name for service_templates_deprecated_callout. |
 | <a id="usercalloutfeaturenameenumsuggest_pipeline"></a>`SUGGEST_PIPELINE` | Callout feature name for suggest_pipeline. |
diff --git a/doc/api/index.md b/doc/api/index.md
index f1059904ac3..d80f4ecf19b 100644
--- a/doc/api/index.md
+++ b/doc/api/index.md
@@ -248,13 +248,13 @@ tries to steal tokens from other jobs.
 #### Limit GitLab CI/CD job token access
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/328553) in GitLab 14.1.
-> - [Deployed behind a feature flag](../user/feature_flags.md), disabled by default.
-> - Disabled on GitLab.com.
-> - Not recommended for production use.
-> - To use in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-ci-job-token-scope-limit). **(FREE SELF)**
+> - [Deployed behind a feature flag](../user/feature_flags.md), enabled by default.
+> - Enabled on GitLab.com.
+> - Recommended for production use.
+> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-ci-job-token-scope-limit). **(FREE SELF)**
 
-This in-development feature might not be available for your use. There can be
-[risks when enabling features still in development](../user/feature_flags.md#risks-when-enabling-features-still-in-development).
+There can be
+[risks when disabling released features](../user/feature_flags.md#risks-when-disabling-released-features).
 Refer to this feature's version history for more details.
 
 You can limit the access scope of a project's CI/CD job token to increase the
@@ -292,21 +292,21 @@ the feature with more strategic control of the access permissions.
 
 ##### Enable or disable CI job token scope limit **(FREE SELF)**
 
-The GitLab CI/CD job token access scope limit is under development and not ready for production
-use. It is deployed behind a feature flag that is **disabled by default**.
+The GitLab CI/CD job token access scope limit is under development but ready for production
+use. It is deployed behind a feature flag that is **enabled by default**.
 [GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md)
-can enable it.
+can disable the feature.
 
-To enable it:
+To disable it:
 
 ```ruby
-Feature.enable(:ci_scoped_job_token)
+Feature.disable(:ci_scoped_job_token)
 ```
 
-To disable it:
+To enable it:
 
 ```ruby
-Feature.disable(:ci_scoped_job_token)
+Feature.enable(:ci_scoped_job_token)
 ```
 
 ### Impersonation tokens
diff --git a/doc/ci/pipeline_editor/index.md b/doc/ci/pipeline_editor/index.md
index 96df93848dc..7132d47d324 100644
--- a/doc/ci/pipeline_editor/index.md
+++ b/doc/ci/pipeline_editor/index.md
@@ -85,9 +85,6 @@ where:
   [extended configuration merged into the job](../yaml/index.md#merge-details).
 - YAML anchors are [replaced with the linked configuration](../yaml/index.md#anchors).
 
-NOTE:
-You can only see the expanded view when editing the [default branch](../../user/project/repository/branches/default.md).
-
 ## Commit changes to CI configuration
 
 The commit form appears at the bottom of each tab in the editor so you can commit
diff --git a/doc/subscriptions/bronze_starter.md b/doc/subscriptions/bronze_starter.md
index 7d3af5979fb..410759aa506 100644
--- a/doc/subscriptions/bronze_starter.md
+++ b/doc/subscriptions/bronze_starter.md
@@ -16,7 +16,7 @@ The following features remain available to Bronze and Starter customers, even th
 the tiers are no longer mentioned in GitLab documentation:
 
 - [Activate GitLab EE with a license](../user/admin_area/license.md)
-- [Adding a help message to the login page](../user/admin_area/settings/help_page.md#adding-a-help-message-to-the-login-page)
+- [Add a help message to the sign-in page](../user/admin_area/settings/help_page.md#add-a-help-message-to-the-sign-in-page)
 - [Burndown and burnup charts](../user/project/milestones/burndown_and_burnup_charts.md),
   including [per-project charts](../user/project/milestones/index.md#project-burndown-charts) and
   [per-group charts](../user/project/milestones/index.md#group-burndown-charts)
diff --git a/doc/user/admin_area/settings/help_page.md b/doc/user/admin_area/settings/help_page.md
index e11527db49a..d2f99a51ec3 100644
--- a/doc/user/admin_area/settings/help_page.md
+++ b/doc/user/admin_area/settings/help_page.md
@@ -5,41 +5,58 @@ info: To determine the technical writer assigned to the Stage/Group associated w
 type: howto
 ---
 
-# Customizing the 'Help' and login page messages
+# Customize the Help and sign-in page messages
 
 In large organizations, it is useful to have information about who to contact or where
-to go for help. You can customize and display this information on the GitLab server's
-`/help` page and on the GitLab login page.
+to go for help. You can customize and display this information on the GitLab  `/help` page and on
+the GitLab sign-in page.
 
-## Adding a help message to the help page
+## Add a help message to the Help page
 
-You can add a help message, which is shown on the GitLab `/help` page (for example,
-<https://gitlab.com/help>) in a new section at the top of the `/help` page:
+You can add a help message, which is shown at the top of the GitLab `/help` page (for example,
+<https://gitlab.com/help>):
 
 1. On the top bar, select **Menu >** **{admin}** **Admin**.
 1. In the left sidebar, select **Settings > Preferences**, then expand **Help page**.
-1. Under **Help page text**, fill in the information you wish to display on `/help`.
-1. Save your changes. You can now see the message on `/help`.
+1. Under **Additional text to show on the Help page**, fill in the information you wish to display on `/help`.
+1. Select **Save changes**. You can now see the message on `/help`.
 
 NOTE:
 By default, `/help` is visible to unauthenticated users. However, if the
 [**Public** visibility level](visibility_and_access_controls.md#restricted-visibility-levels)
 is restricted, `/help` is visible only to signed-in users.
 
-## Adding a help message to the login page **(STARTER)**
+## Add a help message to the sign-in page **(STARTER)**
 
-You can add a help message, which is shown on the GitLab login page in a new section
-titled `Need Help?`, located below the login page message:
+You can add a help message, which is shown on the GitLab sign-in page. The message appears in a new
+section titled **Need Help?**, located below the sign-in page message:
 
 1. On the top bar, select **Menu >** **{admin}** **Admin**.
 1. In the left sidebar, select **Settings > Preferences**, then expand **Help page**.
-1. Under **Help text**, fill in the information you wish to display on the login page.
+1. Under **Additional text to show on the sign-in page**, fill in the information you wish to
+   display on the sign-in page.
+1. Select **Save changes**. You can now see the message on the sign-in page.
 
-   ![help message on login page](img/help_page_help_text_v12_3.png)
+## Hide marketing-related entries from the Help page
 
-1. Save your changes.
+GitLab marketing-related entries are occasionally shown on the Help page. To hide these entries:
 
-![help message on login page example](img/help_page_help_text_ex_v12_3.png)
+1. On the top bar, select **Menu >** **{admin}** **Admin**.
+1. In the left sidebar, select **Settings > Preferences**, then expand **Help page**.
+1. Select the **Hide marketing-related entries from the Help page** checkbox.
+1. Select **Save changes**.
+
+## Set a custom Support page URL
+
+You can specify a custom URL to which users are directed when they:
+
+- Select **Support** from the Help dropdown.
+- Select **See our website for help** on the Help page.
+
+1. On the top bar, select **Menu >** **{admin}** **Admin**.
+1. In the left sidebar, select **Settings > Preferences**, then expand **Help page**.
+1. Enter the URL in the **Support page URL** field.
+1. Select **Save changes**.
 
 <!-- ## Troubleshooting
 
diff --git a/doc/user/admin_area/settings/img/help_page_help_text_ex_v12_3.png b/doc/user/admin_area/settings/img/help_page_help_text_ex_v12_3.png
deleted file mode 100644
index 973be2e8b6e..00000000000
--- a/doc/user/admin_area/settings/img/help_page_help_text_ex_v12_3.png
+++ /dev/null
diff --git a/doc/user/admin_area/settings/img/help_page_help_text_v12_3.png b/doc/user/admin_area/settings/img/help_page_help_text_v12_3.png
deleted file mode 100644
index 8848ea55cf3..00000000000
--- a/doc/user/admin_area/settings/img/help_page_help_text_v12_3.png
+++ /dev/null
diff --git a/doc/user/search/advanced_search.md b/doc/user/search/advanced_search.md
index 20a2a7263c3..bb74a035121 100644
--- a/doc/user/search/advanced_search.md
+++ b/doc/user/search/advanced_search.md
@@ -12,6 +12,7 @@ type: reference
 NOTE:
 This is the user documentation. To configure the Advanced Search,
 visit the [administrator documentation](../../integration/elasticsearch.md).
+Advanced Search is enabled in GitLab.com.
 
 GitLab Advanced Search expands on the Basic Search with an additional set of
 features for faster, more advanced searches across the entire GitLab instance
@@ -34,6 +35,11 @@ The Advanced Search can be useful in various scenarios:
   Advanced Search is based on Elasticsearch, which is a purpose-built full
   text search engine that can be horizontally scaled so that it can provide
   search results in 1-2 seconds in most cases.
+- **Code Maintenance:** 
+  Finding all the code that needs to be updated at once across an entire 
+  instance can save time spent maintaining code. 
+  This is especially helpful for organizations with more than 10 active projects.
+  This can also help build confidence is code refactoring to identify unknown impacts.
 - **Promote innersourcing:**
   Your company may consist of many different developer teams each of which has
   their own group where the various projects are hosted. Some of your applications