Add latest changes from gitlab-org/gitlab@master

7 files changed, 129 insertions, 24 deletions

diff --git a/db/post_migrate/20221115184525_remove_namespaces_tmp_project_id_column.rb b/db/post_migrate/20221115184525_remove_namespaces_tmp_project_id_column.rb
new file mode 100644
index 00000000000..01424f8113f
--- /dev/null
+++ b/db/post_migrate/20221115184525_remove_namespaces_tmp_project_id_column.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class RemoveNamespacesTmpProjectIdColumn < Gitlab::Database::Migration[2.0]
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'tmp_index_on_tmp_project_id_on_namespaces'
+
+  def up
+    with_lock_retries do
+      remove_column :namespaces, :tmp_project_id if column_exists?(:namespaces, :tmp_project_id)
+    end
+  end
+
+  def down
+    unless column_exists?(:namespaces, :tmp_project_id)
+      with_lock_retries do
+        # rubocop:disable Migration/SchemaAdditionMethodsNoPost, Migration/AddColumnsToWideTables
+        add_column :namespaces, :tmp_project_id, :integer
+        # rubocop:enable Migration/SchemaAdditionMethodsNoPost, Migration/AddColumnsToWideTables
+      end
+    end
+
+    add_concurrent_foreign_key :namespaces, :projects, column: :tmp_project_id
+
+    add_concurrent_index :namespaces, :tmp_project_id, name: INDEX_NAME, unique: true
+  end
+end
diff --git a/db/schema_migrations/20221115184525 b/db/schema_migrations/20221115184525
new file mode 100644
index 00000000000..da6620d281c
--- /dev/null
+++ b/db/schema_migrations/20221115184525
@@ -0,0 +1 @@
+7e181636d6fd40eb13a40f596ab442ea7b795748546b58a33b1475b2d5fcd264
\ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index 177b0f158a5..a6d4df68745 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -18173,8 +18173,7 @@ CREATE TABLE namespaces (
     push_rule_id bigint,
     shared_runners_enabled boolean DEFAULT true NOT NULL,
     allow_descendants_override_disabled_shared_runners boolean DEFAULT false NOT NULL,
-    traversal_ids integer[] DEFAULT '{}'::integer[] NOT NULL,
-    tmp_project_id integer
+    traversal_ids integer[] DEFAULT '{}'::integer[] NOT NULL
 );
 
 CREATE SEQUENCE namespaces_id_seq
@@ -31407,8 +31406,6 @@ CREATE INDEX tmp_index_members_on_state ON members USING btree (state) WHERE (st
 
 CREATE INDEX tmp_index_migrated_container_registries ON container_repositories USING btree (project_id) WHERE ((migration_state = 'import_done'::text) OR (created_at >= '2022-01-23 00:00:00'::timestamp without time zone));
 
-CREATE UNIQUE INDEX tmp_index_on_tmp_project_id_on_namespaces ON namespaces USING btree (tmp_project_id);
-
 CREATE INDEX tmp_index_on_vulnerabilities_non_dismissed ON vulnerabilities USING btree (id) WHERE (state <> 2);
 
 CREATE INDEX tmp_index_project_statistics_cont_registry_size ON project_statistics USING btree (project_id) WHERE (container_registry_size = 0);
@@ -33160,9 +33157,6 @@ ALTER TABLE ONLY project_wiki_repository_states
 ALTER TABLE ONLY merge_requests
     ADD CONSTRAINT fk_6a5165a692 FOREIGN KEY (milestone_id) REFERENCES milestones(id) ON DELETE SET NULL;
 
-ALTER TABLE ONLY namespaces
-    ADD CONSTRAINT fk_6a77f66919 FOREIGN KEY (tmp_project_id) REFERENCES projects(id) ON DELETE CASCADE;
-
 ALTER TABLE ONLY geo_event_log
     ADD CONSTRAINT fk_6ada82d42a FOREIGN KEY (container_repository_updated_event_id) REFERENCES geo_container_repository_updated_events(id) ON DELETE CASCADE;
 
diff --git a/doc/administration/gitaly/configure_gitaly.md b/doc/administration/gitaly/configure_gitaly.md
index 17550550062..dd814b7436e 100644
--- a/doc/administration/gitaly/configure_gitaly.md
+++ b/doc/administration/gitaly/configure_gitaly.md
@@ -753,14 +753,25 @@ settings:
 
 ## Limit RPC concurrency
 
-Clone traffic can put a large strain on your Gitaly service. The bulk of the work gets done in the
-either of the following RPCs:
+WARNING:
+Enabling limits on your environment should be done with caution and only
+in select circumstances, such as to protect against unexpected traffic.
+When reached, limits _do_ result in disconnects that negatively impact users.
+For consistent and stable performance, you should first explore other options such as
+adjusting node specifications, and [reviewing large repositories](../../user/project/repository/managing_large_repositories.md) or workloads.
+
+When cloning or pulling repositories, various RPCs run in the background. In particular, the Git pack RPCs:
 
 - `SSHUploadPackWithSidechannel` (for Git SSH).
 - `PostUploadPackWithSidechannel` (for Git HTTP).
 
-To prevent such workloads from overwhelming your Gitaly server, you can set concurrency limits in
-Gitaly's configuration file. For example:
+These RPCs can consume a large amount of resources, which can have a significant impact in situations such as:
+
+- Unexpectedly high traffic.
+- Running against [large repositories](../../user/project/repository/managing_large_repositories.md) that don't follow best practices.
+
+You can limit these processes from overwhelming your Gitaly server in these scenarios using the concurrency limits in Gitaly's configuration file. For
+example:
 
 ```ruby
 # in /etc/gitlab/gitlab.rb
@@ -798,30 +809,40 @@ repository. In the example above:
 - If a request waits in the queue for more than 1 second, it is rejected with an error.
 - If the queue grows beyond 10, subsequent requests are rejected with an error.
 
+NOTE:
+When these limits are reached, users are disconnected.
+
 You can observe the behavior of this queue using the Gitaly logs and Prometheus. For more
 information, see the [relevant documentation](monitoring.md#monitor-gitaly-concurrency-limiting).
 
 ## Control groups
 
+WARNING:
+Enabling limits on your environment should be done with caution and only
+in select circumstances, such as to protect against unexpected traffic.
+When reached, limits _do_ result in disconnects that negatively impact users.
+For consistent and stable performance, you should first explore other options such as
+adjusting node specifications, and [reviewing large repositories](../../user/project/repository/managing_large_repositories.md) or workloads.
+
 FLAG:
 On self-managed GitLab, by default repository cgroups are not available. To make it available, ask an administrator to
 [enable the feature flag](../feature_flags.md) named `gitaly_run_cmds_in_cgroup`.
 
-Control groups (cgroups) in Linux allow limits to be imposed on how much memory and CPU can be consumed.
+When enabling cgroups for memory, you should ensure that no swap is configured on the Gitaly nodes as
+processes may switch to using that instead of being terminated. This situation could lead to notably compromised
+performance.
+
+You can use control groups (cgroups) in Linux to impose limits on how much memory and CPU can be consumed by Gitaly processes.
 See the [`cgroups` Linux man page](https://man7.org/linux/man-pages/man7/cgroups.7.html) for more information.
-cgroups can be useful for protecting the system against resource exhaustion because of over consumption of memory and CPU.
+cgroups can be useful for protecting the system against unexpected resource exhaustion because of over consumption of memory and CPU.
 
-Some Git operations are expensive by nature. `git clone`, for instance,
-spawns a `git-upload-pack` process on the server that can consume a lot of memory
-for large repositories. For example, a client that keeps on cloning a
-large repository over and over again. This situation could potentially use up all of the
-memory on a server, causing other operations to fail for other users.
+Some Git operations can consume notable resources up to the point of exhaustion in situations such as:
 
-A repository can consume large amounts of memory for many reasons when cloned or downloaded.
-Using cgroups allows the kernel to kill these operations before they hog up all system resources.
+- Unexpectedly high traffic.
+- Operations running against large repositories that don't follow best practices.
 
-Gitaly shells out to Git for many of its operations. Git can consume a lot of resources for certain operations,
-especially for large repositories.
+As a hard protection, it's possible to use cgroups that configure the kernel to terminate these operations before they hog up all system resources
+and cause instability.
 
 Gitaly has built-in cgroups control. When configured, Gitaly assigns Git processes to a cgroup based on the repository
 the Git command is operating in. These cgroups are called repository cgroups. Each repository cgroup:
@@ -835,8 +856,8 @@ When a repository cgroup reaches its:
 - Memory limit, the kernel looks through the processes for a candidate to kill.
 - CPU limit, processes are not killed, but the processes are prevented from consuming more CPU than allowed.
 
-You configure repository cgroups for your GitLab installation to protect against system resource starvation from a few
-large repositories or bad actors.
+NOTE:
+When these limits are reached, performance may be reduced and users may be disconnected.
 
 ### Configure repository cgroups (new method)
 
diff --git a/doc/administration/gitaly/troubleshooting.md b/doc/administration/gitaly/troubleshooting.md
index 7edce840396..9b41717bd0b 100644
--- a/doc/administration/gitaly/troubleshooting.md
+++ b/doc/administration/gitaly/troubleshooting.md
@@ -245,6 +245,28 @@ the application might be fetching this secret from a different file. Your Gitaly
 If that setting is missing, GitLab defaults to using `.gitlab_shell_secret` under
 `/opt/gitlab/embedded/service/gitlab-rails/.gitlab_shell_secret`.
 
+### Repository pushes fail
+
+When attempting `git push`, you can see:
+
+- `401 Unauthorized` errors.
+- The following in server logs:
+
+  ```json
+  {
+    ...
+    "exception.class":"JWT::VerificationError",
+    "exception.message":"Signature verification raised",
+    ...
+  }
+  ```
+
+This error occurs when the GitLab server has been upgraded to GitLab 15.5 or later but Gitaly has not yet been upgraded.
+
+From GitLab 15.5, GitLab [authenticates with GitLab Shell using a JWT token instead of a shared secret](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86148).
+You should follow the [recommendations on upgrading external Gitaly](../../update/plan_your_upgrade.md#external-gitaly) and upgrade Gitaly before the GitLab
+server.
+
 ### Repository pushes fail with a `deny updating a hidden ref` error
 
 Due to [a change](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/3426)
diff --git a/doc/ci/environments/external_deployment_tools.md b/doc/ci/environments/external_deployment_tools.md
new file mode 100644
index 00000000000..c22adde2016
--- /dev/null
+++ b/doc/ci/environments/external_deployment_tools.md
@@ -0,0 +1,39 @@
+---
+stage: Release
+group: Release
+info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments
+type: reference
+---
+
+# Track deployments of an external deployment tool **(FREE)**
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22513) in GitLab 12.5.
+
+While GitLab offers a [built-in deployment solution](index.md), you might prefer to use an external deployment tool, such as Heroku or ArgoCD.
+GitLab can receive deployment events from these external tools and allows you to track the deployments within GitLab.
+For example, the following features are available by setting up tracking:
+
+- [See when an merge request has been deployed, and to which environment](../../user/project/merge_requests/widgets.md#post-merge-pipeline-status).
+- [Filter merge requests by environment or deployment date](../../user/project/merge_requests/index.md#filter-merge-requests-by-environment-or-deployment-date).
+- [DevOps Research and Assessment (DORA) metrics](../../user/analytics/dora_metrics.md).
+- [View environments and deployments](index.md#view-environments-and-deployments).
+- [Track newly included merge requests per deployment](index.md#track-newly-included-merge-requests-per-deployment).
+
+NOTE:
+Some of the features are not available because GitLab can't authorize and leverage those external deployments, including
+[Protected Environments](protected_environments.md), [Deployment Approvals](deployment_approvals.md), [Deployment safety](deployment_safety.md), and [Environment rollback](index.md#environment-rollback).
+
+## How to set up deployment tracking
+
+External deployment tools usually offer a [webhook](https://en.wikipedia.org/wiki/Webhook) to execute an additional API request when deployment state is changed.
+You can configure your tool to make a request to the GitLab [Deployment API](../../api/deployments.md). Here is an overview of the event and API request flow:
+
+- When a deployment starts running, [create a deployment with `running` status](../../api/deployments.md#create-a-deployment).
+- When a deployment succeeds, [update the deployment status to `success`](../../api/deployments.md#update-a-deployment).
+- When a deployment fails, [update the deployment status to `failed`](../../api/deployments.md#update-a-deployment).
+
+NOTE:
+You can create a [project access token](../../user/project/settings/project_access_tokens.md) for the GitLab API authentication.
+
+NOTE:
+If you don't have an environment yet, you can [create a new environment](index.md#create-a-static-environment) in the UI or with the [Environment API](../../api/environments.md#create-a-new-environment).
diff --git a/doc/ci/environments/index.md b/doc/ci/environments/index.md
index 383127e651a..749ac60c783 100644
--- a/doc/ci/environments/index.md
+++ b/doc/ci/environments/index.md
@@ -982,6 +982,7 @@ Instead, you need to delete the old environment and create a new one:
 - [Environments Dashboard](../environments/environments_dashboard.md): View a summary of each
   environment's operational health. **(PREMIUM)**
 - [Deployment safety](deployment_safety.md#restrict-write-access-to-a-critical-environment): Secure your deployments.
+- [Track deployments of an external deployment tool](external_deployment_tools.md): Use an external deployment tool instead of built-in deployment solution.
 
 ## Troubleshooting