Add latest changes from gitlab-org/gitlab@master

29 files changed, 527 insertions, 273 deletions

diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION
index 77f935d82a7..66f5311ad3d 100644
--- a/GITALY_SERVER_VERSION
+++ b/GITALY_SERVER_VERSION
@@ -1 +1 @@
-d513d220b183d83ae7219ec52f49aa3b4f7fc551
+fb9de5f27b55e28a5d4737e0aa639a50ccc3dd75
diff --git a/app/assets/javascripts/invite_members/components/invite_members_modal.vue b/app/assets/javascripts/invite_members/components/invite_members_modal.vue
index 4aab1123af9..431900aecf0 100644
--- a/app/assets/javascripts/invite_members/components/invite_members_modal.vue
+++ b/app/assets/javascripts/invite_members/components/invite_members_modal.vue
@@ -86,6 +86,7 @@ export default {
       groupToBeSharedWith: {},
       source: 'unknown',
       invalidFeedbackMessage: '',
+      isLoading: false,
     };
   },
   computed: {
@@ -169,6 +170,7 @@ export default {
       }
     },
     resetFields() {
+      this.isLoading = false;
       this.selectedAccessLevel = this.defaultAccessLevel;
       this.selectedDate = undefined;
       this.newUsersToInvite = [];
@@ -189,6 +191,7 @@ export default {
     },
     submitInviteMembers() {
       this.invalidFeedbackMessage = '';
+      this.isLoading = true;
 
       const [usersToInviteByEmail, usersToAddById] = this.partitionNewUsersToInvite();
       const promises = [];
@@ -247,12 +250,14 @@ export default {
       }
 
       this.invalidFeedbackMessage = message;
+      this.isLoading = false;
     },
     showToastMessageSuccess() {
       this.$toast.show(this.$options.labels.toastMessageSuccessful, this.toastOptions);
       this.closeModal();
     },
     showInvalidFeedbackMessage(response) {
+      this.isLoading = false;
       this.invalidFeedbackMessage =
         responseMessageFromError(response) || this.$options.labels.invalidFeedbackMessageDefault;
     },
@@ -405,6 +410,7 @@ export default {
         <div class="gl-mr-3"></div>
         <gl-button
           :disabled="inviteDisabled"
+          :loading="isLoading"
           variant="success"
           data-qa-selector="invite_button"
           data-testid="invite-button"
diff --git a/app/models/integrations/datadog.rb b/app/models/integrations/datadog.rb
index 27c2fcf266b..8a356cf8abc 100644
--- a/app/models/integrations/datadog.rb
+++ b/app/models/integrations/datadog.rb
@@ -123,6 +123,8 @@ module Integrations
       object_kind = 'job' if object_kind == 'build'
       return unless supported_events.include?(object_kind)
 
+      data = data.with_retried_builds if data.respond_to?(:with_retried_builds)
+
       execute_web_hook!(data, "#{object_kind} hook")
     end
 
diff --git a/app/views/layouts/nav/sidebar/_group_menus.html.haml b/app/views/layouts/nav/sidebar/_group_menus.html.haml
index 72fab67894c..40d994bdf36 100644
--- a/app/views/layouts/nav/sidebar/_group_menus.html.haml
+++ b/app/views/layouts/nav/sidebar/_group_menus.html.haml
@@ -1,5 +1,3 @@
-= render_if_exists "layouts/nav/ee/push_rules_link" # EE-specific
-
 - if group_sidebar_link?(:runners)
   = nav_link(path: 'groups/runners#index') do
     = link_to group_runners_path(@group), title: _('CI/CD'), class: 'has-sub-items' do
diff --git a/db/post_migrate/20210709024048_finalize_push_event_payloads_bigint_conversion_2.rb b/db/post_migrate/20210709024048_finalize_push_event_payloads_bigint_conversion_2.rb
index 717bdb2a887..aa812151164 100644
--- a/db/post_migrate/20210709024048_finalize_push_event_payloads_bigint_conversion_2.rb
+++ b/db/post_migrate/20210709024048_finalize_push_event_payloads_bigint_conversion_2.rb
@@ -3,69 +3,11 @@
 class FinalizePushEventPayloadsBigintConversion2 < ActiveRecord::Migration[6.1]
   include Gitlab::Database::MigrationHelpers
 
-  disable_ddl_transaction!
-
-  TABLE_NAME = 'push_event_payloads'
-  INDEX_NAME = 'index_push_event_payloads_on_event_id_convert_to_bigint'
-
   def up
-    ensure_batched_background_migration_is_finished(
-      job_class_name: 'CopyColumnUsingBackgroundMigrationJob',
-      table_name: TABLE_NAME,
-      column_name: 'event_id',
-      job_arguments: [["event_id"], ["event_id_convert_to_bigint"]]
-    )
-
-    swap_columns
+    # no-op due to https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5256
   end
 
   def down
-    swap_columns
-  end
-
-  private
-
-  def swap_columns
-    add_concurrent_index TABLE_NAME, :event_id_convert_to_bigint, unique: true, name: INDEX_NAME
-
-    # Add a foreign key on `event_id_convert_to_bigint` before we swap the columns and drop the old FK (fk_36c74129da)
-    add_concurrent_foreign_key TABLE_NAME, :events, column: :event_id_convert_to_bigint, on_delete: :cascade
-
-    with_lock_retries(raise_on_exhaustion: true) do
-      # We'll need  ACCESS EXCLUSIVE lock on the related tables,
-      # lets make sure it can be acquired from the start
-      execute "LOCK TABLE #{TABLE_NAME}, events IN ACCESS EXCLUSIVE MODE"
-
-      # Swap column names
-      temp_name = 'event_id_tmp'
-      execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:event_id)} TO #{quote_column_name(temp_name)}"
-      execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:event_id_convert_to_bigint)} TO #{quote_column_name(:event_id)}"
-      execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(temp_name)} TO #{quote_column_name(:event_id_convert_to_bigint)}"
-
-      # We need to update the trigger function in order to make PostgreSQL to
-      # regenerate the execution plan for it. This is to avoid type mismatch errors like
-      # "type of parameter 15 (bigint) does not match that when preparing the plan (integer)"
-      function_name = Gitlab::Database::UnidirectionalCopyTrigger.on_table(TABLE_NAME).name(:event_id, :event_id_convert_to_bigint)
-      execute "ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL"
-
-      # Swap defaults
-      change_column_default TABLE_NAME, :event_id, nil
-      change_column_default TABLE_NAME, :event_id_convert_to_bigint, 0
-
-      # Swap PK constraint
-      execute "ALTER TABLE #{TABLE_NAME} DROP CONSTRAINT push_event_payloads_pkey"
-      rename_index TABLE_NAME, INDEX_NAME, 'push_event_payloads_pkey'
-      execute "ALTER TABLE #{TABLE_NAME} ADD CONSTRAINT push_event_payloads_pkey PRIMARY KEY USING INDEX push_event_payloads_pkey"
-
-      # Drop original FK on the old int4 `event_id` (fk_36c74129da)
-      remove_foreign_key TABLE_NAME, name: concurrent_foreign_key_name(TABLE_NAME, :event_id)
-      # We swapped the columns but the FK for event_id is still using the old name for the event_id_convert_to_bigint column
-      # So we have to also swap the FK name now that we dropped the other one with the same
-      rename_constraint(
-        TABLE_NAME,
-        concurrent_foreign_key_name(TABLE_NAME, :event_id_convert_to_bigint),
-        concurrent_foreign_key_name(TABLE_NAME, :event_id)
-      )
-    end
+    # no-op due to https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5256
   end
 end
diff --git a/db/structure.sql b/db/structure.sql
index 827315a23fb..7517098fc02 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -17561,7 +17561,7 @@ ALTER SEQUENCE protected_tags_id_seq OWNED BY protected_tags.id;
 
 CREATE TABLE push_event_payloads (
     commit_count bigint NOT NULL,
-    event_id_convert_to_bigint integer DEFAULT 0 NOT NULL,
+    event_id integer NOT NULL,
     action smallint NOT NULL,
     ref_type smallint NOT NULL,
     commit_from bytea,
@@ -17569,7 +17569,7 @@ CREATE TABLE push_event_payloads (
     ref text,
     commit_title character varying(70),
     ref_count integer,
-    event_id bigint NOT NULL
+    event_id_convert_to_bigint bigint DEFAULT 0 NOT NULL
 );
 
 CREATE TABLE push_rules (
diff --git a/doc/administration/geo/replication/configuration.md b/doc/administration/geo/replication/configuration.md
index fabca9ca8a3..db5cb597693 100644
--- a/doc/administration/geo/replication/configuration.md
+++ b/doc/administration/geo/replication/configuration.md
@@ -190,6 +190,9 @@ keys must be manually replicated to the **secondary** site.
    gitlab_rails['geo_node_name'] = '<site_name_here>'
    ```
 
+   We recommend mentioning specific names for the `geo_node_name` such as `gitlab-usa` instead of generic names, such as `geo`.
+   This makes the failover process easier because the physical location does not change, but the Geo site role can.
+   
 1. Reconfigure **each Rails and Sidekiq node on your secondary** site for the change to take effect:
 
    ```shell
diff --git a/doc/ci/secrets/index.md b/doc/ci/secrets/index.md
index 52bf30a9b35..9280b03be85 100644
--- a/doc/ci/secrets/index.md
+++ b/doc/ci/secrets/index.md
@@ -181,3 +181,32 @@ You can also specify some attributes for the resulting Vault tokens, such as tim
 IP address range, and number of uses. The full list of options is available in
 [Vault's documentation on creating roles](https://www.vaultproject.io/api/auth/jwt#create-role)
 for the JSON web token method.
+
+## Using a self-signed Vault server
+
+When the Vault server is using a self-signed certificate, you will see the following error in the job logs:
+
+```plaintext
+ERROR: Job failed (system failure): resolving secrets: initializing Vault service: preparing authenticated client: checking Vault server health: Get https://vault.example.com:8000/v1/sys/health?drsecondarycode=299&performancestandbycode=299&sealedcode=299&standbycode=299&uninitcode=299: x509: certificate signed by unknown authority
+```
+
+You have two options to solve this error:
+
+- Add the self-signed certificate to the GitLab Runner server's CA store.
+  If you deployed GitLab Runner using the [Helm chart](https://docs.gitlab.com/runner/install/kubernetes.html), you will have to create your own GitLab Runner image.
+- Use the `VAULT_CACERT` environment variable to configure GitLab Runner to trust the certificate:
+  - If you are using systemd to manage GitLab Runner, see [how to add an environment variable for GitLab Runner](https://docs.gitlab.com/runner/configuration/init.html#setting-custom-environment-variables).
+  - If you deployed GitLab Runner using the [Helm chart](https://docs.gitlab.com/runner/install/kubernetes.html):
+    1. [Provide a custom certificate for accessing GitLab](https://docs.gitlab.com/runner/install/kubernetes.html#providing-a-custom-certificate-for-accessing-gitlab), and make sure to add the certificate for the Vault server instead of the certificate for GitLab. If your GitLab instance is also using a self-signed certificate, you should be able to add both in the same `Secret`.
+    1. Add the following lines in your `values.yaml` file:
+
+       ```yaml
+       ## Replace both the <SECRET_NAME> and the <VAULT_CERTIFICATE>
+       ## with the actual values you used to create the secret
+
+       certsSecretName: <SECRET_NAME>
+
+       envVars:
+         - name: VAULT_CACERT
+           value: "/home/gitlab-runner/.gitlab-runner/certs/<VAULT_CERTIFICATE>"
+       ```
diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md
index e6beef08896..1d69bc8d930 100644
--- a/doc/ci/yaml/index.md
+++ b/doc/ci/yaml/index.md
@@ -419,9 +419,13 @@ configurations. Local configurations in the `.gitlab-ci.yml` file override inclu
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/284883) in GitLab 13.8.
 > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/294294) in GitLab 13.9.
+> - [Custom variable support added](https://gitlab.com/gitlab-org/gitlab/-/issues/219065) in GitLab 14.2.
 
-You can [use some predefined variables in `include` sections](../variables/where_variables_can_be_used.md#gitlab-ciyml-file)
-in your `.gitlab-ci.yml` file:
+In `include` sections in your `.gitlab-ci.yml` file, you can use:
+
+- Project [predefined variables](../variables/predefined_variables.md).
+- [Custom instance, group, and project variables](../variables/index.md#custom-cicd-variables)
+  in GitLab 14.2 and later.
 
 ```yaml
 include:
@@ -680,142 +684,130 @@ For more information, see [Available settings for `services`](../services/index.
 
 ### `script`
 
-Use `script` to specify a shell script for the runner to execute.
+Use `script` to specify commands for the runner to execute.
 
 All jobs except [trigger jobs](#trigger) require a `script` keyword.
 
-For example:
+**Keyword type**: Job keyword. You can use it only as part of a job.
 
-```yaml
-job:
-  script: "bundle exec rspec"
-```
+**Possible inputs**: An array including:
 
-You can use [YAML anchors with `script`](#yaml-anchors-for-scripts).
+- Single line commands.
+- Long commands [split over multiple lines](script.md#split-long-commands).
+- [YAML anchors](#yaml-anchors-for-scripts).
 
-The `script` keyword can also contain several commands in an array:
+**Example of `script`:**
 
 ```yaml
-job:
+job1:
+  script: "bundle exec rspec"
+
+job2:
   script:
     - uname -a
     - bundle exec rspec
 ```
 
-Sometimes, `script` commands must be wrapped in single or double quotes.
-For example, commands that contain a colon (`:`) must be wrapped in single quotes (`'`).
-The YAML parser needs to interpret the text as a string rather than
-a "key: value" pair.
+**Additional details**:
 
-For example, this script uses a colon:
+You might need to use single quotes (`'`) or double quotes (`"`) when using
+[special characters in `script`](script.md#use-special-characters-with-script).
 
-```yaml
-job:
-  script:
-    - curl --request POST --header 'Content-Type: application/json' "https://gitlab/api/v4/projects"
-```
+**Related topics**:
 
-To be considered valid YAML, you must wrap the entire command in single quotes. If
-the command already uses single quotes, you should change them to double quotes (`"`)
-if possible:
+- You can [ignore non-zero exit codes](script.md#ignore-non-zero-exit-codes).
+- [Use color codes with `script`](script.md#add-color-codes-to-script-output)
+  to make job logs easier to review.
+- [Create custom collapsible sections](../jobs/index.md#custom-collapsible-sections)
+  to simplify job log output.
 
-```yaml
-job:
-  script:
-    - 'curl --request POST --header "Content-Type: application/json" "https://gitlab/api/v4/projects"'
-```
+#### `before_script`
+
+Use `before_script` to define an array of commands that should run before each job's
+`script` commands, but after [artifacts](#artifacts) are restored.
 
-You can verify the syntax is valid with the [CI Lint](../lint.md) tool.
+**Keyword type**: Job keyword. You can use it only as part of a job or in the
+[`default:` section](#custom-default-keyword-values).
 
-Be careful when using these characters as well:
+**Possible inputs**: An array including:
 
-- `{`, `}`, `[`, `]`, `,`, `&`, `*`, `#`, `?`, `|`, `-`, `<`, `>`, `=`, `!`, `%`, `@`, `` ` ``.
+- Single line commands.
+- Long commands [split over multiple lines](script.md#split-long-commands).
+- [YAML anchors](#yaml-anchors-for-scripts).
 
-If any of the script commands return an exit code other than zero, the job
-fails and further commands are not executed. Store the exit code in a variable to
-avoid this behavior:
+**Example of `before_script`:**
 
 ```yaml
 job:
+  before_script:
+    - echo "Execute this command before any `script:` commands."
   script:
-    - false || exit_code=$?
-    - if [ $exit_code -ne 0 ]; then echo "Previous command failed"; fi;
+    - echo "This command executes after the job's `before_script` commands."
 ```
 
-#### `before_script`
-
-Use `before_script` to define an array of commands that should run before each job,
-but after [artifacts](#artifacts) are restored.
+**Additional details**:
 
 Scripts you specify in `before_script` are concatenated with any scripts you specify
-in the main [`script`](#script). The combine scripts execute together in a single shell.
+in the main [`script`](#script). The combined scripts execute together in a single shell.
 
-You can overwrite a globally-defined `before_script` if you define it in a job:
+**Related topics**:
 
-```yaml
-default:
-  before_script:
-    - echo "Execute this script in all jobs that don't already have a before_script section."
+- [Use `before_script` with `default`](script.md#set-a-default-before_script-or-after_script-for-all-jobs)
+  to define a default array of commands that should run before the `script` commands in all jobs.
+- You can [ignore non-zero exit codes](script.md#ignore-non-zero-exit-codes).
+- [Use color codes with `before_script`](script.md#add-color-codes-to-script-output)
+  to make job logs easier to review.
+- [Create custom collapsible sections](../jobs/index.md#custom-collapsible-sections)
+  to simplify job log output.
 
-job1:
-  script:
-    - echo "This script executes after the global before_script."
+#### `after_script`
 
-job:
-  before_script:
-    - echo "Execute this script instead of the global before_script."
-  script:
-    - echo "This script executes after the job's `before_script`"
-```
+Use `after_script` to define an array of commands that run after each job, including failed jobs.
 
-You can use [YAML anchors with `before_script`](#yaml-anchors-for-scripts).
+**Keyword type**: Job keyword. You can use it only as part of a job or in the
+[`default:` section](#custom-default-keyword-values).
 
-#### `after_script`
+**Possible inputs**: An array including:
 
-Use `after_script` to define an array of commands that run after each job,
-including failed jobs.
+- Single line commands.
+- Long commands [split over multiple lines](script.md#split-long-commands).
+- [YAML anchors](#yaml-anchors-for-scripts).
 
-If a job times out or is cancelled, the `after_script` commands do not execute.
-An [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/15603) exists to support
-executing `after_script` commands for timed-out or cancelled jobs.
+**Example of `after_script`:**
+
+```yaml
+job:
+  script:
+    - echo "An example script section."
+  after_script:
+    - echo "Execute this command after the `script` section completes."
+```
+
+**Additional details**:
 
 Scripts you specify in `after_script` execute in a new shell, separate from any
-`before_script` or `script` scripts. As a result, they:
+`before_script` or `script` commands. As a result, they:
 
 - Have a current working directory set back to the default.
-- Have no access to changes done by scripts defined in `before_script` or `script`, including:
+- Don't have access to changes done by commands defined in the `before_script` or `script`,
+  including:
   - Command aliases and variables exported in `script` scripts.
   - Changes outside of the working tree (depending on the runner executor), like
     software installed by a `before_script` or `script` script.
-- Have a separate timeout, which is hard coded to 5 minutes. See the
-  [related issue](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2716) for details.
+- Have a separate timeout, which is [hard-coded to 5 minutes](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2716).
 - Don't affect the job's exit code. If the `script` section succeeds and the
   `after_script` times out or fails, the job exits with code `0` (`Job Succeeded`).
 
-```yaml
-default:
-  after_script:
-    - echo "Execute this script in all jobs that don't already have an after_script section."
-
-job1:
-  script:
-    - echo "This script executes first. When it completes, the global after_script executes."
-
-job:
-  script:
-    - echo "This script executes first. When it completes, the job's `after_script` executes."
-  after_script:
-    - echo "Execute this script instead of the global after_script."
-```
-
-You can use [YAML anchors with `after_script`](#yaml-anchors-for-scripts).
-
-#### Script syntax
+If a job times out or is cancelled, the `after_script` commands do not execute.
+[An issue exists](https://gitlab.com/gitlab-org/gitlab/-/issues/15603) to add support for executing `after_script` commands for timed-out or cancelled jobs.
 
-You can use syntax in [`script`](#script) sections to:
+**Related topics**:
 
-- [Split long commands](script.md#split-long-commands) into multiline commands.
-- [Use color codes](script.md#add-color-codes-to-script-output) to make job logs easier to review.
+- [Use `after_script` with `default`](script.md#set-a-default-before_script-or-after_script-for-all-jobs)
+  to define a default array of commands that should run after all jobs.
+- You can [ignore non-zero exit codes](script.md#ignore-non-zero-exit-codes).
+- [Use color codes with `after_script`](script.md#add-color-codes-to-script-output)
+  to make job logs easier to review.
 - [Create custom collapsible sections](../jobs/index.md#custom-collapsible-sections)
   to simplify job log output.
 
diff --git a/doc/ci/yaml/script.md b/doc/ci/yaml/script.md
index 9e118895d7c..93c1a6afe69 100644
--- a/doc/ci/yaml/script.md
+++ b/doc/ci/yaml/script.md
@@ -14,6 +14,85 @@ You can use special syntax in [`script`](index.md#script) sections to:
 - [Create custom collapsible sections](../jobs/index.md#custom-collapsible-sections)
   to simplify job log output.
 
+## Use special characters with `script:`
+
+Sometimes, `script` commands must be wrapped in single or double quotes.
+For example, commands that contain a colon (`:`) must be wrapped in single quotes (`'`).
+The YAML parser needs to interpret the text as a string rather than
+a "key: value" pair.
+
+For example, this script uses a colon:
+
+```yaml
+job:
+  script:
+    - curl --request POST --header 'Content-Type: application/json' "https://gitlab/api/v4/projects"
+```
+
+To be considered valid YAML, you must wrap the entire command in single quotes. If
+the command already uses single quotes, you should change them to double quotes (`"`)
+if possible:
+
+```yaml
+job:
+  script:
+    - 'curl --request POST --header "Content-Type: application/json" "https://gitlab/api/v4/projects"'
+```
+
+You can verify the syntax is valid with the [CI Lint](../lint.md) tool.
+
+Be careful when using these characters as well:
+
+- `{`, `}`, `[`, `]`, `,`, `&`, `*`, `#`, `?`, `|`, `-`, `<`, `>`, `=`, `!`, `%`, `@`, `` ` ``.
+
+## Ignore non-zero exit codes
+
+When script commands return an exit code other than zero, the job fails and further
+commands do not execute.
+
+Store the exit code in a variable to avoid this behavior:
+
+```yaml
+job:
+  script:
+    - false || exit_code=$?
+    - if [ $exit_code -ne 0 ]; then echo "Previous command failed"; fi;
+```
+
+## Set a default `before_script` or `after_script` for all jobs
+
+You can use [`before_script`](index.md#before_script) and [`after_script`](index.md#after_script)
+with [`default`](index.md#custom-default-keyword-values):
+
+- Use `before_script` with `default` to define a default array of commands that
+  should run before the `script` commands in all jobs.
+- Use `after_script` with default to define a default array of commands
+  that should run after the job completes.
+
+You can overwrite a default by defining a different one in a job. To ignore the default
+use `before_script: []` or `after_script: []`:
+
+```yaml
+default:
+  before_script:
+    - echo "Execute this `before_script` in all jobs by default."
+  after_script:
+    - echo "Execute this `after_script` in all jobs by default."
+
+job1:
+  script:
+    - echo "These script commands execute after the default `before_script`,"
+    - echo "and before the default `after_script`."
+
+job2:
+  before_script:
+    - echo "Execute this script instead of the default `before_script`."
+  script:
+    - echo "This script executes after the job's `before_script`,"
+    - echo "but the job does not use the default `after_script`."
+  after_script: []
+```
+
 ## Split long commands
 
 You can split long commands into multiline commands to improve readability with
diff --git a/doc/development/background_migrations.md b/doc/development/background_migrations.md
index df3baa3349e..695c565ca83 100644
--- a/doc/development/background_migrations.md
+++ b/doc/development/background_migrations.md
@@ -8,7 +8,7 @@ info: "See the Technical Writers assigned to Development Guidelines: https://abo
 # Background migrations
 
 Background migrations should be used to perform data migrations whenever a
-migration exceeds [the time limits in our guidelines](database_review.md#timing-guidelines-for-migrations). For example, you can use background
+migration exceeds [the time limits in our guidelines](migration_style_guide.md#how-long-a-migration-should-take). For example, you can use background
 migrations to migrate data that's stored in a single JSON column
 to a separate table instead.
 
@@ -18,7 +18,7 @@ migrations automatically reschedule themselves for a later point in time.
 ## When To Use Background Migrations
 
 You should use a background migration when you migrate _data_ in tables that have
-so many rows that the process would exceed [the time limits in our guidelines](database_review.md#timing-guidelines-for-migrations) if performed using a regular Rails migration.
+so many rows that the process would exceed [the time limits in our guidelines](migration_style_guide.md#how-long-a-migration-should-take) if performed using a regular Rails migration.
 
 - Background migrations should be used when migrating data in [high-traffic tables](migration_style_guide.md#high-traffic-tables).
 - Background migrations may also be used when executing numerous single-row queries
diff --git a/doc/development/database_review.md b/doc/development/database_review.md
index bcb03ff1262..2746d9f6582 100644
--- a/doc/development/database_review.md
+++ b/doc/development/database_review.md
@@ -216,7 +216,7 @@ test its execution using `CREATE INDEX CONCURRENTLY` in the `#database-lab` Slac
     it's suggested to treat background migrations as post migrations:
     place them in `db/post_migrate` instead of `db/migrate`. Keep in mind
     that post migrations are executed post-deployment in production.
-- Check [timing guidelines for migrations](#timing-guidelines-for-migrations)
+- Check [timing guidelines for migrations](migration_style_guide.md#how-long-a-migration-should-take)
 - Check migrations are reversible and implement a `#down` method
 - Check data migrations:
   - Establish a time estimate for execution on GitLab.com.
@@ -234,18 +234,3 @@ test its execution using `CREATE INDEX CONCURRENTLY` in the `#database-lab` Slac
     to queries (changing the query, schema or adding indexes and similar)
   - General guideline is for queries to come in below [100ms execution time](query_performance.md#timing-guidelines-for-queries)
   - Avoid N+1 problems and minimize the [query count](merge_request_performance_guidelines.md#query-counts).
-
-### Timing guidelines for migrations
-
-In general, migrations for a single deploy shouldn't take longer than
-1 hour for GitLab.com. The following guidelines are not hard rules, they were
-estimated to keep migration timing to a minimum.
-
-NOTE:
-Keep in mind that all runtimes should be measured against GitLab.com.
-
-| Migration Type | Execution Time Recommended | Notes |
-|----|----|---|
-| Regular migrations on `db/migrate` | `3 minutes` | A valid exception are index creation as this can take a long time. |
-| Post migrations on `db/post_migrate` | `10 minutes` | |
-| Background migrations | --- | Since these are suitable for larger tables, it's not possible to set a precise timing guideline, however, any single query must stay below [`1 second` execution time](query_performance.md#timing-guidelines-for-queries) with cold caches. |
diff --git a/doc/development/documentation/styleguide/index.md b/doc/development/documentation/styleguide/index.md
index 903c2543de5..1a628297ee8 100644
--- a/doc/development/documentation/styleguide/index.md
+++ b/doc/development/documentation/styleguide/index.md
@@ -1441,11 +1441,12 @@ application:
 Use these verbs for specific uses with user interface
 elements:
 
-| Recommended         | Used for                              | Replaces                   |
-|:--------------------|:--------------------------------------|:---------------------------|
-| _select_            | buttons, links, menu items, dropdowns | "click, "press," "hit"     |
-| _select_ or _clear_ | checkboxes                            | "enable", "click", "press" |
-| _expand_            | expandable sections                   | "open"                     |
+| Recommended             | Used for                              | Replaces                    |
+|:------------------------|:--------------------------------------|:----------------------------|
+| _select_                | buttons, links, menu items, dropdowns | "click, "press," "hit"      |
+| _select_ or _clear_     | checkboxes                            | "enable", "click", "press"  |
+| _expand_                | expandable sections                   | "open"                      |
+| _turn on_ or _turn off_ | toggles                               | "flip", "enable", "disable" |
 
 ### Other Verbs
 
diff --git a/doc/development/migration_style_guide.md b/doc/development/migration_style_guide.md
index f76b053c2bd..cf7283d6831 100644
--- a/doc/development/migration_style_guide.md
+++ b/doc/development/migration_style_guide.md
@@ -31,11 +31,66 @@ Please don't depend on GitLab-specific code since it can change in future
 versions. If needed copy-paste GitLab code into the migration to make it forward
 compatible.
 
-For GitLab.com, please take into consideration that regular migrations (under `db/migrate`)
-are run before [Canary is deployed](https://gitlab.com/gitlab-com/gl-infra/readiness/-/tree/master/library/canary/#configuration-and-deployment),
-and [post-deployment migrations](post_deployment_migrations.md) (`db/post_migrate`) are run after the deployment to production has finished.
+## Choose an appropriate migration type
+
+The first step before adding a new migration should be to decide which type is most appropriate.
+
+There are currently three kinds of migrations you can create, depending on the kind of
+work it needs to perform and how long it takes to complete:
+
+1. [**Regular schema migrations.**](#create-a-regular-schema-migration) These are traditional Rails migrations in `db/migrate` that run _before_ new application code is deployed
+   (for GitLab.com before [Canary is deployed](https://gitlab.com/gitlab-com/gl-infra/readiness/-/tree/master/library/canary/#configuration-and-deployment)).
+   This means that they should be relatively fast, no more than a few minutes, so as not to unnecessarily delay a deployment.
+
+   One exception is a migration that takes longer but is absolutely critical for the application to operate correctly.
+   For example, you might have indices that enforce unique tuples, or that are needed for query performance in critical parts of the application. In cases where the migration would be unacceptably slow, however, a better option might be to guard the feature with a [feature flag](feature_flags/index.md)
+   and perform a post-deployment migration instead. The feature can then be turned on after the migration finishes.
+1. [**Post-deployment migrations.**](post_deployment_migrations.md) These are Rails migrations in `db/post_migrate` and
+   run _after_ new application code has been deployed (for GitLab.com after the production deployment has finished).
+   They can be used for schema changes that aren't critical for the application to operate, or data migrations that take at most a few minutes.
+   Common examples for schema changes that should run post-deploy include:
+     - Clean-ups, like removing unused columns.
+     - Adding non-critical indices on high-traffic tables.
+     - Adding non-critical indices that take a long time to create.
+1. [**Background migrations.**](background_migrations.md) These aren't regular Rails migrations, but application code that is
+   executed via Sidekiq jobs, although a post-deployment migration is used to schedule them. Use them only for data migrations that
+   exceed the timing guidelines for post-deploy migrations. Background migrations should _not_ change the schema.
+
+Use the following diagram to guide your decision, but keep in mind that it is just a tool, and
+the final outcome will always be dependent on the specific changes being made:
+
+```mermaid
+graph LR
+    A{Schema<br/>changed?}
+    A -->|Yes| C{Critical to<br/>speed or<br/>behavior?}
+    A -->|No| D{Is it fast?}
+
+    C -->|Yes| H{Is it fast?}
+    C -->|No| F[Post-deploy migration]
+
+    H -->|Yes| E[Regular migration]
+    H -->|No| I[Post-deploy migration<br/>+ feature flag]
+ 
+    D -->|Yes| F[Post-deploy migration]
+    D -->|No| G[Background migration]
+```
+
+### How long a migration should take
+
+In general, all migrations for a single deploy shouldn't take longer than
+1 hour for GitLab.com. The following guidelines are not hard rules, they were
+estimated to keep migration duration to a minimum.
+
+NOTE:
+Keep in mind that all durations should be measured against GitLab.com.
+
+| Migration Type | Recommended Duration | Notes |
+|----|----|---|
+| Regular migrations | `<= 3 minutes` | A valid exception are changes without which application functionality or performance would be severely degraded and which cannot be delayed. |
+| Post-deployment migrations | `<= 10 minutes` | A valid exception are schema changes, since they must not happen in background migrations. |
+| Background migrations | `> 10 minutes` | Since these are suitable for larger tables, it's not possible to set a precise timing guideline, however, any single query must stay below [`1 second` execution time](query_performance.md#timing-guidelines-for-queries) with cold caches. |
 
-## Create database migrations
+## Create a regular schema migration
 
 To create a migration you can use the following Rails generator:
 
diff --git a/doc/development/query_performance.md b/doc/development/query_performance.md
index 318d9f46aff..2ee4a6d6971 100644
--- a/doc/development/query_performance.md
+++ b/doc/development/query_performance.md
@@ -18,7 +18,7 @@ When you are optimizing your SQL queries, there are two dimensions to pay attent
 | Query Type | Maximum Query Time | Notes |
 |----|----|---|
 | General queries | `100ms` | This is not a hard limit, but if a query is getting above it, it is important to spend time understanding why it can or cannot be optimized. |
-| Queries in a migration | `100ms` | This is different than the total [migration time](database_review.md#timing-guidelines-for-migrations). |
+| Queries in a migration | `100ms` | This is different than the total [migration time](migration_style_guide.md#how-long-a-migration-should-take). |
 | Concurrent operations in a migration | `5min` | Concurrent operations do not block the database, but they block the GitLab update. This includes operations such as `add_concurrent_index` and `add_concurrent_foreign_key`. |
 | Background migrations | `1s` |  |
 | Service Ping | `1s` | See the [Service Ping docs](service_ping/index.md#developing-and-testing-service-ping) for more details. |
diff --git a/doc/user/project/deploy_tokens/index.md b/doc/user/project/deploy_tokens/index.md
index 800aa27f612..2f4eb75fe4c 100644
--- a/doc/user/project/deploy_tokens/index.md
+++ b/doc/user/project/deploy_tokens/index.md
@@ -20,6 +20,8 @@ Deploy tokens can be managed by [maintainers only](../../permissions.md).
 
 Deploy tokens cannot be used with the GitLab API.
 
+Deploy tokens are tied to the project and stay enabled even when the user who created the token is removed from the project.
+
 If you have a key pair, you might want to use [deploy keys](../../project/deploy_keys/index.md)
 instead.
 
diff --git a/doc/user/project/issues/confidential_issues.md b/doc/user/project/issues/confidential_issues.md
index 92c26fb654e..e6705933ae9 100644
--- a/doc/user/project/issues/confidential_issues.md
+++ b/doc/user/project/issues/confidential_issues.md
@@ -67,6 +67,12 @@ There is also an indicator on the sidebar denoting confidentiality.
 | :-----------: | :----------: |
 | ![Sidebar confidential issue](img/sidebar_confidential_issue.png) | ![Sidebar not confidential issue](img/sidebar_not_confidential_issue.png) |
 
+## Merge requests for confidential issues
+
+Although you can make issues be confidential in public projects, you cannot make
+confidential merge requests. Learn how to create [merge requests for confidential issues](../merge_requests/confidential.md)
+that prevent leaks of private data.
+
 ## Permissions and access to confidential issues
 
 There are two kinds of level access for confidential issues. The general rule
@@ -82,48 +88,6 @@ sees in the project's search results respectively.
 |:---------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------|
 | ![Confidential issues search by maintainer](img/confidential_issues_search_master.png) | ![Confidential issues search by guest](img/confidential_issues_search_guest.png) |
 
-## Merge Requests for Confidential Issues
-
-> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/58583) in GitLab 12.1.
-
-To help prevent confidential information being leaked from a public project
-in the process of resolving a confidential issue, confidential issues can be
-resolved by creating a merge request from a private fork.
-
-The created merge request targets the default branch of the private fork,
-not the default branch of the public upstream project. This prevents the merge
-request, branch, and commits entering the public repository, and revealing
-confidential information prematurely. To make a confidential commit public,
-open a merge request from the private fork to the public upstream project.
-
-Permissions are inherited from parent groups. Developers have the same permissions
-for private forks created in the same group or in a subgroup of the original
-Permissions are inherited from parent groups. When private forks are created
-in the same group or subgroup as the original upstream repository, users
-receive the same permissions in both projects. This inheritance ensures
-Developer users have the needed permissions to both view confidential issues and
-resolve them.
-
-### How it works
-
-On a confidential issue, a **Create confidential merge request** button is
-available. Clicking on it opens a dropdown where you can choose to
-**Create confidential merge request and branch** or **Create branch**:
-
-| Create confidential merge request | Create branch |
-| :-------------------------------: | :-----------: |
-| ![Create Confidential Merge Request Dropdown](img/confidential_mr_dropdown_v12_1.png) | ![Create Confidential Branch Dropdown](img/confidential_mr_branch_dropdown_v12_1.png) |
-
-The **Project** dropdown includes the list of private forks the user is a member
-of as at least a Developer and merge requests are enabled.
-
-Whenever the **Branch name** and **Source (branch or tag)** fields change, the
-availability of the target and source branch are checked. Both branches should
-be available in the selected private fork.
-
-By clicking the **Create confidential merge request** button, GitLab creates
-the branch and merge request in the private fork. When you choose
-**Create branch**, GitLab creates only the branch.
+## Related links
 
-After the branch is created in the private fork, developers can push code to
-that branch to fix the confidential issue.
+- [Merge requests for confidential issues](../merge_requests/confidential.md)
diff --git a/doc/user/project/issues/img/confidential_mr_branch_dropdown_v12_1.png b/doc/user/project/issues/img/confidential_mr_branch_dropdown_v12_1.png
deleted file mode 100644
index 1f4ad5c42bb..00000000000
--- a/doc/user/project/issues/img/confidential_mr_branch_dropdown_v12_1.png
+++ /dev/null
diff --git a/doc/user/project/issues/img/confidential_mr_dropdown_v12_1.png b/doc/user/project/issues/img/confidential_mr_dropdown_v12_1.png
deleted file mode 100644
index 7b7bd599a71..00000000000
--- a/doc/user/project/issues/img/confidential_mr_dropdown_v12_1.png
+++ /dev/null
diff --git a/doc/user/project/merge_requests/confidential.md b/doc/user/project/merge_requests/confidential.md
new file mode 100644
index 00000000000..b50c5ee0ea9
--- /dev/null
+++ b/doc/user/project/merge_requests/confidential.md
@@ -0,0 +1,73 @@
+---
+stage: Create
+group: Code Review
+info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
+---
+
+# Merge requests for confidential issues
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/58583) in GitLab 12.1.
+
+Merge requests in a public repository are also public, even when the merge
+request is created for a [confidential issue](../issues/confidential_issues.md).
+To avoid leaking confidential information when working on a confidential issue,
+create your merge request from a private fork.
+
+Roles are inherited from parent groups. If you create your private fork in the
+same group or subgroup as the original (public) repository, developers receive
+the same permissions in your fork. This inheritance ensures:
+
+- Developer users have the needed permissions to view confidential issues and resolve them.
+- You do not need grant individual users access to your fork.
+
+The [security practices for confidential merge requests](https://gitlab.com/gitlab-org/release/docs/blob/master/general/security/developer.md#security-releases-critical-non-critical-as-a-developer) at GitLab are available to read.
+
+## Create a confidential merge request
+
+WARNING:
+To create a confidential merge request, you must create a private fork. This fork
+may expose confidential information, if you create your fork in another namespace
+that may have other members.
+
+Branches are public by default. To protect the confidentiality of your work, you
+must create your changes in a private fork.
+
+Prerequisites:
+
+- You have the Owner or Maintainer role in the public repository, as you need one
+  of these roles to [create a subgroup](../../group/subgroups/index.md).
+- You have [forked](../repository/forking_workflow.md) the public repository.
+- Your fork has a **Visibility level** of _Private_.
+
+To create a confidential merge request:
+
+1. Go to the confidential issue's page. Scroll below the issue description and
+   select **Create confidential merge request**.
+1. Select the item that meets your needs:
+   - *To create both a branch and a merge request,* select
+     **Create confidential merge request and branch**. Your merge request will
+     target the default branch of your fork, *not* the default branch of the
+     public upstream project.
+   - *To create only a branch,* select **Create branch**.
+1. Select a **Project** to use. These projects have merge requests enabled, and
+   you have the Developer role (or greater) in them.
+1. Provide a **Branch name**, and select a **Source (branch or tag)**. GitLab
+   checks whether these branches are available in your private fork, because both
+   branches must be available in your selected fork.
+1. Select **Create**.
+
+If you created a branch in your private fork, users with the Developer role in the
+public repository can push code to that branch in your private fork to fix the
+confidential issue.
+
+As your merge request targets your private fork, not the public upstream project,
+your branch, merge request, and commits do not enter the public repository. This
+prevents prematurely revealing confidential information.
+
+To make a confidential commit public, open a merge request from the private fork
+to the public upstream project.
+
+## Related links
+
+- [Confidential issues](../issues/confidential_issues.md)
+- [Security practices for confidential merge requests](https://gitlab.com/gitlab-org/release/docs/blob/master/general/security/developer.md#security-releases-critical-non-critical-as-a-developer) at GitLab
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 9c6428d701c..24d84ac36bf 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -114,7 +114,22 @@ module Gitlab
           sha: sha || find_sha(project),
           user: user,
           parent_pipeline: parent_pipeline,
-          variables: project&.predefined_variables&.to_runner_variables)
+          variables: build_variables(project: project, ref: sha))
+      end
+
+      def build_variables(project:, ref:)
+        Gitlab::Ci::Variables::Collection.new.tap do |variables|
+          break variables unless project
+
+          # The order of the next 4 lines is important as priority of CI variables is
+          # defined globally within GitLab.
+          #
+          # See more detail in the docs: https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
+          variables.concat(project.predefined_variables)
+          variables.concat(project.ci_instance_variables_for(ref: ref))
+          variables.concat(project.group.ci_variables_for(ref, project)) if project.group
+          variables.concat(project.ci_variables_for(ref: ref))
+        end
       end
 
       def track_and_raise_for_dev_exception(error)
diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb
index e48f1fa8e80..b8ccef03e63 100644
--- a/lib/gitlab/data_builder/pipeline.rb
+++ b/lib/gitlab/data_builder/pipeline.rb
@@ -2,21 +2,35 @@
 
 module Gitlab
   module DataBuilder
-    module Pipeline
-      extend self
+    # Some callers want to include retried builds, so we wrap the payload hash
+    # in a SimpleDelegator with additional methods.
+    class Pipeline < SimpleDelegator
+      def self.build(pipeline)
+        new(pipeline)
+      end
 
-      def build(pipeline)
-        {
+      def initialize(pipeline)
+        @pipeline = pipeline
+
+        super(
           object_kind: 'pipeline',
           object_attributes: hook_attrs(pipeline),
           merge_request: pipeline.merge_request && merge_request_attrs(pipeline.merge_request),
           user: pipeline.user.try(:hook_attrs),
           project: pipeline.project.hook_attrs(backward: false),
           commit: pipeline.commit.try(:hook_attrs),
-          builds: pipeline.builds.latest.map(&method(:build_hook_attrs))
-        }
+          builds: Gitlab::Lazy.new { pipeline.builds.latest.map(&method(:build_hook_attrs)) }
+        )
       end
 
+      def with_retried_builds
+        merge(
+          builds: Gitlab::Lazy.new { @pipeline.builds.map(&method(:build_hook_attrs)) }
+        )
+      end
+
+      private
+
       def hook_attrs(pipeline)
         {
           id: pipeline.id,
diff --git a/lib/gitlab/database/connection.rb b/lib/gitlab/database/connection.rb
index f6183503035..521512f5644 100644
--- a/lib/gitlab/database/connection.rb
+++ b/lib/gitlab/database/connection.rb
@@ -34,12 +34,17 @@ module Gitlab
         Gitlab::Runtime.max_threads + headroom
       end
 
-      def uncached_config
-        scope.connection_db_config.configuration_hash.with_indifferent_access
-      end
-
       def config
-        @config ||= uncached_config
+        # The result of this method must not be cached, as other methods may use
+        # it after making configuration changes and expect those changes to be
+        # present. For example, `disable_prepared_statements` expects the
+        # configuration settings to always be up to date.
+        #
+        # See the following for more information:
+        #
+        # - https://gitlab.com/gitlab-org/release/retrospectives/-/issues/39
+        # - https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5238
+        scope.connection_db_config.configuration_hash.with_indifferent_access
       end
 
       def pool_size
@@ -72,7 +77,7 @@ module Gitlab
 
       # Disables prepared statements for the current database connection.
       def disable_prepared_statements
-        scope.establish_connection(uncached_config.merge(prepared_statements: false))
+        scope.establish_connection(config.merge(prepared_statements: false))
       end
 
       def read_only?
diff --git a/spec/frontend/invite_members/components/invite_members_modal_spec.js b/spec/frontend/invite_members/components/invite_members_modal_spec.js
index b828b5d8a04..267b46d8749 100644
--- a/spec/frontend/invite_members/components/invite_members_modal_spec.js
+++ b/spec/frontend/invite_members/components/invite_members_modal_spec.js
@@ -137,6 +137,10 @@ describe('InviteMembersModal', () => {
       expect(findInviteButton().text()).toBe('Invite');
     });
 
+    it('renders the Invite button modal without isLoading', () => {
+      expect(findInviteButton().props('loading')).toBe(false);
+    });
+
     describe('rendering the access levels dropdown', () => {
       it('sets the default dropdown text to the default access level name', () => {
         expect(findDropdown().attributes('text')).toBe('Guest');
@@ -230,6 +234,16 @@ describe('InviteMembersModal', () => {
           clickInviteButton();
         });
 
+        it('sets isLoading on the Invite button when it is clicked', () => {
+          expect(findInviteButton().props('loading')).toBe(true);
+        });
+
+        it('removes isLoading from the Invite button when request completes', async () => {
+          await waitForPromises();
+
+          expect(findInviteButton().props('loading')).toBe(false);
+        });
+
         it('calls Api addGroupMembersByUserId with the correct params', async () => {
           await waitForPromises;
 
@@ -260,6 +274,7 @@ describe('InviteMembersModal', () => {
           expect(membersFormGroupInvalidFeedback()).toBe('Member already exists');
           expect(findMembersFormGroup().props('state')).toBe(false);
           expect(findMembersSelect().props('validationState')).toBe(false);
+          expect(findInviteButton().props('loading')).toBe(false);
         });
 
         it('clears the invalid state and message once the list of members to invite is cleared', async () => {
@@ -272,6 +287,7 @@ describe('InviteMembersModal', () => {
           expect(membersFormGroupInvalidFeedback()).toBe('Member already exists');
           expect(findMembersFormGroup().props('state')).toBe(false);
           expect(findMembersSelect().props('validationState')).toBe(false);
+          expect(findInviteButton().props('loading')).toBe(false);
 
           findMembersSelect().vm.$emit('clear');
 
@@ -280,6 +296,7 @@ describe('InviteMembersModal', () => {
           expect(membersFormGroupInvalidFeedback()).toBe('');
           expect(findMembersFormGroup().props('state')).not.toBe(false);
           expect(findMembersSelect().props('validationState')).not.toBe(false);
+          expect(findInviteButton().props('loading')).toBe(false);
         });
 
         it('displays the generic error for http server error', async () => {
@@ -375,6 +392,7 @@ describe('InviteMembersModal', () => {
 
           expect(membersFormGroupInvalidFeedback()).toBe(expectedSyntaxError);
           expect(findMembersSelect().props('validationState')).toBe(false);
+          expect(findInviteButton().props('loading')).toBe(false);
         });
 
         it('displays the restricted email error when restricted email is invited', async () => {
@@ -386,6 +404,7 @@ describe('InviteMembersModal', () => {
 
           expect(membersFormGroupInvalidFeedback()).toContain(expectedEmailRestrictedError);
           expect(findMembersSelect().props('validationState')).toBe(false);
+          expect(findInviteButton().props('loading')).toBe(false);
         });
 
         it('displays the successful toast message when email has already been invited', async () => {
diff --git a/spec/lib/gitlab/ci/config_spec.rb b/spec/lib/gitlab/ci/config_spec.rb
index 45ce4cac6c4..1da5881c5d5 100644
--- a/spec/lib/gitlab/ci/config_spec.rb
+++ b/spec/lib/gitlab/ci/config_spec.rb
@@ -286,7 +286,9 @@ RSpec.describe Gitlab::Ci::Config do
   end
 
   context "when using 'include' directive" do
-    let(:project) { create(:project, :repository) }
+    let(:group) { create(:group) }
+    let(:project) { create(:project, :repository, group: group) }
+    let(:main_project) { create(:project, :repository, :public, group: group) }
     let(:remote_location) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' }
     let(:local_location) { 'spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml' }
 
@@ -317,7 +319,9 @@ RSpec.describe Gitlab::Ci::Config do
       include:
         - #{local_location}
         - #{remote_location}
-
+        - project: '$MAIN_PROJECT'
+          ref: '$REF'
+          file: '$FILENAME'
       image: ruby:2.7
       HEREDOC
     end
@@ -331,6 +335,26 @@ RSpec.describe Gitlab::Ci::Config do
 
       allow(project.repository)
         .to receive(:blob_data_at).and_return(local_file_content)
+
+      main_project.repository.create_file(
+        main_project.creator,
+        '.gitlab-ci.yml',
+        local_file_content,
+        message: 'Add README.md',
+        branch_name: 'master'
+      )
+
+      main_project.repository.create_file(
+        main_project.creator,
+        '.another-ci-file.yml',
+        local_file_content,
+        message: 'Add README.md',
+        branch_name: 'master'
+      )
+
+      create(:ci_variable, project: project, key: "REF", value: "HEAD")
+      create(:ci_group_variable, group: group, key: "FILENAME", value: ".gitlab-ci.yml")
+      create(:ci_instance_variable, key: 'MAIN_PROJECT', value: main_project.full_path)
     end
 
     context "when gitlab_ci_yml has valid 'include' defined" do
@@ -344,6 +368,38 @@ RSpec.describe Gitlab::Ci::Config do
 
         expect(config.to_hash).to eq(composed_hash)
       end
+
+      context 'handling variables' do
+        it 'contains all project variables' do
+          ref = config.context.variables.find { |v| v[:key] == 'REF' }
+
+          expect(ref[:value]).to eq("HEAD")
+        end
+
+        it 'contains all group variables' do
+          filename = config.context.variables.find { |v| v[:key] == 'FILENAME' }
+
+          expect(filename[:value]).to eq(".gitlab-ci.yml")
+        end
+
+        it 'contains all instance variables' do
+          project = config.context.variables.find { |v| v[:key] == 'MAIN_PROJECT' }
+
+          expect(project[:value]).to eq(main_project.full_path)
+        end
+
+        context 'overriding a group variable at project level' do
+          before do
+            create(:ci_variable, project: project, key: "FILENAME", value: ".another-ci-file.yml")
+          end
+
+          it 'successfully overrides' do
+            filename = config.context.variables.to_hash[:FILENAME]
+
+            expect(filename).to eq('.another-ci-file.yml')
+          end
+        end
+      end
     end
 
     context "when gitlab_ci.yml has invalid 'include' defined" do
diff --git a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb
index 62de4d2e96d..e8c127f0444 100644
--- a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb
@@ -107,6 +107,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Populate do
     context 'when ref is protected' do
       before do
         allow(project).to receive(:protected_for?).with('master').and_return(true)
+        allow(project).to receive(:protected_for?).with('b83d6e391c22777fca1ed3012fce84f633d7fed0').and_return(true)
         allow(project).to receive(:protected_for?).with('refs/heads/master').and_return(true)
 
         dependencies.map(&:perform!)
diff --git a/spec/lib/gitlab/data_builder/pipeline_spec.rb b/spec/lib/gitlab/data_builder/pipeline_spec.rb
index 8191c8dca45..163de0fd80e 100644
--- a/spec/lib/gitlab/data_builder/pipeline_spec.rb
+++ b/spec/lib/gitlab/data_builder/pipeline_spec.rb
@@ -3,10 +3,10 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::DataBuilder::Pipeline do
-  let(:user) { create(:user) }
-  let(:project) { create(:project, :repository) }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:project) { create(:project, :repository) }
 
-  let(:pipeline) do
+  let_it_be_with_reload(:pipeline) do
     create(:ci_pipeline,
           project: project,
           status: 'success',
@@ -15,12 +15,12 @@ RSpec.describe Gitlab::DataBuilder::Pipeline do
           user: user)
   end
 
-  let!(:build) { create(:ci_build, pipeline: pipeline) }
+  let_it_be(:build) { create(:ci_build, pipeline: pipeline) }
 
   describe '.build' do
     let(:data) { described_class.build(pipeline) }
     let(:attributes) { data[:object_attributes] }
-    let(:build_data) { data[:builds].first }
+    let(:build_data) { data[:builds].last }
     let(:runner_data) { build_data[:runner] }
     let(:project_data) { data[:project] }
 
@@ -51,9 +51,9 @@ RSpec.describe Gitlab::DataBuilder::Pipeline do
     end
 
     context 'build with runner' do
-      let!(:build) { create(:ci_build, pipeline: pipeline, runner: ci_runner) }
-      let!(:tag_names) { %w(tag-1 tag-2) }
-      let(:ci_runner) { create(:ci_runner, tag_list: tag_names.map { |n| ActsAsTaggableOn::Tag.create!(name: n)}) }
+      let_it_be(:tag_names) { %w(tag-1 tag-2) }
+      let_it_be(:ci_runner) { create(:ci_runner, tag_list: tag_names.map { |n| ActsAsTaggableOn::Tag.create!(name: n)}) }
+      let_it_be(:build) { create(:ci_build, pipeline: pipeline, runner: ci_runner) }
 
       it 'has runner attributes', :aggregate_failures do
         expect(runner_data[:id]).to eq(ci_runner.id)
@@ -73,18 +73,15 @@ RSpec.describe Gitlab::DataBuilder::Pipeline do
     end
 
     context 'pipeline with variables' do
-      let(:build) { create(:ci_build, pipeline: pipeline) }
-      let(:data) { described_class.build(pipeline) }
-      let(:attributes) { data[:object_attributes] }
-      let!(:pipeline_variable) { create(:ci_pipeline_variable, pipeline: pipeline, key: 'TRIGGER_KEY_1', value: 'TRIGGER_VALUE_1') }
+      let_it_be(:pipeline_variable) { create(:ci_pipeline_variable, pipeline: pipeline, key: 'TRIGGER_KEY_1', value: 'TRIGGER_VALUE_1') }
 
       it { expect(attributes[:variables]).to be_a(Array) }
       it { expect(attributes[:variables]).to contain_exactly({ key: 'TRIGGER_KEY_1', value: 'TRIGGER_VALUE_1' }) }
     end
 
     context 'when pipeline is a detached merge request pipeline' do
-      let(:merge_request) { create(:merge_request, :with_detached_merge_request_pipeline) }
-      let(:pipeline) { merge_request.all_pipelines.first }
+      let_it_be(:merge_request) { create(:merge_request, :with_detached_merge_request_pipeline) }
+      let_it_be(:pipeline) { merge_request.all_pipelines.first }
 
       it 'returns a source ref' do
         expect(attributes[:ref]).to eq(merge_request.source_branch)
@@ -108,19 +105,25 @@ RSpec.describe Gitlab::DataBuilder::Pipeline do
     end
 
     context 'when pipeline has retried builds' do
-      before do
-        create(:ci_build, :retried, pipeline: pipeline)
-      end
+      let_it_be(:retried_build) { create(:ci_build, :retried, pipeline: pipeline) }
 
       it 'does not contain retried builds in payload' do
-        expect(data[:builds].count).to eq(1)
-        expect(build_data[:id]).to eq(build.id)
+        builds = data[:builds]
+
+        expect(builds.pluck(:id)).to contain_exactly(build.id)
+      end
+
+      it 'contains retried builds if requested' do
+        builds = data.with_retried_builds[:builds]
+
+        expect(builds.pluck(:id)).to contain_exactly(build.id, retried_build.id)
       end
     end
 
     context 'build with environment' do
-      let!(:build) { create(:ci_build, :environment_with_deployment_tier, :with_deployment, pipeline: pipeline) }
-      let!(:build_environment_data) { build_data[:environment] }
+      let_it_be(:build) { create(:ci_build, :environment_with_deployment_tier, :with_deployment, pipeline: pipeline) }
+
+      let(:build_environment_data) { build_data[:environment] }
 
       it 'has environment attributes', :aggregate_failures do
         expect(build_environment_data[:name]).to eq(build.expanded_environment_name)
diff --git a/spec/lib/gitlab/database/connection_spec.rb b/spec/lib/gitlab/database/connection_spec.rb
index fc9ae897a5f..4cbc94660c3 100644
--- a/spec/lib/gitlab/database/connection_spec.rb
+++ b/spec/lib/gitlab/database/connection_spec.rb
@@ -29,12 +29,19 @@ RSpec.describe Gitlab::Database::Connection do
     it 'returns a default pool size' do
       expect(connection.config).to include(pool: connection.default_pool_size)
     end
+
+    it 'does not cache its results' do
+      a = connection.config
+      b = connection.config
+
+      expect(a).not_to equal(b)
+    end
   end
 
   describe '#pool_size' do
     context 'when no explicit size is configured' do
       it 'returns the default pool size' do
-        expect(connection.config).to receive(:[]).with(:pool).and_return(nil)
+        expect(connection).to receive(:config).and_return({ pool: nil })
 
         expect(connection.pool_size).to eq(connection.default_pool_size)
       end
@@ -42,7 +49,7 @@ RSpec.describe Gitlab::Database::Connection do
 
     context 'when an explicit pool size is set' do
       it 'returns the pool size' do
-        expect(connection.config).to receive(:[]).with(:pool).and_return(4)
+        expect(connection).to receive(:config).and_return({ pool: 4 })
 
         expect(connection.pool_size).to eq(4)
       end
diff --git a/spec/models/integrations/datadog_spec.rb b/spec/models/integrations/datadog_spec.rb
index e2749ab1bc1..0920302e386 100644
--- a/spec/models/integrations/datadog_spec.rb
+++ b/spec/models/integrations/datadog_spec.rb
@@ -6,7 +6,8 @@ require 'spec_helper'
 RSpec.describe Integrations::Datadog do
   let_it_be(:project) { create(:project) }
   let_it_be(:pipeline) { create(:ci_pipeline, project: project) }
-  let_it_be(:build) { create(:ci_build, project: project) }
+  let_it_be(:build) { create(:ci_build, pipeline: pipeline) }
+  let_it_be(:retried_build) { create(:ci_build, :retried, pipeline: pipeline) }
 
   let(:active) { true }
   let(:dd_site) { 'datadoghq.com' }
@@ -159,6 +160,10 @@ RSpec.describe Integrations::Datadog do
   end
 
   describe '#execute' do
+    around do |example|
+      freeze_time { example.run }
+    end
+
     before do
       stub_request(:post, expected_hook_url)
       saved_instance.execute(data)
@@ -166,20 +171,18 @@ RSpec.describe Integrations::Datadog do
 
     context 'with pipeline data' do
       let(:data) { pipeline_data }
-      let(:expected_headers) do
-        { WebHookService::GITLAB_EVENT_HEADER => 'Pipeline Hook' }
-      end
+      let(:expected_headers) { { WebHookService::GITLAB_EVENT_HEADER => 'Pipeline Hook' } }
+      let(:expected_body) { data.with_retried_builds.to_json }
 
-      it { expect(a_request(:post, expected_hook_url).with(headers: expected_headers)).to have_been_made }
+      it { expect(a_request(:post, expected_hook_url).with(headers: expected_headers, body: expected_body)).to have_been_made }
     end
 
     context 'with job data' do
       let(:data) { build_data }
-      let(:expected_headers) do
-        { WebHookService::GITLAB_EVENT_HEADER => 'Job Hook' }
-      end
+      let(:expected_headers) { { WebHookService::GITLAB_EVENT_HEADER => 'Job Hook' } }
+      let(:expected_body) { data.to_json }
 
-      it { expect(a_request(:post, expected_hook_url).with(headers: expected_headers)).to have_been_made }
+      it { expect(a_request(:post, expected_hook_url).with(headers: expected_headers, body: expected_body)).to have_been_made }
     end
   end
 end