Add timed incremental rollout to Auto DevOps

Auto DevOps deployment strategies now supports timed incremental
rollout. We are deprecating the usage of INCREMENTAL_ROLLOUT_ENABLED
environment variable in Auto DevOps template.

The new behavior will be driven by the INCREMENTAL_ROLLOUT_MODE variable
that can either be manual (same as INCREMENTAL_ROLLOUT_ENABLED) or
timed.

Rollout deployments will be executed using a 5 minute delay between each
job.

8 files changed, 162 insertions, 68 deletions

diff --git a/app/models/project_auto_devops.rb b/app/models/project_auto_devops.rb
index dc6736dd9cd..2253ad7b543 100644
--- a/app/models/project_auto_devops.rb
+++ b/app/models/project_auto_devops.rb
@@ -5,7 +5,8 @@ class ProjectAutoDevops < ActiveRecord::Base
 
   enum deploy_strategy: {
     continuous: 0,
-    manual: 1
+    manual: 1,
+    timed_incremental: 2
   }
 
   scope :enabled, -> { where(enabled: true) }
@@ -30,10 +31,7 @@ class ProjectAutoDevops < ActiveRecord::Base
                          value: domain.presence || instance_domain)
       end
 
-      if manual?
-        variables.append(key: 'STAGING_ENABLED', value: '1')
-        variables.append(key: 'INCREMENTAL_ROLLOUT_ENABLED', value: '1')
-      end
+      variables.concat(deployment_strategy_default_variables)
     end
   end
 
@@ -51,4 +49,16 @@ class ProjectAutoDevops < ActiveRecord::Base
       !project.public? &&
       !project.deploy_tokens.find_by(name: DeployToken::GITLAB_DEPLOY_TOKEN_NAME).present?
   end
+
+  def deployment_strategy_default_variables
+    Gitlab::Ci::Variables::Collection.new.tap do |variables|
+      if manual?
+        variables.append(key: 'STAGING_ENABLED', value: '1')
+        variables.append(key: 'INCREMENTAL_ROLLOUT_ENABLED', value: '1') # deprecated
+        variables.append(key: 'INCREMENTAL_ROLLOUT_MODE', value: 'manual')
+      elsif timed_incremental?
+        variables.append(key: 'INCREMENTAL_ROLLOUT_MODE', value: 'timed')
+      end
+    end
+  end
 end
diff --git a/app/views/projects/settings/ci_cd/_autodevops_form.html.haml b/app/views/projects/settings/ci_cd/_autodevops_form.html.haml
index ab92b757836..5ec5a06396e 100644
--- a/app/views/projects/settings/ci_cd/_autodevops_form.html.haml
+++ b/app/views/projects/settings/ci_cd/_autodevops_form.html.haml
@@ -39,10 +39,17 @@
                   = form.label :deploy_strategy_continuous, class: 'form-check-label' do
                     = s_('CICD|Continuous deployment to production')
                     = link_to icon('question-circle'), help_page_path('topics/autodevops/index.md', anchor: 'auto-deploy'), target: '_blank'
+
+                .form-check
+                  = form.radio_button :deploy_strategy, 'timed_incremental', class: 'form-check-input'
+                  = form.label :deploy_strategy_timed_incremental, class: 'form-check-label' do
+                    = s_('CICD|Continuous deployment to production using timed incremental rollout')
+                    = link_to icon('question-circle'), help_page_path('topics/autodevops/index.md', anchor: 'timed-incremental-rollout-to-production'), target: '_blank'
+
                 .form-check
                   = form.radio_button :deploy_strategy, 'manual', class: 'form-check-input'
                   = form.label :deploy_strategy_manual, class: 'form-check-label' do
                     = s_('CICD|Automatic deployment to staging, manual deployment to production')
-                    = link_to icon('question-circle'), help_page_path('ci/environments.md', anchor: 'manually-deploying-to-environments'), target: '_blank'
+                    = link_to icon('question-circle'), help_page_path('topics/autodevops/index.md', anchor: 'incremental-rollout-to-production'), target: '_blank'
 
           = f.submit _('Save changes'), class: "btn btn-success prepend-top-15"
diff --git a/changelogs/unreleased/autodevops-timed-incremental-rollout.yml b/changelogs/unreleased/autodevops-timed-incremental-rollout.yml
new file mode 100644
index 00000000000..72c7b41177d
--- /dev/null
+++ b/changelogs/unreleased/autodevops-timed-incremental-rollout.yml
@@ -0,0 +1,5 @@
+---
+title: Add timed incremental rollout to Auto DevOps
+merge_request: 22023
+author:
+type: added
diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md
index b5a9e469965..0d1ba3e8f9a 100644
--- a/doc/topics/autodevops/index.md
+++ b/doc/topics/autodevops/index.md
@@ -239,14 +239,19 @@ project's **Settings > CI/CD > Auto DevOps**.
 
 The available options are:
 
-- **Continuous deployment to production** - enables [Auto Deploy](#auto-deploy)
-  by setting the [`STAGING_ENABLED`](#deploy-policy-for-staging-and-production-environments) and
-  [`INCREMENTAL_ROLLOUT_ENABLED`](#incremental-rollout-to-production) variables
-  to false.
-- **Automatic deployment to staging, manual deployment to production** - sets the
+- **Continuous deployment to production**: Enables [Auto Deploy](#auto-deploy)
+  with `master` branch directly deployed to production.
+- **Continuous deployment to production using timed incremental rollout**: Sets the
+  [`INCREMENTAL_ROLLOUT_MODE`](#timed-incremental-rollout-to-production) variable
+  to `timed`, and production deployment will be executed with a 5 minute delay between
+  each increment in rollout.
+- **Automatic deployment to staging, manual deployment to production**: Sets the
   [`STAGING_ENABLED`](#deploy-policy-for-staging-and-production-environments) and
-  [`INCREMENTAL_ROLLOUT_ENABLED`](#incremental-rollout-to-production) variables
-  to true, and the user is responsible for manually deploying to staging and production.
+  [`INCREMENTAL_ROLLOUT_MODE`](#incremental-rollout-to-production) variables
+  to `1` and `manual`. This means:
+
+  - `master` branch is directly deployed to staging.
+  - Manual actions are provided for incremental rollout to production.
 
 ## Stages of Auto DevOps
 
@@ -609,7 +614,7 @@ also be customized, and you can easily use a [custom buildpack](#custom-buildpac
 | `DB_MIGRATE`                 | From GitLab 11.4, this variable can be used to specify the command to run to migrate the application's PostgreSQL database. It runs inside the application pod. |
 | `STAGING_ENABLED`            | From GitLab 10.8, this variable can be used to define a [deploy policy for staging and production environments](#deploy-policy-for-staging-and-production-environments). |
 | `CANARY_ENABLED`             | From GitLab 11.0, this variable can be used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments). |
-| `INCREMENTAL_ROLLOUT_ENABLED`| From GitLab 10.8, this variable can be used to enable an [incremental rollout](#incremental-rollout-to-production) of your application for the production environment. |
+| `INCREMENTAL_ROLLOUT_MODE`| From GitLab 11.4, this variable, if present, can be used to enable an [incremental rollout](#incremental-rollout-to-production) of your application for the production environment.<br/>Set to: <ul><li>`manual`, for manual deployment jobs.</li><li>`timed`, for automatic rollout deployments with a 5 minute delay each one.</li></ul> |
 | `TEST_DISABLED`              | From GitLab 11.0, this variable can be used to disable the `test` job. If the variable is present, the job will not be created. |
 | `CODE_QUALITY_DISABLED`       | From GitLab 11.0, this variable can be used to disable the `codequality` job. If the variable is present, the job will not be created. |
 | `SAST_DISABLED`              | From GitLab 11.0, this variable can be used to disable the `sast` job. If the variable is present, the job will not be created. |
@@ -730,9 +735,8 @@ to use an incremental rollout to replace just a few pods with the latest code.
 This will allow you to first check how the app is behaving, and later manually
 increasing the rollout up to 100%.
 
-If `INCREMENTAL_ROLLOUT_ENABLED` is defined in your project (e.g., set
-`INCREMENTAL_ROLLOUT_ENABLED` to `1` as a secret variable), then instead of the
-standard `production` job, 4 different
+If `INCREMENTAL_ROLLOUT_MODE` is set to `manual` in your project, then instead
+of the standard `production` job, 4 different
 [manual jobs](../../ci/pipelines.md#manual-actions-from-the-pipeline-graph)
 will be created:
 
@@ -756,21 +760,45 @@ environment page.
 Below, you can see how the pipeline will look if the rollout or staging
 variables are defined.
 
-- **Without `INCREMENTAL_ROLLOUT_ENABLED` and without `STAGING_ENABLED`**
+Without `INCREMENTAL_ROLLOUT_MODE` and without `STAGING_ENABLED`:
+
+![Staging and rollout disabled](img/rollout_staging_disabled.png)
+
+Without `INCREMENTAL_ROLLOUT_MODE` and with `STAGING_ENABLED`:
 
-    ![Staging and rollout disabled](img/rollout_staging_disabled.png)
+![Staging enabled](img/staging_enabled.png)
 
-- **Without `INCREMENTAL_ROLLOUT_ENABLED` and with `STAGING_ENABLED`**
+With `INCREMENTAL_ROLLOUT_MODE` set to `manual` and without `STAGING_ENABLED`:
 
-    ![Staging enabled](img/staging_enabled.png)
+![Rollout enabled](img/rollout_enabled.png)
 
-- **With `INCREMENTAL_ROLLOUT_ENABLED` and without `STAGING_ENABLED`**
+With `INCREMENTAL_ROLLOUT_MODE` set to `manual` and with `STAGING_ENABLED`
+
+![Rollout and staging enabled](img/rollout_staging_enabled.png)
+
+CAUTION: **Caution:**
+Before GitLab 11.4 this feature was enabled by the presence of the
+`INCREMENTAL_ROLLOUT_ENABLED` environment variable.
+This configuration is deprecated and will be removed in the future.
+
+#### Timed incremental rollout to production **[PREMIUM]**
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/7545) in GitLab 11.4.
+
+TIP: **Tip:**
+You can also set this inside your [project's settings](#deployment-strategy).
 
-    ![Rollout enabled](img/rollout_enabled.png)
+This configuration based on
+[incremental rollout to production](#incremental-rollout-to-production).
 
-- **With `INCREMENTAL_ROLLOUT_ENABLED` and with `STAGING_ENABLED`**
+Everything behaves the same way, except:
 
-    ![Rollout and staging enabled](img/rollout_staging_enabled.png)
+- It's enabled by setting the `INCREMENTAL_ROLLOUT_MODE` variable to `timed`.
+- Instead of the standard `production` job, the following jobs with a 5 minute delay between each are created:
+    1. `timed rollout 10%`
+    1. `timed rollout 25%`
+    1. `timed rollout 50%`
+    1. `timed rollout 100%`
 
 ## Currently supported languages
 
diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
index ed4ec7e6385..72547c1b407 100644
--- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
@@ -25,8 +25,9 @@
 # level, or manually added below.
 #
 # Continuous deployment to production is enabled by default.
-# If you want to deploy to staging first, or enable incremental rollouts,
-# set STAGING_ENABLED or INCREMENTAL_ROLLOUT_ENABLED environment variables.
+# If you want to deploy to staging first, set STAGING_ENABLED environment variable.
+# If you want to enable incremental rollout, either manual or time based,
+# set INCREMENTAL_ROLLOUT_TYPE environment variable to "manual" or "timed".
 # If you want to use canary deployments, set CANARY_ENABLED environment variable.
 #
 # If Auto DevOps fails to detect the proper buildpack, or if you want to
@@ -61,6 +62,10 @@ stages:
   - staging
   - canary
   - production
+  - incremental rollout 10%
+  - incremental rollout 25%
+  - incremental rollout 50%
+  - incremental rollout 100%
   - performance
   - cleanup
 
@@ -282,11 +287,6 @@ stop_review:
     variables:
       - $REVIEW_DISABLED
 
-# Keys that start with a dot (.) will not be processed by GitLab CI.
-# Staging and canary jobs are disabled by default, to enable them
-# remove the dot (.) before the job name.
-# https://docs.gitlab.com/ee/ci/yaml/README.html#hidden-keys
-
 # Staging deploys are disabled by default since
 # continuous deployment to production is enabled by default
 # If you prefer to automatically deploy to staging and
@@ -368,6 +368,7 @@ production:
       - $STAGING_ENABLED
       - $CANARY_ENABLED
       - $INCREMENTAL_ROLLOUT_ENABLED
+      - $INCREMENTAL_ROLLOUT_MODE
 
 production_manual:
   <<: *production_template
@@ -383,11 +384,11 @@ production_manual:
   except:
     variables:
       - $INCREMENTAL_ROLLOUT_ENABLED
+      - $INCREMENTAL_ROLLOUT_MODE
 
 # This job implements incremental rollout on for every push to `master`.
 
 .rollout: &rollout_template
-  stage: production
   script:
     - check_kube_domain
     - install_dependencies
@@ -405,52 +406,77 @@ production_manual:
   artifacts:
     paths: [environment_url.txt]
 
-rollout 10%:
+.manual_rollout_template: &manual_rollout_template
   <<: *rollout_template
-  variables:
-    ROLLOUT_PERCENTAGE: 10
+  stage: production
   when: manual
+  # This selectors are backward compatible mode with $INCREMENTAL_ROLLOUT_ENABLED (before 11.4)
   only:
     refs:
       - master
     kubernetes: active
     variables:
+      - $INCREMENTAL_ROLLOUT_MODE == "manual"
       - $INCREMENTAL_ROLLOUT_ENABLED
+  except:
+    variables:
+      - $INCREMENTAL_ROLLOUT_MODE == "timed"
 
-rollout 25%:
+.timed_rollout_template: &timed_rollout_template
   <<: *rollout_template
-  variables:
-    ROLLOUT_PERCENTAGE: 25
-  when: manual
+  when: delayed
+  start_in: 5 minutes
   only:
     refs:
       - master
     kubernetes: active
     variables:
-      - $INCREMENTAL_ROLLOUT_ENABLED
+      - $INCREMENTAL_ROLLOUT_MODE == "timed"
+
+timed rollout 10%:
+  <<: *timed_rollout_template
+  stage: incremental rollout 10%
+  variables:
+    ROLLOUT_PERCENTAGE: 10
+
+timed rollout 25%:
+  <<: *timed_rollout_template
+  stage: incremental rollout 25%
+  variables:
+    ROLLOUT_PERCENTAGE: 25
+
+timed rollout 50%:
+  <<: *timed_rollout_template
+  stage: incremental rollout 50%
+  variables:
+    ROLLOUT_PERCENTAGE: 50
+
+timed rollout 100%:
+  <<: *timed_rollout_template
+  <<: *production_template
+  stage: incremental rollout 100%
+  variables:
+    ROLLOUT_PERCENTAGE: 100
+
+rollout 10%:
+  <<: *manual_rollout_template
+  variables:
+    ROLLOUT_PERCENTAGE: 10
+
+rollout 25%:
+  <<: *manual_rollout_template
+  variables:
+    ROLLOUT_PERCENTAGE: 25
 
 rollout 50%:
-  <<: *rollout_template
+  <<: *manual_rollout_template
   variables:
     ROLLOUT_PERCENTAGE: 50
-  when: manual
-  only:
-    refs:
-      - master
-    kubernetes: active
-    variables:
-      - $INCREMENTAL_ROLLOUT_ENABLED
 
 rollout 100%:
+  <<: *manual_rollout_template
   <<: *production_template
-  when: manual
   allow_failure: false
-  only:
-    refs:
-      - master
-    kubernetes: active
-    variables:
-      - $INCREMENTAL_ROLLOUT_ENABLED
 
 # ---------------------------------------------------------------------------
 
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index d16a72b76b8..20b70bc2cd3 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -1081,6 +1081,9 @@ msgstr ""
 msgid "CICD|Continuous deployment to production"
 msgstr ""
 
+msgid "CICD|Continuous deployment to production using timed incremental rollout"
+msgstr ""
+
 msgid "CICD|Default to Auto DevOps pipeline"
 msgstr ""
 
diff --git a/spec/factories/project_auto_devops.rb b/spec/factories/project_auto_devops.rb
index b77f702f9e1..75ac7cc7687 100644
--- a/spec/factories/project_auto_devops.rb
+++ b/spec/factories/project_auto_devops.rb
@@ -5,8 +5,16 @@ FactoryBot.define do
     domain "example.com"
     deploy_strategy :continuous
 
-    trait :manual do
-      deploy_strategy :manual
+    trait :continuous_deployment do
+      deploy_strategy ProjectAutoDevops.deploy_strategies[:continuous] # rubocop:disable FactoryBot/DynamicAttributeDefinedStatically
+    end
+
+    trait :manual_deployment do
+      deploy_strategy ProjectAutoDevops.deploy_strategies[:manual] # rubocop:disable FactoryBot/DynamicAttributeDefinedStatically
+    end
+
+    trait :timed_incremental_deployment do
+      deploy_strategy ProjectAutoDevops.deploy_strategies[:timed_incremental] # rubocop:disable FactoryBot/DynamicAttributeDefinedStatically
     end
 
     trait :disabled do
diff --git a/spec/models/project_auto_devops_spec.rb b/spec/models/project_auto_devops_spec.rb
index 797d767465a..342798f730b 100644
--- a/spec/models/project_auto_devops_spec.rb
+++ b/spec/models/project_auto_devops_spec.rb
@@ -70,24 +70,31 @@ describe ProjectAutoDevops do
     end
 
     context 'when deploy_strategy is manual' do
-      let(:domain) { 'example.com' }
-
-      before do
-        auto_devops.deploy_strategy = 'manual'
+      let(:auto_devops) { build_stubbed(:project_auto_devops, :manual_deployment, project: project) }
+      let(:expected_variables) do
+        [
+          { key: 'INCREMENTAL_ROLLOUT_MODE', value: 'manual' },
+          { key: 'STAGING_ENABLED', value: '1' },
+          { key: 'INCREMENTAL_ROLLOUT_ENABLED', value: '1' }
+        ]
       end
 
+      it { expect(auto_devops.predefined_variables).to include(*expected_variables) }
+    end
+
+    context 'when deploy_strategy is continuous' do
+      let(:auto_devops) { build_stubbed(:project_auto_devops, :continuous_deployment, project: project) }
+
       it do
         expect(auto_devops.predefined_variables.map { |var| var[:key] })
-          .to include("STAGING_ENABLED", "INCREMENTAL_ROLLOUT_ENABLED")
+          .not_to include("STAGING_ENABLED", "INCREMENTAL_ROLLOUT_ENABLED")
       end
     end
 
-    context 'when deploy_strategy is continuous' do
-      let(:domain) { 'example.com' }
+    context 'when deploy_strategy is timed_incremental' do
+      let(:auto_devops) { build_stubbed(:project_auto_devops, :timed_incremental_deployment, project: project) }
 
-      before do
-        auto_devops.deploy_strategy = 'continuous'
-      end
+      it { expect(auto_devops.predefined_variables).to include(key: 'INCREMENTAL_ROLLOUT_MODE', value: 'timed') }
 
       it do
         expect(auto_devops.predefined_variables.map { |var| var[:key] })