Add latest changes from gitlab-org/gitlab@master

17 files changed, 237 insertions, 60 deletions

diff --git a/app/assets/javascripts/lib/utils/color_utils.js b/app/assets/javascripts/lib/utils/color_utils.js
index a1f56b15631..ff176f11867 100644
--- a/app/assets/javascripts/lib/utils/color_utils.js
+++ b/app/assets/javascripts/lib/utils/color_utils.js
@@ -23,3 +23,23 @@ export const textColorForBackground = (backgroundColor) => {
   }
   return '#FFFFFF';
 };
+
+/**
+ * Check whether a color matches the expected hex format
+ *
+ * This matches any hex (0-9 and A-F) value which is either 3 or 6 characters in length
+ *
+ * An empty string will return `null` which means that this is neither valid nor invalid.
+ * This is useful for forms resetting the validation state
+ *
+ * @param color string = ''
+ *
+ * @returns {null|boolean}
+ */
+export const validateHexColor = (color = '') => {
+  if (!color) {
+    return null;
+  }
+
+  return /^#([0-9A-F]{3}){1,2}$/i.test(color);
+};
diff --git a/app/assets/javascripts/vue_shared/components/color_picker/color_picker.vue b/app/assets/javascripts/vue_shared/components/color_picker/color_picker.vue
index 6977692e30c..8f997149259 100644
--- a/app/assets/javascripts/vue_shared/components/color_picker/color_picker.vue
+++ b/app/assets/javascripts/vue_shared/components/color_picker/color_picker.vue
@@ -3,12 +3,16 @@
  * Renders a color picker input with preset colors to choose from
  *
  * @example
- * <color-picker :label="__('Background color')" set-color="#FF0000" />
+ * <color-picker
+     :invalid-feedback="__('Please enter a valid hex (#RRGGBB or #RGB) color value')"
+     :label="__('Background color')"
+     set-color="#FF0000"
+     state="isValidColor"
+   />
  */
 import { GlFormGroup, GlFormInput, GlFormInputGroup, GlLink, GlTooltipDirective } from '@gitlab/ui';
 import { __ } from '~/locale';
 
-const VALID_RGB_HEX_COLOR = /^#([0-9A-F]{3}){1,2}$/i;
 const PREVIEW_COLOR_DEFAULT_CLASSES =
   'gl-relative gl-w-7 gl-bg-gray-10 gl-rounded-top-left-base gl-rounded-bottom-left-base';
 
@@ -24,6 +28,11 @@ export default {
     GlTooltip: GlTooltipDirective,
   },
   props: {
+    invalidFeedback: {
+      type: String,
+      required: false,
+      default: __('Please enter a valid hex (#RRGGBB or #RGB) color value'),
+    },
     label: {
       type: String,
       required: false,
@@ -34,6 +43,11 @@ export default {
       required: false,
       default: '',
     },
+    state: {
+      type: Boolean,
+      required: false,
+      default: null,
+    },
   },
   data() {
     return {
@@ -50,46 +64,32 @@ export default {
       return gon.suggested_label_colors;
     },
     previewColor() {
-      if (this.isValidColor) {
+      if (this.state) {
         return { backgroundColor: this.selectedColor };
       }
 
       return {};
     },
     previewColorClasses() {
-      const borderStyle = this.isInvalidColor
-        ? 'gl-inset-border-1-red-500'
-        : 'gl-inset-border-1-gray-400';
+      const borderStyle =
+        this.state === false ? 'gl-inset-border-1-red-500' : 'gl-inset-border-1-gray-400';
 
       return `${PREVIEW_COLOR_DEFAULT_CLASSES} ${borderStyle}`;
     },
     hasSuggestedColors() {
       return Object.keys(this.suggestedColors).length;
     },
-    isInvalidColor() {
-      return this.isValidColor === false;
-    },
-    isValidColor() {
-      if (this.selectedColor === '') {
-        return null;
-      }
-
-      return VALID_RGB_HEX_COLOR.test(this.selectedColor);
-    },
   },
   methods: {
     handleColorChange(color) {
       this.selectedColor = color.trim();
 
-      if (this.isValidColor) {
-        this.$emit('input', this.selectedColor);
-      }
+      this.$emit('input', this.selectedColor);
     },
   },
   i18n: {
     fullDescription: __('Choose any color. Or you can choose one of the suggested colors below'),
     shortDescription: __('Choose any color'),
-    invalid: __('Please enter a valid hex (#RRGGBB or #RGB) color value'),
   },
 };
 </script>
@@ -100,17 +100,17 @@ export default {
       :label="label"
       label-for="color-picker"
       :description="description"
-      :invalid-feedback="this.$options.i18n.invalid"
-      :state="isValidColor"
+      :invalid-feedback="invalidFeedback"
+      :state="state"
       :class="{ 'gl-mb-3!': hasSuggestedColors }"
     >
       <gl-form-input-group
         id="color-picker"
-        :state="isValidColor"
         max-length="7"
         type="text"
         class="gl-align-center gl-rounded-0 gl-rounded-top-right-base gl-rounded-bottom-right-base"
         :value="selectedColor"
+        :state="state"
         @input="handleColorChange"
       >
         <template #prepend>
diff --git a/app/models/terraform/state_version.rb b/app/models/terraform/state_version.rb
index 19d708616fc..be0803fee0e 100644
--- a/app/models/terraform/state_version.rb
+++ b/app/models/terraform/state_version.rb
@@ -9,6 +9,8 @@ module Terraform
     belongs_to :build, class_name: 'Ci::Build', optional: true, foreign_key: :ci_build_id
 
     scope :ordered_by_version_desc, -> { order(version: :desc) }
+    scope :with_files_stored_locally, -> { where(file_store: Terraform::StateUploader::Store::LOCAL) }
+    scope :preload_state, -> { includes(:terraform_state) }
 
     default_value_for(:file_store) { StateUploader.default_store }
 
diff --git a/app/uploaders/terraform/state_uploader.rb b/app/uploaders/terraform/state_uploader.rb
index d80725cb051..091b253b0ed 100644
--- a/app/uploaders/terraform/state_uploader.rb
+++ b/app/uploaders/terraform/state_uploader.rb
@@ -6,6 +6,10 @@ module Terraform
 
     storage_options Gitlab.config.terraform_state
 
+    # TODO: Remove this line
+    # See https://gitlab.com/gitlab-org/gitlab/-/issues/232917
+    alias_method :upload, :model
+
     delegate :terraform_state, :project_id, to: :model
 
     # Use Lockbox to encrypt/decrypt the stored file (registers CarrierWave callbacks)
diff --git a/changelogs/unreleased/terraform-states-migrate-task.yml b/changelogs/unreleased/terraform-states-migrate-task.yml
new file mode 100644
index 00000000000..8b1f967be88
--- /dev/null
+++ b/changelogs/unreleased/terraform-states-migrate-task.yml
@@ -0,0 +1,5 @@
+---
+title: Add rake task to migrate Terraform states to object storage
+merge_request: 50740
+author:
+type: added
diff --git a/doc/administration/terraform_state.md b/doc/administration/terraform_state.md
index be5647aa133..5ea5863396f 100644
--- a/doc/administration/terraform_state.md
+++ b/doc/administration/terraform_state.md
@@ -100,6 +100,11 @@ See [the available connection settings for different providers](object_storage.m
    ```
 
 1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect.
+1. Migrate any existing local states to the object storage (GitLab 13.9 and later):
+
+   ```shell
+   gitlab-rake gitlab:terraform_states:migrate
+   ```
 
 **In installations from source:**
 
@@ -120,3 +125,8 @@ See [the available connection settings for different providers](object_storage.m
    ```
 
 1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect.
+1. Migrate any existing local states to the object storage (GitLab 13.9 and later):
+
+   ```shell
+   sudo -u git -H bundle exec rake gitlab:terraform_states:migrate RAILS_ENV=production
+   ```
diff --git a/doc/operations/incident_management/alert_notifications.md b/doc/operations/incident_management/alert_notifications.md
index 6f3b329572b..4f46c2bec71 100644
--- a/doc/operations/incident_management/alert_notifications.md
+++ b/doc/operations/incident_management/alert_notifications.md
@@ -4,5 +4,5 @@ redirect_to: 'paging.md'
 
 This document was moved to [another location](paging.md).
 
-<!-- This redirect file can be deleted after <2022-01-21>. -->
+<!-- This redirect file can be deleted after 2021-04-21 -->
 <!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page -->
diff --git a/lib/gitlab/terraform/state_migration_helper.rb b/lib/gitlab/terraform/state_migration_helper.rb
new file mode 100644
index 00000000000..04c1cbd0373
--- /dev/null
+++ b/lib/gitlab/terraform/state_migration_helper.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Terraform
+    class StateMigrationHelper
+      class << self
+        def migrate_to_remote_storage(&block)
+          migrate_in_batches(
+            ::Terraform::StateVersion.with_files_stored_locally.preload_state,
+            ::Terraform::StateUploader::Store::REMOTE,
+            &block
+          )
+        end
+
+        private
+
+        def batch_size
+          ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i
+        end
+
+        def migrate_in_batches(versions, store, &block)
+          versions.find_each(batch_size: batch_size) do |version| # rubocop:disable CodeReuse/ActiveRecord
+            version.file.migrate!(store)
+
+            yield version if block_given?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb
index 71dfe27dd5a..8662b4ade37 100644
--- a/lib/gitlab/tracking/standard_context.rb
+++ b/lib/gitlab/tracking/standard_context.rb
@@ -11,24 +11,12 @@ module Gitlab
         @data = data
       end
 
-      def namespace_id
-        namespace&.id
-      end
-
-      def project_id
-        @project&.id
-      end
-
       def to_context
         SnowplowTracker::SelfDescribingJson.new(GITLAB_STANDARD_SCHEMA_URL, to_h)
       end
 
       private
 
-      def namespace
-        @namespace || @project&.namespace
-      end
-
       def to_h
         public_methods(false).each_with_object({}) do |method, hash|
           next if method == :to_context
diff --git a/lib/tasks/gitlab/terraform/migrate.rake b/lib/tasks/gitlab/terraform/migrate.rake
new file mode 100644
index 00000000000..a9c16049240
--- /dev/null
+++ b/lib/tasks/gitlab/terraform/migrate.rake
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+desc "GitLab | Terraform | Migrate Terraform states to remote storage"
+namespace :gitlab do
+  namespace :terraform_states do
+    task migrate: :environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Starting transfer of Terraform states to object storage')
+
+      begin
+        Gitlab::Terraform::StateMigrationHelper.migrate_to_remote_storage do |state_version|
+          message = "Transferred Terraform state version ID #{state_version.id} (#{state_version.terraform_state.name}/#{state_version.version}) to object storage"
+
+          logger.info(message)
+        end
+      rescue => e
+        logger.error("Failed to migrate: #{e.message}")
+      end
+    end
+  end
+end
diff --git a/qa/qa/specs/features/browser_ui/6_release/pages/pages_pipeline_spec.rb b/qa/qa/specs/features/browser_ui/6_release/pages/pages_pipeline_spec.rb
index 17c53b3ddc9..616280cfdf4 100644
--- a/qa/qa/specs/features/browser_ui/6_release/pages/pages_pipeline_spec.rb
+++ b/qa/qa/specs/features/browser_ui/6_release/pages/pages_pipeline_spec.rb
@@ -30,7 +30,7 @@ module QA
         pipeline.visit!
       end
 
-      it 'runs a Pages-specific pipeline', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/296937' do
+      it 'runs a Pages-specific pipeline' do
         Page::Project::Pipeline::Show.perform do |show|
           expect(show).to have_job(:pages)
           show.click_job(:pages)
diff --git a/spec/frontend/lib/utils/color_utils_spec.js b/spec/frontend/lib/utils/color_utils_spec.js
index 433e9d5a85e..8c846abd77f 100644
--- a/spec/frontend/lib/utils/color_utils_spec.js
+++ b/spec/frontend/lib/utils/color_utils_spec.js
@@ -1,4 +1,4 @@
-import { textColorForBackground, hexToRgb } from '~/lib/utils/color_utils';
+import { textColorForBackground, hexToRgb, validateHexColor } from '~/lib/utils/color_utils';
 
 describe('Color utils', () => {
   describe('Converting hex code to rgb', () => {
@@ -32,4 +32,19 @@ describe('Color utils', () => {
       expect(textColorForBackground('#000')).toEqual('#FFFFFF');
     });
   });
+
+  describe('Validate hex color', () => {
+    it.each`
+      color        | output
+      ${undefined} | ${null}
+      ${null}      | ${null}
+      ${''}        | ${null}
+      ${'ABC123'}  | ${false}
+      ${'#ZZZ'}    | ${false}
+      ${'#FF0'}    | ${true}
+      ${'#FF0000'} | ${true}
+    `('returns $output when $color is given', ({ color, output }) => {
+      expect(validateHexColor(color)).toEqual(output);
+    });
+  });
 });
diff --git a/spec/frontend/vue_shared/components/color_picker/color_picker_spec.js b/spec/frontend/vue_shared/components/color_picker/color_picker_spec.js
index c8fe6c3131c..22e70a77024 100644
--- a/spec/frontend/vue_shared/components/color_picker/color_picker_spec.js
+++ b/spec/frontend/vue_shared/components/color_picker/color_picker_spec.js
@@ -13,6 +13,7 @@ describe('ColorPicker', () => {
   };
 
   const setColor = '#000000';
+  const invalidText = 'Please enter a valid hex (#RRGGBB or #RGB) color value';
   const label = () => wrapper.find(GlFormGroup).attributes('label');
   const colorPreview = () => wrapper.find('[data-testid="color-preview"]');
   const colorPicker = () => wrapper.find(GlFormInput);
@@ -55,6 +56,7 @@ describe('ColorPicker', () => {
       expect(colorPreview().attributes('style')).toBe(undefined);
       expect(colorPicker().attributes('value')).toBe(undefined);
       expect(colorInput().props('value')).toBe('');
+      expect(colorPreview().attributes('class')).toContain('gl-inset-border-1-gray-400');
     });
 
     it('has a color set on initialization', () => {
@@ -67,7 +69,7 @@ describe('ColorPicker', () => {
       createComponent();
       await colorInput().setValue(setColor);
 
-      expect(wrapper.emitted().input[0]).toEqual([setColor]);
+      expect(wrapper.emitted().input[0]).toStrictEqual([setColor]);
     });
 
     it('trims spaces from submitted colors', async () => {
@@ -75,23 +77,16 @@ describe('ColorPicker', () => {
       await colorInput().setValue(`    ${setColor}    `);
 
       expect(wrapper.vm.$data.selectedColor).toBe(setColor);
+      expect(colorPreview().attributes('class')).toContain('gl-inset-border-1-gray-400');
+      expect(colorInput().attributes('class')).not.toContain('is-invalid');
     });
 
-    it('shows invalid feedback when an invalid color is used', async () => {
-      createComponent();
-      await colorInput().setValue('abcd');
-
-      expect(invalidFeedback().text()).toBe(
-        'Please enter a valid hex (#RRGGBB or #RGB) color value',
-      );
-      expect(wrapper.emitted().input).toBe(undefined);
-    });
-
-    it('shows an invalid feedback border on the preview when an invalid color is used', async () => {
-      createComponent();
-      await colorInput().setValue('abcd');
+    it('shows invalid feedback when the state is marked as invalid', async () => {
+      createComponent(mount, { invalidFeedback: invalidText, state: false });
 
+      expect(invalidFeedback().text()).toBe(invalidText);
       expect(colorPreview().attributes('class')).toContain('gl-inset-border-1-red-500');
+      expect(colorInput().attributes('class')).toContain('is-invalid');
     });
   });
 
diff --git a/spec/lib/gitlab/terraform/state_migration_helper_spec.rb b/spec/lib/gitlab/terraform/state_migration_helper_spec.rb
new file mode 100644
index 00000000000..36c9c060e98
--- /dev/null
+++ b/spec/lib/gitlab/terraform/state_migration_helper_spec.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Terraform::StateMigrationHelper do
+  before do
+    stub_terraform_state_object_storage
+  end
+
+  describe '.migrate_to_remote_storage' do
+    let!(:local_version) { create(:terraform_state_version, file_store: Terraform::StateUploader::Store::LOCAL) }
+
+    subject { described_class.migrate_to_remote_storage }
+
+    it 'migrates remote files to remote storage' do
+      subject
+
+      expect(local_version.reload.file_store).to eq(Terraform::StateUploader::Store::REMOTE)
+    end
+  end
+end
diff --git a/spec/lib/gitlab/tracking/standard_context_spec.rb b/spec/lib/gitlab/tracking/standard_context_spec.rb
index acf7aeb303a..485700da8bc 100644
--- a/spec/lib/gitlab/tracking/standard_context_spec.rb
+++ b/spec/lib/gitlab/tracking/standard_context_spec.rb
@@ -28,8 +28,8 @@ RSpec.describe Gitlab::Tracking::StandardContext do
     context 'with namespace' do
       subject { described_class.new(namespace: namespace) }
 
-      it 'creates a Snowplow context using the given data' do
-        expect(snowplow_context.to_json.dig(:data, :namespace_id)).to eq(namespace.id)
+      it 'creates a Snowplow context without namespace and project' do
+        expect(snowplow_context.to_json.dig(:data, :namespace_id)).to be_nil
         expect(snowplow_context.to_json.dig(:data, :project_id)).to be_nil
       end
     end
@@ -37,18 +37,18 @@ RSpec.describe Gitlab::Tracking::StandardContext do
     context 'with project' do
       subject { described_class.new(project: project) }
 
-      it 'creates a Snowplow context using the given data' do
-        expect(snowplow_context.to_json.dig(:data, :namespace_id)).to eq(project.namespace.id)
-        expect(snowplow_context.to_json.dig(:data, :project_id)).to eq(project.id)
+      it 'creates a Snowplow context without namespace and project' do
+        expect(snowplow_context.to_json.dig(:data, :namespace_id)).to be_nil
+        expect(snowplow_context.to_json.dig(:data, :project_id)).to be_nil
       end
     end
 
     context 'with project and namespace' do
       subject { described_class.new(namespace: namespace, project: project) }
 
-      it 'creates a Snowplow context using the given data' do
-        expect(snowplow_context.to_json.dig(:data, :namespace_id)).to eq(namespace.id)
-        expect(snowplow_context.to_json.dig(:data, :project_id)).to eq(project.id)
+      it 'creates a Snowplow context without namespace and project' do
+        expect(snowplow_context.to_json.dig(:data, :namespace_id)).to be_nil
+        expect(snowplow_context.to_json.dig(:data, :project_id)).to be_nil
       end
     end
   end
diff --git a/spec/models/terraform/state_version_spec.rb b/spec/models/terraform/state_version_spec.rb
index 97ac77d5e7b..ac2e8d167b3 100644
--- a/spec/models/terraform/state_version_spec.rb
+++ b/spec/models/terraform/state_version_spec.rb
@@ -24,6 +24,24 @@ RSpec.describe Terraform::StateVersion do
 
       it { expect(subject.map(&:version)).to eq(versions.sort.reverse) }
     end
+
+    describe '.with_files_stored_locally' do
+      subject { described_class.with_files_stored_locally }
+
+      it 'includes states with local storage' do
+        create_list(:terraform_state_version, 5)
+
+        expect(subject).to have_attributes(count: 5)
+      end
+
+      it 'excludes states without local storage' do
+        stub_terraform_state_object_storage
+
+        create_list(:terraform_state_version, 5)
+
+        expect(subject).to have_attributes(count: 0)
+      end
+    end
   end
 
   context 'file storage' do
diff --git a/spec/tasks/gitlab/terraform/migrate_rake_spec.rb b/spec/tasks/gitlab/terraform/migrate_rake_spec.rb
new file mode 100644
index 00000000000..4188521df8e
--- /dev/null
+++ b/spec/tasks/gitlab/terraform/migrate_rake_spec.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'rake_helper'
+
+RSpec.describe 'gitlab:terraform_states' do
+  let_it_be(:version) { create(:terraform_state_version) }
+
+  let(:logger) { instance_double(Logger) }
+  let(:helper) { double }
+
+  before(:all) do
+    Rake.application.rake_require 'tasks/gitlab/terraform/migrate'
+  end
+
+  before do
+    allow(Logger).to receive(:new).with(STDOUT).and_return(logger)
+  end
+
+  describe 'gitlab:terraform_states:migrate' do
+    subject { run_rake_task('gitlab:terraform_states:migrate') }
+
+    it 'invokes the migration helper to move files to object storage' do
+      expect(Gitlab::Terraform::StateMigrationHelper).to receive(:migrate_to_remote_storage).and_yield(version)
+      expect(logger).to receive(:info).with('Starting transfer of Terraform states to object storage')
+      expect(logger).to receive(:info).with(/Transferred Terraform state version ID #{version.id}/)
+
+      subject
+    end
+
+    context 'an error is raised while migrating' do
+      let(:error_message) { 'Something went wrong' }
+
+      before do
+        allow(Gitlab::Terraform::StateMigrationHelper).to receive(:migrate_to_remote_storage).and_raise(StandardError, error_message)
+      end
+
+      it 'logs the error' do
+        expect(logger).to receive(:info).with('Starting transfer of Terraform states to object storage')
+        expect(logger).to receive(:error).with("Failed to migrate: #{error_message}")
+
+        subject
+      end
+    end
+  end
+end