Add latest changes from gitlab-org/gitlab@master

20 files changed, 241 insertions, 29 deletions

diff --git a/app/assets/javascripts/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue b/app/assets/javascripts/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue
index 7ed51068d42..6f6b9ad025a 100644
--- a/app/assets/javascripts/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue
+++ b/app/assets/javascripts/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue
@@ -1,39 +1,92 @@
 <script>
 import { createNamespacedHelpers, mapState, mapActions } from 'vuex';
+import { sprintf, s__ } from '~/locale';
+import ClusterFormDropdown from './cluster_form_dropdown.vue';
 import RegionDropdown from './region_dropdown.vue';
 import RoleNameDropdown from './role_name_dropdown.vue';
 import SecurityGroupDropdown from './security_group_dropdown.vue';
-import SubnetDropdown from './subnet_dropdown.vue';
-import VpcDropdown from './vpc_dropdown.vue';
 
 const { mapState: mapRegionsState, mapActions: mapRegionsActions } = createNamespacedHelpers(
   'regions',
 );
+const { mapState: mapVpcsState, mapActions: mapVpcActions } = createNamespacedHelpers('vpcs');
+const { mapState: mapSubnetsState, mapActions: mapSubnetActions } = createNamespacedHelpers(
+  'subnets',
+);
 
 export default {
   components: {
+    ClusterFormDropdown,
     RegionDropdown,
     RoleNameDropdown,
     SecurityGroupDropdown,
-    SubnetDropdown,
-    VpcDropdown,
   },
   computed: {
-    ...mapState(['selectedRegion']),
+    ...mapState(['selectedRegion', 'selectedVpc', 'selectedSubnet']),
     ...mapRegionsState({
       regions: 'items',
       isLoadingRegions: 'isLoadingItems',
       loadingRegionsError: 'loadingItemsError',
     }),
+    ...mapVpcsState({
+      vpcs: 'items',
+      isLoadingVpcs: 'isLoadingItems',
+      loadingVpcsError: 'loadingItemsError',
+    }),
+    ...mapSubnetsState({
+      subnets: 'items',
+      isLoadingSubnets: 'isLoadingItems',
+      loadingSubnetsError: 'loadingItemsError',
+    }),
+    vpcDropdownDisabled() {
+      return !this.selectedRegion;
+    },
+    subnetDropdownDisabled() {
+      return !this.selectedVpc;
+    },
+    vpcDropdownHelpText() {
+      return sprintf(
+        s__(
+          'ClusterIntegration|Select a VPC to use for your EKS Cluster resources. To use a new VPC, first create one on %{startLink}Amazon Web Services%{endLink}.',
+        ),
+        {
+          startLink:
+            '<a href="https://console.aws.amazon.com/vpc/home?#vpc" target="_blank" rel="noopener noreferrer">',
+          endLink: '</a>',
+        },
+        false,
+      );
+    },
+    subnetDropdownHelpText() {
+      return sprintf(
+        s__(
+          'ClusterIntegration|Choose the %{startLink}subnets%{endLink} in your VPC where your worker nodes will run.',
+        ),
+        {
+          startLink:
+            '<a href="https://console.aws.amazon.com/vpc/home?#subnets" target="_blank" rel="noopener noreferrer">',
+          endLink: '</a>',
+        },
+        false,
+      );
+    },
   },
   mounted() {
     this.fetchRegions();
   },
   methods: {
-    ...mapActions(['setRegion']),
-    ...mapRegionsActions({
-      fetchRegions: 'fetchItems',
-    }),
+    ...mapActions(['setRegion', 'setVpc', 'setSubnet']),
+    ...mapRegionsActions({ fetchRegions: 'fetchItems' }),
+    ...mapVpcActions({ fetchVpcs: 'fetchItems' }),
+    ...mapSubnetActions({ fetchSubnets: 'fetchItems' }),
+    setRegionAndFetchVpcs(region) {
+      this.setRegion({ region });
+      this.fetchVpcs({ region });
+    },
+    setVpcAndFetchSubnets(vpc) {
+      this.setVpc({ vpc });
+      this.fetchSubnets({ vpc });
+    },
   },
 };
 </script>
@@ -54,8 +107,52 @@ export default {
         :regions="regions"
         :error="loadingRegionsError"
         :loading="isLoadingRegions"
-        @input="setRegion({ region: $event })"
+        @input="setRegionAndFetchVpcs($event)"
+      />
+    </div>
+    <div class="form-group">
+      <label class="label-bold" name="eks-vpc" for="eks-vpc">{{
+        s__('ClusterIntegration|VPC')
+      }}</label>
+      <cluster-form-dropdown
+        field-id="eks-vpc"
+        field-name="eks-vpc"
+        :input="selectedVpc"
+        :items="vpcs"
+        :loading="isLoadingVpcs"
+        :disabled="vpcDropdownDisabled"
+        :disabled-text="s__('ClusterIntegration|Select a region to choose a VPC')"
+        :loading-text="s__('ClusterIntegration|Loading VPCs')"
+        :placeholder="s__('ClusterIntergation|Select a VPC')"
+        :search-field-placeholder="s__('ClusterIntegration|Search VPCs')"
+        :empty-text="s__('ClusterIntegration|No VPCs found')"
+        :has-errors="loadingVpcsError"
+        :error-message="s__('ClusterIntegration|Could not load VPCs for the selected region')"
+        @input="setVpcAndFetchSubnets($event)"
+      />
+      <p class="form-text text-muted" v-html="vpcDropdownHelpText"></p>
+    </div>
+    <div class="form-group">
+      <label class="label-bold" name="eks-subnet" for="eks-subnet">{{
+        s__('ClusterIntegration|Subnet')
+      }}</label>
+      <cluster-form-dropdown
+        field-id="eks-subnet"
+        field-name="eks-subnet"
+        :input="selectedSubnet"
+        :items="subnets"
+        :loading="isLoadingSubnets"
+        :disabled="subnetDropdownDisabled"
+        :disabled-text="s__('ClusterIntegration|Select a VPC to choose a subnet')"
+        :loading-text="s__('ClusterIntegration|Loading subnets')"
+        :placeholder="s__('ClusterIntergation|Select a subnet')"
+        :search-field-placeholder="s__('ClusterIntegration|Search subnets')"
+        :empty-text="s__('ClusterIntegration|No subnet found')"
+        :has-errors="loadingSubnetsError"
+        :error-message="s__('ClusterIntegration|Could not load subnets for the selected VPC')"
+        @input="setSubnet({ subnet: $event })"
       />
+      <p class="form-text text-muted" v-html="subnetDropdownHelpText"></p>
     </div>
   </form>
 </template>
diff --git a/app/assets/javascripts/create_cluster/eks_cluster/components/subnet_dropdown.vue b/app/assets/javascripts/create_cluster/eks_cluster/components/subnet_dropdown.vue
deleted file mode 100644
index e69de29bb2d..00000000000
--- a/app/assets/javascripts/create_cluster/eks_cluster/components/subnet_dropdown.vue
+++ /dev/null
diff --git a/app/assets/javascripts/create_cluster/eks_cluster/components/vpc_dropdown.vue b/app/assets/javascripts/create_cluster/eks_cluster/components/vpc_dropdown.vue
deleted file mode 100644
index e69de29bb2d..00000000000
--- a/app/assets/javascripts/create_cluster/eks_cluster/components/vpc_dropdown.vue
+++ /dev/null
diff --git a/app/assets/javascripts/create_cluster/eks_cluster/services/aws_services_facade.js b/app/assets/javascripts/create_cluster/eks_cluster/services/aws_services_facade.js
index 5a13d32e0d2..030b6b384b1 100644
--- a/app/assets/javascripts/create_cluster/eks_cluster/services/aws_services_facade.js
+++ b/app/assets/javascripts/create_cluster/eks_cluster/services/aws_services_facade.js
@@ -17,4 +17,45 @@ export const fetchRegions = () =>
       .send();
   });
 
+export const fetchVpcs = () =>
+  new Promise((resolve, reject) => {
+    const ec2 = new EC2();
+
+    ec2
+      .describeVpcs()
+      .on('success', ({ data: { Vpcs: vpcs } }) => {
+        const transformedVpcs = vpcs.map(({ VpcId: id }) => ({ id, name: id }));
+
+        resolve(transformedVpcs);
+      })
+      .on('error', error => {
+        reject(error);
+      })
+      .send();
+  });
+
+export const fetchSubnets = ({ vpc }) =>
+  new Promise((resolve, reject) => {
+    const ec2 = new EC2();
+
+    ec2
+      .describeSubnets({
+        Filters: [
+          {
+            Name: 'vpc-id',
+            Values: [vpc.id],
+          },
+        ],
+      })
+      .on('success', ({ data: { Subnets: subnets } }) => {
+        const transformedSubnets = subnets.map(({ SubnetId: id }) => ({ id, name: id }));
+
+        resolve(transformedSubnets);
+      })
+      .on('error', error => {
+        reject(error);
+      })
+      .send();
+  });
+
 export default () => {};
diff --git a/app/assets/javascripts/create_cluster/eks_cluster/store/actions.js b/app/assets/javascripts/create_cluster/eks_cluster/store/actions.js
index 68493be37e0..0809fc2dfc4 100644
--- a/app/assets/javascripts/create_cluster/eks_cluster/store/actions.js
+++ b/app/assets/javascripts/create_cluster/eks_cluster/store/actions.js
@@ -4,4 +4,12 @@ export const setRegion = ({ commit }, payload) => {
   commit(types.SET_REGION, payload);
 };
 
+export const setVpc = ({ commit }, payload) => {
+  commit(types.SET_VPC, payload);
+};
+
+export const setSubnet = ({ commit }, payload) => {
+  commit(types.SET_SUBNET, payload);
+};
+
 export default () => {};
diff --git a/app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/actions.js b/app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/actions.js
index 4f1ecb22719..5d250b2e29e 100644
--- a/app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/actions.js
+++ b/app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/actions.js
@@ -4,10 +4,10 @@ export default fetchItems => ({
   requestItems: ({ commit }) => commit(types.REQUEST_ITEMS),
   receiveItemsSuccess: ({ commit }, payload) => commit(types.RECEIVE_ITEMS_SUCCESS, payload),
   receiveItemsError: ({ commit }, payload) => commit(types.RECEIVE_ITEMS_ERROR, payload),
-  fetchItems: ({ dispatch }) => {
+  fetchItems: ({ dispatch }, payload) => {
     dispatch('requestItems');
 
-    return fetchItems()
+    return fetchItems(payload)
       .then(items => dispatch('receiveItemsSuccess', { items }))
       .catch(error => dispatch('receiveItemsError', { error }));
   },
diff --git a/app/assets/javascripts/create_cluster/eks_cluster/store/index.js b/app/assets/javascripts/create_cluster/eks_cluster/store/index.js
index 608c4ec9e1e..622095f2cc8 100644
--- a/app/assets/javascripts/create_cluster/eks_cluster/store/index.js
+++ b/app/assets/javascripts/create_cluster/eks_cluster/store/index.js
@@ -19,6 +19,14 @@ const createStore = () =>
         namespaced: true,
         ...clusterDropdownStore(awsServices.fetchRegions),
       },
+      vpcs: {
+        namespaced: true,
+        ...clusterDropdownStore(awsServices.fetchVpcs),
+      },
+      subnets: {
+        namespaced: true,
+        ...clusterDropdownStore(awsServices.fetchSubnets),
+      },
     },
   });
 
diff --git a/app/assets/javascripts/create_cluster/eks_cluster/store/mutation_types.js b/app/assets/javascripts/create_cluster/eks_cluster/store/mutation_types.js
index 80c1354482b..ada76b21f18 100644
--- a/app/assets/javascripts/create_cluster/eks_cluster/store/mutation_types.js
+++ b/app/assets/javascripts/create_cluster/eks_cluster/store/mutation_types.js
@@ -1,2 +1,3 @@
-// eslint-disable-next-line import/prefer-default-export
 export const SET_REGION = 'SET_REGION';
+export const SET_VPC = 'SET_VPC';
+export const SET_SUBNET = 'SET_SUBNET';
diff --git a/app/assets/javascripts/create_cluster/eks_cluster/store/mutations.js b/app/assets/javascripts/create_cluster/eks_cluster/store/mutations.js
index 02f34d1bdcd..346716bb0df 100644
--- a/app/assets/javascripts/create_cluster/eks_cluster/store/mutations.js
+++ b/app/assets/javascripts/create_cluster/eks_cluster/store/mutations.js
@@ -4,4 +4,10 @@ export default {
   [types.SET_REGION](state, { region }) {
     state.selectedRegion = region;
   },
+  [types.SET_VPC](state, { vpc }) {
+    state.selectedVpc = vpc;
+  },
+  [types.SET_SUBNET](state, { subnet }) {
+    state.selectedSubnet = subnet;
+  },
 };
diff --git a/app/controllers/concerns/uploads_actions.rb b/app/controllers/concerns/uploads_actions.rb
index 60a68cec3c3..79bee6f89d8 100644
--- a/app/controllers/concerns/uploads_actions.rb
+++ b/app/controllers/concerns/uploads_actions.rb
@@ -91,7 +91,7 @@ module UploadsActions
 
     upload_paths = uploader.upload_paths(params[:filename])
     upload = Upload.find_by(model: model, uploader: uploader_class.to_s, path: upload_paths)
-    upload&.build_uploader
+    upload&.retrieve_uploader
   end
   # rubocop: enable CodeReuse/ActiveRecord
 
diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb
index 42d4e86fe8d..946241b7d4c 100644
--- a/app/models/ci/pipeline_schedule.rb
+++ b/app/models/ci/pipeline_schedule.rb
@@ -86,3 +86,5 @@ module Ci
     end
   end
 end
+
+Ci::PipelineSchedule.prepend_if_ee('EE::Ci::PipelineSchedule')
diff --git a/app/models/ci/trigger.rb b/app/models/ci/trigger.rb
index 8792c5cf98b..68548bd2fdc 100644
--- a/app/models/ci/trigger.rb
+++ b/app/models/ci/trigger.rb
@@ -45,3 +45,5 @@ module Ci
     end
   end
 end
+
+Ci::Trigger.prepend_if_ee('EE::Ci::Trigger')
diff --git a/app/models/clusters/concerns/application_core.rb b/app/models/clusters/concerns/application_core.rb
index e748c0a855d..979cf0645f5 100644
--- a/app/models/clusters/concerns/application_core.rb
+++ b/app/models/clusters/concerns/application_core.rb
@@ -64,3 +64,5 @@ module Clusters
     end
   end
 end
+
+Clusters::Concerns::ApplicationCore.prepend_if_ee('EE::Clusters::Concerns::ApplicationCore')
diff --git a/app/models/commit_collection.rb b/app/models/commit_collection.rb
index 6b303c52283..d4c29aa295b 100644
--- a/app/models/commit_collection.rb
+++ b/app/models/commit_collection.rb
@@ -72,8 +72,15 @@ class CommitCollection
     end.compact]
 
     # Replace the commits, keeping the same order
-    @commits = @commits.map do |c|
-      replacements.fetch(c.id, c)
+    @commits = @commits.map do |original_commit|
+      # Return the original instance: if it didn't need to be batchloaded, it was
+      # already enriched.
+      batch_loaded_commit = replacements.fetch(original_commit.id, original_commit)
+
+      # If batch loading the commit failed, fall back to the original commit.
+      # We need to explicitly check `.nil?` since otherwise a `BatchLoader` instance
+      # that looks like `nil` is returned.
+      batch_loaded_commit.nil? ? original_commit : batch_loaded_commit
     end
 
     self
diff --git a/app/models/upload.rb b/app/models/upload.rb
index 7560002ada8..384949ddb86 100644
--- a/app/models/upload.rb
+++ b/app/models/upload.rb
@@ -15,7 +15,7 @@ class Upload < ApplicationRecord
   scope :with_files_stored_remotely, -> { where(store: ObjectStorage::Store::REMOTE) }
 
   before_save  :calculate_checksum!, if: :foreground_checksummable?
-  after_commit :schedule_checksum,   if: :checksummable?
+  after_commit :schedule_checksum,   if: :needs_checksum?
 
   # as the FileUploader is not mounted, the default CarrierWave ActiveRecord
   # hooks are not executed and the file will not be deleted
@@ -53,20 +53,41 @@ class Upload < ApplicationRecord
 
   def calculate_checksum!
     self.checksum = nil
-    return unless checksummable?
+    return unless needs_checksum?
 
     self.checksum = Digest::SHA256.file(absolute_path).hexdigest
   end
 
+  # Initialize the associated Uploader class with current model
+  #
+  # @param [String] mounted_as
+  # @return [GitlabUploader] one of the subclasses, defined at the model's uploader attribute
   def build_uploader(mounted_as = nil)
     uploader_class.new(model, mounted_as || mount_point).tap do |uploader|
       uploader.upload = self
+    end
+  end
+
+  # Initialize the associated Uploader class with current model and
+  # retrieve existing file from the store to a local cache
+  #
+  # @param [String] mounted_as
+  # @return [GitlabUploader] one of the subclasses, defined at the model's uploader attribute
+  def retrieve_uploader(mounted_as = nil)
+    build_uploader(mounted_as).tap do |uploader|
       uploader.retrieve_from_store!(identifier)
     end
   end
 
+  # This checks for existence of the upload on storage
+  #
+  # @return [Boolean] whether upload exists on storage
   def exist?
-    exist = File.exist?(absolute_path)
+    exist = if local?
+              File.exist?(absolute_path)
+            else
+              retrieve_uploader.exists?
+            end
 
     # Help sysadmins find missing upload files
     if persisted? && !exist
@@ -91,18 +112,24 @@ class Upload < ApplicationRecord
     store == ObjectStorage::Store::LOCAL
   end
 
+  # Returns whether generating checksum is needed
+  #
+  # This takes into account whether file exists, if any checksum exists
+  # or if the storage has checksum generation code implemented
+  #
+  # @return [Boolean] whether generating a checksum is needed
+  def needs_checksum?
+    checksum.nil? && local? && exist?
+  end
+
   private
 
   def delete_file!
-    build_uploader.remove!
-  end
-
-  def checksummable?
-    checksum.nil? && local? && exist?
+    retrieve_uploader.remove!
   end
 
   def foreground_checksummable?
-    checksummable? && size <= CHECKSUM_THRESHOLD
+    needs_checksum? && size <= CHECKSUM_THRESHOLD
   end
 
   def schedule_checksum
diff --git a/app/uploaders/avatar_uploader.rb b/app/uploaders/avatar_uploader.rb
index 9af59b0aceb..d42c9dbedf4 100644
--- a/app/uploaders/avatar_uploader.rb
+++ b/app/uploaders/avatar_uploader.rb
@@ -19,7 +19,7 @@ class AvatarUploader < GitlabUploader
   end
 
   def absolute_path
-    self.class.absolute_path(model.avatar.upload)
+    self.class.absolute_path(upload)
   end
 
   private
diff --git a/app/uploaders/gitlab_uploader.rb b/app/uploaders/gitlab_uploader.rb
index cefcd3d3f5a..7dc211b14e4 100644
--- a/app/uploaders/gitlab_uploader.rb
+++ b/app/uploaders/gitlab_uploader.rb
@@ -99,6 +99,17 @@ class GitlabUploader < CarrierWave::Uploader::Base
     end
   end
 
+  # Used to replace an existing upload with another +file+ without modifying stored metadata
+  # Use this method only to repair/replace an existing upload, or to upload to a Geo secondary node
+  #
+  # @param [CarrierWave::SanitizedFile] file that will replace existing upload
+  # @return CarrierWave::SanitizedFile
+  def replace_file_without_saving!(file)
+    raise ArgumentError, 'should be a CarrierWave::SanitizedFile' unless file.is_a? CarrierWave::SanitizedFile
+
+    storage.store!(file)
+  end
+
   private
 
   # Designed to be overridden by child uploaders that have a dynamic path
diff --git a/app/workers/import_issues_csv_worker.rb b/app/workers/import_issues_csv_worker.rb
index d44fdfec8ae..b9d7099af71 100644
--- a/app/workers/import_issues_csv_worker.rb
+++ b/app/workers/import_issues_csv_worker.rb
@@ -12,7 +12,7 @@ class ImportIssuesCsvWorker
     @project = Project.find(project_id)
     @upload = Upload.find(upload_id)
 
-    importer = Issues::ImportCsvService.new(@user, @project, @upload.build_uploader)
+    importer = Issues::ImportCsvService.new(@user, @project, @upload.retrieve_uploader)
     importer.execute
 
     @upload.destroy
diff --git a/app/workers/object_storage/background_move_worker.rb b/app/workers/object_storage/background_move_worker.rb
index 8dff65e46e3..19ccae7739c 100644
--- a/app/workers/object_storage/background_move_worker.rb
+++ b/app/workers/object_storage/background_move_worker.rb
@@ -22,7 +22,7 @@ module ObjectStorage
 
     def build_uploader(subject, mount_point)
       case subject
-      when Upload then subject.build_uploader(mount_point)
+      when Upload then subject.retrieve_uploader(mount_point)
       else
         subject.send(mount_point) # rubocop:disable GitlabSecurity/PublicSend
       end
diff --git a/app/workers/object_storage/migrate_uploads_worker.rb b/app/workers/object_storage/migrate_uploads_worker.rb
index 55ac7cd9b3c..c9fd19cf9d7 100644
--- a/app/workers/object_storage/migrate_uploads_worker.rb
+++ b/app/workers/object_storage/migrate_uploads_worker.rb
@@ -119,7 +119,7 @@ module ObjectStorage
     end
 
     def build_uploaders(uploads)
-      uploads.map { |upload| upload.build_uploader(@mounted_as) }
+      uploads.map { |upload| upload.retrieve_uploader(@mounted_as) }
     end
 
     def migrate(uploads)