diff options
Diffstat (limited to 'app')
20 files changed, 241 insertions, 29 deletions
diff --git a/app/assets/javascripts/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue b/app/assets/javascripts/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue index 7ed51068d42..6f6b9ad025a 100644 --- a/app/assets/javascripts/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue +++ b/app/assets/javascripts/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue @@ -1,39 +1,92 @@ <script> import { createNamespacedHelpers, mapState, mapActions } from 'vuex'; +import { sprintf, s__ } from '~/locale'; +import ClusterFormDropdown from './cluster_form_dropdown.vue'; import RegionDropdown from './region_dropdown.vue'; import RoleNameDropdown from './role_name_dropdown.vue'; import SecurityGroupDropdown from './security_group_dropdown.vue'; -import SubnetDropdown from './subnet_dropdown.vue'; -import VpcDropdown from './vpc_dropdown.vue'; const { mapState: mapRegionsState, mapActions: mapRegionsActions } = createNamespacedHelpers( 'regions', ); +const { mapState: mapVpcsState, mapActions: mapVpcActions } = createNamespacedHelpers('vpcs'); +const { mapState: mapSubnetsState, mapActions: mapSubnetActions } = createNamespacedHelpers( + 'subnets', +); export default { components: { + ClusterFormDropdown, RegionDropdown, RoleNameDropdown, SecurityGroupDropdown, - SubnetDropdown, - VpcDropdown, }, computed: { - ...mapState(['selectedRegion']), + ...mapState(['selectedRegion', 'selectedVpc', 'selectedSubnet']), ...mapRegionsState({ regions: 'items', isLoadingRegions: 'isLoadingItems', loadingRegionsError: 'loadingItemsError', }), + ...mapVpcsState({ + vpcs: 'items', + isLoadingVpcs: 'isLoadingItems', + loadingVpcsError: 'loadingItemsError', + }), + ...mapSubnetsState({ + subnets: 'items', + isLoadingSubnets: 'isLoadingItems', + loadingSubnetsError: 'loadingItemsError', + }), + vpcDropdownDisabled() { + return !this.selectedRegion; + }, + subnetDropdownDisabled() { + return !this.selectedVpc; + }, + vpcDropdownHelpText() { + return sprintf( + s__( + 'ClusterIntegration|Select a VPC to use for your EKS Cluster resources. To use a new VPC, first create one on %{startLink}Amazon Web Services%{endLink}.', + ), + { + startLink: + '<a href="https://console.aws.amazon.com/vpc/home?#vpc" target="_blank" rel="noopener noreferrer">', + endLink: '</a>', + }, + false, + ); + }, + subnetDropdownHelpText() { + return sprintf( + s__( + 'ClusterIntegration|Choose the %{startLink}subnets%{endLink} in your VPC where your worker nodes will run.', + ), + { + startLink: + '<a href="https://console.aws.amazon.com/vpc/home?#subnets" target="_blank" rel="noopener noreferrer">', + endLink: '</a>', + }, + false, + ); + }, }, mounted() { this.fetchRegions(); }, methods: { - ...mapActions(['setRegion']), - ...mapRegionsActions({ - fetchRegions: 'fetchItems', - }), + ...mapActions(['setRegion', 'setVpc', 'setSubnet']), + ...mapRegionsActions({ fetchRegions: 'fetchItems' }), + ...mapVpcActions({ fetchVpcs: 'fetchItems' }), + ...mapSubnetActions({ fetchSubnets: 'fetchItems' }), + setRegionAndFetchVpcs(region) { + this.setRegion({ region }); + this.fetchVpcs({ region }); + }, + setVpcAndFetchSubnets(vpc) { + this.setVpc({ vpc }); + this.fetchSubnets({ vpc }); + }, }, }; </script> @@ -54,8 +107,52 @@ export default { :regions="regions" :error="loadingRegionsError" :loading="isLoadingRegions" - @input="setRegion({ region: $event })" + @input="setRegionAndFetchVpcs($event)" + /> + </div> + <div class="form-group"> + <label class="label-bold" name="eks-vpc" for="eks-vpc">{{ + s__('ClusterIntegration|VPC') + }}</label> + <cluster-form-dropdown + field-id="eks-vpc" + field-name="eks-vpc" + :input="selectedVpc" + :items="vpcs" + :loading="isLoadingVpcs" + :disabled="vpcDropdownDisabled" + :disabled-text="s__('ClusterIntegration|Select a region to choose a VPC')" + :loading-text="s__('ClusterIntegration|Loading VPCs')" + :placeholder="s__('ClusterIntergation|Select a VPC')" + :search-field-placeholder="s__('ClusterIntegration|Search VPCs')" + :empty-text="s__('ClusterIntegration|No VPCs found')" + :has-errors="loadingVpcsError" + :error-message="s__('ClusterIntegration|Could not load VPCs for the selected region')" + @input="setVpcAndFetchSubnets($event)" + /> + <p class="form-text text-muted" v-html="vpcDropdownHelpText"></p> + </div> + <div class="form-group"> + <label class="label-bold" name="eks-subnet" for="eks-subnet">{{ + s__('ClusterIntegration|Subnet') + }}</label> + <cluster-form-dropdown + field-id="eks-subnet" + field-name="eks-subnet" + :input="selectedSubnet" + :items="subnets" + :loading="isLoadingSubnets" + :disabled="subnetDropdownDisabled" + :disabled-text="s__('ClusterIntegration|Select a VPC to choose a subnet')" + :loading-text="s__('ClusterIntegration|Loading subnets')" + :placeholder="s__('ClusterIntergation|Select a subnet')" + :search-field-placeholder="s__('ClusterIntegration|Search subnets')" + :empty-text="s__('ClusterIntegration|No subnet found')" + :has-errors="loadingSubnetsError" + :error-message="s__('ClusterIntegration|Could not load subnets for the selected VPC')" + @input="setSubnet({ subnet: $event })" /> + <p class="form-text text-muted" v-html="subnetDropdownHelpText"></p> </div> </form> </template> diff --git a/app/assets/javascripts/create_cluster/eks_cluster/components/subnet_dropdown.vue b/app/assets/javascripts/create_cluster/eks_cluster/components/subnet_dropdown.vue deleted file mode 100644 index e69de29bb2d..00000000000 --- a/app/assets/javascripts/create_cluster/eks_cluster/components/subnet_dropdown.vue +++ /dev/null diff --git a/app/assets/javascripts/create_cluster/eks_cluster/components/vpc_dropdown.vue b/app/assets/javascripts/create_cluster/eks_cluster/components/vpc_dropdown.vue deleted file mode 100644 index e69de29bb2d..00000000000 --- a/app/assets/javascripts/create_cluster/eks_cluster/components/vpc_dropdown.vue +++ /dev/null diff --git a/app/assets/javascripts/create_cluster/eks_cluster/services/aws_services_facade.js b/app/assets/javascripts/create_cluster/eks_cluster/services/aws_services_facade.js index 5a13d32e0d2..030b6b384b1 100644 --- a/app/assets/javascripts/create_cluster/eks_cluster/services/aws_services_facade.js +++ b/app/assets/javascripts/create_cluster/eks_cluster/services/aws_services_facade.js @@ -17,4 +17,45 @@ export const fetchRegions = () => .send(); }); +export const fetchVpcs = () => + new Promise((resolve, reject) => { + const ec2 = new EC2(); + + ec2 + .describeVpcs() + .on('success', ({ data: { Vpcs: vpcs } }) => { + const transformedVpcs = vpcs.map(({ VpcId: id }) => ({ id, name: id })); + + resolve(transformedVpcs); + }) + .on('error', error => { + reject(error); + }) + .send(); + }); + +export const fetchSubnets = ({ vpc }) => + new Promise((resolve, reject) => { + const ec2 = new EC2(); + + ec2 + .describeSubnets({ + Filters: [ + { + Name: 'vpc-id', + Values: [vpc.id], + }, + ], + }) + .on('success', ({ data: { Subnets: subnets } }) => { + const transformedSubnets = subnets.map(({ SubnetId: id }) => ({ id, name: id })); + + resolve(transformedSubnets); + }) + .on('error', error => { + reject(error); + }) + .send(); + }); + export default () => {}; diff --git a/app/assets/javascripts/create_cluster/eks_cluster/store/actions.js b/app/assets/javascripts/create_cluster/eks_cluster/store/actions.js index 68493be37e0..0809fc2dfc4 100644 --- a/app/assets/javascripts/create_cluster/eks_cluster/store/actions.js +++ b/app/assets/javascripts/create_cluster/eks_cluster/store/actions.js @@ -4,4 +4,12 @@ export const setRegion = ({ commit }, payload) => { commit(types.SET_REGION, payload); }; +export const setVpc = ({ commit }, payload) => { + commit(types.SET_VPC, payload); +}; + +export const setSubnet = ({ commit }, payload) => { + commit(types.SET_SUBNET, payload); +}; + export default () => {}; diff --git a/app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/actions.js b/app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/actions.js index 4f1ecb22719..5d250b2e29e 100644 --- a/app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/actions.js +++ b/app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/actions.js @@ -4,10 +4,10 @@ export default fetchItems => ({ requestItems: ({ commit }) => commit(types.REQUEST_ITEMS), receiveItemsSuccess: ({ commit }, payload) => commit(types.RECEIVE_ITEMS_SUCCESS, payload), receiveItemsError: ({ commit }, payload) => commit(types.RECEIVE_ITEMS_ERROR, payload), - fetchItems: ({ dispatch }) => { + fetchItems: ({ dispatch }, payload) => { dispatch('requestItems'); - return fetchItems() + return fetchItems(payload) .then(items => dispatch('receiveItemsSuccess', { items })) .catch(error => dispatch('receiveItemsError', { error })); }, diff --git a/app/assets/javascripts/create_cluster/eks_cluster/store/index.js b/app/assets/javascripts/create_cluster/eks_cluster/store/index.js index 608c4ec9e1e..622095f2cc8 100644 --- a/app/assets/javascripts/create_cluster/eks_cluster/store/index.js +++ b/app/assets/javascripts/create_cluster/eks_cluster/store/index.js @@ -19,6 +19,14 @@ const createStore = () => namespaced: true, ...clusterDropdownStore(awsServices.fetchRegions), }, + vpcs: { + namespaced: true, + ...clusterDropdownStore(awsServices.fetchVpcs), + }, + subnets: { + namespaced: true, + ...clusterDropdownStore(awsServices.fetchSubnets), + }, }, }); diff --git a/app/assets/javascripts/create_cluster/eks_cluster/store/mutation_types.js b/app/assets/javascripts/create_cluster/eks_cluster/store/mutation_types.js index 80c1354482b..ada76b21f18 100644 --- a/app/assets/javascripts/create_cluster/eks_cluster/store/mutation_types.js +++ b/app/assets/javascripts/create_cluster/eks_cluster/store/mutation_types.js @@ -1,2 +1,3 @@ -// eslint-disable-next-line import/prefer-default-export export const SET_REGION = 'SET_REGION'; +export const SET_VPC = 'SET_VPC'; +export const SET_SUBNET = 'SET_SUBNET'; diff --git a/app/assets/javascripts/create_cluster/eks_cluster/store/mutations.js b/app/assets/javascripts/create_cluster/eks_cluster/store/mutations.js index 02f34d1bdcd..346716bb0df 100644 --- a/app/assets/javascripts/create_cluster/eks_cluster/store/mutations.js +++ b/app/assets/javascripts/create_cluster/eks_cluster/store/mutations.js @@ -4,4 +4,10 @@ export default { [types.SET_REGION](state, { region }) { state.selectedRegion = region; }, + [types.SET_VPC](state, { vpc }) { + state.selectedVpc = vpc; + }, + [types.SET_SUBNET](state, { subnet }) { + state.selectedSubnet = subnet; + }, }; diff --git a/app/controllers/concerns/uploads_actions.rb b/app/controllers/concerns/uploads_actions.rb index 60a68cec3c3..79bee6f89d8 100644 --- a/app/controllers/concerns/uploads_actions.rb +++ b/app/controllers/concerns/uploads_actions.rb @@ -91,7 +91,7 @@ module UploadsActions upload_paths = uploader.upload_paths(params[:filename]) upload = Upload.find_by(model: model, uploader: uploader_class.to_s, path: upload_paths) - upload&.build_uploader + upload&.retrieve_uploader end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb index 42d4e86fe8d..946241b7d4c 100644 --- a/app/models/ci/pipeline_schedule.rb +++ b/app/models/ci/pipeline_schedule.rb @@ -86,3 +86,5 @@ module Ci end end end + +Ci::PipelineSchedule.prepend_if_ee('EE::Ci::PipelineSchedule') diff --git a/app/models/ci/trigger.rb b/app/models/ci/trigger.rb index 8792c5cf98b..68548bd2fdc 100644 --- a/app/models/ci/trigger.rb +++ b/app/models/ci/trigger.rb @@ -45,3 +45,5 @@ module Ci end end end + +Ci::Trigger.prepend_if_ee('EE::Ci::Trigger') diff --git a/app/models/clusters/concerns/application_core.rb b/app/models/clusters/concerns/application_core.rb index e748c0a855d..979cf0645f5 100644 --- a/app/models/clusters/concerns/application_core.rb +++ b/app/models/clusters/concerns/application_core.rb @@ -64,3 +64,5 @@ module Clusters end end end + +Clusters::Concerns::ApplicationCore.prepend_if_ee('EE::Clusters::Concerns::ApplicationCore') diff --git a/app/models/commit_collection.rb b/app/models/commit_collection.rb index 6b303c52283..d4c29aa295b 100644 --- a/app/models/commit_collection.rb +++ b/app/models/commit_collection.rb @@ -72,8 +72,15 @@ class CommitCollection end.compact] # Replace the commits, keeping the same order - @commits = @commits.map do |c| - replacements.fetch(c.id, c) + @commits = @commits.map do |original_commit| + # Return the original instance: if it didn't need to be batchloaded, it was + # already enriched. + batch_loaded_commit = replacements.fetch(original_commit.id, original_commit) + + # If batch loading the commit failed, fall back to the original commit. + # We need to explicitly check `.nil?` since otherwise a `BatchLoader` instance + # that looks like `nil` is returned. + batch_loaded_commit.nil? ? original_commit : batch_loaded_commit end self diff --git a/app/models/upload.rb b/app/models/upload.rb index 7560002ada8..384949ddb86 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -15,7 +15,7 @@ class Upload < ApplicationRecord scope :with_files_stored_remotely, -> { where(store: ObjectStorage::Store::REMOTE) } before_save :calculate_checksum!, if: :foreground_checksummable? - after_commit :schedule_checksum, if: :checksummable? + after_commit :schedule_checksum, if: :needs_checksum? # as the FileUploader is not mounted, the default CarrierWave ActiveRecord # hooks are not executed and the file will not be deleted @@ -53,20 +53,41 @@ class Upload < ApplicationRecord def calculate_checksum! self.checksum = nil - return unless checksummable? + return unless needs_checksum? self.checksum = Digest::SHA256.file(absolute_path).hexdigest end + # Initialize the associated Uploader class with current model + # + # @param [String] mounted_as + # @return [GitlabUploader] one of the subclasses, defined at the model's uploader attribute def build_uploader(mounted_as = nil) uploader_class.new(model, mounted_as || mount_point).tap do |uploader| uploader.upload = self + end + end + + # Initialize the associated Uploader class with current model and + # retrieve existing file from the store to a local cache + # + # @param [String] mounted_as + # @return [GitlabUploader] one of the subclasses, defined at the model's uploader attribute + def retrieve_uploader(mounted_as = nil) + build_uploader(mounted_as).tap do |uploader| uploader.retrieve_from_store!(identifier) end end + # This checks for existence of the upload on storage + # + # @return [Boolean] whether upload exists on storage def exist? - exist = File.exist?(absolute_path) + exist = if local? + File.exist?(absolute_path) + else + retrieve_uploader.exists? + end # Help sysadmins find missing upload files if persisted? && !exist @@ -91,18 +112,24 @@ class Upload < ApplicationRecord store == ObjectStorage::Store::LOCAL end + # Returns whether generating checksum is needed + # + # This takes into account whether file exists, if any checksum exists + # or if the storage has checksum generation code implemented + # + # @return [Boolean] whether generating a checksum is needed + def needs_checksum? + checksum.nil? && local? && exist? + end + private def delete_file! - build_uploader.remove! - end - - def checksummable? - checksum.nil? && local? && exist? + retrieve_uploader.remove! end def foreground_checksummable? - checksummable? && size <= CHECKSUM_THRESHOLD + needs_checksum? && size <= CHECKSUM_THRESHOLD end def schedule_checksum diff --git a/app/uploaders/avatar_uploader.rb b/app/uploaders/avatar_uploader.rb index 9af59b0aceb..d42c9dbedf4 100644 --- a/app/uploaders/avatar_uploader.rb +++ b/app/uploaders/avatar_uploader.rb @@ -19,7 +19,7 @@ class AvatarUploader < GitlabUploader end def absolute_path - self.class.absolute_path(model.avatar.upload) + self.class.absolute_path(upload) end private diff --git a/app/uploaders/gitlab_uploader.rb b/app/uploaders/gitlab_uploader.rb index cefcd3d3f5a..7dc211b14e4 100644 --- a/app/uploaders/gitlab_uploader.rb +++ b/app/uploaders/gitlab_uploader.rb @@ -99,6 +99,17 @@ class GitlabUploader < CarrierWave::Uploader::Base end end + # Used to replace an existing upload with another +file+ without modifying stored metadata + # Use this method only to repair/replace an existing upload, or to upload to a Geo secondary node + # + # @param [CarrierWave::SanitizedFile] file that will replace existing upload + # @return CarrierWave::SanitizedFile + def replace_file_without_saving!(file) + raise ArgumentError, 'should be a CarrierWave::SanitizedFile' unless file.is_a? CarrierWave::SanitizedFile + + storage.store!(file) + end + private # Designed to be overridden by child uploaders that have a dynamic path diff --git a/app/workers/import_issues_csv_worker.rb b/app/workers/import_issues_csv_worker.rb index d44fdfec8ae..b9d7099af71 100644 --- a/app/workers/import_issues_csv_worker.rb +++ b/app/workers/import_issues_csv_worker.rb @@ -12,7 +12,7 @@ class ImportIssuesCsvWorker @project = Project.find(project_id) @upload = Upload.find(upload_id) - importer = Issues::ImportCsvService.new(@user, @project, @upload.build_uploader) + importer = Issues::ImportCsvService.new(@user, @project, @upload.retrieve_uploader) importer.execute @upload.destroy diff --git a/app/workers/object_storage/background_move_worker.rb b/app/workers/object_storage/background_move_worker.rb index 8dff65e46e3..19ccae7739c 100644 --- a/app/workers/object_storage/background_move_worker.rb +++ b/app/workers/object_storage/background_move_worker.rb @@ -22,7 +22,7 @@ module ObjectStorage def build_uploader(subject, mount_point) case subject - when Upload then subject.build_uploader(mount_point) + when Upload then subject.retrieve_uploader(mount_point) else subject.send(mount_point) # rubocop:disable GitlabSecurity/PublicSend end diff --git a/app/workers/object_storage/migrate_uploads_worker.rb b/app/workers/object_storage/migrate_uploads_worker.rb index 55ac7cd9b3c..c9fd19cf9d7 100644 --- a/app/workers/object_storage/migrate_uploads_worker.rb +++ b/app/workers/object_storage/migrate_uploads_worker.rb @@ -119,7 +119,7 @@ module ObjectStorage end def build_uploaders(uploads) - uploads.map { |upload| upload.build_uploader(@mounted_as) } + uploads.map { |upload| upload.retrieve_uploader(@mounted_as) } end def migrate(uploads) |