diff options
63 files changed, 1008 insertions, 81 deletions
diff --git a/app/assets/javascripts/diffs/components/diff_line_note_form.vue b/app/assets/javascripts/diffs/components/diff_line_note_form.vue index 2f09f2e24b2..3f249567daa 100644 --- a/app/assets/javascripts/diffs/components/diff_line_note_form.vue +++ b/app/assets/javascripts/diffs/components/diff_line_note_form.vue @@ -10,7 +10,12 @@ import { } from '../../notes/components/multiline_comment_utils'; import noteForm from '../../notes/components/note_form.vue'; import autosave from '../../notes/mixins/autosave'; -import { DIFF_NOTE_TYPE, INLINE_DIFF_LINES_KEY, PARALLEL_DIFF_VIEW_TYPE } from '../constants'; +import { + DIFF_NOTE_TYPE, + INLINE_DIFF_LINES_KEY, + PARALLEL_DIFF_VIEW_TYPE, + OLD_LINE_TYPE, +} from '../constants'; export default { components: { @@ -113,6 +118,34 @@ export default { const lines = getDiffLines(); return commentLineOptions(lines, this.line, this.line.line_code, side); }, + commentLines() { + if (!this.selectedCommentPosition) return []; + + const lines = []; + const { start, end } = this.selectedCommentPosition; + const diffLines = this.diffFile[INLINE_DIFF_LINES_KEY]; + let isAdding = false; + + for (let i = 0, diffLinesLength = diffLines.length - 1; i < diffLinesLength; i += 1) { + const line = diffLines[i]; + + if (start.line_code === line.line_code) { + isAdding = true; + } + + if (isAdding) { + if (line.type !== OLD_LINE_TYPE) { + lines.push(line); + } + + if (end.line_code === line.line_code) { + break; + } + } + } + + return lines; + }, }, mounted() { if (this.isLoggedIn) { @@ -177,6 +210,7 @@ export default { :is-editing="true" :line-code="line.line_code" :line="line" + :lines="commentLines" :help-page-path="helpPagePath" :diff-file="diffFile" :show-suggest-popover="showSuggestPopover" diff --git a/app/assets/javascripts/lib/utils/text_markdown.js b/app/assets/javascripts/lib/utils/text_markdown.js index 345dfaf895b..1593a363dd1 100644 --- a/app/assets/javascripts/lib/utils/text_markdown.js +++ b/app/assets/javascripts/lib/utils/text_markdown.js @@ -232,7 +232,7 @@ export function insertMarkdownText({ .join('\n'); } } else if (tag.indexOf(textPlaceholder) > -1) { - textToInsert = tag.replace(textPlaceholder, selected); + textToInsert = tag.replace(textPlaceholder, selected.replace(/\\n/g, '\n')); } else { textToInsert = String(startChar) + tag + selected + (wrap ? tag : ''); } @@ -322,7 +322,7 @@ export function updateTextForToolbarBtn($toolbarBtn) { blockTag: $toolbarBtn.data('mdBlock'), wrap: !$toolbarBtn.data('mdPrepend'), select: $toolbarBtn.data('mdSelect'), - tagContent: $toolbarBtn.data('mdTagContent'), + tagContent: $toolbarBtn.attr('data-md-tag-content'), }); } diff --git a/app/assets/javascripts/notes/components/note_form.vue b/app/assets/javascripts/notes/components/note_form.vue index d74ade15de1..6a60e4de518 100644 --- a/app/assets/javascripts/notes/components/note_form.vue +++ b/app/assets/javascripts/notes/components/note_form.vue @@ -60,6 +60,11 @@ export default { required: false, default: null, }, + lines: { + type: Array, + required: false, + default: () => [], + }, note: { type: Object, required: false, @@ -333,6 +338,7 @@ export default { :help-page-path="helpPagePath" :show-suggest-popover="showSuggestPopover" :textarea-value="updatedNoteBody" + :lines="lines" @handleSuggestDismissed="() => $emit('handleSuggestDismissed')" > <template #textarea> diff --git a/app/assets/javascripts/packages/list/constants.js b/app/assets/javascripts/packages/list/constants.js index 25a55200df2..b4fe3c70dea 100644 --- a/app/assets/javascripts/packages/list/constants.js +++ b/app/assets/javascripts/packages/list/constants.js @@ -82,6 +82,10 @@ export const PACKAGE_TYPES = [ title: s__('PackageRegistry|PyPI'), type: PackageType.PYPI, }, + { + title: s__('PackageRegistry|RubyGems'), + type: PackageType.RUBYGEMS, + }, ]; export const LIST_TITLE_TEXT = s__('PackageRegistry|Package Registry'); diff --git a/app/assets/javascripts/packages/shared/constants.js b/app/assets/javascripts/packages/shared/constants.js index c0f7f150337..f7de31c2c86 100644 --- a/app/assets/javascripts/packages/shared/constants.js +++ b/app/assets/javascripts/packages/shared/constants.js @@ -7,6 +7,7 @@ export const PackageType = { NUGET: 'nuget', PYPI: 'pypi', COMPOSER: 'composer', + RUBYGEMS: 'rubygems', GENERIC: 'generic', }; diff --git a/app/assets/javascripts/packages/shared/utils.js b/app/assets/javascripts/packages/shared/utils.js index d34372e89b6..1f9cb8bf477 100644 --- a/app/assets/javascripts/packages/shared/utils.js +++ b/app/assets/javascripts/packages/shared/utils.js @@ -19,6 +19,8 @@ export const getPackageTypeLabel = (packageType) => { return s__('PackageType|NuGet'); case PackageType.PYPI: return s__('PackageType|PyPI'); + case PackageType.RUBYGEMS: + return s__('PackageType|RubyGems'); case PackageType.COMPOSER: return s__('PackageType|Composer'); case PackageType.GENERIC: diff --git a/app/assets/javascripts/repository/components/upload_blob_modal.vue b/app/assets/javascripts/repository/components/upload_blob_modal.vue index ec7ba469ca0..d2ff01e7fc1 100644 --- a/app/assets/javascripts/repository/components/upload_blob_modal.vue +++ b/app/assets/javascripts/repository/components/upload_blob_modal.vue @@ -168,6 +168,7 @@ export default { }); }, }, + validFileMimetypes: [], }; </script> <template> @@ -179,7 +180,12 @@ export default { :action-cancel="cancelOptions" @primary.prevent="uploadFile" > - <upload-dropzone class="gl-h-200! gl-mb-4" single-file-selection @change="setFile"> + <upload-dropzone + class="gl-h-200! gl-mb-4" + single-file-selection + :valid-file-mimetypes="$options.validFileMimetypes" + @change="setFile" + > <div v-if="file" class="card upload-dropzone-card upload-dropzone-border gl-w-full gl-h-full gl-align-items-center gl-justify-content-center gl-p-3" diff --git a/app/assets/javascripts/vue_shared/components/markdown/field.vue b/app/assets/javascripts/vue_shared/components/markdown/field.vue index 220570cd8a0..8f6d94a172a 100644 --- a/app/assets/javascripts/vue_shared/components/markdown/field.vue +++ b/app/assets/javascripts/vue_shared/components/markdown/field.vue @@ -77,6 +77,11 @@ export default { required: false, default: null, }, + lines: { + type: Array, + required: false, + default: () => [], + }, note: { type: Object, required: false, @@ -115,6 +120,20 @@ export default { return this.referencedUsers.length >= referencedUsersThreshold; }, lineContent() { + if (this.lines.length) { + return this.lines + .map((line) => { + const { rich_text: richText, text } = line; + + if (text) { + return text; + } + + return unescape(stripHtml(richText).replace(/\n/g, '')); + }) + .join('\\n'); + } + if (this.line) { const { rich_text: richText, text } = this.line; @@ -241,6 +260,7 @@ export default { :line-content="lineContent" :can-suggest="canSuggest" :show-suggest-popover="showSuggestPopover" + :suggestion-start-index="lines.length" @preview-markdown="showPreviewTab" @write-markdown="showWriteTab" @handleSuggestDismissed="() => $emit('handleSuggestDismissed')" diff --git a/app/assets/javascripts/vue_shared/components/markdown/header.vue b/app/assets/javascripts/vue_shared/components/markdown/header.vue index f52dc43aaff..01cf0beea3a 100644 --- a/app/assets/javascripts/vue_shared/components/markdown/header.vue +++ b/app/assets/javascripts/vue_shared/components/markdown/header.vue @@ -37,6 +37,11 @@ export default { required: false, default: false, }, + suggestionStartIndex: { + type: Number, + required: false, + default: 0, + }, }, data() { return { @@ -54,7 +59,9 @@ export default { ].join('\n'); }, mdSuggestion() { - return ['```suggestion:-0+0', `{text}`, '```'].join('\n'); + return [['```', `suggestion:-${this.suggestionStartIndex}+0`].join(''), `{text}`, '```'].join( + '\n', + ); }, isMac() { // Accessing properties using ?. to allow tests to use diff --git a/app/assets/javascripts/vue_shared/security_reports/queries/security_report_download_paths.query.graphql b/app/assets/javascripts/vue_shared/security_reports/queries/security_report_download_paths.query.graphql index 310d8d88904..4ce13827da2 100644 --- a/app/assets/javascripts/vue_shared/security_reports/queries/security_report_download_paths.query.graphql +++ b/app/assets/javascripts/vue_shared/security_reports/queries/security_report_download_paths.query.graphql @@ -6,6 +6,7 @@ query securityReportDownloadPaths( project(fullPath: $projectPath) { mergeRequest(iid: $iid) { headPipeline { + id jobs(securityReportTypes: $reportTypes) { nodes { name diff --git a/app/serializers/discussion_entity.rb b/app/serializers/discussion_entity.rb index bcf6b331192..0dbfe0f0772 100644 --- a/app/serializers/discussion_entity.rb +++ b/app/serializers/discussion_entity.rb @@ -2,7 +2,13 @@ class DiscussionEntity < BaseDiscussionEntity expose :notes do |discussion, opts| - request.note_entity.represent(discussion.notes, opts.merge(with_base_discussion: false)) + request.note_entity.represent( + discussion.notes, + opts.merge( + with_base_discussion: false, + discussion: discussion + ) + ) end expose :positions, if: -> (d, _) { display_merge_ref_discussions?(d) } do |discussion| diff --git a/app/serializers/note_entity.rb b/app/serializers/note_entity.rb index 9a96778786b..d44958bc0c4 100644 --- a/app/serializers/note_entity.rb +++ b/app/serializers/note_entity.rb @@ -36,7 +36,8 @@ class NoteEntity < API::Entities::Note end expose :can_resolve_discussion do |note| - note.discussion.resolvable? && note.discussion.can_resolve?(current_user) + discussion = options.fetch(:discussion, nil) || note.discussion + discussion.resolvable? && discussion.can_resolve?(current_user) end end diff --git a/app/services/packages/rubygems/create_dependencies_service.rb b/app/services/packages/rubygems/create_dependencies_service.rb new file mode 100644 index 00000000000..dea429148cf --- /dev/null +++ b/app/services/packages/rubygems/create_dependencies_service.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Packages + module Rubygems + class CreateDependenciesService + include BulkInsertSafe + + def initialize(package, gemspec) + @package = package + @gemspec = gemspec + end + + def execute + set_dependencies + end + + private + + attr_reader :package, :gemspec + + def set_dependencies + Packages::Dependency.transaction do + dependency_type_rows = gemspec.dependencies.map do |dependency| + dependency = Packages::Dependency.safe_find_or_create_by!( + name: dependency.name, + version_pattern: dependency.requirement.to_s + ) + + { + dependency_id: dependency.id, + package_id: package.id, + dependency_type: :dependencies + } + end + + package.dependency_links.upsert_all( + dependency_type_rows, + unique_by: %i[package_id dependency_id dependency_type] + ) + end + end + end + end +end diff --git a/app/services/packages/rubygems/create_gemspec_service.rb b/app/services/packages/rubygems/create_gemspec_service.rb new file mode 100644 index 00000000000..22533264480 --- /dev/null +++ b/app/services/packages/rubygems/create_gemspec_service.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Packages + module Rubygems + class CreateGemspecService + def initialize(package, gemspec) + @package = package + @gemspec = gemspec + end + + def execute + write_gemspec_to_file + end + + private + + attr_reader :package, :gemspec + + def write_gemspec_to_file + file = Tempfile.new + + begin + content = gemspec.to_ruby + file.write(content) + file.flush + + package.package_files.create!( + file: file, + size: file.size, + file_name: "#{gemspec.name}.gemspec", + file_sha1: Digest::SHA1.hexdigest(content), + file_md5: Digest::MD5.hexdigest(content), + file_sha256: Digest::SHA256.hexdigest(content) + ) + ensure + file.close + file.unlink + end + end + end + end +end diff --git a/app/services/packages/rubygems/metadata_extraction_service.rb b/app/services/packages/rubygems/metadata_extraction_service.rb new file mode 100644 index 00000000000..b3bac1854d7 --- /dev/null +++ b/app/services/packages/rubygems/metadata_extraction_service.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Packages + module Rubygems + class MetadataExtractionService + def initialize(package, gemspec) + @package = package + @gemspec = gemspec + end + + def execute + write_metadata + end + + private + + attr_reader :package, :gemspec + + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/PerceivedComplexity + # rubocop:disable Metrics/CyclomaticComplexity + def write_metadata + metadatum.update!( + authors: gemspec&.authors, + files: gemspec&.files&.to_json, + summary: gemspec&.summary, + description: gemspec&.description, + email: gemspec&.email, + homepage: gemspec&.homepage, + licenses: gemspec&.licenses&.to_json, + metadata: gemspec&.metadata&.to_json, + author: gemspec&.author, + bindir: gemspec&.bindir, + executables: gemspec&.executables&.to_json, + extensions: gemspec&.extensions&.to_json, + extra_rdoc_files: gemspec&.extra_rdoc_files&.to_json, + platform: gemspec&.platform, + post_install_message: gemspec&.post_install_message, + rdoc_options: gemspec&.rdoc_options&.to_json, + require_paths: gemspec&.require_paths&.to_json, + required_ruby_version: gemspec&.required_ruby_version&.to_s, + required_rubygems_version: gemspec&.required_rubygems_version&.to_s, + requirements: gemspec&.requirements&.to_json, + rubygems_version: gemspec&.rubygems_version + ) + end + # rubocop:enable Metrics/AbcSize + # rubocop:enable Metrics/PerceivedComplexity + # rubocop:enable Metrics/CyclomaticComplexity + + def metadatum + Packages::Rubygems::Metadatum.safe_find_or_create_by!(package: package) + end + end + end +end diff --git a/app/services/packages/rubygems/process_gem_service.rb b/app/services/packages/rubygems/process_gem_service.rb new file mode 100644 index 00000000000..59bf2a1ec28 --- /dev/null +++ b/app/services/packages/rubygems/process_gem_service.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +require 'rubygems/package' + +module Packages + module Rubygems + class ProcessGemService + include Gitlab::Utils::StrongMemoize + include ExclusiveLeaseGuard + + ExtractionError = Class.new(StandardError) + DEFAULT_LEASE_TIMEOUT = 1.hour.to_i.freeze + + def initialize(package_file) + @package_file = package_file + end + + def execute + return success if process_gem + + error('Gem was not processed') + end + + private + + attr_reader :package_file + + def process_gem + return false unless package_file + + try_obtain_lease do + package.transaction do + rename_package_and_set_version + rename_package_file + ::Packages::Rubygems::MetadataExtractionService.new(package, gemspec).execute + ::Packages::Rubygems::CreateGemspecService.new(package, gemspec).execute + ::Packages::Rubygems::CreateDependenciesService.new(package, gemspec).execute + cleanup_temp_package + end + end + + true + end + + def rename_package_and_set_version + package.update!( + name: gemspec.name, + version: gemspec.version, + status: :default + ) + end + + def rename_package_file + # Updating file_name updates the path where the file is stored. + # We must pass the file again so that CarrierWave can handle the update + package_file.update!( + file_name: "#{gemspec.name}-#{gemspec.version}.gem", + file: package_file.file, + package_id: package.id + ) + end + + def cleanup_temp_package + temp_package.destroy if package.id != temp_package.id + end + + def gemspec + strong_memoize(:gemspec) do + gem.spec + end + end + + def success + ServiceResponse.success(payload: { package: package }) + end + + def error(message) + ServiceResponse.error(message: message) + end + + def temp_package + strong_memoize(:temp_package) do + package_file.package + end + end + + def package + strong_memoize(:package) do + # if package with name/version already exists, use that package + package = temp_package.project + .packages + .rubygems + .with_name(gemspec.name) + .with_version(gemspec.version.to_s) + .last + package || temp_package + end + end + + def gem + # use_file will set an exclusive lease on the file for as long as + # the resulting gem object is being used. This means we are not + # able to rename the package_file while also using the gem object. + # We need to use a separate AR object to create the gem file to allow + # `package_file` to be free for update so we re-find the file here. + Packages::PackageFile.find(package_file.id).file.use_file do |file_path| + Gem::Package.new(File.open(file_path)) + end + rescue + raise ExtractionError.new('Unable to read gem file') + end + + # used by ExclusiveLeaseGuard + def lease_key + "packages:rubygems:process_gem_service:package:#{package.id}" + end + + # used by ExclusiveLeaseGuard + def lease_timeout + DEFAULT_LEASE_TIMEOUT + end + end + end +end diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 3dbb087d8d8..eeae5de2637 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -1107,6 +1107,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: package_repositories:packages_rubygems_extraction + :feature_category: :package_registry + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: pipeline_background:archive_trace :feature_category: :continuous_integration :has_external_dependencies: diff --git a/app/workers/packages/rubygems/extraction_worker.rb b/app/workers/packages/rubygems/extraction_worker.rb new file mode 100644 index 00000000000..1e5cd0b54ce --- /dev/null +++ b/app/workers/packages/rubygems/extraction_worker.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Packages + module Rubygems + class ExtractionWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + queue_namespace :package_repositories + feature_category :package_registry + deduplicate :until_executing + + idempotent! + + def perform(package_file_id) + package_file = ::Packages::PackageFile.find_by_id(package_file_id) + + return unless package_file + + ::Packages::Rubygems::ProcessGemService.new(package_file).execute + + rescue ::Packages::Rubygems::ProcessGemService::ExtractionError => e + Gitlab::ErrorTracking.log_exception(e, project_id: package_file.project_id) + package_file.package.destroy! + end + end + end +end diff --git a/changelogs/unreleased/300403-replace-commit-box-minipipeline-default-true.yml b/changelogs/unreleased/300403-replace-commit-box-minipipeline-default-true.yml new file mode 100644 index 00000000000..084cc49635b --- /dev/null +++ b/changelogs/unreleased/300403-replace-commit-box-minipipeline-default-true.yml @@ -0,0 +1,5 @@ +--- +title: Center the pipeline stages dropdown in the commit details page +merge_request: 56505 +author: +type: changed diff --git a/changelogs/unreleased/301175-gemfile-extraction-service.yml b/changelogs/unreleased/301175-gemfile-extraction-service.yml new file mode 100644 index 00000000000..143fe565dbb --- /dev/null +++ b/changelogs/unreleased/301175-gemfile-extraction-service.yml @@ -0,0 +1,5 @@ +--- +title: Update RubyGems metadata constraints and add gem metadata extraction +merge_request: 53673 +author: +type: changed diff --git a/changelogs/unreleased/326071-fix-security-mr-widget.yml b/changelogs/unreleased/326071-fix-security-mr-widget.yml new file mode 100644 index 00000000000..2c929f2042c --- /dev/null +++ b/changelogs/unreleased/326071-fix-security-mr-widget.yml @@ -0,0 +1,5 @@ +--- +title: Fix security report fetching in Merge Requests +merge_request: 57574 +author: +type: fixed diff --git a/changelogs/unreleased/discussions_nplusone.yml b/changelogs/unreleased/discussions_nplusone.yml new file mode 100644 index 00000000000..751079bbde3 --- /dev/null +++ b/changelogs/unreleased/discussions_nplusone.yml @@ -0,0 +1,5 @@ +--- +title: Fix N+1 issue when loading merge request comments +merge_request: 57374 +author: +type: performance diff --git a/changelogs/unreleased/ph-232339-codeSuggestionsMultiLineComment.yml b/changelogs/unreleased/ph-232339-codeSuggestionsMultiLineComment.yml new file mode 100644 index 00000000000..0ee349d28a9 --- /dev/null +++ b/changelogs/unreleased/ph-232339-codeSuggestionsMultiLineComment.yml @@ -0,0 +1,5 @@ +--- +title: Code suggestions correctly add based on multi-line comments +merge_request: 57125 +author: +type: added diff --git a/changelogs/unreleased/unlock-all-file-types-for-file-repo-uploads.yml b/changelogs/unreleased/unlock-all-file-types-for-file-repo-uploads.yml new file mode 100644 index 00000000000..c7ef4fb86c8 --- /dev/null +++ b/changelogs/unreleased/unlock-all-file-types-for-file-repo-uploads.yml @@ -0,0 +1,5 @@ +--- +title: Allow all file types to be uploaded from the repo file upload tool +merge_request: 57498 +author: +type: fixed diff --git a/config/feature_flags/development/ci_commit_pipeline_mini_graph_vue.yml b/config/feature_flags/development/ci_commit_pipeline_mini_graph_vue.yml index 22a58977f0f..4a0feee04dc 100644 --- a/config/feature_flags/development/ci_commit_pipeline_mini_graph_vue.yml +++ b/config/feature_flags/development/ci_commit_pipeline_mini_graph_vue.yml @@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/323356 milestone: '13.10' type: development group: group::pipeline authoring -default_enabled: false +default_enabled: true diff --git a/config/metrics/counts_all/20210216175442_ingress_modsecurity_packets_processed.yml b/config/metrics/counts_all/20210216175442_ingress_modsecurity_packets_processed.yml index 7ef825975a2..2257a325aa6 100644 --- a/config/metrics/counts_all/20210216175442_ingress_modsecurity_packets_processed.yml +++ b/config/metrics/counts_all/20210216175442_ingress_modsecurity_packets_processed.yml @@ -7,7 +7,7 @@ product_stage: protect product_group: group::container security product_category: web_firewall value_type: number -status: data_available +status: deprecated time_frame: all data_source: database distribution: @@ -17,4 +17,3 @@ tier: - free - premium - ultimate -skip_validation: true diff --git a/config/metrics/counts_all/20210216175444_ingress_modsecurity_packets_anomalous.yml b/config/metrics/counts_all/20210216175444_ingress_modsecurity_packets_anomalous.yml index 2aad13de693..b716862c512 100644 --- a/config/metrics/counts_all/20210216175444_ingress_modsecurity_packets_anomalous.yml +++ b/config/metrics/counts_all/20210216175444_ingress_modsecurity_packets_anomalous.yml @@ -7,7 +7,7 @@ product_stage: protect product_group: group::container security product_category: web_firewall value_type: number -status: data_available +status: deprecated time_frame: all data_source: database distribution: @@ -17,4 +17,3 @@ tier: - free - premium - ultimate -skip_validation: true diff --git a/config/metrics/counts_all/20210216175450_ingress_modsecurity_logging.yml b/config/metrics/counts_all/20210216175450_ingress_modsecurity_logging.yml index 6fc4f6178bb..74d2c99c5aa 100644 --- a/config/metrics/counts_all/20210216175450_ingress_modsecurity_logging.yml +++ b/config/metrics/counts_all/20210216175450_ingress_modsecurity_logging.yml @@ -6,7 +6,7 @@ product_stage: protect product_group: group::container security product_category: web_firewall value_type: number -status: data_available +status: deprecated time_frame: all data_source: database distribution: @@ -16,4 +16,3 @@ tier: - free - premium - ultimate -skip_validation: true diff --git a/config/metrics/counts_all/20210216175452_ingress_modsecurity_blocking.yml b/config/metrics/counts_all/20210216175452_ingress_modsecurity_blocking.yml index 7f2e91c88eb..7fce44b02cd 100644 --- a/config/metrics/counts_all/20210216175452_ingress_modsecurity_blocking.yml +++ b/config/metrics/counts_all/20210216175452_ingress_modsecurity_blocking.yml @@ -6,7 +6,7 @@ product_stage: protect product_group: group::container security product_category: web_firewall value_type: number -status: data_available +status: deprecated time_frame: all data_source: database distribution: @@ -16,4 +16,3 @@ tier: - free - premium - ultimate -skip_validation: true diff --git a/config/metrics/counts_all/20210216175454_ingress_modsecurity_disabled.yml b/config/metrics/counts_all/20210216175454_ingress_modsecurity_disabled.yml index 5c028ab30e1..838837e223d 100644 --- a/config/metrics/counts_all/20210216175454_ingress_modsecurity_disabled.yml +++ b/config/metrics/counts_all/20210216175454_ingress_modsecurity_disabled.yml @@ -6,7 +6,7 @@ product_stage: protect product_group: group::container security product_category: web_firewall value_type: number -status: data_available +status: deprecated time_frame: all data_source: database distribution: @@ -16,4 +16,3 @@ tier: - free - premium - ultimate -skip_validation: true diff --git a/config/metrics/counts_all/20210216175456_ingress_modsecurity_not_installed.yml b/config/metrics/counts_all/20210216175456_ingress_modsecurity_not_installed.yml index e0c49c6b070..1e6ee0d40cb 100644 --- a/config/metrics/counts_all/20210216175456_ingress_modsecurity_not_installed.yml +++ b/config/metrics/counts_all/20210216175456_ingress_modsecurity_not_installed.yml @@ -6,7 +6,7 @@ product_stage: protect product_group: group::container security product_category: web_firewall value_type: number -status: data_available +status: deprecated time_frame: all data_source: database distribution: @@ -16,4 +16,3 @@ tier: - free - premium - ultimate -skip_validation: true diff --git a/config/metrics/settings/20210216175459_ingress_modsecurity_enabled.yml b/config/metrics/settings/20210216175459_ingress_modsecurity_enabled.yml index 6bb5795c971..ee3741581b4 100644 --- a/config/metrics/settings/20210216175459_ingress_modsecurity_enabled.yml +++ b/config/metrics/settings/20210216175459_ingress_modsecurity_enabled.yml @@ -6,9 +6,9 @@ product_stage: protect product_group: group::container security product_category: web_firewall value_type: boolean -status: data_available +status: deprecated time_frame: none -data_source: +data_source: database distribution: - ce - ee @@ -16,4 +16,3 @@ tier: - free - premium - ultimate -skip_validation: true diff --git a/db/migrate/20210223230600_update_rubygems_metadata_metadata.rb b/db/migrate/20210223230600_update_rubygems_metadata_metadata.rb new file mode 100644 index 00000000000..39e79be301e --- /dev/null +++ b/db/migrate/20210223230600_update_rubygems_metadata_metadata.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class UpdateRubygemsMetadataMetadata < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + remove_text_limit :packages_rubygems_metadata, :metadata + add_text_limit :packages_rubygems_metadata, :metadata, 30000 + end + + def down + remove_text_limit :packages_rubygems_metadata, :metadata + add_text_limit :packages_rubygems_metadata, :metadata, 255, validate: false + end +end diff --git a/db/schema_migrations/20210223230600 b/db/schema_migrations/20210223230600 new file mode 100644 index 00000000000..be6e0621771 --- /dev/null +++ b/db/schema_migrations/20210223230600 @@ -0,0 +1 @@ +18d64af208338baec9d56a6ac9d7fc35aaeb79d3f8036d3cf5bcc72879827299
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 78f959a71f1..04d82926404 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -15658,7 +15658,7 @@ CREATE TABLE packages_rubygems_metadata ( CONSTRAINT check_b7b296b420 CHECK ((char_length(author) <= 255)), CONSTRAINT check_bf16b21a47 CHECK ((char_length(rdoc_options) <= 255)), CONSTRAINT check_ca641a3354 CHECK ((char_length(required_ruby_version) <= 255)), - CONSTRAINT check_ea02f4800f CHECK ((char_length(metadata) <= 255)), + CONSTRAINT check_ea02f4800f CHECK ((char_length(metadata) <= 30000)), CONSTRAINT check_f76bad1a9a CHECK ((char_length(require_paths) <= 255)) ); diff --git a/doc/development/fe_guide/graphql.md b/doc/development/fe_guide/graphql.md index 16ce7d9a0c0..6b3675cacfd 100644 --- a/doc/development/fe_guide/graphql.md +++ b/doc/development/fe_guide/graphql.md @@ -230,17 +230,33 @@ Read more about [Vue Apollo](https://github.com/vuejs/vue-apollo) in the [Vue Ap It is possible to manage an application state with Apollo by passing in a resolvers object when creating the default client. The default state can be set by writing -to the cache after setting up the default client. +to the cache after setting up the default client. In the example below, we are using query with `@client` Apollo directive to write the initial data to Apollo cache and then get this state in the Vue component: ```javascript +// user.query.graphql + +query User { + user @client { + name + surname + age + } +} +``` + +```javascript +// index.js + import Vue from 'vue'; import VueApollo from 'vue-apollo'; import createDefaultClient from '~/lib/graphql'; +import userQuery from '~/user/user.query.graphql' Vue.use(VueApollo); const defaultClient = createDefaultClient(); -defaultClient.cache.writeData({ +defaultClient.cache.writeQuery({ + query: userQuery, data: { user: { name: 'John', @@ -255,16 +271,15 @@ const apolloProvider = new VueApollo({ }); ``` -We can query local data with `@client` Apollo directive: - ```javascript -// user.query.graphql +// App.vue +import userQuery from '~/user/user.query.graphql' -query User { - user @client { - name - surname - age +export default { + apollo: { + user: { + query: userQuery + } } } ``` diff --git a/doc/development/i18n/externalization.md b/doc/development/i18n/externalization.md index 6d848934a17..779b203fd73 100644 --- a/doc/development/i18n/externalization.md +++ b/doc/development/i18n/externalization.md @@ -256,6 +256,7 @@ For example use `%{created_at}` in Ruby but `%{createdAt}` in JavaScript. Make s - In Vue: Use the [`GlSprintf`](https://gitlab-org.gitlab.io/gitlab-ui/?path=/docs/utilities-sprintf--sentence-with-link) component if: + - you need to include child components in the translation string. - you need to include HTML in your translation string. - you are using `sprintf` and need to pass `false` as the third argument to diff --git a/doc/development/usage_ping/dictionary.md b/doc/development/usage_ping/dictionary.md index 8f0c961cdf6..b4eacf01bec 100644 --- a/doc/development/usage_ping/dictionary.md +++ b/doc/development/usage_ping/dictionary.md @@ -2056,7 +2056,7 @@ Whether or not ModSecurity is set to blocking mode Group: `group::container security` -Status: `data_available` +Status: `deprecated` Tiers: `free`, `premium`, `ultimate` @@ -2068,7 +2068,7 @@ Whether or not ModSecurity is disabled within Ingress Group: `group::container security` -Status: `data_available` +Status: `deprecated` Tiers: `free`, `premium`, `ultimate` @@ -2080,7 +2080,7 @@ Whether or not ModSecurity is set to logging mode Group: `group::container security` -Status: `data_available` +Status: `deprecated` Tiers: `free`, `premium`, `ultimate` @@ -2092,7 +2092,7 @@ Whether or not ModSecurity has not been installed into the cluster Group: `group::container security` -Status: `data_available` +Status: `deprecated` Tiers: `free`, `premium`, `ultimate` @@ -2104,7 +2104,7 @@ Cumulative count of packets identified as anomalous by ModSecurity since Usage P Group: `group::container security` -Status: `data_available` +Status: `deprecated` Tiers: `free`, `premium`, `ultimate` @@ -2116,7 +2116,7 @@ Cumulative count of packets processed by ModSecurity since Usage Ping was last r Group: `group::container security` -Status: `data_available` +Status: `deprecated` Tiers: `free`, `premium`, `ultimate` @@ -2128,7 +2128,7 @@ Whether or not ModSecurity statistics are unavailable Group: `group::container security` -Status: `data_available` +Status: `deprecated` Tiers: `ultimate` @@ -6592,7 +6592,7 @@ Whether or not ModSecurity is enabled within Ingress Group: `group::container security` -Status: `data_available` +Status: `deprecated` Tiers: `free`, `premium`, `ultimate` diff --git a/doc/topics/git/troubleshooting_git.md b/doc/topics/git/troubleshooting_git.md index dc6d053e301..8db683f6291 100644 --- a/doc/topics/git/troubleshooting_git.md +++ b/doc/topics/git/troubleshooting_git.md @@ -47,8 +47,8 @@ errors can sometimes be caused by underlying issues with SSH (such as authentication). Make sure that SSH is correctly configured by following the instructions in the [SSH troubleshooting](../../ssh/README.md#troubleshooting-ssh-connections) documentation. -If you're a GitLab administrator and have access to the server, you can also prevent -session timeouts by configuring SSH `keep alive` either on the client or on the server. +If you're a GitLab administrator with server access, you can also prevent +session timeouts by configuring SSH `keep-alive` on the client or the server. NOTE: Configuring both the client and the server is unnecessary. @@ -153,7 +153,7 @@ and provide GitLab with more information on how to improve the service. ## `git clone` over HTTP fails with `transfer closed with outstanding read data remaining` error -If the buffer size is lower than what is allowed in the request, the action fails with an error similar to the one below: +Sometimes, when cloning old or large repositories, the following error is thrown: ```plaintext error: RPC failed; curl 18 transfer closed with outstanding read data remaining @@ -162,11 +162,59 @@ fatal: early EOF fatal: index-pack failed ``` -This can be fixed by increasing the existing `http.postBuffer` value to one greater than the repository size. For example, if `git clone` fails when cloning a 500M repository, you should set `http.postBuffer` to `524288000`. That setting ensures the request only starts buffering after the first 524288000 bytes. +This is a common problem with Git itself, due to its inability to handle large files or large quantities of files. +[Git LFS](https://about.gitlab.com/blog/2017/01/30/getting-started-with-git-lfs-tutorial/) was created to work around this problem; however, even it has limitations. It's usually due to one of these reasons: -NOTE: -The default value of `http.postBuffer`, 1 MiB, is applied if the setting is not configured. +- The number of files in the repository. +- The number of revisions in the history. +- The existence of large files in the repository. -```shell -git config http.postBuffer 524288000 -``` +The root causes vary, so multiple potential solutions exist, and you may need to +apply more than one: + +- If this error occurs when cloning a large repository, you can + [decrease the cloning depth](../../ci/large_repositories/index.md#shallow-cloning) + to a value of `1`. For example: + + ```shell + variables: + GIT_DEPTH: 1 + ``` + +- You can increase the + [http.postBuffer](https://git-scm.com/docs/git-config#Documentation/git-config.txt-httppostBuffer) + value in your local Git configuration from the default 1 MB value to a value greater + than the repository size. For example, if `git clone` fails when cloning a 500 MB + repository, you should set `http.postBuffer` to `524288000`: + + ```shell + # Set the http.postBuffer size, in bytes + git config http.postBuffer 524288000 + ``` + +- You can increase the `http.postBuffer` on the server side: + + 1. Modify the GitLab instance's + [`gitlab.rb`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/13.5.1+ee.0/files/gitlab-config-template/gitlab.rb.template#L1435-1455) file: + + ```shell + omnibus_gitconfig['system'] = { + # Set the http.postBuffer size, in bytes + "http" => ["postBuffer" => 524288000] + } + ``` + + 1. After applying this change, apply the configuration change: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +For example, if a repository has a very long history and no large files, changing +the depth should fix the problem. However, if a repository has very large files, +even a depth of 1 may be too large, thus requiring the `postBuffer` change. +If you increase your local `postBuffer` but the NGINX value on the backend is still +too small, the error persists. + +Modifying the server is not always an option, and introduces more potential risk. +Attempt local changes first. diff --git a/doc/update/index.md b/doc/update/index.md index 5c26ddb7ec6..1ecc3b106af 100644 --- a/doc/update/index.md +++ b/doc/update/index.md @@ -177,14 +177,14 @@ Find where your version sits in the upgrade path below, and upgrade GitLab accordingly, while also consulting the [version-specific upgrade instructions](#version-specific-upgrading-instructions): -`8.11.Z` -> `8.12.0` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> `11.11.8` -> `12.0.12` -> `12.1.17` -> `12.10.14` -> `13.0.14` -> `13.1.11` - > [latest `13.Y.Z`](https://about.gitlab.com/releases/categories/releases/) +`8.11.Z` -> `8.12.0` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> `11.11.8` -> `12.0.12` -> `12.1.17` -> `12.10.14` -> `13.0.14` -> `13.1.11` -> `13.5.4` - > [latest `13.Y.Z`](https://about.gitlab.com/releases/categories/releases/) The following table, while not exhaustive, shows some examples of the supported upgrade paths. | Target version | Your version | Supported upgrade path | Note | | --------------------- | ------------ | ------------------------ | ---- | -| `13.4.3` | `12.9.2` | `12.9.2` -> `12.10.14` -> `13.0.14` -> `13.4.3` | Two intermediate versions are required: the final `12.10` release, plus `13.0`. | +| `13.5.4` | `12.9.2` | `12.9.2` -> `12.10.14` -> `13.0.14` -> `13.1.11` -> `13.5.4` | Three intermediate versions are required: the final `12.10` release, plus `13.0` and `13.1`. | | `13.2.10` | `11.5.0` | `11.5.0` -> `11.11.8` -> `12.0.12` -> `12.1.17` -> `12.10.14` -> `13.0.14` -> `13.2.10` | Five intermediate versions are required: the final `11.11`, `12.0`, `12.1` and `12.10` releases, plus `13.0`. | | `12.10.14` | `11.3.4` | `11.3.4` -> `11.11.8` -> `12.0.12` -> `12.1.17` -> `12.10.14` | Three intermediate versions are required: the final `11.11` and `12.0` releases, plus `12.1` | | `12.9.5` | `10.4.5` | `10.4.5` -> `10.8.7` -> `11.11.8` -> `12.0.12` -> `12.1.17` -> `12.9.5` | Four intermediate versions are required: `10.8`, `11.11`, `12.0` and `12.1`, then `12.9.5` | @@ -358,6 +358,10 @@ Ruby 2.7.2 is required. GitLab will not start with Ruby 2.6.6 or older versions. The required Git version is Git v2.29 or higher. +### 13.4.0 + +GitLab 13.4.0 includes a background migration to [move all remaining repositories in legacy storage to hashed storage](../administration/raketasks/storage.md#migrate-to-hashed-storage). There are [known issues with this migration](https://gitlab.com/gitlab-org/gitlab/-/issues/259605) which are fixed in GitLab 13.5.4 and later. If possible, skip 13.4.0 and upgrade to 13.5.4 or higher instead. Note that the migration can take quite a while to run, depending on how many repositories must be moved. Be sure to check that all background migrations have completed before upgrading further. + ### 13.3.0 The recommended Git version is Git v2.28. The minimum required version of Git diff --git a/lib/api/rubygem_packages.rb b/lib/api/rubygem_packages.rb index 8d2d4586d8d..1d17148e0df 100644 --- a/lib/api/rubygem_packages.rb +++ b/lib/api/rubygem_packages.rb @@ -99,6 +99,8 @@ module API track_package_event('push_package', :rubygems) + package_file = nil + ActiveRecord::Base.transaction do package = ::Packages::CreateTemporaryPackageService.new( user_project, current_user, declared_params.merge(build: current_authenticated_job) @@ -109,12 +111,18 @@ module API file_name: PACKAGE_FILENAME } - ::Packages::CreatePackageFileService.new( + package_file = ::Packages::CreatePackageFileService.new( package, file_params.merge(build: current_authenticated_job) ).execute end - created! + if package_file + ::Packages::Rubygems::ExtractionWorker.perform_async(package_file.id) # rubocop:disable CodeReuse/Worker + + created! + else + bad_request!('Package creation failed') + end rescue ObjectStorage::RemoteStoreError => e Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: user_project.id }) diff --git a/locale/gitlab.pot b/locale/gitlab.pot index ce9dadc16e8..2b62445f182 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -6440,7 +6440,7 @@ msgstr "" msgid "CloudLicense|Paste your activation code below" msgstr "" -msgid "CloudLicense|This instance is currently using the Core plan." +msgid "CloudLicense|This instance is currently using the %{planName} plan." msgstr "" msgid "Cluster" @@ -22051,6 +22051,9 @@ msgstr "" msgid "PackageRegistry|Remove package" msgstr "" +msgid "PackageRegistry|RubyGems" +msgstr "" + msgid "PackageRegistry|Settings for Maven packages" msgstr "" @@ -22147,6 +22150,9 @@ msgstr "" msgid "PackageType|PyPI" msgstr "" +msgid "PackageType|RubyGems" +msgstr "" + msgid "PackageType|npm" msgstr "" diff --git a/spec/factories/packages.rb b/spec/factories/packages.rb index 882bac1daa9..599b31d9ed5 100644 --- a/spec/factories/packages.rb +++ b/spec/factories/packages.rb @@ -36,8 +36,8 @@ FactoryBot.define do package_type { :rubygems } after :create do |package| - create :package_file, :gem, package: package - create :package_file, :gemspec, package: package + create :package_file, package.processing? ? :unprocessed_gem : :gem, package: package + create :package_file, :gemspec, package: package unless package.processing? end trait(:with_metadatum) do diff --git a/spec/factories/packages/package_file.rb b/spec/factories/packages/package_file.rb index 6cbe9e6219d..74400975670 100644 --- a/spec/factories/packages/package_file.rb +++ b/spec/factories/packages/package_file.rb @@ -247,6 +247,14 @@ FactoryBot.define do size { 4.kilobytes } end + trait(:unprocessed_gem) do + package + file_fixture { 'spec/fixtures/packages/rubygems/package.gem' } + file_name { 'package.gem' } + file_sha1 { '5fe852b2a6abd96c22c11fa1ff2fb19d9ce58b57' } + size { 4.kilobytes } + end + trait(:gemspec) do package file_fixture { 'spec/fixtures/packages/rubygems/package.gemspec' } diff --git a/spec/features/projects/files/user_uploads_files_spec.rb b/spec/features/projects/files/user_uploads_files_spec.rb index e8f51837ec9..54e816d3d13 100644 --- a/spec/features/projects/files/user_uploads_files_spec.rb +++ b/spec/features/projects/files/user_uploads_files_spec.rb @@ -19,9 +19,11 @@ RSpec.describe 'Projects > Files > User uploads files' do wait_for_requests end - include_examples 'it uploads and commit a new text file' + include_examples 'it uploads and commits a new text file' - include_examples 'it uploads and commit a new image file' + include_examples 'it uploads and commits a new image file' + + include_examples 'it uploads and commits a new pdf file' include_examples 'it uploads a file to a sub-directory' end @@ -33,6 +35,6 @@ RSpec.describe 'Projects > Files > User uploads files' do visit(project_tree_path(project2)) end - include_examples 'it uploads and commit a new file to a forked project' + include_examples 'it uploads and commits a new file to a forked project' end end diff --git a/spec/features/projects/show/user_uploads_files_spec.rb b/spec/features/projects/show/user_uploads_files_spec.rb index 00181bc22f5..eb230082bfa 100644 --- a/spec/features/projects/show/user_uploads_files_spec.rb +++ b/spec/features/projects/show/user_uploads_files_spec.rb @@ -21,9 +21,11 @@ RSpec.describe 'Projects > Show > User uploads files' do wait_for_requests end - include_examples 'it uploads and commit a new text file' + include_examples 'it uploads and commits a new text file' - include_examples 'it uploads and commit a new image file' + include_examples 'it uploads and commits a new image file' + + include_examples 'it uploads and commits a new pdf file' include_examples 'it uploads a file to a sub-directory' end @@ -35,7 +37,7 @@ RSpec.describe 'Projects > Show > User uploads files' do visit(project_path(project2)) end - include_examples 'it uploads and commit a new file to a forked project' + include_examples 'it uploads and commits a new file to a forked project' end context 'when in the empty_repo_upload experiment' do diff --git a/spec/fixtures/api/schemas/graphql/packages/package_details.json b/spec/fixtures/api/schemas/graphql/packages/package_details.json index d2e2e65db54..cf624f9c644 100644 --- a/spec/fixtures/api/schemas/graphql/packages/package_details.json +++ b/spec/fixtures/api/schemas/graphql/packages/package_details.json @@ -23,7 +23,7 @@ }, "packageType": { "type": ["string"], - "enum": ["MAVEN", "NPM", "CONAN", "NUGET", "PYPI", "COMPOSER", "GENERIC", "GOLANG", "DEBIAN"] + "enum": ["MAVEN", "NPM", "CONAN", "NUGET", "PYPI", "COMPOSER", "GENERIC", "GOLANG", "RUBYGEMS", "DEBIAN"] }, "tags": { "type": "object", diff --git a/spec/fixtures/packages/rubygems/package-0.0.1.gem b/spec/fixtures/packages/rubygems/package-0.0.1.gem Binary files differindex 2143ef408ac..658ef4ee25f 100644 --- a/spec/fixtures/packages/rubygems/package-0.0.1.gem +++ b/spec/fixtures/packages/rubygems/package-0.0.1.gem diff --git a/spec/fixtures/packages/rubygems/package.gem b/spec/fixtures/packages/rubygems/package.gem Binary files differnew file mode 100644 index 00000000000..658ef4ee25f --- /dev/null +++ b/spec/fixtures/packages/rubygems/package.gem diff --git a/spec/fixtures/packages/rubygems/package.gemspec b/spec/fixtures/packages/rubygems/package.gemspec index bb87c47f5dc..ea03414cc6f 100644 --- a/spec/fixtures/packages/rubygems/package.gemspec +++ b/spec/fixtures/packages/rubygems/package.gemspec @@ -1,15 +1,42 @@ # frozen_string_literal: true Gem::Specification.new do |s| - s.name = %q{package} - s.authors = ["Tanuki Steve"] - s.version = "0.0.1" - s.date = %q{2011-09-29} - s.summary = %q{package is the best} - s.files = [ - "lib/package.rb" - ] + s.name = 'package' + s.authors = ['Tanuki Steve', 'Hal 9000'] + s.author = 'Tanuki Steve' + s.version = '0.0.1' + s.date = '2011-09-29' + s.summary = 'package is the best' + s.files = ['lib/test_gem.rb'] + s.require_paths = ['lib'] + + s.description = 'A test package for GitLab.' + s.email = 'tanuki@not_real.com' + s.homepage = 'https://gitlab.com/ruby-co/my-package' + s.license = 'MIT' + + s.metadata = { + 'bug_tracker_uri' => 'https://gitlab.com/ruby-co/my-package/issues', + 'changelog_uri' => 'https://gitlab.com/ruby-co/my-package/CHANGELOG.md', + 'documentation_uri' => 'https://gitlab.com/ruby-co/my-package/docs', + 'mailing_list_uri' => 'https://gitlab.com/ruby-co/my-package/mailme', + 'source_code_uri' => 'https://gitlab.com/ruby-co/my-package' + } + + s.bindir = 'bin' + s.executables = ['rake'] + s.extensions = ['ext/foo.rb'] + s.extra_rdoc_files = ['README.md', 'doc/userguide.md'] + s.platform = Gem::Platform::RUBY + s.post_install_message = 'Installed, thank you!' + s.rdoc_options = ['--main', 'README.md'] s.required_ruby_version = '>= 2.7.0' - s.rubygems_version = '>= 1.8.11' - s.require_paths = ["lib"] + s.required_rubygems_version = '>= 1.8.11' + s.requirements = 'A high powered server or calculator' + s.rubygems_version = '1.8.09' + + s.add_dependency 'dependency_1', '~> 1.2.3' + s.add_dependency 'dependency_2', '3.0.0' + s.add_dependency 'dependency_3', '>= 1.0.0' + s.add_dependency 'dependency_4' end diff --git a/spec/frontend/packages/details/store/getters_spec.js b/spec/frontend/packages/details/store/getters_spec.js index f12b75d3b70..005adece56e 100644 --- a/spec/frontend/packages/details/store/getters_spec.js +++ b/spec/frontend/packages/details/store/getters_spec.js @@ -27,6 +27,7 @@ import { mockPipelineInfo, mavenPackage as packageWithoutBuildInfo, pypiPackage, + rubygemsPackage, } from '../../mock_data'; import { generateMavenCommand, @@ -104,6 +105,7 @@ describe('Getters PackageDetails Store', () => { ${npmPackage} | ${'npm'} ${nugetPackage} | ${'NuGet'} ${pypiPackage} | ${'PyPI'} + ${rubygemsPackage} | ${'RubyGems'} `(`package type`, ({ packageEntity, expectedResult }) => { beforeEach(() => setupState({ packageEntity })); diff --git a/spec/frontend/packages/mock_data.js b/spec/frontend/packages/mock_data.js index fbc167729d9..06009daba54 100644 --- a/spec/frontend/packages/mock_data.js +++ b/spec/frontend/packages/mock_data.js @@ -134,6 +134,23 @@ export const nugetPackage = { }, }; +export const rubygemsPackage = { + created_at: '2015-12-10', + id: 4, + name: 'RubyGem1', + package_files: [], + package_type: 'rubygems', + project_id: 1, + tags: [], + updated_at: '2015-12-10', + version: '1.0.0', + rubygems_metadatum: { + author: 'Fake Name', + summary: 'My gem', + email: 'tanuki@fake.com', + }, +}; + export const pypiPackage = { created_at: '2015-12-10', id: 5, diff --git a/spec/frontend/packages/shared/utils_spec.js b/spec/frontend/packages/shared/utils_spec.js index 4a95def1bef..463e4a4febb 100644 --- a/spec/frontend/packages/shared/utils_spec.js +++ b/spec/frontend/packages/shared/utils_spec.js @@ -38,6 +38,7 @@ describe('Packages shared utils', () => { ${'npm'} | ${'npm'} ${'nuget'} | ${'NuGet'} ${'pypi'} | ${'PyPI'} + ${'rubygems'} | ${'RubyGems'} ${'composer'} | ${'Composer'} ${'foo'} | ${null} `(`package type`, ({ packageType, expectedResult }) => { diff --git a/spec/frontend/snippets/components/__snapshots__/snippet_description_edit_spec.js.snap b/spec/frontend/snippets/components/__snapshots__/snippet_description_edit_spec.js.snap index 885f42cc065..22e206bb483 100644 --- a/spec/frontend/snippets/components/__snapshots__/snippet_description_edit_spec.js.snap +++ b/spec/frontend/snippets/components/__snapshots__/snippet_description_edit_spec.js.snap @@ -29,6 +29,7 @@ exports[`Snippet Description Edit component rendering matches the snapshot 1`] = > <markdown-header-stub linecontent="" + suggestionstartindex="0" /> <div diff --git a/spec/requests/projects/merge_requests_discussions_spec.rb b/spec/requests/projects/merge_requests_discussions_spec.rb index 6ec586ed22c..eb8cf9f797d 100644 --- a/spec/requests/projects/merge_requests_discussions_spec.rb +++ b/spec/requests/projects/merge_requests_discussions_spec.rb @@ -26,6 +26,10 @@ RSpec.describe 'merge requests discussions' do # https://docs.gitlab.com/ee/development/query_recorder.html#use-request-specs-instead-of-controller-specs it 'avoids N+1 DB queries', :request_store do + send_request # warm up + + create(:diff_note_on_merge_request, noteable: merge_request, + project: merge_request.project) control = ActiveRecord::QueryRecorder.new { send_request } create(:diff_note_on_merge_request, noteable: merge_request, diff --git a/spec/services/packages/rubygems/create_dependencies_service_spec.rb b/spec/services/packages/rubygems/create_dependencies_service_spec.rb new file mode 100644 index 00000000000..b6e12b1cc61 --- /dev/null +++ b/spec/services/packages/rubygems/create_dependencies_service_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe Packages::Rubygems::CreateDependenciesService do + include RubygemsHelpers + + let_it_be(:package) { create(:rubygems_package) } + let_it_be(:package_file) { create(:package_file, :gem) } + let_it_be(:gem) { gem_from_file(package_file.file) } + let_it_be(:gemspec) { gem.spec } + + let(:service) { described_class.new(package, gemspec) } + + describe '#execute' do + subject { service.execute } + + it 'creates dependencies', :aggregate_failures do + expect { subject }.to change { Packages::Dependency.count }.by(4) + + gemspec.dependencies.each do |dependency| + persisted_dependency = Packages::Dependency.find_by(name: dependency.name) + + expect(persisted_dependency.version_pattern).to eq dependency.requirement.to_s + end + end + + it 'links dependencies to the package' do + expect { subject }.to change { package.dependency_links.count }.by(4) + + expect(package.dependency_links.first).to be_dependencies + end + end +end diff --git a/spec/services/packages/rubygems/create_gemspec_service_spec.rb b/spec/services/packages/rubygems/create_gemspec_service_spec.rb new file mode 100644 index 00000000000..4d5061933e0 --- /dev/null +++ b/spec/services/packages/rubygems/create_gemspec_service_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe Packages::Rubygems::CreateGemspecService do + include RubygemsHelpers + + let_it_be(:package) { create(:rubygems_package) } + let_it_be(:package_file) { create(:package_file, :gem) } + let_it_be(:gem) { gem_from_file(package_file.file) } + let_it_be(:gemspec) { gem.spec } + + let(:service) { described_class.new(package, gemspec) } + + describe '#execute' do + subject { service.execute } + + it 'creates a new package file', :aggregate_failures do + expect { subject }.to change { package.package_files.count }.by(1) + + gemspec_file = package.package_files.find_by(file_name: "#{gemspec.name}.gemspec") + expect(gemspec_file.file).not_to be_nil + expect(gemspec_file.size).not_to be_nil + expect(gemspec_file.file_md5).not_to be_nil + expect(gemspec_file.file_sha1).not_to be_nil + expect(gemspec_file.file_sha256).not_to be_nil + end + end +end diff --git a/spec/services/packages/rubygems/metadata_extraction_service_spec.rb b/spec/services/packages/rubygems/metadata_extraction_service_spec.rb new file mode 100644 index 00000000000..b308daad8f5 --- /dev/null +++ b/spec/services/packages/rubygems/metadata_extraction_service_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true +require 'spec_helper' +require 'rubygems/package' + +RSpec.describe Packages::Rubygems::MetadataExtractionService do + include RubygemsHelpers + + let_it_be(:package) { create(:rubygems_package) } + let_it_be(:package_file) { create(:package_file, :gem) } + let_it_be(:gem) { gem_from_file(package_file.file) } + let_it_be(:gemspec) { gem.spec } + + let(:service) { described_class.new(package, gemspec) } + + describe '#execute' do + subject { service.execute } + + it 'creates the metadata' do + expect { subject }.to change { Packages::Rubygems::Metadatum.count }.by(1) + end + + it 'stores the metadata', :aggregate_failures do + subject + + metadata = package.rubygems_metadatum + + expect(metadata.authors).to eq(gemspec.authors.to_json) + expect(metadata.files).to eq(gemspec.files.to_json) + expect(metadata.summary).to eq(gemspec.summary) + expect(metadata.description).to eq(gemspec.description) + expect(metadata.email).to eq(gemspec.email) + expect(metadata.homepage).to eq(gemspec.homepage) + expect(metadata.licenses).to eq(gemspec.licenses.to_json) + expect(metadata.metadata).to eq(gemspec.metadata.to_json) + expect(metadata.author).to eq(gemspec.author) + expect(metadata.bindir).to eq(gemspec.bindir) + expect(metadata.executables).to eq(gemspec.executables.to_json) + expect(metadata.extensions).to eq(gemspec.extensions.to_json) + expect(metadata.extra_rdoc_files).to eq(gemspec.extra_rdoc_files.to_json) + expect(metadata.platform).to eq(gemspec.platform) + expect(metadata.post_install_message).to eq(gemspec.post_install_message) + expect(metadata.rdoc_options).to eq(gemspec.rdoc_options.to_json) + expect(metadata.require_paths).to eq(gemspec.require_paths.to_json) + expect(metadata.required_ruby_version).to eq(gemspec.required_ruby_version.to_s) + expect(metadata.required_rubygems_version).to eq(gemspec.required_rubygems_version.to_s) + expect(metadata.requirements).to eq(gemspec.requirements.to_json) + expect(metadata.rubygems_version).to eq(gemspec.rubygems_version) + end + end +end diff --git a/spec/services/packages/rubygems/process_gem_service_spec.rb b/spec/services/packages/rubygems/process_gem_service_spec.rb new file mode 100644 index 00000000000..83e868d9579 --- /dev/null +++ b/spec/services/packages/rubygems/process_gem_service_spec.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Packages::Rubygems::ProcessGemService do + include ExclusiveLeaseHelpers + include RubygemsHelpers + + let_it_be_with_reload(:package) { create(:rubygems_package, :processing, name: 'temp_name', version: '0.0.0') } + + let(:package_file) { create(:package_file, :unprocessed_gem, package: package) } + let(:gem) { gem_from_file(package_file.file) } + let(:gemspec) { gem.spec } + let(:service) { described_class.new(package_file) } + + describe '#execute' do + subject { service.execute } + + context 'no gem file', :aggregate_failures do + let(:package_file) { nil } + + it 'returns an error' do + expect(subject.error?).to be(true) + expect(subject.message).to eq('Gem was not processed') + end + end + + context 'success' do + let(:sub_service) { double } + + before do + expect(Packages::Rubygems::MetadataExtractionService).to receive(:new).with(package, gemspec).and_return(sub_service) + expect(Packages::Rubygems::CreateGemspecService).to receive(:new).with(package, gemspec).and_return(sub_service) + expect(Packages::Rubygems::CreateDependenciesService).to receive(:new).with(package, gemspec).and_return(sub_service) + + expect(sub_service).to receive(:execute).exactly(3).times.and_return(true) + end + + it 'returns successfully', :aggregate_failures do + result = subject + + expect(result.success?).to be true + expect(result.payload[:package]).to eq(package) + end + + it 'updates the package name and version', :aggregate_failures do + expect(package.name).to eq('temp_name') + expect(package.version).to eq('0.0.0') + expect(package).to be_processing + + subject + + expect(package.reload.name).to eq('package') + expect(package.version).to eq('0.0.1') + expect(package).to be_default + end + + it 'updates the package file name', :aggregate_failures do + expect(package_file.file_name).to eq('package.gem') + + subject + + expect(package_file.reload.file_name).to eq('package-0.0.1.gem') + end + end + + context 'when the package already exists' do + let_it_be(:existing_package) { create(:rubygems_package, name: 'package', version: '0.0.1', project: package.project) } + + let(:sub_service) { double } + + before do + expect(Packages::Rubygems::MetadataExtractionService).to receive(:new).with(existing_package, gemspec).and_return(sub_service) + expect(Packages::Rubygems::CreateGemspecService).to receive(:new).with(existing_package, gemspec).and_return(sub_service) + expect(Packages::Rubygems::CreateDependenciesService).to receive(:new).with(existing_package, gemspec).and_return(sub_service) + + expect(sub_service).to receive(:execute).exactly(3).times.and_return(true) + end + + it 'assigns the package_file to the existing package and deletes the temporary package', :aggregate_failures do + expect(package).to receive(:destroy) + + expect { subject }.to change { existing_package.package_files.count }.by(1) + + expect(package_file.reload.package).to eq(existing_package) + end + end + + context 'sub-service failure' do + before do + expect(Packages::Rubygems::MetadataExtractionService).to receive(:new).with(package, gemspec).and_raise(::Packages::Rubygems::ProcessGemService::ExtractionError.new('failure')) + end + + it 'returns an error' do + expect { subject }.to raise_error(::Packages::Rubygems::ProcessGemService::ExtractionError, 'failure') + end + end + + context 'bad gem file' do + before do + expect(Gem::Package).to receive(:new).and_raise(ArgumentError) + end + + it 'returns an error' do + expect { subject }.to raise_error(::Packages::Rubygems::ProcessGemService::ExtractionError, 'Unable to read gem file') + end + end + + context 'without obtaining an exclusive lease' do + let(:lease_key) { "packages:rubygems:process_gem_service:package:#{package.id}" } + + before do + stub_exclusive_lease_taken(lease_key, timeout: 1.hour) + end + + it 'does not perform the services', :aggregate_failures do + # The #use_file call triggers a separate lease on the package file being opened + # for use with the gem. We don't want to test that here, so we allow the call to proceed + expect(Gitlab::ExclusiveLease).to receive(:new).with("object_storage_migrate:Packages::PackageFile:#{package_file.id}", anything).and_call_original + + expect(Packages::Rubygems::MetadataExtractionService).not_to receive(:new) + expect(Packages::Rubygems::CreateGemspecService).not_to receive(:new) + expect(Packages::Rubygems::CreateDependenciesService).not_to receive(:new) + + subject + + expect(package.reload.name).to eq('temp_name') + expect(package.version).to eq('0.0.0') + expect(package).to be_processing + expect(package_file.reload.file_name).to eq('package.gem') + end + end + end +end diff --git a/spec/support/helpers/rubygems_helpers.rb b/spec/support/helpers/rubygems_helpers.rb new file mode 100644 index 00000000000..6a808f52e97 --- /dev/null +++ b/spec/support/helpers/rubygems_helpers.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module RubygemsHelpers + def gem_from_file(file) + full_path = File.expand_path( + Rails.root.join('spec', 'fixtures', 'packages', 'rubygems', file.filename) + ) + + Gem::Package.new(File.open(full_path)) + end +end diff --git a/spec/support/shared_examples/features/project_upload_files_shared_examples.rb b/spec/support/shared_examples/features/project_upload_files_shared_examples.rb index d6f25aa228d..7adf303bde4 100644 --- a/spec/support/shared_examples/features/project_upload_files_shared_examples.rb +++ b/spec/support/shared_examples/features/project_upload_files_shared_examples.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -RSpec.shared_examples 'it uploads and commit a new text file' do - it 'uploads and commit a new text file', :js do +RSpec.shared_examples 'it uploads and commits a new text file' do + it 'uploads and commits a new text file', :js do find('.add-to-tree').click page.within('.dropdown-menu') do @@ -32,8 +32,8 @@ RSpec.shared_examples 'it uploads and commit a new text file' do end end -RSpec.shared_examples 'it uploads and commit a new image file' do - it 'uploads and commit a new image file', :js do +RSpec.shared_examples 'it uploads and commits a new image file' do + it 'uploads and commits a new image file', :js do find('.add-to-tree').click page.within('.dropdown-menu') do @@ -58,13 +58,39 @@ RSpec.shared_examples 'it uploads and commit a new image file' do end end -RSpec.shared_examples 'it uploads and commit a new file to a forked project' do +RSpec.shared_examples 'it uploads and commits a new pdf file' do + it 'uploads and commits a new pdf file', :js do + find('.add-to-tree').click + + page.within('.dropdown-menu') do + click_link('Upload file') + + wait_for_requests + end + + attach_file('upload_file', File.join(Rails.root, 'spec', 'fixtures', 'git-cheat-sheet.pdf'), make_visible: true) + + page.within('#modal-upload-blob') do + fill_in(:commit_message, with: 'New commit message') + fill_in(:branch_name, with: 'upload_image', visible: true) + click_button('Upload file') + end + + wait_for_all_requests + + visit(project_blob_path(project, 'upload_image/git-cheat-sheet.pdf')) + + expect(page).to have_css('.js-pdf-viewer') + end +end + +RSpec.shared_examples 'it uploads and commits a new file to a forked project' do let(:fork_message) do "You're not allowed to make changes to this project directly. "\ "A fork of this project has been created that you can make changes in, so you can submit a merge request." end - it 'uploads and commit a new file to a forked project', :js, :sidekiq_might_not_need_inline do + it 'uploads and commits a new file to a forked project', :js, :sidekiq_might_not_need_inline do find('.add-to-tree').click click_link('Upload file') diff --git a/spec/support/shared_examples/requests/api/rubygems_packages_shared_examples.rb b/spec/support/shared_examples/requests/api/rubygems_packages_shared_examples.rb index 15fb6611b90..abdb468353a 100644 --- a/spec/support/shared_examples/requests/api/rubygems_packages_shared_examples.rb +++ b/spec/support/shared_examples/requests/api/rubygems_packages_shared_examples.rb @@ -43,6 +43,8 @@ end RSpec.shared_examples 'process rubygems upload' do |user_type, status, add_member = true| RSpec.shared_examples 'creates rubygems package files' do it 'creates package files', :aggregate_failures do + expect(::Packages::Rubygems::ExtractionWorker).to receive(:perform_async).once + expect { subject } .to change { project.packages.count }.by(1) .and change { Packages::PackageFile.count }.by(1) @@ -51,6 +53,17 @@ RSpec.shared_examples 'process rubygems upload' do |user_type, status, add_membe package_file = project.packages.last.package_files.reload.last expect(package_file.file_name).to eq('package.gem') end + + it 'returns bad request if package creation fails' do + file_service = double('file_service', execute: nil) + + expect(::Packages::CreatePackageFileService).to receive(:new).and_return(file_service) + expect(::Packages::Rubygems::ExtractionWorker).not_to receive(:perform_async) + + subject + + expect(response).to have_gitlab_http_status(:bad_request) + end end context "for user type #{user_type}" do diff --git a/spec/workers/packages/rubygems/extraction_worker_spec.rb b/spec/workers/packages/rubygems/extraction_worker_spec.rb new file mode 100644 index 00000000000..15c0a3be90c --- /dev/null +++ b/spec/workers/packages/rubygems/extraction_worker_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Packages::Rubygems::ExtractionWorker, type: :worker do + describe '#perform' do + let_it_be(:package) { create(:rubygems_package) } + + let(:package_file) { package.package_files.first } + let(:package_file_id) { package_file.id } + let(:package_name) { 'TempProject.TempPackage' } + let(:package_version) { '1.0.0' } + let(:job_args) { package_file_id } + + subject { described_class.new.perform(*job_args) } + + include_examples 'an idempotent worker' do + it 'processes the gem', :aggregate_failures do + expect { subject } + .to change { Packages::Package.count }.by(0) + .and change { Packages::PackageFile.count }.by(2) + + expect(Packages::Package.last.id).to be(package.id) + expect(package.name).not_to be(package_name) + end + end + + it 'handles a processing failure', :aggregate_failures do + expect(::Packages::Rubygems::ProcessGemService).to receive(:new) + .and_raise(::Packages::Rubygems::ProcessGemService::ExtractionError) + + expect(Gitlab::ErrorTracking).to receive(:log_exception).with( + instance_of(::Packages::Rubygems::ProcessGemService::ExtractionError), + project_id: package.project_id + ) + + expect { subject } + .to change { Packages::Package.count }.by(-1) + .and change { Packages::PackageFile.count }.by(-2) + end + + context 'returns when there is no package file' do + let(:package_file_id) { 999999 } + + it 'returns without action' do + expect(::Packages::Rubygems::ProcessGemService).not_to receive(:new) + + expect { subject } + .to change { Packages::Package.count }.by(0) + .and change { Packages::PackageFile.count }.by(0) + end + end + end +end |