diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-03-10 21:12:41 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-03-10 21:12:41 +0300 |
commit | 73fe31a692af05918e234b1acc915e487f194d23 (patch) | |
tree | 9f011371fb4667d5027a571969345b9588b3901d | |
parent | ad2d90fb2475c9660b04951cd93ee969cf78c09b (diff) |
Add latest changes from gitlab-org/gitlab@master
81 files changed, 1561 insertions, 464 deletions
diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 1dc88f733ee..5687640cd8a 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -768280df636bd606b76c43fab1a0c334251536e5 +37fd51c54395a06dc39322606f9a3e4ba2dafa3d @@ -194,10 +194,10 @@ gem 'rdoc', '~> 6.3.2' gem 'org-ruby', '~> 0.9.12' gem 'creole', '~> 0.5.0' gem 'wikicloth', '0.8.1' -gem 'asciidoctor', '~> 2.0.17' +gem 'asciidoctor', '~> 2.0.18' gem 'asciidoctor-include-ext', '~> 0.4.0', require: false gem 'asciidoctor-plantuml', '~> 0.0.16' -gem 'asciidoctor-kroki', '~> 0.7.0', require: false +gem 'asciidoctor-kroki', '~> 0.8.0', require: false gem 'rouge', '~> 3.30.0' gem 'truncato', '~> 0.7.12' gem 'nokogiri', '~> 1.14.2' diff --git a/Gemfile.checksum b/Gemfile.checksum index cd822ba89c4..aed7cde551a 100644 --- a/Gemfile.checksum +++ b/Gemfile.checksum @@ -24,9 +24,9 @@ {"name":"app_store_connect","version":"0.29.0","platform":"ruby","checksum":"01d7a923825a4221892099acb5a72f86f6ee7d8aa95815d3c459ba6816ea430f"}, {"name":"arr-pm","version":"0.0.12","platform":"ruby","checksum":"fdff482f75239239201f4d667d93424412639aad0b3b0ad4d827e7c637e0ad39"}, {"name":"asana","version":"0.10.13","platform":"ruby","checksum":"36d0d37f8dd6118a54580f1b80224875d7b6a9027598938e1722a508bfc2d7ac"}, -{"name":"asciidoctor","version":"2.0.17","platform":"ruby","checksum":"ed5b5e399e8d64994cc16f0983f993d6e33990909a8415b6fc8b786cdeb00f3d"}, +{"name":"asciidoctor","version":"2.0.18","platform":"ruby","checksum":"bbd1e1d16deed8db94bf9624b9f4474fac32d9ca7225d377f076c08d9adde387"}, {"name":"asciidoctor-include-ext","version":"0.4.0","platform":"ruby","checksum":"406adb9d2fbfc25536609ca13b787ed704dc06a4e49d6709b83f3bad578f7878"}, -{"name":"asciidoctor-kroki","version":"0.7.0","platform":"ruby","checksum":"528ae4e49cae10e98c76e91f9aa40c67bf8540aa5ce4bbd44c5cd57af9f0b121"}, +{"name":"asciidoctor-kroki","version":"0.8.0","platform":"ruby","checksum":"e53b3f349167cebde990b0098863e8fe98fd235e35263a78c88cc4e0268b1a36"}, {"name":"asciidoctor-plantuml","version":"0.0.16","platform":"ruby","checksum":"407e47cd1186ded5ccc75f0c812e5524c26c571d542247c5132abb8f47bd1793"}, {"name":"ast","version":"2.4.2","platform":"ruby","checksum":"1e280232e6a33754cde542bc5ef85520b74db2aac73ec14acef453784447cc12"}, {"name":"atlassian-jwt","version":"0.2.0","platform":"ruby","checksum":"52e653e9d6062d7a740c3675b0e79fa08367927c6fc17f5476d1b6b3798c6eb2"}, diff --git a/Gemfile.lock b/Gemfile.lock index 0a95518e60b..40a2739b744 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -184,10 +184,10 @@ GEM faraday_middleware (~> 1.0) faraday_middleware-multi_json (~> 0.0) oauth2 (>= 1.4, < 3) - asciidoctor (2.0.17) + asciidoctor (2.0.18) asciidoctor-include-ext (0.4.0) asciidoctor (>= 1.5.6, < 3.0.0) - asciidoctor-kroki (0.7.0) + asciidoctor-kroki (0.8.0) asciidoctor (~> 2.0) asciidoctor-plantuml (0.0.16) asciidoctor (>= 2.0.17, < 3.0.0) @@ -1632,9 +1632,9 @@ DEPENDENCIES app_store_connect arr-pm (~> 0.0.12) asana (~> 0.10.13) - asciidoctor (~> 2.0.17) + asciidoctor (~> 2.0.18) asciidoctor-include-ext (~> 0.4.0) - asciidoctor-kroki (~> 0.7.0) + asciidoctor-kroki (~> 0.8.0) asciidoctor-plantuml (~> 0.0.16) atlassian-jwt (~> 0.2.0) attr_encrypted (~> 3.2.4)! diff --git a/app/assets/javascripts/commit/components/signature_badge.vue b/app/assets/javascripts/commit/components/signature_badge.vue new file mode 100644 index 00000000000..344536df093 --- /dev/null +++ b/app/assets/javascripts/commit/components/signature_badge.vue @@ -0,0 +1,94 @@ +<script> +import { GlBadge, GlLink, GlPopover } from '@gitlab/ui'; +import { helpPagePath } from '~/helpers/help_page_helper'; +import { typeConfig, statusConfig } from '../constants'; +import X509CertificateDetails from './x509_certificate_details.vue'; + +export default { + components: { + GlBadge, + GlPopover, + GlLink, + X509CertificateDetails, + }, + props: { + signature: { + type: Object, + required: true, + }, + }, + computed: { + statusConfig() { + return this.$options.statusConfig?.[this.signature?.verificationStatus]; + }, + typeConfig() { + // eslint-disable-next-line no-underscore-dangle + return this.$options.typeConfig?.[this.signature?.__typename]; + }, + }, + methods: { + helpPagePath, + getSubjectKeyIdentifierToDisplay(subjectKeyIdentifier) { + // we need to remove : to not trigger secret detection scan + return subjectKeyIdentifier.replaceAll(':', ' '); + }, + }, + typeConfig, + statusConfig, +}; +</script> +<template> + <span + v-if="statusConfig && typeConfig" + class="gl-display-flex gl-align-items-center gl-hover-cursor-pointer gl-ml-2" + > + <button + id="signature" + tabindex="0" + data-testid="signature-badge" + role="button" + variant="link" + class="gl-border-0 gl-outline-0! gl-p-0 gl-bg-transparent" + :aria-label="statusConfig.label" + > + <gl-badge :variant="statusConfig.variant" size="md" data-testid="signature-status"> + {{ statusConfig.label }} + </gl-badge> + </button> + <gl-popover target="signature" triggers="focus" data-testid="signature-info"> + <template #title> + {{ statusConfig.title }} + </template> + <p data-testid="signature-description"> + {{ statusConfig.description }} + </p> + <p v-if="typeConfig.keyLabel" data-testid="signature-key-label"> + {{ typeConfig.keyLabel }} + <span class="gl-font-monospace" data-testid="signature-key"> + {{ signature[typeConfig.keyNamespace] || __('Unknown') }} + </span> + </p> + <x509-certificate-details + v-if="signature.x509Certificate" + :title="typeConfig.subjectTitle" + :subject="signature.x509Certificate.subject" + :subject-key-identifier=" + getSubjectKeyIdentifierToDisplay(signature.x509Certificate.subjectKeyIdentifier) + " + /> + <x509-certificate-details + v-if="signature.x509Certificate && signature.x509Certificate.x509Issuer" + :title="typeConfig.issuerTitle" + :subject="signature.x509Certificate.x509Issuer.subject" + :subject-key-identifier=" + getSubjectKeyIdentifierToDisplay( + signature.x509Certificate.x509Issuer.subjectKeyIdentifier, + ) + " + /> + <gl-link :href="helpPagePath(typeConfig.helpLink.path)"> + {{ typeConfig.helpLink.label }} + </gl-link> + </gl-popover> + </span> +</template> diff --git a/app/assets/javascripts/commit/components/x509_certificate_details.vue b/app/assets/javascripts/commit/components/x509_certificate_details.vue new file mode 100644 index 00000000000..6880fab9043 --- /dev/null +++ b/app/assets/javascripts/commit/components/x509_certificate_details.vue @@ -0,0 +1,45 @@ +<script> +import { X509_CERTIFICATE_KEY_IDENTIFIER_TITLE } from '../constants'; + +export default { + props: { + subject: { + type: String, + required: true, + }, + title: { + type: String, + required: true, + }, + subjectKeyIdentifier: { + type: String, + required: true, + }, + }, + computed: { + subjectValues() { + return this.subject.split(','); + }, + subjectKeyIdentifierToDisplay() { + return this.subjectKeyIdentifier.replaceAll(':', ' '); + }, + }, + i18n: { + keyIdentifierTitle: X509_CERTIFICATE_KEY_IDENTIFIER_TITLE, + }, +}; +</script> + +<template> + <div> + <strong>{{ title }}</strong> + <ul class="gl-pl-5"> + <li v-for="value in subjectValues" :key="value" data-testid="subject-value"> + {{ value }} + </li> + <li data-testid="key-identifier"> + {{ $options.i18n.keyIdentifierTitle }} {{ subjectKeyIdentifierToDisplay }} + </li> + </ul> + </div> +</template> diff --git a/app/assets/javascripts/commit/constants.js b/app/assets/javascripts/commit/constants.js new file mode 100644 index 00000000000..4f865e99e46 --- /dev/null +++ b/app/assets/javascripts/commit/constants.js @@ -0,0 +1,104 @@ +import { __, s__ } from '~/locale'; + +export const X509_CERTIFICATE_KEY_IDENTIFIER_TITLE = __('Subject Key Identifier:'); + +export const verificationStatuses = { + VERIFIED: 'VERIFIED', + UNVERIFIED: 'UNVERIFIED', + UNVERIFIED_KEY: 'UNVERIFIED_KEY', + UNKNOWN_KEY: 'UNKNOWN_KEY', + OTHER_USER: 'OTHER_USER', + SAME_USER_DIFFERENT_EMAIL: 'SAME_USER_DIFFERENT_EMAIL', + MULTIPLE_SIGNATURES: 'MULTIPLE_SIGNATURES', + REVOKED_KEY: 'REVOKED_KEY', +}; + +export const signatureTypes = { + /* eslint-disable @gitlab/require-i18n-strings */ + GPG: 'GpgSignature', + X509: 'X509Signature', + SSH: 'SshSignature', + /* eslint-enable @gitlab/require-i18n-strings */ +}; + +const UNVERIFIED_CONFIG = { + variant: 'muted', + label: __('Unverified'), + title: __('Unverified signature'), + description: __('This commit was signed with an unverified signature.'), +}; + +export const statusConfig = { + [verificationStatuses.VERIFIED]: { + variant: 'success', + label: __('Verified'), + title: __('Verified commit'), + description: __( + 'This commit was signed with a verified signature and the committer email was verified to belong to the same user.', + ), + }, + [verificationStatuses.UNVERIFIED]: { + ...UNVERIFIED_CONFIG, + }, + [verificationStatuses.UNVERIFIED_KEY]: { + ...UNVERIFIED_CONFIG, + }, + [verificationStatuses.UNKNOWN_KEY]: { + ...UNVERIFIED_CONFIG, + }, + [verificationStatuses.OTHER_USER]: { + variant: 'muted', + label: __('Unverified'), + title: __("Different user's signature"), + description: __('This commit was signed with an unverified signature.'), + }, + [verificationStatuses.SAME_USER_DIFFERENT_EMAIL]: { + variant: 'muted', + label: __('Unverified'), + title: __('GPG key mismatch'), + description: __( + 'This commit was signed with a verified signature, but the committer email is not associated with the GPG Key.', + ), + }, + [verificationStatuses.MULTIPLE_SIGNATURES]: { + variant: 'muted', + label: __('Unverified'), + title: __('Multiple signatures'), + description: __('This commit was signed with multiple signatures.'), + }, + [verificationStatuses.REVOKED_KEY]: { + variant: 'muted', + label: __('Unverified'), + title: s__('CommitSignature|Unverified signature'), + description: s__('CommitSignature|This commit was signed with a key that was revoked.'), + }, +}; + +export const typeConfig = { + [signatureTypes.GPG]: { + keyLabel: __('GPG Key ID:'), + keyNamespace: 'gpgKeyPrimaryKeyid', + helpLink: { + label: __('Learn about signing commits'), + path: 'user/project/repository/gpg_signed_commits/index.md', + }, + }, + [signatureTypes.X509]: { + keyLabel: '', + helpLink: { + label: __('Learn more about X.509 signed commits'), + path: '/user/project/repository/x509_signed_commits/index.md', + }, + subjectTitle: __('Certificate Subject'), + issuerTitle: __('Certificate Issuer'), + keyIdentifierTitle: __('Subject Key Identifier:'), + }, + [signatureTypes.SSH]: { + keyLabel: __('SSH key fingerprint:'), + keyNamespace: 'keyFingerprintSha256', + helpLink: { + label: __('Learn about signing commits with SSH keys.'), + path: '/user/project/repository/ssh_signed_commits/index.md', + }, + }, +}; diff --git a/app/assets/javascripts/repository/components/last_commit.vue b/app/assets/javascripts/repository/components/last_commit.vue index 4d3c1521559..2d2e21dfd92 100644 --- a/app/assets/javascripts/repository/components/last_commit.vue +++ b/app/assets/javascripts/repository/components/last_commit.vue @@ -9,6 +9,7 @@ import ClipboardButton from '~/vue_shared/components/clipboard_button.vue'; import TimeagoTooltip from '~/vue_shared/components/time_ago_tooltip.vue'; import UserAvatarLink from '~/vue_shared/components/user_avatar/user_avatar_link.vue'; import UserAvatarImage from '~/vue_shared/components/user_avatar/user_avatar_image.vue'; +import SignatureBadge from '~/commit/components/signature_badge.vue'; import getRefMixin from '../mixins/get_ref'; import projectPathQuery from '../queries/project_path.query.graphql'; @@ -23,6 +24,7 @@ export default { GlLink, GlLoadingIcon, UserAvatarImage, + SignatureBadge, }, directives: { GlTooltip: GlTooltipDirective, @@ -170,10 +172,7 @@ export default { <div class="commit-actions gl-display-flex gl-flex-align gl-align-items-center gl-flex-direction-row" > - <div - v-if="commit.signatureHtml" - v-html="commit.signatureHtml /* eslint-disable-line vue/no-v-html */" - ></div> + <signature-badge v-if="commit.signature" :signature="commit.signature" /> <div v-if="commit.pipeline" class="ci-status-link"> <gl-link v-gl-tooltip.left diff --git a/app/controllers/groups/group_links_controller.rb b/app/controllers/groups/group_links_controller.rb index cc2ca728592..c74c48a960d 100644 --- a/app/controllers/groups/group_links_controller.rb +++ b/app/controllers/groups/group_links_controller.rb @@ -7,7 +7,7 @@ class Groups::GroupLinksController < Groups::ApplicationController feature_category :subgroups def update - Groups::GroupLinks::UpdateService.new(@group_link).execute(group_link_params) + Groups::GroupLinks::UpdateService.new(@group_link, current_user).execute(group_link_params) if @group_link.expires? render json: { diff --git a/app/graphql/queries/repository/path_last_commit.query.graphql b/app/graphql/queries/repository/path_last_commit.query.graphql index 914be3a72c1..facbf1555fc 100644 --- a/app/graphql/queries/repository/path_last_commit.query.graphql +++ b/app/graphql/queries/repository/path_last_commit.query.graphql @@ -27,7 +27,30 @@ query pathLastCommit($projectPath: ID!, $path: String, $ref: String!) { avatarUrl webPath } - signatureHtml + signature { + __typename + ... on GpgSignature { + gpgKeyPrimaryKeyid + verificationStatus + } + ... on X509Signature { + verificationStatus + x509Certificate { + id + subject + subjectKeyIdentifier + x509Issuer { + id + subject + subjectKeyIdentifier + } + } + } + ... on SshSignature { + verificationStatus + keyFingerprintSha256 + } + } pipelines(ref: $ref, first: 1) { __typename edges { diff --git a/app/helpers/users_helper.rb b/app/helpers/users_helper.rb index e0cf7aa61ee..4e5c437ebaf 100644 --- a/app/helpers/users_helper.rb +++ b/app/helpers/users_helper.rb @@ -58,12 +58,13 @@ module UsersHelper end # Used to preload when you are rendering many projects and checking access - # - # rubocop: disable CodeReuse/ActiveRecord: `projects` can be array which also responds to pluck def load_max_project_member_accesses(projects) - current_user&.max_member_access_for_project_ids(projects.pluck(:id)) + return unless current_user + + Preloaders::UsersMaxAccessLevelInProjectsPreloader + .new(projects: projects, users: [current_user]) + .execute end - # rubocop: enable CodeReuse/ActiveRecord def max_project_member_access(project) current_user&.max_member_access_for_project(project.id) || Gitlab::Access::NO_ACCESS diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb index 14be924f9da..ec4ee7985fe 100644 --- a/app/models/concerns/atomic_internal_id.rb +++ b/app/models/concerns/atomic_internal_id.rb @@ -61,6 +61,8 @@ module AtomicInternalId AtomicInternalId.project_init(self) when :group AtomicInternalId.group_init(self) + when :namespace + AtomicInternalId.namespace_init(self) else # We require init here to retain the ability to recalculate in the absence of a # InternalId record (we may delete records in `internal_ids` for example). @@ -241,6 +243,16 @@ module AtomicInternalId end end + def self.namespace_init(klass, column_name = :iid) + ->(instance, scope) do + if instance + klass.where(namespace_id: instance.namespace_id).maximum(column_name) + elsif scope.present? + klass.where(**scope).maximum(column_name) + end + end + end + def internal_id_read_scope(scope) association(scope).reader end diff --git a/app/models/concerns/packages/debian/component_file.rb b/app/models/concerns/packages/debian/component_file.rb index 77409549e85..5905670227c 100644 --- a/app/models/concerns/packages/debian/component_file.rb +++ b/app/models/concerns/packages/debian/component_file.rb @@ -88,6 +88,10 @@ module Packages end end + def empty? + size == 0 + end + private def extension diff --git a/app/models/issue.rb b/app/models/issue.rb index 352aa89b4c8..a19b5809ff8 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -63,7 +63,24 @@ class Issue < ApplicationRecord belongs_to :moved_to, class_name: 'Issue' has_one :moved_from, class_name: 'Issue', foreign_key: :moved_to_id - has_internal_id :iid, scope: :project, track_if: -> { !importing? } + has_internal_id :iid, scope: :namespace, track_if: -> { !importing? }, init: ->(issue, scope) do + # we need this init for the case where the IID allocation in internal_ids#last_value + # is higher than the actual issues.max(iid) value for a given project. For instance + # in case of an import where a batch of IIDs may be prealocated + # + # TODO: remove this once the UpdateIssuesInternalIdScope migration completes + if issue + [ + InternalId.where(project: issue.project, usage: :issues).pick(:last_value).to_i, + issue.namespace&.issues&.maximum(:iid).to_i + ].max + else + [ + InternalId.where(**scope, usage: :issues).pick(:last_value).to_i, + where(**scope).maximum(:iid).to_i + ].max + end + end has_many :events, as: :target, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent @@ -104,7 +121,7 @@ class Issue < ApplicationRecord accepts_nested_attributes_for :sentry_issue accepts_nested_attributes_for :incident_management_issuable_escalation_status, update_only: true - validates :project, presence: true + validates :project, presence: true, if: -> { !namespace || namespace.is_a?(Namespaces::ProjectNamespace) } validates :issue_type, presence: true validates :namespace, presence: true validates :work_item_type, presence: true @@ -137,7 +154,7 @@ class Issue < ApplicationRecord scope :order_due_date_asc, -> { reorder(arel_table[:due_date].asc.nulls_last) } scope :order_due_date_desc, -> { reorder(arel_table[:due_date].desc.nulls_last) } - scope :order_closest_future_date, -> { reorder(Arel.sql('CASE WHEN issues.due_date >= CURRENT_DATE THEN 0 ELSE 1 END ASC, ABS(CURRENT_DATE - issues.due_date) ASC')) } + scope :order_closest_future_date, -> { reorder(Arel.sql("CASE WHEN issues.due_date >= CURRENT_DATE THEN 0 ELSE 1 END ASC, ABS(CURRENT_DATE - issues.due_date) ASC")) } scope :order_created_at_desc, -> { reorder(created_at: :desc) } scope :order_severity_asc, -> do build_keyset_order_on_joined_column( diff --git a/app/models/packages/debian.rb b/app/models/packages/debian.rb index 9c615c20250..887a5695530 100644 --- a/app/models/packages/debian.rb +++ b/app/models/packages/debian.rb @@ -10,6 +10,8 @@ module Packages LETTER_REGEX = %r{(lib)?[a-z0-9]}.freeze + EMPTY_FILE_SHA256 = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'.freeze + def self.table_name_prefix 'packages_debian_' end diff --git a/app/services/groups/group_links/create_service.rb b/app/services/groups/group_links/create_service.rb index 9c1a003ff36..a6e2c0b952e 100644 --- a/app/services/groups/group_links/create_service.rb +++ b/app/services/groups/group_links/create_service.rb @@ -36,3 +36,5 @@ module Groups end end end + +Groups::GroupLinks::CreateService.prepend_mod diff --git a/app/services/groups/group_links/destroy_service.rb b/app/services/groups/group_links/destroy_service.rb index dc3cab927be..8eed46b28ca 100644 --- a/app/services/groups/group_links/destroy_service.rb +++ b/app/services/groups/group_links/destroy_service.rb @@ -24,7 +24,11 @@ module Groups Gitlab::AppLogger.info( "Failed to delete GroupGroupLinks with ids: #{links.map(&:id)}.") end + + links end end end end + +Groups::GroupLinks::DestroyService.prepend_mod diff --git a/app/services/groups/group_links/update_service.rb b/app/services/groups/group_links/update_service.rb index 66d0d63cb9b..913bf2bfce7 100644 --- a/app/services/groups/group_links/update_service.rb +++ b/app/services/groups/group_links/update_service.rb @@ -15,6 +15,8 @@ module Groups if requires_authorization_refresh?(group_link_params) group_link.shared_with_group.refresh_members_authorized_projects(direct_members_only: true) end + + group_link end private @@ -27,3 +29,5 @@ module Groups end end end + +Groups::GroupLinks::UpdateService.prepend_mod diff --git a/app/services/packages/debian/generate_distribution_service.rb b/app/services/packages/debian/generate_distribution_service.rb index 12ae6c68918..2ced2e5f275 100644 --- a/app/services/packages/debian/generate_distribution_service.rb +++ b/app/services/packages/debian/generate_distribution_service.rb @@ -165,16 +165,29 @@ module Packages def reuse_or_create_component_file(component, component_file_type, architecture, content) file_md5 = Digest::MD5.hexdigest(content) file_sha256 = Digest::SHA256.hexdigest(content) - component_file = component.files - .with_file_type(component_file_type) - .with_architecture(architecture) - .with_compression_type(nil) - .with_file_sha256(file_sha256) - .last - - if component_file + component_files = component.files + .with_file_type(component_file_type) + .with_architecture(architecture) + .with_compression_type(nil) + .order_updated_asc + component_file = component_files.with_file_sha256(file_sha256).last + last_component_file = component_files.last + + if content.empty? && (!last_component_file || last_component_file.file_sha256 == file_sha256) + # Do not create empty component file for empty content + # when there is no last component file or when the last component file is empty too + component_file = last_component_file || component.files.build( + updated_at: release_date, + file_type: component_file_type, + architecture: architecture, + compression_type: nil, + size: 0 + ) + elsif component_file + # Reuse existing component file component_file.touch(time: release_date) else + # Create a new component file component_file = component.files.create!( updated_at: release_date, file_type: component_file_type, @@ -182,7 +195,8 @@ module Packages compression_type: nil, file: CarrierWaveStringFile.new(content), file_md5: file_md5, - file_sha256: file_sha256 + file_sha256: file_sha256, + size: content.size ) end diff --git a/app/services/users/build_service.rb b/app/services/users/build_service.rb index 934dccf2f76..9d221119985 100644 --- a/app/services/users/build_service.rb +++ b/app/services/users/build_service.rb @@ -70,7 +70,7 @@ module Users @user_params[:created_by_id] = current_user&.id @user_params[:external] = user_external? if set_external_param? - @user_params.delete(:user_type) unless project_bot? + @user_params.delete(:user_type) unless allowed_user_type? end def set_external_param? @@ -81,7 +81,7 @@ module Users user_default_internal_regex_instance.match(params[:email]).nil? end - def project_bot? + def allowed_user_type? user_params[:user_type]&.to_sym == :project_bot end diff --git a/app/workers/gitlab/github_import/stage/import_repository_worker.rb b/app/workers/gitlab/github_import/stage/import_repository_worker.rb index 8c1a2cd2677..e13f43ee1f3 100644 --- a/app/workers/gitlab/github_import/stage/import_repository_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_repository_worker.rb @@ -72,7 +72,7 @@ module Gitlab return unless last_github_issue - Issue.track_project_iid!(project, last_github_issue[:number]) + Issue.track_namespace_iid!(project.project_namespace, last_github_issue[:number]) end end end diff --git a/config/feature_flags/development/github_import_gists.yml b/config/feature_flags/development/github_import_gists.yml index 8e6e5825362..a8d1483f26f 100644 --- a/config/feature_flags/development/github_import_gists.yml +++ b/config/feature_flags/development/github_import_gists.yml @@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/386579 milestone: '15.8' type: development group: group::import -default_enabled: false +default_enabled: true diff --git a/config/metrics/counts_28d/20210216184454_code_review_total_unique_counts_monthly.yml b/config/metrics/counts_28d/20210216184454_code_review_total_unique_counts_monthly.yml index 90c053612cf..ac6500672c2 100644 --- a/config/metrics/counts_28d/20210216184454_code_review_total_unique_counts_monthly.yml +++ b/config/metrics/counts_28d/20210216184454_code_review_total_unique_counts_monthly.yml @@ -7,7 +7,9 @@ product_group: code_review product_category: code_review product_section: 'TBD' value_type: number -status: active +status: removed +removed_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/113422 +milestone_removed: 15.10 time_frame: 28d data_source: redis_hll instrumentation_class: AggregatedMetric diff --git a/config/metrics/counts_7d/20210216184452_code_review_total_unique_counts_weekly.yml b/config/metrics/counts_7d/20210216184452_code_review_total_unique_counts_weekly.yml index 07985c3e56e..4c12bd72f94 100644 --- a/config/metrics/counts_7d/20210216184452_code_review_total_unique_counts_weekly.yml +++ b/config/metrics/counts_7d/20210216184452_code_review_total_unique_counts_weekly.yml @@ -7,7 +7,9 @@ product_group: code_review product_category: code_review product_section: 'TBD' value_type: number -status: active +status: removed +removed_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/113422 +milestone_removed: 15.10 time_frame: 7d data_source: redis_hll instrumentation_class: AggregatedMetric diff --git a/db/post_migrate/20230224085743_update_issues_internal_id_scope.rb b/db/post_migrate/20230224085743_update_issues_internal_id_scope.rb new file mode 100644 index 00000000000..71d16ccf2a6 --- /dev/null +++ b/db/post_migrate/20230224085743_update_issues_internal_id_scope.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class UpdateIssuesInternalIdScope < Gitlab::Database::Migration[2.1] + MIGRATION = 'IssuesInternalIdScopeUpdater' + INTERVAL = 2.minutes + BATCH_SIZE = 5_000 + MAX_BATCH_SIZE = 20_000 + SUB_BATCH_SIZE = 100 + + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :internal_ids, + :id, + job_interval: INTERVAL, + batch_size: BATCH_SIZE, + max_batch_size: MAX_BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :internal_ids, :id, []) + end +end diff --git a/db/post_migrate/20230306195007_queue_backfill_project_wiki_repositories.rb b/db/post_migrate/20230306195007_queue_backfill_project_wiki_repositories.rb new file mode 100644 index 00000000000..fd2dc0d16da --- /dev/null +++ b/db/post_migrate/20230306195007_queue_backfill_project_wiki_repositories.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class QueueBackfillProjectWikiRepositories < Gitlab::Database::Migration[2.1] + MIGRATION = "BackfillProjectWikiRepositories" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :projects, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :projects, :id, []) + end +end diff --git a/db/schema_migrations/20230224085743 b/db/schema_migrations/20230224085743 new file mode 100644 index 00000000000..bda82e5e10c --- /dev/null +++ b/db/schema_migrations/20230224085743 @@ -0,0 +1 @@ +e6deb8645468ab4e90487211b14d5432b26fb4c06635b333776c1ac175187444
\ No newline at end of file diff --git a/db/schema_migrations/20230306195007 b/db/schema_migrations/20230306195007 new file mode 100644 index 00000000000..bb28fbc5586 --- /dev/null +++ b/db/schema_migrations/20230306195007 @@ -0,0 +1 @@ +f799b921663f3de04e0b8f5017305e186c4e418392256adf33f2408ea6d8d2ca
\ No newline at end of file diff --git a/doc/.vale/gitlab/spelling-exceptions.txt b/doc/.vale/gitlab/spelling-exceptions.txt index 8277ffe96ea..83e187fe1b5 100644 --- a/doc/.vale/gitlab/spelling-exceptions.txt +++ b/doc/.vale/gitlab/spelling-exceptions.txt @@ -497,6 +497,7 @@ Kibana Kinesis Klar Knative +KPIs Kramdown Kroki kubeconfig @@ -838,8 +839,8 @@ sanitization SBOMs sbt SBT -scalers scalar's +scalers scatterplot scatterplots schedulable @@ -1187,7 +1188,7 @@ ZAProxy Zeitwerk Zendesk ZenTao +Zoekt zsh Zstandard Zuora -Zoekt diff --git a/doc/administration/clusters/kas.md b/doc/administration/clusters/kas.md index a7f8f8e712b..1f549898a80 100644 --- a/doc/administration/clusters/kas.md +++ b/doc/administration/clusters/kas.md @@ -105,6 +105,24 @@ For GitLab [Helm Chart](https://docs.gitlab.com/charts/) installations: For details, see [how to use the GitLab-KAS chart](https://docs.gitlab.com/charts/charts/gitlab/kas/). +## Kubernetes API proxy cookie + +> Introduced in GitLab 15.10 [with feature flags](../feature_flags.md) named `kas_user_access` and `kas_user_access_project`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flags](../feature_flags.md) named `kas_user_access` and `kas_user_access_project`. + +KAS proxies Kubernetes API requests to the GitLab agent with either: + +- A [CI/CD job](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/kubernetes_ci_access.md). +- [GitLab user credentials](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/kubernetes_user_access.md). + +To authenticate with user credentials, Rails sets a cookie for the GitLab frontend. +This cookie is called `_gitlab_kas` and it contains an encrypted +session ID, like the [`_gitlab_session` cookie](../../user/profile/index.md#cookies-used-for-sign-in). +The `_gitlab_kas` cookie must be sent to the KAS proxy endpoint with every request +to authenticate and authorize the user. + ## Troubleshooting If you have issues while using the agent server for Kubernetes, view the diff --git a/doc/administration/integration/kroki.md b/doc/administration/integration/kroki.md index 6655657adcb..f90458200b3 100644 --- a/doc/administration/integration/kroki.md +++ b/doc/administration/integration/kroki.md @@ -29,11 +29,16 @@ docker run -d --name kroki -p 8080:8000 yuzutech/kroki The **Kroki URL** is the hostname of the server running the container. -The [`yuzutech/kroki`](https://hub.docker.com/r/yuzutech/kroki) image contains the following diagrams libraries out-of-the-box: +The [`yuzutech/kroki`](https://hub.docker.com/r/yuzutech/kroki) Docker image contains several diagram +libraries out of the box. For a complete list, see the +[`asciidoctor-kroki` README](https://github.com/ggrossetie/asciidoctor-kroki/blob/master/README.md#supported-diagram-types). +Supported libraries include: <!-- vale gitlab.Spelling = NO --> - [Bytefield](https://bytefield-svg.deepsymmetry.org/) +- [D2](https://d2lang.com/tour/intro/) +- [DBML](https://www.dbml.org/home/) - [Ditaa](https://ditaa.sourceforge.net) - [Erd](https://github.com/BurntSushi/erd) - [GraphViz](https://www.graphviz.org/) diff --git a/doc/api/draft_notes.md b/doc/api/draft_notes.md index e0d00517291..079b08781ae 100644 --- a/doc/api/draft_notes.md +++ b/doc/api/draft_notes.md @@ -115,6 +115,25 @@ POST /projects/:id/merge_requests/:merge_request_iid/draft_notes curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/14/merge_requests/11/draft_notes?note=note ``` +## Modify existing draft note + +Modify a draft note for a given merge request. + +```plaintext +PUT /projects/:id/merge_requests/:merge_request_iid/draft_notes/:draft_note_id +``` + +| Attribute | Type | Required | Description | +| ------------------- | ----------------- | ----------- | --------------------- | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). +| `draft_note_id` | integer | yes | The ID of a draft note. +| `merge_request_iid` | integer | yes | The IID of a project merge request. +| `note` | string | no | The content of a note. + +```shell +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/14/merge_requests/11/draft_notes/5" +``` + ## Delete a draft note Deletes an existing draft note for a given merge request. diff --git a/doc/api/import.md b/doc/api/import.md index fc6fa4aeec3..87bbb56869d 100644 --- a/doc/api/import.md +++ b/doc/api/import.md @@ -139,11 +139,12 @@ Returns the following status codes: ### Import GitHub gists into GitLab snippets -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/371099) in GitLab 15.8 [with a flag](../administration/feature_flags.md) named `github_import_gists`. Disabled by default. Enabled on GitLab.com. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/371099) in GitLab 15.8 [with a flag](../administration/feature_flags.md) named `github_import_gists`. Disabled by default. Enabled on GitLab.com. +> - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/386579) in GitLab 15.10. FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, -ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `github_import_gists`. +On self-managed GitLab, this feature is available by default. To hide the feature, +ask an administrator to [disable the feature flag](../administration/feature_flags.md) named `github_import_gists`. On GitLab.com, this feature is available. You can use the GitLab API to import personal GitHub gists (with up to 10 files) into personal GitLab snippets. diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index 024593b2c6b..3e5982faee8 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -1174,7 +1174,8 @@ Example response: ## List merge request pipelines -Get a list of merge request pipelines. +Get a list of merge request pipelines. The pagination parameters `page` and +`per_page` can be used to restrict the list of merge request pipelines. ```plaintext GET /projects/:id/merge_requests/:merge_request_iid/pipelines diff --git a/doc/ci/runners/saas/linux_saas_runner.md b/doc/ci/runners/saas/linux_saas_runner.md index 8d5f976fa0e..e9ac91409af 100644 --- a/doc/ci/runners/saas/linux_saas_runner.md +++ b/doc/ci/runners/saas/linux_saas_runner.md @@ -96,7 +96,7 @@ This means that the available free disk space that your jobs can use is **less t WARNING: This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/391896) in GitLab 15.9 -and is planned for removal in 15.11. Use [`pre_get_sources_script`](../../../ci/yaml/index.md#hookspre_get_sources_script) instead. This change is a breaking change. +and is planned for removal in 16.0. Use [`pre_get_sources_script`](../../../ci/yaml/index.md#hookspre_get_sources_script) instead. This change is a breaking change. With SaaS runners on Linux, you can run commands in a CI/CD job before the runner attempts to run `git init` and `git fetch` to download a GitLab repository. The diff --git a/doc/development/documentation/feature_flags.md b/doc/development/documentation/feature_flags.md index 010058350ba..986252eedac 100644 --- a/doc/development/documentation/feature_flags.md +++ b/doc/development/documentation/feature_flags.md @@ -17,7 +17,7 @@ When the state of a feature flag changes, the developer who made the change Every feature introduced to the codebase, even if it's behind a disabled feature flag, must be documented. For more information, see -[the discussion that led to this decision](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47917#note_459984428). +[the discussion that led to this decision](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47917#note_459984428). [Alpha, Beta, or Limited Availability](../../policy/alpha-beta-support.md) features are usually behind a feature flag, and must also be documented. For more information, see [Document Alpha, Beta, LA features](alpha_beta.md). When the feature is [implemented in multiple merge requests](../feature_flags/index.md#feature-flags-in-gitlab-development), discuss the plan with your technical writer. diff --git a/doc/user/okrs.md b/doc/user/okrs.md index 0d3be8474fe..7ca102402cc 100644 --- a/doc/user/okrs.md +++ b/doc/user/okrs.md @@ -17,12 +17,11 @@ On self-managed GitLab, by default this feature is not available. To make it ava On GitLab.com, this feature is not available. The feature is not ready for production use. -Use objectives and key results to align your workforce towards common goals and track the progress. -Set a big goal with an objective and use [child objectives and key results](#child-objectives-and-key-results) -to measure the big goal's completion. +[Objectives and key results](https:://en.wikipedia.org/wiki/OKR) (OKRs) are a framework for setting +and tracking goals that are aligned with your organization's overall strategy and vision. The objective and the key result in GitLab share many features. In the documentation, the term -**OKR** refers to either an objective or a key result. +**OKRs** refers to both objectives and key results. OKRs are a type of work item, a step towards [default issue types](https://gitlab.com/gitlab-org/gitlab/-/issues/323404) in GitLab. @@ -31,6 +30,29 @@ to work items and adding custom work item types, see [epic 6033](https://gitlab.com/groups/gitlab-org/-/epics/6033) or the [Plan direction page](https://about.gitlab.com/direction/plan/). +## Designing effective OKRs + +Use objectives and key results to align your workforce towards common goals and track the progress. +Set a big goal with an objective and use [child objectives and key results](#child-objectives-and-key-results) +to measure the big goal's completion. + +**Objectives** are aspirational goals to be achieved and define **what you're aiming to do**. +They show how an individual's, team's, or department's work impacts overall direction of the +organization by connecting their work to overall company strategy. + +**Key results** are measures of progress against aligned objectives. They express +**how you know if you have reached your goal** (objective). +By achieving a specific outcome (key result), you create progress for the linked objective. + +To know if your OKR makes sense, you can use this sentence: + +<!-- vale gitlab.FutureTense = NO --> +> I/we will accomplish (objective) by (date) through attaining and achieving the following metrics (key results). +<!-- vale gitlab.FutureTense = YES --> + +To learn how to create better OKRs and how we use them at GitLab, see the +[Objectives and Key Results handbook page](https://about.gitlab.com/company/okrs/). + ## Create an objective Prerequisites: diff --git a/doc/user/project/code_owners.md b/doc/user/project/code_owners.md index 5838f2c05c0..2574c90982e 100644 --- a/doc/user/project/code_owners.md +++ b/doc/user/project/code_owners.md @@ -45,24 +45,21 @@ For example: ## Code Owners file -A `CODEOWNERS` file (with no extension) can specify users or [shared groups](members/share_project_with_groups.md) -that are responsible for specific files and directories in a repository. Each repository -can have a single `CODEOWNERS` file, and it must be found one of these three locations: +A `CODEOWNERS` file (with no extension) specifies the users or +[shared groups](members/share_project_with_groups.md) responsible for +specific files and directories in a repository. -- In the root directory of the repository. -- In the `.gitlab/` directory. -- In the `docs/` directory. +Each repository uses a single `CODEOWNERS` file. GitLab checks these locations +in your repository in this order. The first `CODEOWNERS` file found is used, and +all others are ignored: -A CODEOWNERS file in any other location is ignored. +1. In the root directory: `./CODEOWNERS`. +1. In the `docs` directory: `./docs/CODEOWNERS`. +1. In the `.gitlab` directory: `./.gitlab/CODEOWNERS`. ## Set up Code Owners -1. Create a file named `CODEOWNERS` (with no extension) in one of these locations: - -- In the root directory of the repository -- In the `.gitlab/` directory -- In the `docs/` directory - +1. Create a `CODEOWNERS` file in your [preferred location](#code-owners-file). 1. In the file, enter text that follows one of these patterns: ```plaintext @@ -88,8 +85,7 @@ Next steps: ### Groups as Code Owners -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53182) in GitLab 12.1. -> - Group and subgroup hierarchy support was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32432) in GitLab 13.0. +> Group and subgroup hierarchy support was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32432) in GitLab 13.0. You can use members of groups and subgroups as Code Owners for projects: @@ -188,7 +184,7 @@ README.md @user3 The Code Owners for the `README.md` in the root directory are `@user1`, `@user2`, and `@user3`. The Code Owners for `internal/README.md` are `@user4` and `@user3`. -Only one CODEOWNERS pattern per section will be matched to a file path. +Only one CODEOWNERS pattern per section is matched to a file path. ### Organize Code Owners by putting them into sections @@ -403,6 +399,7 @@ if any of these conditions are true: Check the project [merge request approval](merge_requests/approvals/settings.md#edit-merge-request-approval-settings) settings. - A Code Owner group has a visibility of **private**, and the current user is not a member of the Code Owner group. +- Current user is an external user who does not have permission to the internal Code Owner group. ### Approval rule is invalid. GitLab has approved this rule automatically to unblock the merge request diff --git a/doc/user/search/advanced_search.md b/doc/user/search/advanced_search.md index 464c44a6f14..1444e5385f9 100644 --- a/doc/user/search/advanced_search.md +++ b/doc/user/search/advanced_search.md @@ -64,11 +64,12 @@ In user search, a [fuzzy query](https://www.elastic.co/guide/en/elasticsearch/re | Syntax | Description | Example | |--------------|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------| | `filename:` | Filename | [`filename:*spec.rb`](https://gitlab.com/search?snippets=&scope=blobs&repository_ref=&search=filename%3A*spec.rb&group_id=9970&project_id=278964) | -| `path:` | Repository location | [`path:spec/workers/`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=path%3Aspec%2Fworkers&snippets=) | -| `extension:` | File extension without `.` | [`extension:js`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=extension%3Ajs&snippets=) | -| `blob:` | Git object ID | [`blob:998707*`](https://gitlab.com/search?snippets=false&scope=blobs&repository_ref=&search=blob%3A998707*&group_id=9970) | +| `path:` | Repository location <sup>1</sup> | [`path:spec/workers/`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=path%3Aspec%2Fworkers&snippets=) | +| `extension:` | File extension without `.` <sup>2</sup> | [`extension:js`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=extension%3Ajs&snippets=) | +| `blob:` | Git object ID <sup>2</sup> | [`blob:998707*`](https://gitlab.com/search?snippets=false&scope=blobs&repository_ref=&search=blob%3A998707*&group_id=9970) | -`extension:` and `blob:` return exact matches only. +1. `path:` returns matches for full paths or subpaths. +1. `extension:` and `blob:` return exact matches only. ### Examples @@ -78,6 +79,8 @@ In user search, a [fuzzy query](https://www.elastic.co/guide/en/elasticsearch/re | [`RSpec.describe Resolvers -*builder`](https://gitlab.com/search?group_id=9970&project_id=278964&scope=blobs&search=RSpec.describe+Resolvers+-*builder) | Returns `RSpec.describe Resolvers` that does not start with `builder`. | | [<code>bug | (display +banner)</code>](https://gitlab.com/search?snippets=&scope=issues&repository_ref=&search=bug+%7C+%28display+%2Bbanner%29&group_id=9970&project_id=278964) | Returns `bug` or both `display` and `banner`. | | [<code>helper -extension:yml -extension:js</code>](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=helper+-extension%3Ayml+-extension%3Ajs&snippets=) | Returns `helper` in all files except files with a `.yml` or `.js` extension. | +| [<code>helper path:lib/git</code>](https://gitlab.com/search?group_id=9970&project_id=278964&scope=blobs&search=helper+path%3Alib%2Fgit) | Returns `helper` in all files with a `lib/git*` path (for example, `spec/lib/gitlab`). | + <!-- markdownlint-enable --> diff --git a/lib/api/concerns/packages/debian_package_endpoints.rb b/lib/api/concerns/packages/debian_package_endpoints.rb index db31f2e35f1..7969a49909a 100644 --- a/lib/api/concerns/packages/debian_package_endpoints.rb +++ b/lib/api/concerns/packages/debian_package_endpoints.rb @@ -58,9 +58,19 @@ module API .with_compression_type(nil) .order_created_asc + # Empty component files are not persisted in DB + no_content! if params[:file_sha256] == ::Packages::Debian::EMPTY_FILE_SHA256 + relation = relation.with_file_sha256(params[:file_sha256]) if params[:file_sha256] - present_carrierwave_file!(relation.last!.file) + component_file = relation.last + + if component_file.nil? || component_file.empty? + not_found! if params[:file_sha256] # asking for a non existing component file. + no_content! # empty component files are not always persisted in DB + end + + present_carrierwave_file!(component_file.file) end end @@ -156,7 +166,10 @@ module API # https://wiki.debian.org/DebianRepository/Format#A.22Packages.22_Indices desc 'The installer (udeb) binary files index' do detail 'This feature was introduced in GitLab 15.4' - success code: 200 + success [ + { code: 200 }, + { code: 202 } + ] failure [ { code: 400, message: 'Bad Request' }, { code: 401, message: 'Unauthorized' }, @@ -175,7 +188,10 @@ module API # https://wiki.debian.org/DebianRepository/Format?action=show&redirect=RepositoryFormat#indices_acquisition_via_hashsums_.28by-hash.29 desc 'The installer (udeb) binary files index by hash' do detail 'This feature was introduced in GitLab 15.4' - success code: 200 + success [ + { code: 200 }, + { code: 202 } + ] failure [ { code: 400, message: 'Bad Request' }, { code: 401, message: 'Unauthorized' }, @@ -196,7 +212,10 @@ module API # https://wiki.debian.org/DebianRepository/Format#A.22Sources.22_Indices desc 'The source files index' do detail 'This feature was introduced in GitLab 15.4' - success code: 200 + success [ + { code: 200 }, + { code: 202 } + ] failure [ { code: 400, message: 'Bad Request' }, { code: 401, message: 'Unauthorized' }, @@ -215,7 +234,10 @@ module API # https://wiki.debian.org/DebianRepository/Format?action=show&redirect=RepositoryFormat#indices_acquisition_via_hashsums_.28by-hash.29 desc 'The source files index by hash' do detail 'This feature was introduced in GitLab 15.4' - success code: 200 + success [ + { code: 200 }, + { code: 202 } + ] failure [ { code: 400, message: 'Bad Request' }, { code: 401, message: 'Unauthorized' }, @@ -240,7 +262,10 @@ module API # https://wiki.debian.org/DebianRepository/Format#A.22Packages.22_Indices desc 'The binary files index' do detail 'This feature was introduced in GitLab 13.5' - success code: 200 + success [ + { code: 200 }, + { code: 202 } + ] failure [ { code: 400, message: 'Bad Request' }, { code: 401, message: 'Unauthorized' }, @@ -259,7 +284,10 @@ module API # https://wiki.debian.org/DebianRepository/Format?action=show&redirect=RepositoryFormat#indices_acquisition_via_hashsums_.28by-hash.29 desc 'The binary files index by hash' do detail 'This feature was introduced in GitLab 15.4' - success code: 200 + success [ + { code: 200 }, + { code: 202 } + ] failure [ { code: 400, message: 'Bad Request' }, { code: 401, message: 'Unauthorized' }, diff --git a/lib/gitlab/background_migration/backfill_project_wiki_repositories.rb b/lib/gitlab/background_migration/backfill_project_wiki_repositories.rb new file mode 100644 index 00000000000..8d6df905f15 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_wiki_repositories.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill project_wiki_repositories table for a range of projects + class BackfillProjectWikiRepositories < BatchedMigrationJob + operation_name :backfill_project_wiki_repositories + feature_category :geo_replication + + scope_to ->(relation) do + relation + .joins('LEFT OUTER JOIN project_wiki_repositories ON project_wiki_repositories.project_id = projects.id') + .where(project_wiki_repositories: { project_id: nil }) + end + + def perform + each_sub_batch do |sub_batch| + backfill_project_wiki_repositories(sub_batch) + end + end + + def backfill_project_wiki_repositories(relation) + connection.execute( + <<~SQL + INSERT INTO project_wiki_repositories (project_id, created_at, updated_at) + SELECT projects.id, now(), now() + FROM projects + WHERE projects.id IN(#{relation.select(:id).to_sql}) + ON CONFLICT (project_id) DO NOTHING; + SQL + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/issues_internal_id_scope_updater.rb b/lib/gitlab/background_migration/issues_internal_id_scope_updater.rb new file mode 100644 index 00000000000..485fb28405d --- /dev/null +++ b/lib/gitlab/background_migration/issues_internal_id_scope_updater.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Migrates internal_ids records for `usage: issues` from project to namespace scope. + # For project issues it will be project namespace, for group issues it will be group namespace. + class IssuesInternalIdScopeUpdater < ::Gitlab::BackgroundMigration::BatchedMigrationJob + operation_name :issues_internal_id_scope_updater + feature_category :database + + ISSUES_USAGE = 0 # see Enums::InternalId#usage_resources[:issues] + + scope_to ->(relation) do + relation.where(usage: ISSUES_USAGE).where.not(project_id: nil) + end + + def perform + each_sub_batch do |sub_batch| + create_namespace_scoped_records(sub_batch) + delete_project_scoped_records(sub_batch) + end + end + + private + + def delete_project_scoped_records(sub_batch) + # There is no need to keep the project scoped issues usage as we move to scoping issues to namespace. + # Also in case we do decide to move back to scoping issues usage to project, we are better off if the + # project record is not present as that would result in overlapping IIDs because project scoped issues + # usage will have outdated IIDs left in the DB + log_info("Deleted internal_ids records", ids: sub_batch.pluck(:id)) + + connection.execute( + <<~SQL + DELETE FROM internal_ids WHERE id IN (#{sub_batch.select(:id).to_sql}) + SQL + ) + end + + def create_namespace_scoped_records(sub_batch) + # Creates a corresponding namespace scoped record for every `issues` usage scoped to a project. + # On conflict there is nothing to do as it means the record was already created when + # a new issue is created with the newlly namespace scoped Issue model, see Issue#has_internal_id + # definition. + created_records_ids = connection.execute( + <<~SQL + INSERT INTO internal_ids (usage, last_value, namespace_id) + SELECT #{ISSUES_USAGE}, last_value, project_namespace_id + FROM internal_ids + INNER JOIN projects ON projects.id = internal_ids.project_id + WHERE internal_ids.id IN(#{sub_batch.select(:id).to_sql}) + ON CONFLICT (usage, namespace_id) WHERE namespace_id IS NOT NULL DO NOTHING + RETURNING id; + SQL + ) + + log_info("Created internal_ids records", ids: created_records_ids.field_values('id')) + end + + def log_info(message, **extra) + ::Gitlab::BackgroundMigration::Logger.info(migrator: self.class.to_s, message: message, **extra) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 3dafe7c8962..592e75b1430 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -72,7 +72,7 @@ module Gitlab return unless last_bitbucket_issue - Issue.track_project_iid!(project, last_bitbucket_issue.iid) + Issue.track_namespace_iid!(project.project_namespace, last_bitbucket_issue.iid) end def repo diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index ef2e2d8cccc..7060754a670 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -105,6 +105,19 @@ module Gitlab protected + def content_result + strong_memoize(:content_hash) do + ::Gitlab::Ci::Config::Yaml + .load_result!(content, project: context.project) + end + end + + def content_hash + return unless content_result.valid? + + content_result.content + end + def expanded_content_hash return unless content_hash @@ -113,14 +126,6 @@ module Gitlab end end - def content_hash - strong_memoize(:content_hash) do - ::Gitlab::Ci::Config::Yaml.load!(content, project: context.project) - end - rescue Gitlab::Config::Loader::FormatError - nil - end - def validate_hash! if to_hash.blank? errors.push("Included file `#{masked_location}` does not have valid YAML syntax!") diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb index 31efe6ab6d9..11165570e81 100644 --- a/lib/gitlab/ci/config/yaml.rb +++ b/lib/gitlab/ci/config/yaml.rb @@ -17,16 +17,24 @@ module Gitlab ensure_custom_tags if project.present? && ::Feature.enabled?(:ci_multi_doc_yaml, project) - Gitlab::Config::Loader::MultiDocYaml.new( + ::Gitlab::Config::Loader::MultiDocYaml.new( content, max_documents: MAX_DOCUMENTS, additional_permitted_classes: AVAILABLE_TAGS - ).load!.first + ).load! else - Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load! + ::Gitlab::Config::Loader::Yaml + .new(content, additional_permitted_classes: AVAILABLE_TAGS) + .load! end end + def to_result + Yaml::Result.new(load!) + rescue ::Gitlab::Config::Loader::FormatError => e + Yaml::Result.new(error: e) + end + private attr_reader :content, :project @@ -42,7 +50,18 @@ module Gitlab class << self def load!(content, project: nil) - Loader.new(content, project: project).load! + Loader.new(content, project: project).to_result.then do |result| + ## + # raise an error for backwards compatibility + # + raise result.error unless result.valid? + + result.content + end + end + + def load_result!(content, project: nil) + Loader.new(content, project: project).to_result end end end diff --git a/lib/gitlab/ci/config/yaml/result.rb b/lib/gitlab/ci/config/yaml/result.rb new file mode 100644 index 00000000000..5e37876d73d --- /dev/null +++ b/lib/gitlab/ci/config/yaml/result.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + class Result + attr_reader :error + + def initialize(config = nil, error: nil) + @config = Array.wrap(config) + @error = error + end + + def valid? + error.nil? + end + + def has_header? + @config.size > 1 + end + + def header + raise ArgumentError unless has_header? + + @config.first + end + + def content + @config.last + end + end + end + end + end +end diff --git a/lib/gitlab/database/async_constraints.rb b/lib/gitlab/database/async_constraints.rb index 6952115eca5..026197c7e40 100644 --- a/lib/gitlab/database/async_constraints.rb +++ b/lib/gitlab/database/async_constraints.rb @@ -6,8 +6,8 @@ module Gitlab DEFAULT_ENTRIES_PER_INVOCATION = 2 def self.validate_pending_entries!(how_many: DEFAULT_ENTRIES_PER_INVOCATION) - PostgresAsyncConstraintValidation.ordered.foreign_key_type.limit(how_many).each do |record| - ForeignKeyValidator.new(record).perform + PostgresAsyncConstraintValidation.ordered.limit(how_many).each do |record| + AsyncConstraints::Validators.for(record).perform end end end diff --git a/lib/gitlab/database/async_constraints/foreign_key_validator.rb b/lib/gitlab/database/async_constraints/foreign_key_validator.rb deleted file mode 100644 index a535a86913c..00000000000 --- a/lib/gitlab/database/async_constraints/foreign_key_validator.rb +++ /dev/null @@ -1,90 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module AsyncConstraints - class ForeignKeyValidator - include AsyncDdlExclusiveLeaseGuard - - TIMEOUT_PER_ACTION = 1.day - STATEMENT_TIMEOUT = 12.hours - - def initialize(async_validation) - @async_validation = async_validation - end - - def perform - try_obtain_lease do - if foreign_key_exists? - log_index_info("Starting to validate foreign key") - validate_foreign_with_error_handling - log_index_info("Finished validating foreign key") - else - log_index_info(skip_log_message) - async_validation.destroy! - end - end - end - - private - - attr_reader :async_validation - - delegate :connection, :name, :table_name, :connection_db_config, to: :async_validation - - def foreign_key_exists? - relation = Gitlab::Database::PostgresForeignKey.by_constrained_table_name_or_identifier(table_name) - - relation.by_name(name).exists? - end - - def validate_foreign_with_error_handling - validate_foreign_key - async_validation.destroy! - rescue StandardError => error - async_validation.handle_exception!(error) - - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - Gitlab::AppLogger.error(message: error.message, **logging_options) - end - - def validate_foreign_key - set_statement_timeout do - connection.execute(<<~SQL.squish) - ALTER TABLE #{connection.quote_table_name(table_name)} - VALIDATE CONSTRAINT #{connection.quote_column_name(name)}; - SQL - end - end - - def set_statement_timeout - connection.execute(format("SET statement_timeout TO '%ds'", STATEMENT_TIMEOUT)) - yield - ensure - connection.execute('RESET statement_timeout') - end - - def lease_timeout - TIMEOUT_PER_ACTION - end - - def log_index_info(message) - Gitlab::AppLogger.info(message: message, **logging_options) - end - - def skip_log_message - "Skipping #{name} validation since it does not exist. " \ - "The queuing entry will be deleted" - end - - def logging_options - { - fk_name: name, - table_name: table_name, - class: self.class.name.to_s - } - end - end - end - end -end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index bcc03ca08c9..5c2ae7fefd0 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -306,18 +306,18 @@ module Gitlab end def search_files_by_name(ref, query, limit: 0, offset: 0) - request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: query, limit: limit, offset: offset) + request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: encode_binary(ref), query: query, limit: limit, offset: offset) gitaly_client_call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) end def search_files_by_content(ref, query, options = {}) - request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query) + request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: encode_binary(ref), query: query) response = gitaly_client_call(@storage, :repository_service, :search_files_by_content, request, timeout: GitalyClient.default_timeout) search_results_from_response(response, options) end def search_files_by_regexp(ref, filter, limit: 0, offset: 0) - request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: '.', filter: filter, limit: limit, offset: offset) + request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: encode_binary(ref), query: '.', filter: filter, limit: limit, offset: offset) gitaly_client_call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) end diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb index 7b031c26b72..458f7c3f470 100644 --- a/lib/gitlab/jira_import/issues_importer.rb +++ b/lib/gitlab/jira_import/issues_importer.rb @@ -48,7 +48,7 @@ module Gitlab end def schedule_issue_import_workers(issues) - next_iid = Issue.with_project_iid_supply(project, &:next_value) + next_iid = Issue.with_namespace_iid_supply(project.project_namespace, &:next_value) issues.each do |jira_issue| # Technically it's possible that the same work is performed multiple @@ -71,7 +71,7 @@ module Gitlab job_waiter.jobs_remaining += 1 - next_iid = Issue.with_project_iid_supply(project, &:next_value) + next_iid = Issue.with_namespace_iid_supply(project.project_namespace, &:next_value) # Mark the issue as imported immediately so we don't end up # importing it multiple times within same import. diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 6edcc18f239..06b3318e269 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -39935,6 +39935,12 @@ msgstr "" msgid "Service usage data" msgstr "" +msgid "ServiceAccount|User does not have permission to create a service account in this namespace." +msgstr "" + +msgid "ServiceAccount|User does not have permission to create a service account." +msgstr "" + msgid "ServiceDesk|Enable Service Desk" msgstr "" diff --git a/spec/factories/packages/debian/component_file.rb b/spec/factories/packages/debian/component_file.rb index a2422e4a126..eefc6ab5966 100644 --- a/spec/factories/packages/debian/component_file.rb +++ b/spec/factories/packages/debian/component_file.rb @@ -47,5 +47,12 @@ FactoryBot.define do trait(:object_storage) do file_store { Packages::PackageFileUploader::Store::REMOTE } end + + trait(:empty) do + file_md5 { 'd41d8cd98f00b204e9800998ecf8427e' } + file_sha256 { 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' } + file_fixture { nil } + size { 0 } + end end end diff --git a/spec/frontend/commit/components/signature_badge_spec.js b/spec/frontend/commit/components/signature_badge_spec.js new file mode 100644 index 00000000000..d52ad2b43e2 --- /dev/null +++ b/spec/frontend/commit/components/signature_badge_spec.js @@ -0,0 +1,134 @@ +import { GlBadge, GlLink, GlPopover } from '@gitlab/ui'; +import { stubComponent, RENDER_ALL_SLOTS_TEMPLATE } from 'helpers/stub_component'; +import SignatureBadge from '~/commit/components/signature_badge.vue'; +import X509CertificateDetails from '~/commit/components/x509_certificate_details.vue'; +import { typeConfig, statusConfig, verificationStatuses, signatureTypes } from '~/commit/constants'; +import { mountExtended } from 'helpers/vue_test_utils_helper'; +import { helpPagePath } from '~/helpers/help_page_helper'; +import { sshSignatureProp, gpgSignatureProp, x509SignatureProp } from '../mock_data'; + +describe('Commit signature', () => { + let wrapper; + + const createComponent = (props = {}) => { + wrapper = mountExtended(SignatureBadge, { + propsData: { + signature: { + ...props, + }, + stubs: { + GlBadge, + GlLink, + X509CertificateDetails, + GlPopover: stubComponent(GlPopover, { template: RENDER_ALL_SLOTS_TEMPLATE }), + }, + }, + }); + }; + + const signatureBadge = () => wrapper.findComponent(GlBadge); + const signaturePopover = () => wrapper.findComponent(GlPopover); + const signatureDescription = () => wrapper.findByTestId('signature-description'); + const signatureKeyLabel = () => wrapper.findByTestId('signature-key-label'); + const signatureKey = () => wrapper.findByTestId('signature-key'); + const helpLink = () => wrapper.findComponent(GlLink); + const X509CertificateDetailsComponents = () => wrapper.findAllComponents(X509CertificateDetails); + + describe.each` + signatureType | verificationStatus + ${signatureTypes.GPG} | ${verificationStatuses.VERIFIED} + ${signatureTypes.GPG} | ${verificationStatuses.UNVERIFIED} + ${signatureTypes.GPG} | ${verificationStatuses.UNVERIFIED_KEY} + ${signatureTypes.GPG} | ${verificationStatuses.UNKNOWN_KEY} + ${signatureTypes.GPG} | ${verificationStatuses.OTHER_USER} + ${signatureTypes.GPG} | ${verificationStatuses.SAME_USER_DIFFERENT_EMAIL} + ${signatureTypes.GPG} | ${verificationStatuses.MULTIPLE_SIGNATURES} + ${signatureTypes.X509} | ${verificationStatuses.VERIFIED} + ${signatureTypes.SSH} | ${verificationStatuses.VERIFIED} + ${signatureTypes.SSH} | ${verificationStatuses.REVOKED_KEY} + `( + 'For a specified `$signatureType` and `$verificationStatus` it renders component correctly', + ({ signatureType, verificationStatus }) => { + beforeEach(() => { + createComponent({ __typename: signatureType, verificationStatus }); + }); + it('renders correct badge class', () => { + expect(signatureBadge().props('variant')).toBe(statusConfig[verificationStatus].variant); + }); + it('renders badge text', () => { + expect(signatureBadge().text()).toBe(statusConfig[verificationStatus].label); + }); + it('renders popover header text', () => { + expect(signaturePopover().text()).toMatch(statusConfig[verificationStatus].title); + }); + it('renders signature description', () => { + expect(signatureDescription().text()).toBe(statusConfig[verificationStatus].description); + }); + it('renders help link with correct path', () => { + expect(helpLink().text()).toBe(typeConfig[signatureType].helpLink.label); + expect(helpLink().attributes('href')).toBe( + helpPagePath(typeConfig[signatureType].helpLink.path), + ); + }); + }, + ); + + describe('SSH signature', () => { + beforeEach(() => { + createComponent(sshSignatureProp); + }); + + it('renders key label', () => { + expect(signatureKeyLabel().text()).toMatch(typeConfig[signatureTypes.SSH].keyLabel); + }); + + it('renders key signature', () => { + expect(signatureKey().text()).toBe(sshSignatureProp.keyFingerprintSha256); + }); + }); + + describe('GPG signature', () => { + beforeEach(() => { + createComponent(gpgSignatureProp); + }); + + it('renders key label', () => { + expect(signatureKeyLabel().text()).toMatch(typeConfig[signatureTypes.GPG].keyLabel); + }); + + it('renders key signature for GGP signature', () => { + expect(signatureKey().text()).toBe(gpgSignatureProp.gpgKeyPrimaryKeyid); + }); + }); + + describe('X509 signature', () => { + beforeEach(() => { + createComponent(x509SignatureProp); + }); + + it('does not render key label', () => { + expect(signatureKeyLabel().exists()).toBe(false); + }); + + it('renders X509 certificate details components', () => { + expect(X509CertificateDetailsComponents()).toHaveLength(2); + }); + + it('passes correct props', () => { + expect(X509CertificateDetailsComponents().at(0).props()).toStrictEqual({ + subject: x509SignatureProp.x509Certificate.subject, + title: typeConfig[signatureTypes.X509].subjectTitle, + subjectKeyIdentifier: wrapper.vm.getSubjectKeyIdentifierToDisplay( + x509SignatureProp.x509Certificate.subjectKeyIdentifier, + ), + }); + expect(X509CertificateDetailsComponents().at(1).props()).toStrictEqual({ + subject: x509SignatureProp.x509Certificate.x509Issuer.subject, + title: typeConfig[signatureTypes.X509].issuerTitle, + subjectKeyIdentifier: wrapper.vm.getSubjectKeyIdentifierToDisplay( + x509SignatureProp.x509Certificate.x509Issuer.subjectKeyIdentifier, + ), + }); + }); + }); +}); diff --git a/spec/frontend/commit/components/x509_certificate_details_spec.js b/spec/frontend/commit/components/x509_certificate_details_spec.js new file mode 100644 index 00000000000..5d9398b572b --- /dev/null +++ b/spec/frontend/commit/components/x509_certificate_details_spec.js @@ -0,0 +1,36 @@ +import { shallowMount } from '@vue/test-utils'; +import X509CertificateDetails from '~/commit/components/x509_certificate_details.vue'; +import { X509_CERTIFICATE_KEY_IDENTIFIER_TITLE } from '~/commit/constants'; +import { x509CertificateDetailsProp } from '../mock_data'; + +describe('X509 certificate details', () => { + let wrapper; + + const createComponent = () => { + wrapper = shallowMount(X509CertificateDetails, { + propsData: x509CertificateDetailsProp, + }); + }; + + beforeEach(() => { + createComponent(); + }); + + const findTitle = () => wrapper.find('strong'); + const findSubjectValues = () => wrapper.findAll("[data-testid='subject-value']"); + const findKeyIdentifier = () => wrapper.find("[data-testid='key-identifier']"); + + it('renders a title', () => { + expect(findTitle().text()).toBe(x509CertificateDetailsProp.title); + }); + + it('renders subject values', () => { + expect(findSubjectValues()).toHaveLength(3); + }); + + it('renders key identifier', () => { + expect(findKeyIdentifier().text()).toBe( + `${X509_CERTIFICATE_KEY_IDENTIFIER_TITLE} ${x509CertificateDetailsProp.subjectKeyIdentifier}`, + ); + }); +}); diff --git a/spec/frontend/commit/mock_data.js b/spec/frontend/commit/mock_data.js index a13ef9c563e..3b6971d9607 100644 --- a/spec/frontend/commit/mock_data.js +++ b/spec/frontend/commit/mock_data.js @@ -201,3 +201,34 @@ export const mockUpstreamQueryResponse = { }, }, }; + +export const sshSignatureProp = { + __typename: 'SshSignature', + verificationStatus: 'VERIFIED', + keyFingerprintSha256: 'xxx', +}; + +export const gpgSignatureProp = { + __typename: 'GpgSignature', + verificationStatus: 'VERIFIED', + gpgKeyPrimaryKeyid: 'yyy', +}; + +export const x509SignatureProp = { + __typename: 'X509Signature', + verificationStatus: 'VERIFIED', + x509Certificate: { + subject: 'CN=gitlab@example.org,OU=Example,O=World', + subjectKeyIdentifier: 'BC:BC:BC:BC:BC:BC:BC:BC', + x509Issuer: { + subject: 'CN=PKI,OU=Example,O=World', + subjectKeyIdentifier: 'AB:AB:AB:AB:AB:AB:AB:AB:', + }, + }, +}; + +export const x509CertificateDetailsProp = { + title: 'Title', + subject: 'CN=gitlab@example.org,OU=Example,O=World', + subjectKeyIdentifier: 'BC BC BC BC BC BC BC BC', +}; diff --git a/spec/frontend/repository/components/last_commit_spec.js b/spec/frontend/repository/components/last_commit_spec.js index 2ef93a4565a..f16edcb0b7c 100644 --- a/spec/frontend/repository/components/last_commit_spec.js +++ b/spec/frontend/repository/components/last_commit_spec.js @@ -6,6 +6,7 @@ import waitForPromises from 'helpers/wait_for_promises'; import { shallowMountExtended } from 'helpers/vue_test_utils_helper'; import LastCommit from '~/repository/components/last_commit.vue'; +import SignatureBadge from '~/commit/components/signature_badge.vue'; import UserAvatarLink from '~/vue_shared/components/user_avatar/user_avatar_link.vue'; import pathLastCommitQuery from 'shared_queries/repository/path_last_commit.query.graphql'; import { refMock } from '../mock_data'; @@ -20,7 +21,7 @@ const findUserAvatarLink = () => wrapper.findComponent(UserAvatarLink); const findLastCommitLabel = () => wrapper.findByTestId('last-commit-id-label'); const findLoadingIcon = () => wrapper.findComponent(GlLoadingIcon); const findCommitRowDescription = () => wrapper.find('.commit-row-description'); -const findStatusBox = () => wrapper.find('.signature-badge'); +const findStatusBox = () => wrapper.findComponent(SignatureBadge); const findItemTitle = () => wrapper.find('.item-title'); const defaultPipelineEdges = [ @@ -56,7 +57,7 @@ const createCommitData = ({ pipelineEdges = defaultPipelineEdges, author = defaultAuthor, descriptionHtml = '', - signatureHtml = null, + signature = null, message = defaultMessage, }) => { return { @@ -84,7 +85,7 @@ const createCommitData = ({ authorName: 'Test', authorGravatar: 'https://test.com', author, - signatureHtml, + signature, pipelines: { __typename: 'PipelineConnection', edges: pipelineEdges, @@ -110,6 +111,9 @@ const createComponent = async (data = {}) => { apolloProvider: createMockApollo([[pathLastCommitQuery, mockResolver]]), propsData: { currentPath }, mixins: [{ data: () => ({ ref: refMock }) }], + stubs: { + SignatureBadge, + }, }); }; @@ -203,23 +207,19 @@ describe('Repository last commit component', () => { }); it('renders the signature HTML as returned by the backend', async () => { + const signatureResponse = { + __typename: 'GpgSignature', + gpgKeyPrimaryKeyid: 'xxx', + verificationStatus: 'VERIFIED', + }; createComponent({ - signatureHtml: `<a - class="btn signature-badge" - data-content="signature-content" - data-html="true" - data-placement="top" - data-title="signature-title" - data-toggle="popover" - role="button" - tabindex="0" - ><span class="gl-badge badge badge-pill badge-success md">Verified</span></a>`, + signature: { + ...signatureResponse, + }, }); await waitForPromises(); - expect(findStatusBox().html()).toBe( - `<a class="btn signature-badge" data-content="signature-content" data-html="true" data-placement="top" data-title="signature-title" data-toggle="popover" role="button" tabindex="0"><span class="gl-badge badge badge-pill badge-success md">Verified</span></a>`, - ); + expect(findStatusBox().props()).toMatchObject({ signature: signatureResponse }); }); it('sets correct CSS class if the commit message is empty', async () => { diff --git a/spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb new file mode 100644 index 00000000000..e81bd0604e6 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe( + Gitlab::BackgroundMigration::BackfillProjectWikiRepositories, + schema: 20230306195007, + feature_category: :geo_replication) do + let!(:namespaces) { table(:namespaces) } + let!(:projects) { table(:projects) } + let!(:project_wiki_repositories) { table(:project_wiki_repositories) } + + subject(:migration) do + described_class.new( + start_id: projects.minimum(:id), + end_id: projects.maximum(:id), + batch_table: :projects, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ) + end + + describe '#perform' do + it 'creates project_wiki_repositories entries for all projects in range' do + namespace1 = create_namespace('test1') + namespace2 = create_namespace('test2') + project1 = create_project(namespace1, 'test1') + project2 = create_project(namespace2, 'test2') + project_wiki_repositories.create!(project_id: project2.id) + + expect { migration.perform } + .to change { project_wiki_repositories.pluck(:project_id) } + .from([project2.id]) + .to match_array([project1.id, project2.id]) + end + + it 'does nothing if project_id already exist in project_wiki_repositories' do + namespace = create_namespace('test1') + project = create_project(namespace, 'test1') + project_wiki_repositories.create!(project_id: project.id) + + expect { migration.perform } + .not_to change { project_wiki_repositories.pluck(:project_id) } + end + + def create_namespace(name) + namespaces.create!( + name: name, + path: name, + type: 'Project' + ) + end + + def create_project(namespace, name) + projects.create!( + namespace_id: namespace.id, + project_namespace_id: namespace.id, + name: name, + path: name + ) + end + end +end diff --git a/spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb b/spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb new file mode 100644 index 00000000000..8830af52730 --- /dev/null +++ b/spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +require 'spec_helper' +# this needs the schema to be before we introduce the not null constraint on routes#namespace_id +RSpec.describe Gitlab::BackgroundMigration::IssuesInternalIdScopeUpdater, feature_category: :team_planning do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:internal_ids) { table(:internal_ids) } + + let(:gr1) { namespaces.create!(name: 'batchtest1', type: 'Group', path: 'space1') } + let(:gr2) { namespaces.create!(name: 'batchtest2', type: 'Group', parent_id: gr1.id, path: 'space2') } + + let(:pr_nmsp1) { namespaces.create!(name: 'proj1', path: 'proj1', type: 'Project', parent_id: gr1.id) } + let(:pr_nmsp2) { namespaces.create!(name: 'proj2', path: 'proj2', type: 'Project', parent_id: gr1.id) } + let(:pr_nmsp3) { namespaces.create!(name: 'proj3', path: 'proj3', type: 'Project', parent_id: gr2.id) } + let(:pr_nmsp4) { namespaces.create!(name: 'proj4', path: 'proj4', type: 'Project', parent_id: gr2.id) } + let(:pr_nmsp5) { namespaces.create!(name: 'proj5', path: 'proj5', type: 'Project', parent_id: gr2.id) } + + # rubocop:disable Layout/LineLength + let(:p1) { projects.create!(name: 'proj1', path: 'proj1', namespace_id: gr1.id, project_namespace_id: pr_nmsp1.id) } + let(:p2) { projects.create!(name: 'proj2', path: 'proj2', namespace_id: gr1.id, project_namespace_id: pr_nmsp2.id) } + let(:p3) { projects.create!(name: 'proj3', path: 'proj3', namespace_id: gr2.id, project_namespace_id: pr_nmsp3.id) } + let(:p4) { projects.create!(name: 'proj4', path: 'proj4', namespace_id: gr2.id, project_namespace_id: pr_nmsp4.id) } + let(:p5) { projects.create!(name: 'proj5', path: 'proj5', namespace_id: gr2.id, project_namespace_id: pr_nmsp5.id) } + # rubocop:enable Layout/LineLength + + # a project that already is covered by a record for its namespace. This should result in no new record added and + # project related record deleted + let!(:issues_internal_ids_p1) { internal_ids.create!(project_id: p1.id, usage: 0, last_value: 100) } + let!(:issues_internal_ids_pr_nmsp1) { internal_ids.create!(namespace_id: pr_nmsp1.id, usage: 0, last_value: 111) } + + # project records that do not have a corresponding namespace record. This should result 2 new records + # scoped to corresponding project namespaces being added and the project related records being deleted. + let!(:issues_internal_ids_p2) { internal_ids.create!(project_id: p2.id, usage: 0, last_value: 200) } + let!(:issues_internal_ids_p3) { internal_ids.create!(project_id: p3.id, usage: 0, last_value: 300) } + + # a project record on a different usage, should not be affected by the migration and + # no new record should be created for this case + let!(:issues_internal_ids_p4) { internal_ids.create!(project_id: p4.id, usage: 4, last_value: 400) } + + # a project namespace scoped record without a corresponding project record, should not affect anything. + let!(:issues_internal_ids_pr_nmsp5) { internal_ids.create!(namespace_id: pr_nmsp5.id, usage: 0, last_value: 500) } + + # a record scoped to a group, should not affect anything. + let!(:issues_internal_ids_gr1) { internal_ids.create!(namespace_id: gr1.id, usage: 0, last_value: 600) } + + subject(:perform_migration) do + described_class.new( + start_id: internal_ids.minimum(:id), + end_id: internal_ids.maximum(:id), + batch_table: :internal_ids, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform + end + + it 'backfills internal_ids records and removes related project records', :aggregate_failures do + perform_migration + + expected_recs = [pr_nmsp1.id, pr_nmsp2.id, pr_nmsp3.id, pr_nmsp5.id, gr1.id] + + # all namespace scoped records for issues(0) usage + expect(internal_ids.where.not(namespace_id: nil).where(usage: 0).count).to eq(5) + # all namespace_ids for issues(0) usage + expect(internal_ids.where.not(namespace_id: nil).where(usage: 0).pluck(:namespace_id)).to match_array(expected_recs) + # this is the record with usage: 4 + expect(internal_ids.where.not(project_id: nil).count).to eq(1) + # no project scoped records for issues usage left + expect(internal_ids.where.not(project_id: nil).where(usage: 0).count).to eq(0) + end +end diff --git a/spec/lib/gitlab/bitbucket_import/importer_spec.rb b/spec/lib/gitlab/bitbucket_import/importer_spec.rb index 1526a1a9f2d..48ceda9e8d8 100644 --- a/spec/lib/gitlab/bitbucket_import/importer_spec.rb +++ b/spec/lib/gitlab/bitbucket_import/importer_spec.rb @@ -358,7 +358,7 @@ RSpec.describe Gitlab::BitbucketImport::Importer, feature_category: :integration describe 'issue import' do it 'allocates internal ids' do - expect(Issue).to receive(:track_project_iid!).with(project, 6) + expect(Issue).to receive(:track_namespace_iid!).with(project.project_namespace, 6) importer.execute end diff --git a/spec/lib/gitlab/ci/config/yaml/result_spec.rb b/spec/lib/gitlab/ci/config/yaml/result_spec.rb new file mode 100644 index 00000000000..9a47738e18e --- /dev/null +++ b/spec/lib/gitlab/ci/config/yaml/result_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::Yaml::Result, feature_category: :pipeline_composition do + it 'does not have a header when config is a single hash' do + result = described_class.new({ a: 1, b: 2 }) + + expect(result).not_to have_header + end + + it 'has a header when config is an array of hashes' do + result = described_class.new([{ a: 1 }, { b: 2 }]) + + expect(result).to have_header + expect(result.header).to eq({ a: 1 }) + end + + it 'raises an error when reading a header when there is none' do + result = described_class.new({ b: 2 }) + + expect { result.header }.to raise_error(ArgumentError) + end + + it 'stores an error / exception when initialized with it' do + result = described_class.new(error: ArgumentError.new('abc')) + + expect(result).not_to be_valid + expect(result.error).to be_a ArgumentError + end +end diff --git a/spec/lib/gitlab/ci/config/yaml_spec.rb b/spec/lib/gitlab/ci/config/yaml_spec.rb index 6911ae4ed83..f4b70069bbe 100644 --- a/spec/lib/gitlab/ci/config/yaml_spec.rb +++ b/spec/lib/gitlab/ci/config/yaml_spec.rb @@ -50,6 +50,15 @@ RSpec.describe Gitlab::Ci::Config::Yaml, feature_category: :pipeline_composition }) end + context 'when YAML is invalid' do + let(:yaml) { 'some: invalid: syntax' } + + it 'raises an error' do + expect { described_class.load!(yaml) } + .to raise_error ::Gitlab::Config::Loader::FormatError, /mapping values are not allowed in this context/ + end + end + context 'when ci_multi_doc_yaml is disabled' do before do stub_feature_flags(ci_multi_doc_yaml: false) @@ -102,4 +111,38 @@ RSpec.describe Gitlab::Ci::Config::Yaml, feature_category: :pipeline_composition end end end + + describe '.load_result!' do + context 'when syntax is invalid' do + let(:yaml) { 'some: invalid: syntax' } + + it 'returns an invalid result object' do + result = described_class.load_result!(yaml) + + expect(result).not_to be_valid + expect(result.error).to be_a ::Gitlab::Config::Loader::FormatError + end + end + + context 'when syntax is valid and contains a header document' do + let(:yaml) do + <<~YAML + a: 1 + --- + b: 2 + YAML + end + + let(:project) { create(:project) } + + it 'returns a result object' do + result = described_class.load_result!(yaml, project: project) + + expect(result).to be_valid + expect(result.error).to be_nil + expect(result.header).to eq({ a: 1 }) + expect(result.content).to eq({ b: 2 }) + end + end + end end diff --git a/spec/lib/gitlab/database/async_constraints/foreign_key_validator_spec.rb b/spec/lib/gitlab/database/async_constraints/foreign_key_validator_spec.rb deleted file mode 100644 index 15474912d9a..00000000000 --- a/spec/lib/gitlab/database/async_constraints/foreign_key_validator_spec.rb +++ /dev/null @@ -1,152 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::AsyncConstraints::ForeignKeyValidator, feature_category: :database do - include ExclusiveLeaseHelpers - - describe '#perform' do - let!(:lease) { stub_exclusive_lease(lease_key, :uuid, timeout: lease_timeout) } - let(:lease_key) { "gitlab/database/asyncddl/actions/#{Gitlab::Database::PRIMARY_DATABASE_NAME}" } - let(:lease_timeout) { described_class::TIMEOUT_PER_ACTION } - - let(:fk_model) { Gitlab::Database::AsyncConstraints::PostgresAsyncConstraintValidation } - let(:table_name) { '_test_async_fks' } - let(:fk_name) { 'fk_parent_id' } - let(:validation) { create(:postgres_async_constraint_validation, table_name: table_name, name: fk_name) } - let(:connection) { validation.connection } - - subject { described_class.new(validation) } - - before do - connection.create_table(table_name) do |t| - t.references :parent, foreign_key: { to_table: table_name, validate: false, name: fk_name } - end - end - - it 'validates the FK while controlling statement timeout' do - allow(connection).to receive(:execute).and_call_original - expect(connection).to receive(:execute) - .with("SET statement_timeout TO '43200s'").ordered.and_call_original - expect(connection).to receive(:execute) - .with('ALTER TABLE "_test_async_fks" VALIDATE CONSTRAINT "fk_parent_id";').ordered.and_call_original - expect(connection).to receive(:execute) - .with("RESET statement_timeout").ordered.and_call_original - - subject.perform - end - - context 'with fully qualified table names' do - let(:validation) do - create(:postgres_async_constraint_validation, - table_name: "public.#{table_name}", - name: fk_name - ) - end - - it 'validates the FK' do - allow(connection).to receive(:execute).and_call_original - - expect(connection).to receive(:execute) - .with('ALTER TABLE "public"."_test_async_fks" VALIDATE CONSTRAINT "fk_parent_id";').ordered.and_call_original - - subject.perform - end - end - - it 'removes the FK validation record from table' do - expect(validation).to receive(:destroy!).and_call_original - - expect { subject.perform }.to change { fk_model.count }.by(-1) - end - - it 'skips logic if not able to acquire exclusive lease' do - expect(lease).to receive(:try_obtain).ordered.and_return(false) - expect(connection).not_to receive(:execute).with(/ALTER TABLE/) - expect(validation).not_to receive(:destroy!) - - expect { subject.perform }.not_to change { fk_model.count } - end - - it 'logs messages around execution' do - allow(Gitlab::AppLogger).to receive(:info).and_call_original - - subject.perform - - expect(Gitlab::AppLogger) - .to have_received(:info) - .with(a_hash_including(message: 'Starting to validate foreign key')) - - expect(Gitlab::AppLogger) - .to have_received(:info) - .with(a_hash_including(message: 'Finished validating foreign key')) - end - - context 'when the FK does not exist' do - before do - connection.create_table(table_name, force: true) - end - - it 'skips validation and removes the record' do - expect(connection).not_to receive(:execute).with(/ALTER TABLE/) - - expect { subject.perform }.to change { fk_model.count }.by(-1) - end - - it 'logs an appropriate message' do - expected_message = "Skipping #{fk_name} validation since it does not exist. The queuing entry will be deleted" - - allow(Gitlab::AppLogger).to receive(:info).and_call_original - - subject.perform - - expect(Gitlab::AppLogger) - .to have_received(:info) - .with(a_hash_including(message: expected_message)) - end - end - - context 'with error handling' do - before do - allow(connection).to receive(:execute).and_call_original - - allow(connection).to receive(:execute) - .with('ALTER TABLE "_test_async_fks" VALIDATE CONSTRAINT "fk_parent_id";') - .and_raise(ActiveRecord::StatementInvalid) - end - - context 'on production' do - before do - allow(Gitlab::ErrorTracking).to receive(:should_raise_for_dev?).and_return(false) - end - - it 'increases execution attempts' do - expect { subject.perform }.to change { validation.attempts }.by(1) - - expect(validation.last_error).to be_present - expect(validation).not_to be_destroyed - end - - it 'logs an error message including the fk_name' do - expect(Gitlab::AppLogger) - .to receive(:error) - .with(a_hash_including(:message, :fk_name)) - .and_call_original - - subject.perform - end - end - - context 'on development' do - it 'also raises errors' do - expect { subject.perform } - .to raise_error(ActiveRecord::StatementInvalid) - .and change { validation.attempts }.by(1) - - expect(validation.last_error).to be_present - expect(validation).not_to be_destroyed - end - end - end - end -end diff --git a/spec/lib/gitlab/database/async_constraints_spec.rb b/spec/lib/gitlab/database/async_constraints_spec.rb index 2141131479d..e5cf782485f 100644 --- a/spec/lib/gitlab/database/async_constraints_spec.rb +++ b/spec/lib/gitlab/database/async_constraints_spec.rb @@ -6,14 +6,20 @@ RSpec.describe Gitlab::Database::AsyncConstraints, feature_category: :database d describe '.validate_pending_entries!' do subject { described_class.validate_pending_entries! } - before do - create_list(:postgres_async_constraint_validation, 3) + let!(:fk_validation) do + create(:postgres_async_constraint_validation, :foreign_key, attempts: 2) end - it 'takes 2 pending FK validations and executes them' do - validations = described_class::PostgresAsyncConstraintValidation.ordered.limit(2).to_a + let(:check_validation) do + create(:postgres_async_constraint_validation, :check_constraint, attempts: 1) + end + + it 'executes pending validations' do + expect_next_instance_of(described_class::Validators::ForeignKey, fk_validation) do |validator| + expect(validator).to receive(:perform) + end - expect_next_instances_of(described_class::ForeignKeyValidator, 2, validations) do |validator| + expect_next_instance_of(described_class::Validators::CheckConstraint, check_validation) do |validator| expect(validator).to receive(:perform) end diff --git a/spec/lib/gitlab/gitaly_client/repository_service_spec.rb b/spec/lib/gitlab/gitaly_client/repository_service_spec.rb index 434550186c1..671f1ee1a0a 100644 --- a/spec/lib/gitlab/gitaly_client/repository_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/repository_service_spec.rb @@ -314,17 +314,31 @@ RSpec.describe Gitlab::GitalyClient::RepositoryService do end describe '#search_files_by_regexp' do - subject(:result) { client.search_files_by_regexp('master', '.*') } + subject(:result) { client.search_files_by_regexp(ref, '.*') } before do expect_any_instance_of(Gitaly::RepositoryService::Stub) .to receive(:search_files_by_name) - .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) - .and_return([double(files: ['file1.txt']), double(files: ['file2.txt'])]) + .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) + .and_return([double(files: ['file1.txt']), double(files: ['file2.txt'])]) + end + + shared_examples 'a search for files by regexp' do + it 'sends a search_files_by_name message and returns a flatten array' do + expect(result).to contain_exactly('file1.txt', 'file2.txt') + end end - it 'sends a search_files_by_name message and returns a flatten array' do - expect(result).to contain_exactly('file1.txt', 'file2.txt') + context 'with ASCII ref' do + let(:ref) { 'master' } + + it_behaves_like 'a search for files by regexp' + end + + context 'with non-ASCII ref' do + let(:ref) { 'ref-ñéüçæøß-val' } + + it_behaves_like 'a search for files by regexp' end end diff --git a/spec/lib/gitlab/jira_import/issues_importer_spec.rb b/spec/lib/gitlab/jira_import/issues_importer_spec.rb index 9f654bbcd15..36135c56dd9 100644 --- a/spec/lib/gitlab/jira_import/issues_importer_spec.rb +++ b/spec/lib/gitlab/jira_import/issues_importer_spec.rb @@ -44,7 +44,7 @@ RSpec.describe Gitlab::JiraImport::IssuesImporter do def mock_issue_serializer(count, raise_exception_on_even_mocks: false) serializer = instance_double(Gitlab::JiraImport::IssueSerializer, execute: { key: 'data' }) - allow(Issue).to receive(:with_project_iid_supply).and_return('issue_iid') + allow(Issue).to receive(:with_namespace_iid_supply).and_return('issue_iid') count.times do |i| if raise_exception_on_even_mocks && i.even? diff --git a/spec/lib/gitlab/kroki_spec.rb b/spec/lib/gitlab/kroki_spec.rb index 3d6ecf20377..6d8e6ecbf54 100644 --- a/spec/lib/gitlab/kroki_spec.rb +++ b/spec/lib/gitlab/kroki_spec.rb @@ -6,7 +6,7 @@ RSpec.describe Gitlab::Kroki do describe '.formats' do def default_formats - %w[bytefield c4plantuml ditaa erd graphviz nomnoml pikchr plantuml + %w[bytefield c4plantuml d2 dbml diagramsnet ditaa erd graphviz nomnoml pikchr plantuml structurizr svgbob umlet vega vegalite wavedrom].freeze end diff --git a/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb b/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb index 7d180ed13a0..8da86e4fae5 100644 --- a/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb @@ -6,7 +6,7 @@ require 'spec_helper' # NOTE: ONLY user related metrics to be added to the aggregates - otherwise add it to the exception list RSpec.describe 'Code review events' do it 'the aggregated metrics contain all the code review metrics' do - user_related_events = %w[i_code_review_create_mr i_code_review_mr_diffs i_code_review_mr_with_invalid_approvers i_code_review_mr_single_file_diffs i_code_review_total_suggestions_applied i_code_review_total_suggestions_added i_code_review_create_note_in_ipynb_diff i_code_review_create_note_in_ipynb_diff_mr i_code_review_create_note_in_ipynb_diff_commit] + mr_related_events = %w[i_code_review_create_mr i_code_review_mr_diffs i_code_review_mr_with_invalid_approvers i_code_review_mr_single_file_diffs i_code_review_total_suggestions_applied i_code_review_total_suggestions_added i_code_review_create_note_in_ipynb_diff i_code_review_create_note_in_ipynb_diff_mr i_code_review_create_note_in_ipynb_diff_commit i_code_review_merge_request_widget_license_compliance_warning] all_code_review_events = Gitlab::Usage::MetricDefinition.all.flat_map do |definition| next [] unless definition.attributes[:key_path].include?('.code_review.') && @@ -22,7 +22,7 @@ RSpec.describe 'Code review events' do definition.attributes.dig(:options, :events) end.uniq - expect(all_code_review_events - (code_review_aggregated_events + user_related_events)).to be_empty + expect(all_code_review_events - (code_review_aggregated_events + mr_related_events)).to be_empty end def code_review_aggregated_metric?(attributes) diff --git a/spec/migrations/20230224085743_update_issues_internal_id_scope_spec.rb b/spec/migrations/20230224085743_update_issues_internal_id_scope_spec.rb new file mode 100644 index 00000000000..7c7b58c7f0e --- /dev/null +++ b/spec/migrations/20230224085743_update_issues_internal_id_scope_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe UpdateIssuesInternalIdScope, feature_category: :team_planning do + describe '#up' do + it 'schedules background migration' do + migrate! + + expect(described_class::MIGRATION).to have_scheduled_batched_migration( + table_name: :internal_ids, + column_name: :id, + interval: described_class::INTERVAL) + end + end + + describe '#down' do + it 'does not schedule background migration' do + schema_migrate_down! + + expect(described_class::MIGRATION).not_to have_scheduled_batched_migration( + table_name: :internal_ids, + column_name: :id, + interval: described_class::INTERVAL) + end + end +end diff --git a/spec/migrations/20230306195007_queue_backfill_project_wiki_repositories_spec.rb b/spec/migrations/20230306195007_queue_backfill_project_wiki_repositories_spec.rb new file mode 100644 index 00000000000..07f501a3f98 --- /dev/null +++ b/spec/migrations/20230306195007_queue_backfill_project_wiki_repositories_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillProjectWikiRepositories, feature_category: :geo_replication do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :projects, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + } + end + end +end diff --git a/spec/models/concerns/atomic_internal_id_spec.rb b/spec/models/concerns/atomic_internal_id_spec.rb index 5fe3141eb17..625d8fec0fb 100644 --- a/spec/models/concerns/atomic_internal_id_spec.rb +++ b/spec/models/concerns/atomic_internal_id_spec.rb @@ -250,11 +250,104 @@ RSpec.describe AtomicInternalId do end end - describe '.track_project_iid!' do + describe '.track_namespace_iid!' do it 'tracks the present value' do expect do - ::Issue.track_project_iid!(milestone.project, external_iid) - end.to change { InternalId.find_by(project: milestone.project, usage: :issues)&.last_value.to_i }.to(external_iid) + ::Issue.track_namespace_iid!(milestone.project.project_namespace, external_iid) + end.to change { + InternalId.find_by(namespace: milestone.project.project_namespace, usage: :issues)&.last_value.to_i + }.to(external_iid) + end + end + + context 'when transitioning a model from one scope to another' do + let!(:issue) { build(:issue, project: project) } + let(:old_issue_model) do + Class.new(ApplicationRecord) do + include AtomicInternalId + + self.table_name = :issues + + belongs_to :project + belongs_to :namespace + + has_internal_id :iid, scope: :project + + def self.name + 'TestClassA' + end + end + end + + let(:old_issue_instance) { old_issue_model.new(issue.attributes) } + let(:new_issue_instance) { Issue.new(issue.attributes) } + + it 'generates the iid on the new scope' do + # set a random iid, just so that it does not start at 1 + old_issue_instance.iid = 123 + old_issue_instance.save! + + # creating a new old_issue_instance increments the iid. + expect { old_issue_model.new(issue.attributes).save! }.to change { + InternalId.find_by(project: project, usage: :issues)&.last_value.to_i + }.from(123).to(124).and(not_change { InternalId.count }) + + # creating a new Issue creates a new record in internal_ids, scoped to the namespace. + # Given the Issue#has_internal_id -> init definition the internal_ids#last_value would be the + # maximum between the old iid value in internal_ids, scoped to the project and max(iid) value from issues + # table by namespace_id. + # see Issue#has_internal_id + expect { new_issue_instance.save! }.to change { + InternalId.find_by(namespace: project.project_namespace, usage: :issues)&.last_value.to_i + }.to(125).and(change { InternalId.count }.by(1)) + + # transition back to project scope would generate overlapping IIDs and raise a duplicate key value error, unless + # we cleanup the issues usage scoped to the project first + expect { old_issue_model.new(issue.attributes).save! }.to raise_error(ActiveRecord::RecordNotUnique) + + # delete issues usage scoped to te project + InternalId.where(project: project, usage: :issues).delete_all + + expect { old_issue_model.new(issue.attributes).save! }.to change { + InternalId.find_by(project: project, usage: :issues)&.last_value.to_i + }.to(126).and(change { InternalId.count }.by(1)) + end + end + + context 'when models is scoped to namespace and does not have an init proc' do + let!(:issue) { build(:issue, namespace: create(:group)) } + + let(:issue_model) do + Class.new(ApplicationRecord) do + include AtomicInternalId + + self.table_name = :issues + + belongs_to :project + belongs_to :namespace + + has_internal_id :iid, scope: :namespace + + def self.name + 'TestClass' + end + end + end + + let(:model_instance) { issue_model.new(issue.attributes) } + + it 'generates the iid on the new scope' do + expect { model_instance.save! }.to change { + InternalId.find_by(namespace: model_instance.namespace, usage: :issues)&.last_value.to_i + }.to(1).and(change { InternalId.count }.by(1)) + end + + it 'supplies a stream of iid values' do + expect do + issue_model.with_namespace_iid_supply(model_instance.namespace) do |supply| + 4.times { supply.next_value } + end + end.to change { InternalId.find_by(namespace: model_instance.namespace, usage: :issues)&.last_value.to_i }.by(4) end end end diff --git a/spec/models/internal_id_spec.rb b/spec/models/internal_id_spec.rb index f0007e1203c..59ade8783e5 100644 --- a/spec/models/internal_id_spec.rb +++ b/spec/models/internal_id_spec.rb @@ -7,7 +7,7 @@ RSpec.describe InternalId do let(:usage) { :issues } let(:issue) { build(:issue, project: project) } let(:id_subject) { issue } - let(:scope) { { project: project } } + let(:scope) { { namespace: project.project_namespace } } let(:init) { ->(issue, scope) { issue&.project&.issues&.size || Issue.where(**scope).count } } it_behaves_like 'having unique enum values' @@ -17,7 +17,7 @@ RSpec.describe InternalId do end describe '.flush_records!' do - subject { described_class.flush_records!(project: project) } + subject { described_class.flush_records!(namespace: project.project_namespace) } let(:another_project) { create(:project) } @@ -27,11 +27,11 @@ RSpec.describe InternalId do end it 'deletes all records for the given project' do - expect { subject }.to change { described_class.where(project: project).count }.from(1).to(0) + expect { subject }.to change { described_class.where(namespace: project.project_namespace).count }.from(1).to(0) end it 'retains records for other projects' do - expect { subject }.not_to change { described_class.where(project: another_project).count } + expect { subject }.not_to change { described_class.where(namespace: another_project.project_namespace).count } end it 'does not allow an empty filter' do @@ -51,7 +51,7 @@ RSpec.describe InternalId do subject described_class.first.tap do |record| - expect(record.project).to eq(project) + expect(record.namespace).to eq(project.project_namespace) expect(record.usage).to eq(usage.to_s) end end @@ -182,7 +182,7 @@ RSpec.describe InternalId do subject described_class.first.tap do |record| - expect(record.project).to eq(project) + expect(record.namespace).to eq(project.project_namespace) expect(record.usage).to eq(usage.to_s) expect(record.last_value).to eq(value) end diff --git a/spec/models/issue_spec.rb b/spec/models/issue_spec.rb index a61a3c5e2ab..8072a60326c 100644 --- a/spec/models/issue_spec.rb +++ b/spec/models/issue_spec.rb @@ -63,8 +63,8 @@ RSpec.describe Issue, feature_category: :team_planning do it_behaves_like 'AtomicInternalId' do let(:internal_id_attribute) { :iid } let(:instance) { build(:issue) } - let(:scope) { :project } - let(:scope_attrs) { { project: instance.project } } + let(:scope) { :namespace } + let(:scope_attrs) { { namespace: instance.project.project_namespace } } let(:usage) { :issues } end end diff --git a/spec/requests/api/debian_group_packages_spec.rb b/spec/requests/api/debian_group_packages_spec.rb index 0c80b7d830f..67600f422ed 100644 --- a/spec/requests/api/debian_group_packages_spec.rb +++ b/spec/requests/api/debian_group_packages_spec.rb @@ -31,9 +31,11 @@ RSpec.describe API::DebianGroupPackages, feature_category: :package_registry do end describe 'GET groups/:id/-/packages/debian/dists/*distribution/:component/binary-:architecture/Packages' do - let(:url) { "/groups/#{container.id}/-/packages/debian/dists/#{distribution.codename}/#{component.name}/binary-#{architecture.name}/Packages" } + let(:target_component_file) { component_file } + let(:target_component_name) { component.name } + let(:url) { "/groups/#{container.id}/-/packages/debian/dists/#{distribution.codename}/#{target_component_name}/binary-#{architecture.name}/Packages" } - it_behaves_like 'Debian packages read endpoint', 'GET', :success, /Description: This is an incomplete Packages file/ + it_behaves_like 'Debian packages index endpoint', /Description: This is an incomplete Packages file/ end describe 'GET groups/:id/-/packages/debian/dists/*distribution/:component/binary-:architecture/Packages.gz' do @@ -43,27 +45,37 @@ RSpec.describe API::DebianGroupPackages, feature_category: :package_registry do end describe 'GET groups/:id/-/packages/debian/dists/*distribution/:component/binary-:architecture/by-hash/SHA256/:file_sha256' do - let(:url) { "/groups/#{container.id}/-/packages/debian/dists/#{distribution.codename}/#{component.name}/binary-#{architecture.name}/by-hash/SHA256/#{component_file_older_sha256.file_sha256}" } + let(:target_component_file) { component_file_older_sha256 } + let(:target_component_name) { component.name } + let(:target_sha256) { target_component_file.file_sha256 } + let(:url) { "/groups/#{container.id}/-/packages/debian/dists/#{distribution.codename}/#{target_component_name}/binary-#{architecture.name}/by-hash/SHA256/#{target_sha256}" } - it_behaves_like 'Debian packages read endpoint', 'GET', :success, /^Other SHA256$/ + it_behaves_like 'Debian packages index sha256 endpoint', /^Other SHA256$/ end describe 'GET groups/:id/-/packages/debian/dists/*distribution/:component/source/Sources' do - let(:url) { "/groups/#{container.id}/-/packages/debian/dists/#{distribution.codename}/#{component.name}/source/Sources" } + let(:target_component_file) { component_file_sources } + let(:target_component_name) { component.name } + let(:url) { "/groups/#{container.id}/-/packages/debian/dists/#{distribution.codename}/#{target_component_name}/source/Sources" } - it_behaves_like 'Debian packages read endpoint', 'GET', :success, /Description: This is an incomplete Sources file/ + it_behaves_like 'Debian packages index endpoint', /^Description: This is an incomplete Sources file$/ end describe 'GET groups/:id/-/packages/debian/dists/*distribution/:component/source/by-hash/SHA256/:file_sha256' do - let(:url) { "/groups/#{container.id}/-/packages/debian/dists/#{distribution.codename}/#{component.name}/source/by-hash/SHA256/#{component_file_sources_older_sha256.file_sha256}" } + let(:target_component_file) { component_file_sources_older_sha256 } + let(:target_component_name) { component.name } + let(:target_sha256) { target_component_file.file_sha256 } + let(:url) { "/groups/#{container.id}/-/packages/debian/dists/#{distribution.codename}/#{target_component_name}/source/by-hash/SHA256/#{target_sha256}" } - it_behaves_like 'Debian packages read endpoint', 'GET', :success, /^Other SHA256$/ + it_behaves_like 'Debian packages index sha256 endpoint', /^Other SHA256$/ end describe 'GET groups/:id/-/packages/debian/dists/*distribution/:component/debian-installer/binary-:architecture/Packages' do - let(:url) { "/groups/#{container.id}/-/packages/debian/dists/#{distribution.codename}/#{component.name}/debian-installer/binary-#{architecture.name}/Packages" } + let(:target_component_file) { component_file_di } + let(:target_component_name) { component.name } + let(:url) { "/groups/#{container.id}/-/packages/debian/dists/#{distribution.codename}/#{target_component_name}/debian-installer/binary-#{architecture.name}/Packages" } - it_behaves_like 'Debian packages read endpoint', 'GET', :success, /Description: This is an incomplete D-I Packages file/ + it_behaves_like 'Debian packages index endpoint', /Description: This is an incomplete D-I Packages file/ end describe 'GET groups/:id/-/packages/debian/dists/*distribution/:component/debian-installer/binary-:architecture/Packages.gz' do @@ -73,9 +85,12 @@ RSpec.describe API::DebianGroupPackages, feature_category: :package_registry do end describe 'GET groups/:id/-/packages/debian/dists/*distribution/:component/debian-installer/binary-:architecture/by-hash/SHA256/:file_sha256' do - let(:url) { "/groups/#{container.id}/-/packages/debian/dists/#{distribution.codename}/#{component.name}/debian-installer/binary-#{architecture.name}/by-hash/SHA256/#{component_file_di_older_sha256.file_sha256}" } + let(:target_component_file) { component_file_di_older_sha256 } + let(:target_component_name) { component.name } + let(:target_sha256) { target_component_file.file_sha256 } + let(:url) { "/groups/#{container.id}/-/packages/debian/dists/#{distribution.codename}/#{target_component_name}/debian-installer/binary-#{architecture.name}/by-hash/SHA256/#{target_sha256}" } - it_behaves_like 'Debian packages read endpoint', 'GET', :success, /^Other SHA256$/ + it_behaves_like 'Debian packages index sha256 endpoint', /^Other SHA256$/ end describe 'GET groups/:id/-/packages/debian/pool/:codename/:project_id/:letter/:package_name/:package_version/:file_name' do diff --git a/spec/requests/api/debian_project_packages_spec.rb b/spec/requests/api/debian_project_packages_spec.rb index 46f79efd928..e31cf236654 100644 --- a/spec/requests/api/debian_project_packages_spec.rb +++ b/spec/requests/api/debian_project_packages_spec.rb @@ -44,9 +44,11 @@ RSpec.describe API::DebianProjectPackages, feature_category: :package_registry d end describe 'GET projects/:id/packages/debian/dists/*distribution/:component/binary-:architecture/Packages' do - let(:url) { "/projects/#{container.id}/packages/debian/dists/#{distribution.codename}/#{component.name}/binary-#{architecture.name}/Packages" } + let(:target_component_file) { component_file } + let(:target_component_name) { component.name } + let(:url) { "/projects/#{container.id}/packages/debian/dists/#{distribution.codename}/#{target_component_name}/binary-#{architecture.name}/Packages" } - it_behaves_like 'Debian packages read endpoint', 'GET', :success, /Description: This is an incomplete Packages file/ + it_behaves_like 'Debian packages index endpoint', /Description: This is an incomplete Packages file/ it_behaves_like 'accept GET request on private project with access to package registry for everyone' end @@ -57,30 +59,40 @@ RSpec.describe API::DebianProjectPackages, feature_category: :package_registry d end describe 'GET projects/:id/packages/debian/dists/*distribution/:component/binary-:architecture/by-hash/SHA256/:file_sha256' do - let(:url) { "/projects/#{container.id}/packages/debian/dists/#{distribution.codename}/#{component.name}/binary-#{architecture.name}/by-hash/SHA256/#{component_file_older_sha256.file_sha256}" } + let(:target_component_file) { component_file_older_sha256 } + let(:target_component_name) { component.name } + let(:target_sha256) { target_component_file.file_sha256 } + let(:url) { "/projects/#{container.id}/packages/debian/dists/#{distribution.codename}/#{target_component_name}/binary-#{architecture.name}/by-hash/SHA256/#{target_sha256}" } - it_behaves_like 'Debian packages read endpoint', 'GET', :success, /^Other SHA256$/ + it_behaves_like 'Debian packages index sha256 endpoint', /^Other SHA256$/ it_behaves_like 'accept GET request on private project with access to package registry for everyone' end describe 'GET projects/:id/packages/debian/dists/*distribution/:component/source/Sources' do - let(:url) { "/projects/#{container.id}/packages/debian/dists/#{distribution.codename}/#{component.name}/source/Sources" } + let(:target_component_file) { component_file_sources } + let(:target_component_name) { component.name } + let(:url) { "/projects/#{container.id}/packages/debian/dists/#{distribution.codename}/#{target_component_name}/source/Sources" } - it_behaves_like 'Debian packages read endpoint', 'GET', :success, /Description: This is an incomplete Sources file/ + it_behaves_like 'Debian packages index endpoint', /^Description: This is an incomplete Sources file$/ it_behaves_like 'accept GET request on private project with access to package registry for everyone' end describe 'GET projects/:id/packages/debian/dists/*distribution/:component/source/by-hash/SHA256/:file_sha256' do - let(:url) { "/projects/#{container.id}/packages/debian/dists/#{distribution.codename}/#{component.name}/source/by-hash/SHA256/#{component_file_sources_older_sha256.file_sha256}" } + let(:target_component_file) { component_file_sources_older_sha256 } + let(:target_component_name) { component.name } + let(:target_sha256) { target_component_file.file_sha256 } + let(:url) { "/projects/#{container.id}/packages/debian/dists/#{distribution.codename}/#{target_component_name}/source/by-hash/SHA256/#{target_sha256}" } - it_behaves_like 'Debian packages read endpoint', 'GET', :success, /^Other SHA256$/ + it_behaves_like 'Debian packages index sha256 endpoint', /^Other SHA256$/ it_behaves_like 'accept GET request on private project with access to package registry for everyone' end describe 'GET projects/:id/packages/debian/dists/*distribution/:component/debian-installer/binary-:architecture/Packages' do - let(:url) { "/projects/#{container.id}/packages/debian/dists/#{distribution.codename}/#{component.name}/debian-installer/binary-#{architecture.name}/Packages" } + let(:target_component_file) { component_file_di } + let(:target_component_name) { component.name } + let(:url) { "/projects/#{container.id}/packages/debian/dists/#{distribution.codename}/#{target_component_name}/debian-installer/binary-#{architecture.name}/Packages" } - it_behaves_like 'Debian packages read endpoint', 'GET', :success, /Description: This is an incomplete D-I Packages file/ + it_behaves_like 'Debian packages index endpoint', /Description: This is an incomplete D-I Packages file/ it_behaves_like 'accept GET request on private project with access to package registry for everyone' end @@ -91,9 +103,12 @@ RSpec.describe API::DebianProjectPackages, feature_category: :package_registry d end describe 'GET projects/:id/packages/debian/dists/*distribution/:component/debian-installer/binary-:architecture/by-hash/SHA256/:file_sha256' do - let(:url) { "/projects/#{container.id}/packages/debian/dists/#{distribution.codename}/#{component.name}/debian-installer/binary-#{architecture.name}/by-hash/SHA256/#{component_file_di_older_sha256.file_sha256}" } + let(:target_component_file) { component_file_di_older_sha256 } + let(:target_component_name) { component.name } + let(:target_sha256) { target_component_file.file_sha256 } + let(:url) { "/projects/#{container.id}/packages/debian/dists/#{distribution.codename}/#{target_component_name}/debian-installer/binary-#{architecture.name}/by-hash/SHA256/#{target_sha256}" } - it_behaves_like 'Debian packages read endpoint', 'GET', :success, /^Other SHA256$/ + it_behaves_like 'Debian packages index sha256 endpoint', /^Other SHA256$/ it_behaves_like 'accept GET request on private project with access to package registry for everyone' end diff --git a/spec/services/groups/group_links/update_service_spec.rb b/spec/services/groups/group_links/update_service_spec.rb index 42f622811d4..f17d2f50a02 100644 --- a/spec/services/groups/group_links/update_service_spec.rb +++ b/spec/services/groups/group_links/update_service_spec.rb @@ -18,7 +18,7 @@ RSpec.describe Groups::GroupLinks::UpdateService, '#execute', feature_category: expires_at: expiry_date } end - subject { described_class.new(link).execute(group_link_params) } + subject { described_class.new(link, user).execute(group_link_params) } before do group.add_developer(group_member_user) diff --git a/spec/support/shared_contexts/requests/api/debian_repository_shared_context.rb b/spec/support/shared_contexts/requests/api/debian_repository_shared_context.rb index 57967fb9414..ad64e4d5be5 100644 --- a/spec/support/shared_contexts/requests/api/debian_repository_shared_context.rb +++ b/spec/support/shared_contexts/requests/api/debian_repository_shared_context.rb @@ -58,6 +58,9 @@ RSpec.shared_context 'Debian repository shared context' do |container_type, can_ let(:distribution) { { private: private_distribution, public: public_distribution }[visibility_level] } let(:architecture) { { private: private_architecture, public: public_architecture }[visibility_level] } let(:component) { { private: private_component, public: public_component }[visibility_level] } + let(:component_file) { { private: private_component_file, public: public_component_file }[visibility_level] } + let(:component_file_sources) { { private: private_component_file_sources, public: public_component_file_sources }[visibility_level] } + let(:component_file_di) { { private: private_component_file_di, public: public_component_file_di }[visibility_level] } let(:component_file_older_sha256) { { private: private_component_file_older_sha256, public: public_component_file_older_sha256 }[visibility_level] } let(:component_file_sources_older_sha256) { { private: private_component_file_sources_older_sha256, public: public_component_file_sources_older_sha256 }[visibility_level] } let(:component_file_di_older_sha256) { { private: private_component_file_di_older_sha256, public: public_component_file_di_older_sha256 }[visibility_level] } diff --git a/spec/support/shared_examples/models/packages/debian/component_file_shared_example.rb b/spec/support/shared_examples/models/packages/debian/component_file_shared_example.rb index 5be0f6349ea..78591482696 100644 --- a/spec/support/shared_examples/models/packages/debian/component_file_shared_example.rb +++ b/spec/support/shared_examples/models/packages/debian/component_file_shared_example.rb @@ -231,4 +231,20 @@ RSpec.shared_examples 'Debian Component File' do |container_type, can_freeze| it { is_expected.to eq("#{component1_1.name}/binary-#{architecture1_1.name}/Packages.xz") } end end + + describe '#empty?' do + subject { component_file_with_architecture.empty? } + + context 'with a non-empty component' do + it { is_expected.to be_falsey } + end + + context 'with an empty component' do + before do + component_file_with_architecture.update! size: 0 + end + + it { is_expected.to be_truthy } + end + end end diff --git a/spec/support/shared_examples/requests/api/debian_packages_shared_examples.rb b/spec/support/shared_examples/requests/api/debian_packages_shared_examples.rb index 6d29076da0f..66554f18e80 100644 --- a/spec/support/shared_examples/requests/api/debian_packages_shared_examples.rb +++ b/spec/support/shared_examples/requests/api/debian_packages_shared_examples.rb @@ -165,3 +165,29 @@ RSpec.shared_examples 'Debian packages write endpoint' do |desired_behavior, suc it_behaves_like 'rejects Debian access with unknown container id', :unauthorized, :basic end + +RSpec.shared_examples 'Debian packages index endpoint' do |success_body| + it_behaves_like 'Debian packages read endpoint', 'GET', :success, success_body + + context 'when no ComponentFile is found' do + let(:target_component_name) { component.name + FFaker::Lorem.word } + + it_behaves_like 'Debian packages read endpoint', 'GET', :no_content, /^$/ + end +end + +RSpec.shared_examples 'Debian packages index sha256 endpoint' do |success_body| + it_behaves_like 'Debian packages read endpoint', 'GET', :success, success_body + + context 'with empty checksum' do + let(:target_sha256) { 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' } + + it_behaves_like 'Debian packages read endpoint', 'GET', :no_content, /^$/ + end + + context 'when ComponentFile is not found' do + let(:target_component_name) { component.name + FFaker::Lorem.word } + + it_behaves_like 'Debian packages read endpoint', 'GET', :not_found, /^{"message":"404 Not Found"}$/ + end +end diff --git a/spec/support/shared_examples/services/packages/debian/generate_distribution_shared_examples.rb b/spec/support/shared_examples/services/packages/debian/generate_distribution_shared_examples.rb index a3042ac2e26..fe5c9032dab 100644 --- a/spec/support/shared_examples/services/packages/debian/generate_distribution_shared_examples.rb +++ b/spec/support/shared_examples/services/packages/debian/generate_distribution_shared_examples.rb @@ -29,26 +29,76 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do let_it_be(:architecture_amd64) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'amd64') } let_it_be(:architecture_arm64) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'arm64') } - let_it_be(:component_file1) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all, updated_at: '2020-01-24T08:00:00Z', file_sha256: 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', file_md5: 'd41d8cd98f00b204e9800998ecf8427e', file_fixture: nil, size: 0) } # updated - let_it_be(:component_file2) { create("debian_#{container_type}_component_file", component: component_main, architecture: architecture_all, updated_at: '2020-01-24T09:00:00Z', file_sha256: 'a') } # destroyed - let_it_be(:component_file3) { create("debian_#{container_type}_component_file", component: component_main, architecture: architecture_amd64, updated_at: '2020-01-24T10:54:59Z', file_sha256: 'b') } # destroyed, 1 second before last generation - let_it_be(:component_file4) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all, updated_at: '2020-01-24T10:55:00Z', file_sha256: 'c') } # kept, last generation - let_it_be(:component_file5) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all, updated_at: '2020-01-24T10:55:00Z', file_sha256: 'd') } # kept, last generation - let_it_be(:component_file6) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_amd64, updated_at: '2020-01-25T15:17:18Z', file_sha256: 'e') } # kept, less than 1 hour ago - - def check_component_file(release_date, component_name, component_file_type, architecture_name, expected_content) + let_it_be(:component_file_old_main_amd64) { create("debian_#{container_type}_component_file", component: component_main, architecture: architecture_amd64, updated_at: '2020-01-24T08:00:00Z', file_sha256: 'a') } # destroyed + + let_it_be(:component_file_oldest_kept_contrib_all) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all, updated_at: '2020-01-24T10:55:00Z', file_sha256: 'b') } # oldest kept + let_it_be(:component_file_oldest_kept_contrib_amd64) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_amd64, updated_at: '2020-01-24T10:55:00Z', file_sha256: 'c') } # oldest kept + let_it_be(:component_file_recent_contrib_amd64) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_amd64, updated_at: '2020-01-25T15:17:18Z', file_sha256: 'd') } # kept, less than 1 hour ago + + let_it_be(:component_file_empty_contrib_all_di) { create("debian_#{container_type}_component_file", :di_packages, :empty, component: component_contrib, architecture: architecture_all, updated_at: '2020-01-24T10:55:00Z') } # oldest kept + let_it_be(:component_file_empty_contrib_amd64_di) { create("debian_#{container_type}_component_file", :di_packages, :empty, component: component_contrib, architecture: architecture_amd64, updated_at: '2020-01-24T10:55:00Z') } # touched, as last empty + let_it_be(:component_file_recent_contrib_amd64_di) { create("debian_#{container_type}_component_file", :di_packages, component: component_contrib, architecture: architecture_amd64, updated_at: '2020-01-25T15:17:18Z', file_sha256: 'f') } # kept, less than 1 hour ago + + let(:pool_prefix) do + prefix = "pool/#{distribution.codename}" + prefix += "/#{project.id}" if container_type == :group + prefix += "/#{package.name[0]}/#{package.name}/#{package.version}" + prefix + end + + let(:expected_main_amd64_di_content) do + <<~MAIN_AMD64_DI_CONTENT + Section: misc + Priority: extra + Filename: #{pool_prefix}/sample-udeb_1.2.3~alpha2_amd64.udeb + Size: 409600 + SHA256: #{package.package_files.with_debian_file_type(:udeb).first.file_sha256} + MAIN_AMD64_DI_CONTENT + end + + let(:expected_main_amd64_di_sha256) { Digest::SHA256.hexdigest(expected_main_amd64_di_content) } + let!(:component_file_old_main_amd64_di) do # touched + create("debian_#{container_type}_component_file", :di_packages, component: component_main, architecture: architecture_amd64, updated_at: '2020-01-24T08:00:00Z', file_sha256: expected_main_amd64_di_sha256).tap do |cf| + cf.update! file: CarrierWaveStringFile.new(expected_main_amd64_di_content), size: expected_main_amd64_di_content.size + end + end + + def check_component_file( + release_date, component_name, component_file_type, architecture_name, expected_content, + updated: true, id_of: nil + ) component_file = distribution .component_files .with_component_name(component_name) .with_file_type(component_file_type) .with_architecture_name(architecture_name) + .with_compression_type(nil) .order_updated_asc .last + if expected_content.nil? + expect(component_file).to be_nil + return + end + expect(component_file).not_to be_nil - expect(component_file.updated_at).to eq(release_date) - unless expected_content.nil? + if id_of + expect(component_file&.id).to eq(id_of.id) + else + # created + expect(component_file&.id).to be > component_file_old_main_amd64_di.id + end + + if updated + expect(component_file.updated_at).to eq(release_date) + else + expect(component_file.updated_at).not_to eq(release_date) + end + + if expected_content == '' + expect(component_file.size).to eq(0) + else expect(expected_content).not_to include('MD5') component_file.file.use_file do |file_path| expect(File.read(file_path)).to eq(expected_content) @@ -57,30 +107,23 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do end it 'generates Debian distribution and component files', :aggregate_failures do - current_time = Time.utc(2020, 01, 25, 15, 17, 18, 123456) + current_time = Time.utc(2020, 1, 25, 15, 17, 19) travel_to(current_time) do expect(Gitlab::ErrorTracking).not_to receive(:log_exception) - components_count = 2 - architectures_count = 3 - - initial_count = 6 - destroyed_count = 2 - updated_count = 1 - created_count = components_count * (architectures_count * 2 + 1) - updated_count + initial_count = 8 + destroyed_count = 1 + created_count = 4 # main_amd64 + main_sources + empty contrib_all + empty contrib_amd64 expect { subject } .to not_change { Packages::Package.count } .and not_change { Packages::PackageFile.count } .and change { distribution.reload.updated_at }.to(current_time.round) .and change { distribution.component_files.reset.count }.from(initial_count).to(initial_count - destroyed_count + created_count) - .and change { component_file1.reload.updated_at }.to(current_time.round) + .and change { component_file_old_main_amd64_di.reload.updated_at }.to(current_time.round) package_files = package.package_files.order(id: :asc).preload_debian_file_metadata.to_a - pool_prefix = "pool/#{distribution.codename}" - pool_prefix += "/#{project.id}" if container_type == :group - pool_prefix += "/#{package.name[0]}/#{package.name}/#{package.version}" expected_main_amd64_content = <<~EOF Package: libsample0 Source: #{package.name} @@ -120,14 +163,6 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do SHA256: #{package_files[3].file_sha256} EOF - expected_main_amd64_di_content = <<~EOF - Section: misc - Priority: extra - Filename: #{pool_prefix}/sample-udeb_1.2.3~alpha2_amd64.udeb - Size: 409600 - SHA256: #{package_files[4].file_sha256} - EOF - expected_main_sources_content = <<~EOF Package: #{package.name} Binary: sample-dev, libsample0, sample-udeb @@ -157,42 +192,38 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do check_component_file(current_time.round, 'main', :packages, 'arm64', nil) check_component_file(current_time.round, 'main', :di_packages, 'all', nil) - check_component_file(current_time.round, 'main', :di_packages, 'amd64', expected_main_amd64_di_content) + check_component_file(current_time.round, 'main', :di_packages, 'amd64', expected_main_amd64_di_content, id_of: component_file_old_main_amd64_di) check_component_file(current_time.round, 'main', :di_packages, 'arm64', nil) check_component_file(current_time.round, 'main', :sources, nil, expected_main_sources_content) - check_component_file(current_time.round, 'contrib', :packages, 'all', nil) - check_component_file(current_time.round, 'contrib', :packages, 'amd64', nil) + check_component_file(current_time.round, 'contrib', :packages, 'all', '') + check_component_file(current_time.round, 'contrib', :packages, 'amd64', '') check_component_file(current_time.round, 'contrib', :packages, 'arm64', nil) - check_component_file(current_time.round, 'contrib', :di_packages, 'all', nil) - check_component_file(current_time.round, 'contrib', :di_packages, 'amd64', nil) + check_component_file(current_time.round, 'contrib', :di_packages, 'all', '', updated: false, id_of: component_file_empty_contrib_all_di) + check_component_file(current_time.round, 'contrib', :di_packages, 'amd64', '', id_of: component_file_empty_contrib_amd64_di) check_component_file(current_time.round, 'contrib', :di_packages, 'arm64', nil) check_component_file(current_time.round, 'contrib', :sources, nil, nil) - main_amd64_size = expected_main_amd64_content.length - main_amd64_sha256 = Digest::SHA256.hexdigest(expected_main_amd64_content) + expected_main_amd64_size = expected_main_amd64_content.length + expected_main_amd64_sha256 = Digest::SHA256.hexdigest(expected_main_amd64_content) - contrib_all_size = component_file1.size - contrib_all_sha256 = component_file1.file_sha256 + expected_main_amd64_di_size = expected_main_amd64_di_content.length - main_amd64_di_size = expected_main_amd64_di_content.length - main_amd64_di_sha256 = Digest::SHA256.hexdigest(expected_main_amd64_di_content) - - main_sources_size = expected_main_sources_content.length - main_sources_sha256 = Digest::SHA256.hexdigest(expected_main_sources_content) + expected_main_sources_size = expected_main_sources_content.length + expected_main_sources_sha256 = Digest::SHA256.hexdigest(expected_main_sources_content) expected_release_content = <<~EOF Codename: #{distribution.codename} - Date: Sat, 25 Jan 2020 15:17:18 +0000 - Valid-Until: Mon, 27 Jan 2020 15:17:18 +0000 + Date: Sat, 25 Jan 2020 15:17:19 +0000 + Valid-Until: Mon, 27 Jan 2020 15:17:19 +0000 Acquire-By-Hash: yes Architectures: all amd64 arm64 Components: contrib main SHA256: - #{contrib_all_sha256} #{contrib_all_size.to_s.rjust(8)} contrib/binary-all/Packages + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 contrib/binary-all/Packages e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 contrib/debian-installer/binary-all/Packages e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 contrib/binary-amd64/Packages e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 contrib/debian-installer/binary-amd64/Packages @@ -201,11 +232,11 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 contrib/source/Sources e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 main/binary-all/Packages e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 main/debian-installer/binary-all/Packages - #{main_amd64_sha256} #{main_amd64_size.to_s.rjust(8)} main/binary-amd64/Packages - #{main_amd64_di_sha256} #{main_amd64_di_size.to_s.rjust(8)} main/debian-installer/binary-amd64/Packages + #{expected_main_amd64_sha256} #{expected_main_amd64_size.to_s.rjust(8)} main/binary-amd64/Packages + #{expected_main_amd64_di_sha256} #{expected_main_amd64_di_size.to_s.rjust(8)} main/debian-installer/binary-amd64/Packages e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 main/binary-arm64/Packages e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 main/debian-installer/binary-arm64/Packages - #{main_sources_sha256} #{main_sources_size.to_s.rjust(8)} main/source/Sources + #{expected_main_sources_sha256} #{expected_main_sources_size.to_s.rjust(8)} main/source/Sources EOF expected_release_content = "Suite: #{distribution.suite}\n#{expected_release_content}" if distribution.suite @@ -222,7 +253,7 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do context 'without components and architectures' do it 'generates minimal distribution', :aggregate_failures do - travel_to(Time.utc(2020, 01, 25, 15, 17, 18, 123456)) do + travel_to(Time.utc(2020, 1, 25, 15, 17, 18, 123456)) do expect(Gitlab::ErrorTracking).not_to receive(:log_exception) expect { subject } diff --git a/spec/workers/gitlab/github_import/stage/import_repository_worker_spec.rb b/spec/workers/gitlab/github_import/stage/import_repository_worker_spec.rb index 24fca3b7c73..0d36864f274 100644 --- a/spec/workers/gitlab/github_import/stage/import_repository_worker_spec.rb +++ b/spec/workers/gitlab/github_import/stage/import_repository_worker_spec.rb @@ -33,7 +33,7 @@ RSpec.describe Gitlab::GithubImport::Stage::ImportRepositoryWorker do :issues, project.import_source, options ).and_return([{ number: 5 }].each) - expect(Issue).to receive(:track_project_iid!).with(project, 5) + expect(Issue).to receive(:track_namespace_iid!).with(project.project_namespace, 5) expect(Gitlab::GithubImport::Stage::ImportBaseDataWorker) .to receive(:perform_async) @@ -54,7 +54,7 @@ RSpec.describe Gitlab::GithubImport::Stage::ImportRepositoryWorker do expect(InternalId).to receive(:exists?).and_return(false) expect(client).to receive(:each_object).with(:issues, project.import_source, options).and_return([nil].each) - expect(Issue).not_to receive(:track_project_iid!) + expect(Issue).not_to receive(:track_namespace_iid!) expect(Gitlab::GithubImport::Stage::ImportBaseDataWorker) .to receive(:perform_async) @@ -74,7 +74,7 @@ RSpec.describe Gitlab::GithubImport::Stage::ImportRepositoryWorker do expect(InternalId).to receive(:exists?).and_return(true) expect(client).not_to receive(:each_object) - expect(Issue).not_to receive(:track_project_iid!) + expect(Issue).not_to receive(:track_namespace_iid!) expect(Gitlab::GithubImport::Stage::ImportBaseDataWorker) .to receive(:perform_async) @@ -96,7 +96,7 @@ RSpec.describe Gitlab::GithubImport::Stage::ImportRepositoryWorker do expect(InternalId).to receive(:exists?).and_return(false) expect(client).to receive(:each_object).and_return([nil].each) - expect(Issue).not_to receive(:track_project_iid!) + expect(Issue).not_to receive(:track_namespace_iid!) expect(Gitlab::Import::ImportFailureService).to receive(:track) .with( diff --git a/yarn.lock b/yarn.lock index 9c8b2566549..70556d0941b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12094,9 +12094,9 @@ undefsafe@^2.0.5: integrity sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA== undici@^5.0.0: - version "5.8.0" - resolved "https://registry.yarnpkg.com/undici/-/undici-5.8.0.tgz#dec9a8ccd90e5a1d81d43c0eab6503146d649a4f" - integrity sha512-1F7Vtcez5w/LwH2G2tGnFIihuWUlc58YidwLiCv+jR2Z50x0tNXpRRw7eOIJ+GvqCqIkg9SB7NWAJ/T9TLfv8Q== + version "5.8.2" + resolved "https://registry.yarnpkg.com/undici/-/undici-5.8.2.tgz#071fc8a6a5d24db0ad510ad442f607d9b09d5eec" + integrity sha512-3KLq3pXMS0Y4IELV045fTxqz04Nk9Ms7yfBBHum3yxsTR4XNn+ZCaUbf/mWitgYDAhsplQ0B1G4S5D345lMO3A== unicode-canonical-property-names-ecmascript@^2.0.0: version "2.0.0" |