Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--app/assets/javascripts/breadcrumb.js4
-rw-r--r--app/assets/javascripts/vue_shared/issuable/show/components/issuable_body.vue9
-rw-r--r--app/assets/javascripts/vue_shared/issuable/show/components/issuable_edit_form.vue5
-rw-r--r--app/assets/javascripts/vue_shared/issuable/show/components/issuable_header.vue2
-rw-r--r--app/assets/javascripts/vue_shared/issuable/show/components/issuable_show_root.vue9
-rw-r--r--app/assets/javascripts/vue_shared/issuable/show/components/issuable_title.vue6
-rw-r--r--app/assets/javascripts/vue_shared/issuable/sidebar/components/issuable_sidebar_root.vue55
-rw-r--r--app/assets/stylesheets/components/detail_page.scss6
-rw-r--r--app/assets/stylesheets/framework/header.scss73
-rw-r--r--app/assets/stylesheets/framework/sidebar.scss9
-rw-r--r--app/controllers/projects/deploy_keys_controller.rb41
-rw-r--r--app/finders/deploy_keys/deploy_keys_finder.rb50
-rw-r--r--app/helpers/breadcrumbs_helper.rb4
-rw-r--r--app/models/ci/catalog/resource.rb19
-rw-r--r--app/models/ci/catalog/resources/sync_event.rb88
-rw-r--r--app/models/deploy_key.rb1
-rw-r--r--app/models/namespaces/sync_event.rb5
-rw-r--r--app/models/project.rb15
-rw-r--r--app/models/projects/sync_event.rb5
-rw-r--r--app/serializers/deploy_keys/deploy_key_serializer.rb1
-rw-r--r--app/services/ci/process_sync_events_service.rb8
-rw-r--r--app/views/layouts/nav/breadcrumbs/_breadcrumbs.html.haml8
-rw-r--r--app/views/layouts/nav/breadcrumbs/_collapsed_inline_list.html.haml6
-rw-r--r--app/workers/all_queues.yml9
-rw-r--r--app/workers/ci/catalog/resources/process_sync_events_worker.rb31
-rw-r--r--config/feature_flags/development/ci_process_catalog_resource_sync_events.yml8
-rw-r--r--config/gitlab.yml.example6
-rw-r--r--config/initializers/postgres_partitioning.rb3
-rw-r--r--config/routes/project.rb6
-rw-r--r--config/sidekiq_queues.yml2
-rw-r--r--db/docs/p_catalog_resource_sync_events.yml13
-rw-r--r--db/migrate/20231124191759_add_catalog_resource_sync_events_table.rb39
-rw-r--r--db/migrate/20231124282441_add_catalog_resource_sync_event_triggers.rb44
-rw-r--r--db/schema_migrations/202311241917591
-rw-r--r--db/schema_migrations/202311242824411
-rw-r--r--db/structure.sql41
-rw-r--r--doc/administration/geo/index.md8
-rw-r--r--doc/architecture/blueprints/cells/routing-service.md102
-rw-r--r--doc/integration/auth0.md2
-rw-r--r--doc/update/versions/gitlab_16_changes.md2
-rw-r--r--doc/user/custom_roles.md3
-rw-r--r--doc/user/project/repository/code_suggestions/saas.md4
-rw-r--r--locale/gitlab.pot6
-rw-r--r--package.json2
-rw-r--r--spec/controllers/projects/deploy_keys_controller_spec.rb104
-rw-r--r--spec/db/schema_spec.rb1
-rw-r--r--spec/factories/ci/catalog/resources/sync_events.rb8
-rw-r--r--spec/features/explore/user_explores_projects_spec.rb2
-rw-r--r--spec/features/projects/commits/user_browses_commits_spec.rb4
-rw-r--r--spec/finders/deploy_keys/deploy_keys_finder_spec.rb78
-rw-r--r--spec/frontend/vue_shared/issuable/sidebar/components/issuable_sidebar_root_spec.js16
-rw-r--r--spec/helpers/groups_helper_spec.rb2
-rw-r--r--spec/lib/gitlab/import_export/all_models.yml2
-rw-r--r--spec/models/ci/catalog/resource_spec.rb179
-rw-r--r--spec/models/ci/catalog/resources/sync_event_spec.rb190
-rw-r--r--spec/models/project_spec.rb79
-rw-r--r--spec/services/ci/process_sync_events_service_spec.rb79
-rw-r--r--spec/workers/ci/catalog/resources/process_sync_events_worker_spec.rb52
-rw-r--r--yarn.lock8
59 files changed, 1285 insertions, 281 deletions
diff --git a/app/assets/javascripts/breadcrumb.js b/app/assets/javascripts/breadcrumb.js
index 113840dbc52..0dacd5af5cc 100644
--- a/app/assets/javascripts/breadcrumb.js
+++ b/app/assets/javascripts/breadcrumb.js
@@ -23,11 +23,11 @@ export default () => {
topLevelLinks.forEach((el) => addTooltipToEl(el));
$expanderBtn.on('click', () => {
- const detailItems = $('.breadcrumbs-detail-item');
+ const detailItems = $('.gl-breadcrumb-item');
const hiddenClass = 'gl-display-none!';
$.each(detailItems, (_key, item) => {
- $(item).toggleClass(hiddenClass);
+ $(item).removeClass(hiddenClass);
});
// remove the ellipsis
diff --git a/app/assets/javascripts/vue_shared/issuable/show/components/issuable_body.vue b/app/assets/javascripts/vue_shared/issuable/show/components/issuable_body.vue
index dae3ddfe016..bac71c1eda2 100644
--- a/app/assets/javascripts/vue_shared/issuable/show/components/issuable_body.vue
+++ b/app/assets/javascripts/vue_shared/issuable/show/components/issuable_body.vue
@@ -31,6 +31,10 @@ export default {
type: Boolean,
required: true,
},
+ hideEditButton: {
+ type: Boolean,
+ required: false,
+ },
enableAutocomplete: {
type: Boolean,
required: true,
@@ -166,6 +170,7 @@ export default {
:issuable="issuable"
:status-icon="statusIcon"
:enable-edit="enableEdit"
+ :hide-edit-button="hideEditButton"
:workspace-type="workspaceType"
@edit-issuable="$emit('edit-issuable', $event)"
>
@@ -181,12 +186,12 @@ export default {
:task-list-update-path="taskListUpdatePath"
/>
<slot name="secondary-content"></slot>
- <small v-if="isUpdated" class="edited-text gl-font-sm!">
+ <small v-if="isUpdated" class="edited-text gl-font-sm! gl-text-secondary">
{{ __('Edited') }}
<time-ago-tooltip :time="issuable.updatedAt" tooltip-placement="bottom" />
<span v-if="updatedBy">
{{ __('by') }}
- <gl-link :href="updatedBy.webUrl" class="author-link gl-font-sm!">
+ <gl-link :href="updatedBy.webUrl" class="author-link gl-font-sm! gl-text-secondary">
<span>{{ updatedBy.name }}</span>
</gl-link>
</span>
diff --git a/app/assets/javascripts/vue_shared/issuable/show/components/issuable_edit_form.vue b/app/assets/javascripts/vue_shared/issuable/show/components/issuable_edit_form.vue
index 7c3dd5c3623..3353374310f 100644
--- a/app/assets/javascripts/vue_shared/issuable/show/components/issuable_edit_form.vue
+++ b/app/assets/javascripts/vue_shared/issuable/show/components/issuable_edit_form.vue
@@ -153,7 +153,10 @@ export default {
</template>
</markdown-field>
</gl-form-group>
- <div data-testid="actions" class="col-12 gl-mt-3 gl-mb-3 gl-px-0 clearfix">
+ <div
+ data-testid="actions"
+ class="col-12 gl-mt-3 gl-mb-3 gl-px-0 clearfix gl-display-flex gl-gap-3"
+ >
<slot
name="edit-form-actions"
:issuable-title="title"
diff --git a/app/assets/javascripts/vue_shared/issuable/show/components/issuable_header.vue b/app/assets/javascripts/vue_shared/issuable/show/components/issuable_header.vue
index 760a3e01d6f..1b95a2abdf9 100644
--- a/app/assets/javascripts/vue_shared/issuable/show/components/issuable_header.vue
+++ b/app/assets/javascripts/vue_shared/issuable/show/components/issuable_header.vue
@@ -221,7 +221,7 @@ export default {
@click="handleRightSidebarToggleClick"
/>
</div>
- <div class="detail-page-header-actions gl-align-self-center gl-display-flex">
+ <div class="detail-page-header-actions gl-align-self-center gl-display-flex gl-gap-3">
<slot name="header-actions"></slot>
</div>
</div>
diff --git a/app/assets/javascripts/vue_shared/issuable/show/components/issuable_show_root.vue b/app/assets/javascripts/vue_shared/issuable/show/components/issuable_show_root.vue
index 040f49c7c25..1d44c4a1c14 100644
--- a/app/assets/javascripts/vue_shared/issuable/show/components/issuable_show_root.vue
+++ b/app/assets/javascripts/vue_shared/issuable/show/components/issuable_show_root.vue
@@ -32,6 +32,11 @@ export default {
required: false,
default: false,
},
+ hideEditButton: {
+ type: Boolean,
+ required: false,
+ default: false,
+ },
enableAutocomplete: {
type: Boolean,
required: false,
@@ -137,6 +142,7 @@ export default {
:status-icon="statusIcon"
:status-icon-class="statusIconClass"
:enable-edit="enableEdit"
+ :hide-edit-button="hideEditButton"
:enable-autocomplete="enableAutocomplete"
:enable-autosave="enableAutosave"
:enable-zen-mode="enableZenMode"
@@ -169,6 +175,9 @@ export default {
</issuable-discussion>
<issuable-sidebar>
+ <template #right-sidebar-top-items="{ sidebarExpanded, toggleSidebar }">
+ <slot name="right-sidebar-top-items" v-bind="{ sidebarExpanded, toggleSidebar }"></slot>
+ </template>
<template #right-sidebar-items="{ sidebarExpanded, toggleSidebar }">
<slot name="right-sidebar-items" v-bind="{ sidebarExpanded, toggleSidebar }"></slot>
</template>
diff --git a/app/assets/javascripts/vue_shared/issuable/show/components/issuable_title.vue b/app/assets/javascripts/vue_shared/issuable/show/components/issuable_title.vue
index 5387e39e3eb..3dae894b127 100644
--- a/app/assets/javascripts/vue_shared/issuable/show/components/issuable_title.vue
+++ b/app/assets/javascripts/vue_shared/issuable/show/components/issuable_title.vue
@@ -33,6 +33,10 @@ export default {
type: Boolean,
required: true,
},
+ hideEditButton: {
+ type: Boolean,
+ required: false,
+ },
workspaceType: {
type: String,
required: false,
@@ -70,7 +74,7 @@ export default {
data-testid="issuable-title"
></h1>
<gl-button
- v-if="enableEdit"
+ v-if="enableEdit && !hideEditButton"
v-gl-tooltip.bottom
:title="$options.i18n.editTitleAndDescription"
:aria-label="$options.i18n.editTitleAndDescription"
diff --git a/app/assets/javascripts/vue_shared/issuable/sidebar/components/issuable_sidebar_root.vue b/app/assets/javascripts/vue_shared/issuable/sidebar/components/issuable_sidebar_root.vue
index 774267639fc..cb9ad6418a4 100644
--- a/app/assets/javascripts/vue_shared/issuable/sidebar/components/issuable_sidebar_root.vue
+++ b/app/assets/javascripts/vue_shared/issuable/sidebar/components/issuable_sidebar_root.vue
@@ -1,13 +1,17 @@
<script>
-import { GlIcon } from '@gitlab/ui';
+import { GlButton, GlTooltipDirective } from '@gitlab/ui';
import { GlBreakpointInstance as bp } from '@gitlab/ui/dist/utils';
import { getCookie, setCookie, parseBoolean } from '~/lib/utils/common_utils';
+import { __ } from '~/locale';
import { USER_COLLAPSED_GUTTER_COOKIE } from '../constants';
export default {
components: {
- GlIcon,
+ GlButton,
+ },
+ directives: {
+ GlTooltip: GlTooltipDirective,
},
data() {
const userExpanded = !parseBoolean(getCookie(USER_COLLAPSED_GUTTER_COOKIE));
@@ -20,6 +24,20 @@ export default {
isExpanded: userExpanded ? bp.isDesktop() : userExpanded,
};
},
+ computed: {
+ toggleLabel() {
+ return this.isExpanded ? __('Collapse sidebar') : __('Expand sidebar');
+ },
+ toggleIcon() {
+ return this.isExpanded ? 'chevron-double-lg-right' : 'chevron-double-lg-left';
+ },
+ expandedToggleClass() {
+ return this.isExpanded ? 'block' : '';
+ },
+ collapsedToggleClass() {
+ return !this.isExpanded ? 'block' : '';
+ },
+ },
mounted() {
window.addEventListener('resize', this.handleWindowResize);
this.updatePageContainerClass();
@@ -59,23 +77,24 @@ export default {
class="right-sidebar"
aria-live="polite"
>
- <button
- class="toggle-right-sidebar-button js-toggle-right-sidebar-button w-100 gl-text-decoration-none! gl-display-flex gl-outline-0!"
- data-testid="toggle-right-sidebar-button"
- :title="__('Toggle sidebar')"
- @click="toggleSidebar"
- >
- <span v-if="isExpanded" class="collapse-text gl-flex-grow-1 gl-text-left">{{
- __('Collapse sidebar')
- }}</span>
- <gl-icon v-show="isExpanded" data-testid="icon-collapse" name="chevron-double-lg-right" />
- <gl-icon
- v-show="!isExpanded"
- data-testid="icon-expand"
- name="chevron-double-lg-left"
- class="gl-ml-2"
+ <div class="right-sidebar-header" :class="expandedToggleClass">
+ <gl-button
+ v-gl-tooltip.hover.left
+ category="tertiary"
+ size="small"
+ class="gl-float-right gutter-toggle toggle-right-sidebar-button js-toggle-right-sidebar-button gl-shadow-none!"
+ :class="collapsedToggleClass"
+ data-testid="toggle-right-sidebar-button"
+ :icon="toggleIcon"
+ :title="toggleLabel"
+ :aria-label="toggleLabel"
+ @click="toggleSidebar"
/>
- </button>
+ <slot
+ name="right-sidebar-top-items"
+ v-bind="{ sidebarExpanded: isExpanded, toggleSidebar }"
+ ></slot>
+ </div>
<div data-testid="sidebar-items" class="issuable-sidebar">
<slot
name="right-sidebar-items"
diff --git a/app/assets/stylesheets/components/detail_page.scss b/app/assets/stylesheets/components/detail_page.scss
index 56214040fdd..98ed7f590ea 100644
--- a/app/assets/stylesheets/components/detail_page.scss
+++ b/app/assets/stylesheets/components/detail_page.scss
@@ -53,13 +53,17 @@
margin: 0 0 $gl-spacing-scale-4;
color: $gl-text-color;
padding: 0 0 0.3em;
- border-bottom: 1px solid $white-dark;
}
.description {
@include clearfix;
margin-top: 6px;
+
+ + .edited-text {
+ display: inline-block;
+ margin-top: $gl-spacing-scale-4;
+ }
}
.author-link {
diff --git a/app/assets/stylesheets/framework/header.scss b/app/assets/stylesheets/framework/header.scss
index 3e97343c3fe..ce2296e319a 100644
--- a/app/assets/stylesheets/framework/header.scss
+++ b/app/assets/stylesheets/framework/header.scss
@@ -260,12 +260,6 @@
@media (prefers-reduced-motion: no-preference) {
transition: left $gl-transition-duration-medium, right $gl-transition-duration-medium;
}
-
- .breadcrumbs-list {
- @include media-breakpoint-down(xs) {
- flex-wrap: nowrap;
- }
- }
}
.breadcrumbs {
@@ -280,59 +274,6 @@
border-radius: 50%;
vertical-align: sub;
}
-
- .text-expander {
- margin-left: 0;
- margin-right: 2px;
-
- > i {
- position: relative;
- top: 1px;
- }
- }
-}
-
-.breadcrumbs-list {
- display: flex;
- margin-bottom: 0;
- line-height: 16px;
-
- @include media-breakpoint-down(xs) {
- flex-wrap: wrap;
- }
-
- > li {
- display: flex;
- align-items: center;
- position: relative;
- min-width: 0;
- padding: 2px 0;
-
- &:not(:last-child) {
- padding-right: 20px;
- }
-
- &:last-child {
- > a {
- font-weight: 600;
- line-height: 16px;
- color: $gl-text-color;
- }
- }
-
- > a {
- font-size: 12px;
- color: currentColor;
- white-space: nowrap;
- overflow: hidden;
- text-overflow: ellipsis;
- flex: 0 1 auto;
- }
-
- &:last-of-type > .breadcrumbs-list-angle {
- display: none;
- }
- }
}
.breadcrumb-item-text {
@@ -343,20 +284,6 @@
}
}
-.breadcrumbs-list-angle {
- position: absolute;
- right: 7px;
- top: 50%;
- color: $gl-text-color-tertiary;
- transform: translateY(-50%);
-}
-
-.breadcrumbs-extra {
- display: flex;
- flex: 0 0 auto;
- margin-left: auto;
-}
-
@include media-breakpoint-down(xs) {
.navbar-gitlab.legacy-top-bar .container-fluid {
font-size: 18px;
diff --git a/app/assets/stylesheets/framework/sidebar.scss b/app/assets/stylesheets/framework/sidebar.scss
index 168aa704a69..30f6d39f70b 100644
--- a/app/assets/stylesheets/framework/sidebar.scss
+++ b/app/assets/stylesheets/framework/sidebar.scss
@@ -53,12 +53,21 @@
height: $gl-padding;
}
}
+
+ .right-sidebar-header {
+ flex-wrap: wrap;
+ }
}
.right-sidebar-expanded {
padding-right: 0;
z-index: $zindex-dropdown-menu;
+ .right-sidebar-header {
+ padding-block: $gl-spacing-scale-4;
+ margin-left: 20px;
+ }
+
.inline-block {
@include gl-display-inline-block;
}
diff --git a/app/controllers/projects/deploy_keys_controller.rb b/app/controllers/projects/deploy_keys_controller.rb
index 9cdbd2a30f6..65aa836c562 100644
--- a/app/controllers/projects/deploy_keys_controller.rb
+++ b/app/controllers/projects/deploy_keys_controller.rb
@@ -22,6 +22,34 @@ class Projects::DeployKeysController < Projects::ApplicationController
end
end
+ def enabled_keys
+ respond_to do |format|
+ format.json do
+ enabled_keys = find_keys(filter: :enabled_keys)
+ render json: serialize(enabled_keys)
+ end
+ end
+ end
+
+ def available_project_keys
+ respond_to do |format|
+ format.json do
+ available_project_keys = find_keys(filter: :available_project_keys)
+ render json: serialize(available_project_keys)
+ end
+ end
+ end
+
+ def available_public_keys
+ respond_to do |format|
+ format.json do
+ available_public_keys = find_keys(filter: :available_public_keys)
+
+ render json: serialize(available_public_keys)
+ end
+ end
+ end
+
def new
redirect_to_repository
end
@@ -108,4 +136,17 @@ class Projects::DeployKeysController < Projects::ApplicationController
def redirect_to_repository
redirect_to_repository_settings(@project, anchor: 'js-deploy-keys-settings')
end
+
+ def find_keys(params)
+ DeployKeys::DeployKeysFinder.new(@project, current_user, params)
+ .execute
+ end
+
+ def serialize(keys)
+ opts = { user: current_user, project: project }
+
+ DeployKeys::DeployKeySerializer.new
+ .with_pagination(request, response)
+ .represent(keys, opts)
+ end
end
diff --git a/app/finders/deploy_keys/deploy_keys_finder.rb b/app/finders/deploy_keys/deploy_keys_finder.rb
new file mode 100644
index 00000000000..5924a656801
--- /dev/null
+++ b/app/finders/deploy_keys/deploy_keys_finder.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module DeployKeys
+ class DeployKeysFinder
+ attr_reader :project, :current_user, :params
+
+ def initialize(project, current_user, params = {})
+ @project = project
+ @current_user = current_user
+ @params = params
+ end
+
+ def execute
+ return empty unless can_admin_project?
+
+ case params[:filter]
+ when :enabled_keys
+ enabled_keys
+ when :available_project_keys
+ available_project_keys
+ when :available_public_keys
+ available_public_keys
+ else
+ empty
+ end
+ end
+
+ private
+
+ def enabled_keys
+ project.deploy_keys.with_projects
+ end
+
+ def available_project_keys
+ current_user.project_deploy_keys.with_projects.not_in(enabled_keys)
+ end
+
+ def available_public_keys
+ DeployKey.are_public.with_projects.not_in(enabled_keys)
+ end
+
+ def empty
+ DeployKey.none
+ end
+
+ def can_admin_project?
+ current_user.can?(:admin_project, project)
+ end
+ end
+end
diff --git a/app/helpers/breadcrumbs_helper.rb b/app/helpers/breadcrumbs_helper.rb
index 6996c7a1766..da8310995cc 100644
--- a/app/helpers/breadcrumbs_helper.rb
+++ b/app/helpers/breadcrumbs_helper.rb
@@ -22,9 +22,7 @@ module BreadcrumbsHelper
end
def breadcrumb_list_item(link)
- content_tag "li" do
- link + sprite_icon("chevron-lg-right", size: 8, css_class: "breadcrumbs-list-angle")
- end
+ content_tag :li, link, class: 'gl-breadcrumb-item'
end
def add_to_breadcrumb_collapsed_links(link, location: :before)
diff --git a/app/models/ci/catalog/resource.rb b/app/models/ci/catalog/resource.rb
index 8739450fb8e..4be48f1bd1f 100644
--- a/app/models/ci/catalog/resource.rb
+++ b/app/models/ci/catalog/resource.rb
@@ -17,6 +17,8 @@ module Ci
inverse_of: :catalog_resource
has_many :versions, class_name: 'Ci::Catalog::Resources::Version', foreign_key: :catalog_resource_id,
inverse_of: :catalog_resource
+ has_many :sync_events, class_name: 'Ci::Catalog::Resources::SyncEvent', foreign_key: :catalog_resource_id,
+ inverse_of: :catalog_resource
scope :for_projects, ->(project_ids) { where(project_id: project_ids) }
@@ -37,6 +39,14 @@ module Ci
before_create :sync_with_project
+ class << self
+ # Used by Ci::ProcessSyncEventsService
+ def sync!(event)
+ # There may be orphaned records since this table does not enforce FKs
+ event.catalog_resource&.sync_with_project!
+ end
+ end
+
def to_param
full_path
end
@@ -54,17 +64,16 @@ module Ci
save!
end
- # Triggered in Ci::Catalog::Resources::Version and Release model callbacks.
+ # Triggered in Ci::Catalog::Resources::Version and Release model callbacks
def update_latest_released_at!
update!(latest_released_at: versions.latest&.released_at)
end
private
- # These columns are denormalized from the `projects` table. We first sync these
- # columns when the catalog resource record is created. Then any updates to the
- # `projects` columns will be synced to the `catalog_resources` table by a worker
- # (to be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/429376.)
+ # These denormalized columns are first synced when a new catalog resource is created.
+ # A PG trigger adds a SyncEvent when the associated project updates any of these columns.
+ # A worker processes the SyncEvents with Ci::ProcessSyncEventsService.
def sync_with_project
self.name = project.name
self.description = project.description
diff --git a/app/models/ci/catalog/resources/sync_event.rb b/app/models/ci/catalog/resources/sync_event.rb
new file mode 100644
index 00000000000..a2819102875
--- /dev/null
+++ b/app/models/ci/catalog/resources/sync_event.rb
@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Ci
+ module Catalog
+ module Resources
+ # This table is used as a queue of catalog resources that need to be synchronized with `projects`.
+ # A PG trigger adds a SyncEvent when the associated `projects` record of a catalog resource
+ # updates any of the relevant columns referenced in `Ci::Catalog::Resource#sync_with_project`
+ # (DB function name: `insert_catalog_resource_sync_event`).
+ class SyncEvent < ::ApplicationRecord
+ include PartitionedTable
+ include IgnorableColumns
+
+ PARTITION_DURATION = 1.day
+
+ self.table_name = 'p_catalog_resource_sync_events'
+ self.primary_key = :id
+ self.sequence_name = :p_catalog_resource_sync_events_id_seq
+
+ ignore_column :partition_id, remove_with: '3000.0', remove_after: '3000-01-01'
+
+ belongs_to :catalog_resource, class_name: 'Ci::Catalog::Resource', inverse_of: :sync_events
+ belongs_to :project, inverse_of: :catalog_resource_sync_events
+
+ scope :for_partition, ->(partition) { where(partition_id: partition) }
+ scope :select_with_partition,
+ -> { select(:id, :catalog_resource_id, arel_table[:partition_id].as('partition')) }
+
+ scope :unprocessed_events, -> { select_with_partition.status_pending }
+ scope :preload_synced_relation, -> { preload(catalog_resource: :project) }
+
+ enum status: { pending: 1, processed: 2 }, _prefix: :status
+
+ partitioned_by :partition_id, strategy: :sliding_list,
+ next_partition_if: ->(active_partition) do
+ oldest_record_in_partition = Ci::Catalog::Resources::SyncEvent
+ .select(:id, :created_at)
+ .for_partition(active_partition.value)
+ .order(:id)
+ .limit(1)
+ .take
+
+ oldest_record_in_partition.present? &&
+ oldest_record_in_partition.created_at < PARTITION_DURATION.ago
+ end,
+ detach_partition_if: ->(partition) do
+ !Ci::Catalog::Resources::SyncEvent
+ .for_partition(partition.value)
+ .status_pending
+ .exists?
+ end
+
+ class << self
+ def mark_records_processed(records)
+ update_by_partition(records) do |partitioned_scope|
+ partitioned_scope.update_all(status: :processed)
+ end
+ end
+
+ def enqueue_worker
+ return unless Feature.enabled?(:ci_process_catalog_resource_sync_events)
+
+ ::Ci::Catalog::Resources::ProcessSyncEventsWorker.perform_async # rubocop:disable CodeReuse/Worker -- Worker is scheduled in model callback functions
+ end
+
+ def upper_bound_count
+ select('COALESCE(MAX(id) - MIN(id) + 1, 0) AS upper_bound_count')
+ .status_pending.to_a.first.upper_bound_count
+ end
+
+ private
+
+ # You must use .select_with_partition before calling this method
+ # as it requires the partition to be explicitly selected.
+ def update_by_partition(records)
+ records.group_by(&:partition).each do |partition, records_within_partition|
+ partitioned_scope = status_pending
+ .for_partition(partition)
+ .where(id: records_within_partition.map(&:id))
+
+ yield(partitioned_scope)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/app/models/deploy_key.rb b/app/models/deploy_key.rb
index 9ede494cfcc..5923497a1a3 100644
--- a/app/models/deploy_key.rb
+++ b/app/models/deploy_key.rb
@@ -25,6 +25,7 @@ class DeployKey < Key
scope :with_projects, -> { includes(deploy_keys_projects: { project: [:route, namespace: :route] }) }
scope :including_projects_with_write_access, -> { includes(:projects_with_write_access) }
scope :including_projects_with_readonly_access, -> { includes(:projects_with_readonly_access) }
+ scope :not_in, ->(keys) { where.not(id: keys.select(:id)) }
accepts_nested_attributes_for :deploy_keys_projects, reject_if: :reject_deploy_keys_projects?
diff --git a/app/models/namespaces/sync_event.rb b/app/models/namespaces/sync_event.rb
index fbe047f2c5a..c06d86cb297 100644
--- a/app/models/namespaces/sync_event.rb
+++ b/app/models/namespaces/sync_event.rb
@@ -7,9 +7,14 @@ class Namespaces::SyncEvent < ApplicationRecord
belongs_to :namespace
+ scope :unprocessed_events, -> { all }
scope :preload_synced_relation, -> { preload(:namespace) }
scope :order_by_id_asc, -> { order(id: :asc) }
+ def self.mark_records_processed(records)
+ id_in(records).delete_all
+ end
+
def self.enqueue_worker
::Namespaces::ProcessSyncEventsWorker.perform_async # rubocop:disable CodeReuse/Worker
end
diff --git a/app/models/project.rb b/app/models/project.rb
index bb421fc7dc6..35e38cd37d1 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -142,9 +142,8 @@ class Project < ApplicationRecord
after_create :set_timestamps_for_create
after_create :check_repository_absence!
- # TODO: Remove this callback after background syncing is implemented. See https://gitlab.com/gitlab-org/gitlab/-/issues/429376.
- after_update :update_catalog_resource,
- if: -> { (saved_change_to_name? || saved_change_to_description? || saved_change_to_visibility_level?) && catalog_resource }
+ after_update :enqueue_catalog_resource_sync_event_worker,
+ if: -> { catalog_resource && (saved_change_to_name? || saved_change_to_description? || saved_change_to_visibility_level?) }
before_destroy :remove_private_deploy_keys
after_destroy :remove_exports
@@ -187,6 +186,7 @@ class Project < ApplicationRecord
has_one :catalog_resource, class_name: 'Ci::Catalog::Resource', inverse_of: :project
has_many :ci_components, class_name: 'Ci::Catalog::Resources::Component', inverse_of: :project
has_many :catalog_resource_versions, class_name: 'Ci::Catalog::Resources::Version', inverse_of: :project
+ has_many :catalog_resource_sync_events, class_name: 'Ci::Catalog::Resources::SyncEvent', inverse_of: :project
has_one :last_event, -> { order 'events.created_at DESC' }, class_name: 'Event'
has_many :boards
@@ -3468,8 +3468,13 @@ class Project < ApplicationRecord
pool_repository_shard == repository_storage
end
- def update_catalog_resource
- catalog_resource.sync_with_project!
+ # Catalog resource SyncEvents are created by PG triggers
+ def enqueue_catalog_resource_sync_event_worker
+ catalog_resource.sync_with_project! if Feature.disabled?(:ci_process_catalog_resource_sync_events)
+
+ run_after_commit do
+ ::Ci::Catalog::Resources::SyncEvent.enqueue_worker
+ end
end
end
diff --git a/app/models/projects/sync_event.rb b/app/models/projects/sync_event.rb
index 7af863c0cf0..f1688bcd19d 100644
--- a/app/models/projects/sync_event.rb
+++ b/app/models/projects/sync_event.rb
@@ -7,9 +7,14 @@ class Projects::SyncEvent < ApplicationRecord
belongs_to :project
+ scope :unprocessed_events, -> { all }
scope :preload_synced_relation, -> { preload(:project) }
scope :order_by_id_asc, -> { order(id: :asc) }
+ def self.mark_records_processed(records)
+ id_in(records).delete_all
+ end
+
def self.enqueue_worker
::Projects::ProcessSyncEventsWorker.perform_async # rubocop:disable CodeReuse/Worker
end
diff --git a/app/serializers/deploy_keys/deploy_key_serializer.rb b/app/serializers/deploy_keys/deploy_key_serializer.rb
index b00ef65696f..2e6291a95f2 100644
--- a/app/serializers/deploy_keys/deploy_key_serializer.rb
+++ b/app/serializers/deploy_keys/deploy_key_serializer.rb
@@ -3,5 +3,6 @@
module DeployKeys
class DeployKeySerializer < BaseSerializer
entity DeployKeyEntity
+ include WithPagination
end
end
diff --git a/app/services/ci/process_sync_events_service.rb b/app/services/ci/process_sync_events_service.rb
index d90ee02b1c6..d3c699597b6 100644
--- a/app/services/ci/process_sync_events_service.rb
+++ b/app/services/ci/process_sync_events_service.rb
@@ -13,7 +13,7 @@ module Ci
end
def execute
- # preventing parallel processing over the same event table
+ # To prevent parallel processing over the same event table
try_obtain_lease { process_events }
enqueue_worker_if_there_still_event
@@ -26,7 +26,7 @@ module Ci
def process_events
add_result(estimated_total_events: @sync_event_class.upper_bound_count)
- events = @sync_event_class.preload_synced_relation.first(BATCH_SIZE)
+ events = @sync_event_class.unprocessed_events.preload_synced_relation.first(BATCH_SIZE)
add_result(consumable_events: events.size)
@@ -42,12 +42,12 @@ module Ci
end
ensure
add_result(processed_events: processed_events.size)
- @sync_event_class.id_in(processed_events).delete_all
+ @sync_event_class.mark_records_processed(processed_events)
end
end
def enqueue_worker_if_there_still_event
- @sync_event_class.enqueue_worker if @sync_event_class.exists?
+ @sync_event_class.enqueue_worker if @sync_event_class.unprocessed_events.exists?
end
def lease_key
diff --git a/app/views/layouts/nav/breadcrumbs/_breadcrumbs.html.haml b/app/views/layouts/nav/breadcrumbs/_breadcrumbs.html.haml
index 040793d616f..417df51e984 100644
--- a/app/views/layouts/nav/breadcrumbs/_breadcrumbs.html.haml
+++ b/app/views/layouts/nav/breadcrumbs/_breadcrumbs.html.haml
@@ -2,8 +2,8 @@
- unless @skip_current_level_breadcrumb
- push_to_schema_breadcrumb(@breadcrumb_title, breadcrumb_title_link)
-%nav.breadcrumbs{ 'aria-label': _('Breadcrumbs'), data: { testid: 'breadcrumb-links' } }
- %ul.list-unstyled.breadcrumbs-list.js-breadcrumbs-list
+%nav.breadcrumbs.gl-breadcrumbs{ 'aria-label': _('Breadcrumbs'), data: { testid: 'breadcrumb-links' } }
+ %ul.breadcrumb.gl-breadcrumb-list.js-breadcrumbs-list
- unless hide_top_links
= header_title
- if @breadcrumbs_extra_links
@@ -11,8 +11,8 @@
= breadcrumb_list_item link_to(extra[:text], extra[:link])
= render "layouts/nav/breadcrumbs/collapsed_inline_list", location: :after
- unless @skip_current_level_breadcrumb
- %li{ data: { testid: 'breadcrumb-current-link' } }
- = link_to @breadcrumb_title, breadcrumb_title_link
+ %li.gl-breadcrumb-item{ data: { testid: 'breadcrumb-current-link' } }
+ = link_to(@breadcrumb_title, breadcrumb_title_link)
-# haml-lint:disable InlineJavaScript
%script{ type: 'application/ld+json' }
:plain
diff --git a/app/views/layouts/nav/breadcrumbs/_collapsed_inline_list.html.haml b/app/views/layouts/nav/breadcrumbs/_collapsed_inline_list.html.haml
index dd2d23320be..3894501bbbb 100644
--- a/app/views/layouts/nav/breadcrumbs/_collapsed_inline_list.html.haml
+++ b/app/views/layouts/nav/breadcrumbs/_collapsed_inline_list.html.haml
@@ -1,11 +1,9 @@
- dropdown_location = local_assigns.fetch(:location, nil)
- button_tooltip = local_assigns.fetch(:title, _("Show all breadcrumbs"))
- if defined?(@breadcrumb_collapsed_links) && @breadcrumb_collapsed_links.key?(dropdown_location)
- %li.expander
+ %li.expander.gl-breadcrumb-item
%button.text-expander.has-tooltip.js-breadcrumbs-collapsed-expander{ type: "button", data: { container: "body" }, "aria-label": button_tooltip, title: button_tooltip }
= sprite_icon("ellipsis_h", size: 12)
- = sprite_icon("chevron-lg-right", size: 8, css_class: "breadcrumbs-list-angle")
- @breadcrumb_collapsed_links[dropdown_location].each_with_index do |link, index|
- %li{ :class => "gl-display-none! breadcrumbs-detail-item" }
+ %li.gl-breadcrumb-item{ :class => "gl-display-none!" }
= link
- = sprite_icon("chevron-lg-right", size: 8, css_class: "breadcrumbs-list-angle")
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index db255f222e0..89047bda445 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -2694,6 +2694,15 @@
:weight: 1
:idempotent: true
:tags: []
+- :name: ci_catalog_resources_process_sync_events
+ :worker_name: Ci::Catalog::Resources::ProcessSyncEventsWorker
+ :feature_category: :pipeline_composition
+ :has_external_dependencies: false
+ :urgency: :high
+ :resource_boundary: :unknown
+ :weight: 1
+ :idempotent: true
+ :tags: []
- :name: ci_delete_objects
:worker_name: Ci::DeleteObjectsWorker
:feature_category: :continuous_integration
diff --git a/app/workers/ci/catalog/resources/process_sync_events_worker.rb b/app/workers/ci/catalog/resources/process_sync_events_worker.rb
new file mode 100644
index 00000000000..a577f36858d
--- /dev/null
+++ b/app/workers/ci/catalog/resources/process_sync_events_worker.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Ci
+ module Catalog
+ module Resources
+ # This worker can be called multiple times simultaneously but only one can process events
+ # at a time. This is ensured by `try_obtain_lease` in `Ci::ProcessSyncEventsService`.
+ class ProcessSyncEventsWorker
+ include ApplicationWorker
+
+ feature_category :pipeline_composition
+
+ data_consistency :always # rubocop:disable SidekiqLoadBalancing/WorkerDataConsistency -- We should not sync stale data
+ urgency :high
+
+ idempotent!
+ deduplicate :until_executed, if_deduplicated: :reschedule_once, ttl: 1.minute
+
+ def perform
+ results = ::Ci::ProcessSyncEventsService.new(
+ ::Ci::Catalog::Resources::SyncEvent, ::Ci::Catalog::Resource
+ ).execute
+
+ results.each do |key, value|
+ log_extra_metadata_on_done(key, value)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/config/feature_flags/development/ci_process_catalog_resource_sync_events.yml b/config/feature_flags/development/ci_process_catalog_resource_sync_events.yml
new file mode 100644
index 00000000000..374ccc4409a
--- /dev/null
+++ b/config/feature_flags/development/ci_process_catalog_resource_sync_events.yml
@@ -0,0 +1,8 @@
+---
+name: ci_process_catalog_resource_sync_events
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137238
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/432963
+milestone: '16.7'
+type: development
+group: group::pipeline authoring
+default_enabled: false
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index 244af6b0ccf..14fb285f4f8 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -1134,7 +1134,8 @@ production: &base
# args: {
# client_id: 'YOUR_AUTH0_CLIENT_ID',
# client_secret: 'YOUR_AUTH0_CLIENT_SECRET',
- # namespace: 'YOUR_AUTH0_DOMAIN' } }
+ # domain: 'YOUR_AUTH0_DOMAIN',
+ # scope: 'openid profile email' } }
# FortiAuthenticator settings
forti_authenticator:
@@ -1618,7 +1619,8 @@ test:
args: {
client_id: 'YOUR_AUTH0_CLIENT_ID',
client_secret: 'YOUR_AUTH0_CLIENT_SECRET',
- namespace: 'YOUR_AUTH0_DOMAIN' } }
+ domain: 'YOUR_AUTH0_DOMAIN',
+ scope: 'openid profile email' } }
- { name: 'salesforce',
app_id: 'YOUR_CLIENT_ID',
app_secret: 'YOUR_CLIENT_SECRET'
diff --git a/config/initializers/postgres_partitioning.rb b/config/initializers/postgres_partitioning.rb
index 458feacba0d..5086f6f7da2 100644
--- a/config/initializers/postgres_partitioning.rb
+++ b/config/initializers/postgres_partitioning.rb
@@ -12,7 +12,8 @@ Gitlab::Database::Partitioning.register_models(
CommitStatus,
BatchedGitRefUpdates::Deletion,
Users::ProjectVisit,
- Users::GroupVisit
+ Users::GroupVisit,
+ Ci::Catalog::Resources::SyncEvent
])
if Gitlab.ee?
diff --git a/config/routes/project.rb b/config/routes/project.rb
index 9016a683404..7064e1e19db 100644
--- a/config/routes/project.rb
+++ b/config/routes/project.rb
@@ -186,6 +186,12 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
end
resources :deploy_keys, constraints: { id: /\d+/ }, only: [:index, :new, :create, :edit, :update] do
+ collection do
+ get :enabled_keys
+ get :available_project_keys
+ get :available_public_keys
+ end
+
member do
put :enable
put :disable
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index 77503814158..25a4299ab70 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -157,6 +157,8 @@
- 1
- - ci_cancel_redundant_pipelines
- 1
+- - ci_catalog_resources_process_sync_events
+ - 1
- - ci_delete_objects
- 1
- - ci_initialize_pipelines_iid_sequence
diff --git a/db/docs/p_catalog_resource_sync_events.yml b/db/docs/p_catalog_resource_sync_events.yml
new file mode 100644
index 00000000000..b74a644fd74
--- /dev/null
+++ b/db/docs/p_catalog_resource_sync_events.yml
@@ -0,0 +1,13 @@
+---
+table_name: p_catalog_resource_sync_events
+classes:
+- Ci::Catalog::Resources::SyncEvent
+feature_categories:
+- pipeline_composition
+description: A queue of catalog resources that need to be synchronized with data from
+ their associated `projects` records.
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137238
+milestone: '16.7'
+gitlab_schema: gitlab_main_cell
+sharding_key:
+ project_id: projects
diff --git a/db/migrate/20231124191759_add_catalog_resource_sync_events_table.rb b/db/migrate/20231124191759_add_catalog_resource_sync_events_table.rb
new file mode 100644
index 00000000000..d4c628a1770
--- /dev/null
+++ b/db/migrate/20231124191759_add_catalog_resource_sync_events_table.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class AddCatalogResourceSyncEventsTable < Gitlab::Database::Migration[2.2]
+ milestone '16.7'
+
+ enable_lock_retries!
+
+ def up
+ options = {
+ primary_key: [:id, :partition_id],
+ options: 'PARTITION BY LIST (partition_id)',
+ if_not_exists: true
+ }
+
+ create_table(:p_catalog_resource_sync_events, **options) do |t|
+ t.bigserial :id, null: false
+ # We will not bother with foreign keys as they come with a performance cost; they will get cleaned up over time.
+ t.bigint :catalog_resource_id, null: false
+ t.bigint :project_id, null: false
+ t.bigint :partition_id, null: false, default: 1
+ t.integer :status, null: false, default: 1, limit: 2
+ t.timestamps_with_timezone null: false, default: -> { 'NOW()' }
+
+ t.index :id,
+ where: 'status = 1',
+ name: :index_p_catalog_resource_sync_events_on_id_where_pending
+ end
+
+ connection.execute(<<~SQL)
+ CREATE TABLE IF NOT EXISTS gitlab_partitions_dynamic.p_catalog_resource_sync_events_1
+ PARTITION OF p_catalog_resource_sync_events
+ FOR VALUES IN (1);
+ SQL
+ end
+
+ def down
+ drop_table :p_catalog_resource_sync_events
+ end
+end
diff --git a/db/migrate/20231124282441_add_catalog_resource_sync_event_triggers.rb b/db/migrate/20231124282441_add_catalog_resource_sync_event_triggers.rb
new file mode 100644
index 00000000000..01f87d61e02
--- /dev/null
+++ b/db/migrate/20231124282441_add_catalog_resource_sync_event_triggers.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class AddCatalogResourceSyncEventTriggers < Gitlab::Database::Migration[2.2]
+ milestone '16.7'
+
+ include Gitlab::Database::SchemaHelpers
+
+ enable_lock_retries!
+
+ EVENTS_TABLE_NAME = 'p_catalog_resource_sync_events'
+ RESOURCES_TABLE_NAME = 'catalog_resources'
+ PROJECTS_TABLE_NAME = 'projects'
+
+ TRIGGER_FUNCTION_NAME = 'insert_catalog_resource_sync_event'
+ TRIGGER_NAME = 'trigger_catalog_resource_sync_event_on_project_update'
+
+ def up
+ create_trigger_function(TRIGGER_FUNCTION_NAME, replace: true) do
+ <<~SQL
+ INSERT INTO #{EVENTS_TABLE_NAME} (catalog_resource_id, project_id)
+ SELECT id, OLD.id FROM #{RESOURCES_TABLE_NAME}
+ WHERE project_id = OLD.id;
+ RETURN NULL;
+ SQL
+ end
+
+ create_trigger(
+ PROJECTS_TABLE_NAME, TRIGGER_NAME, TRIGGER_FUNCTION_NAME, fires: 'AFTER UPDATE'
+ ) do
+ <<~SQL
+ WHEN (
+ OLD.name IS DISTINCT FROM NEW.name OR
+ OLD.description IS DISTINCT FROM NEW.description OR
+ OLD.visibility_level IS DISTINCT FROM NEW.visibility_level
+ )
+ SQL
+ end
+ end
+
+ def down
+ drop_trigger(PROJECTS_TABLE_NAME, TRIGGER_NAME)
+ drop_function(TRIGGER_FUNCTION_NAME)
+ end
+end
diff --git a/db/schema_migrations/20231124191759 b/db/schema_migrations/20231124191759
new file mode 100644
index 00000000000..adbafd9b2bd
--- /dev/null
+++ b/db/schema_migrations/20231124191759
@@ -0,0 +1 @@
+32a80f29a5a3511a8dfdea203874aecde5a58eab6665ba127379c9c2e01d254f \ No newline at end of file
diff --git a/db/schema_migrations/20231124282441 b/db/schema_migrations/20231124282441
new file mode 100644
index 00000000000..78c0636635f
--- /dev/null
+++ b/db/schema_migrations/20231124282441
@@ -0,0 +1 @@
+2bdaabfe2fa23ce334af1878b1234618b4717f05a9b68f7f9839f48c7f38f410 \ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index 286cff2b88c..a6a80d9e773 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -290,6 +290,18 @@ BEGIN
END
$$;
+CREATE FUNCTION insert_catalog_resource_sync_event() RETURNS trigger
+ LANGUAGE plpgsql
+ AS $$
+BEGIN
+INSERT INTO p_catalog_resource_sync_events (catalog_resource_id, project_id)
+SELECT id, OLD.id FROM catalog_resources
+WHERE project_id = OLD.id;
+RETURN NULL;
+
+END
+$$;
+
CREATE FUNCTION insert_into_loose_foreign_keys_deleted_records() RETURNS trigger
LANGUAGE plpgsql
AS $$
@@ -909,6 +921,17 @@ CREATE TABLE p_batched_git_ref_updates_deletions (
)
PARTITION BY LIST (partition_id);
+CREATE TABLE p_catalog_resource_sync_events (
+ id bigint NOT NULL,
+ catalog_resource_id bigint NOT NULL,
+ project_id bigint NOT NULL,
+ partition_id bigint DEFAULT 1 NOT NULL,
+ status smallint DEFAULT 1 NOT NULL,
+ created_at timestamp with time zone DEFAULT now() NOT NULL,
+ updated_at timestamp with time zone DEFAULT now() NOT NULL
+)
+PARTITION BY LIST (partition_id);
+
CREATE TABLE p_ci_finished_build_ch_sync_events (
build_id bigint NOT NULL,
partition bigint DEFAULT 1 NOT NULL,
@@ -20069,6 +20092,15 @@ CREATE SEQUENCE p_batched_git_ref_updates_deletions_id_seq
ALTER SEQUENCE p_batched_git_ref_updates_deletions_id_seq OWNED BY p_batched_git_ref_updates_deletions.id;
+CREATE SEQUENCE p_catalog_resource_sync_events_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+ALTER SEQUENCE p_catalog_resource_sync_events_id_seq OWNED BY p_catalog_resource_sync_events.id;
+
CREATE SEQUENCE p_ci_job_annotations_id_seq
START WITH 1
INCREMENT BY 1
@@ -26906,6 +26938,8 @@ ALTER TABLE ONLY organizations ALTER COLUMN id SET DEFAULT nextval('organization
ALTER TABLE ONLY p_batched_git_ref_updates_deletions ALTER COLUMN id SET DEFAULT nextval('p_batched_git_ref_updates_deletions_id_seq'::regclass);
+ALTER TABLE ONLY p_catalog_resource_sync_events ALTER COLUMN id SET DEFAULT nextval('p_catalog_resource_sync_events_id_seq'::regclass);
+
ALTER TABLE ONLY p_ci_builds_metadata ALTER COLUMN id SET DEFAULT nextval('ci_builds_metadata_id_seq'::regclass);
ALTER TABLE ONLY p_ci_job_annotations ALTER COLUMN id SET DEFAULT nextval('p_ci_job_annotations_id_seq'::regclass);
@@ -29265,6 +29299,9 @@ ALTER TABLE ONLY organizations
ALTER TABLE ONLY p_batched_git_ref_updates_deletions
ADD CONSTRAINT p_batched_git_ref_updates_deletions_pkey PRIMARY KEY (id, partition_id);
+ALTER TABLE ONLY p_catalog_resource_sync_events
+ ADD CONSTRAINT p_catalog_resource_sync_events_pkey PRIMARY KEY (id, partition_id);
+
ALTER TABLE ONLY p_ci_finished_build_ch_sync_events
ADD CONSTRAINT p_ci_finished_build_ch_sync_events_pkey PRIMARY KEY (build_id, partition);
@@ -33770,6 +33807,8 @@ CREATE INDEX index_organization_users_on_user_id ON organization_users USING btr
CREATE UNIQUE INDEX index_organizations_on_unique_name_per_group ON customer_relations_organizations USING btree (group_id, lower(name), id);
+CREATE INDEX index_p_catalog_resource_sync_events_on_id_where_pending ON ONLY p_catalog_resource_sync_events USING btree (id) WHERE (status = 1);
+
CREATE UNIQUE INDEX index_p_ci_job_annotations_on_partition_id_job_id_name ON ONLY p_ci_job_annotations USING btree (partition_id, job_id, name);
CREATE INDEX index_p_ci_runner_machine_builds_on_runner_machine_id ON ONLY p_ci_runner_machine_builds USING btree (runner_machine_id);
@@ -37032,6 +37071,8 @@ CREATE TRIGGER trigger_10ee1357e825 BEFORE INSERT OR UPDATE ON p_ci_builds FOR E
CREATE TRIGGER trigger_b2d852e1e2cb BEFORE INSERT OR UPDATE ON ci_pipelines FOR EACH ROW EXECUTE FUNCTION trigger_b2d852e1e2cb();
+CREATE TRIGGER trigger_catalog_resource_sync_event_on_project_update AFTER UPDATE ON projects FOR EACH ROW WHEN ((((old.name)::text IS DISTINCT FROM (new.name)::text) OR (old.description IS DISTINCT FROM new.description) OR (old.visibility_level IS DISTINCT FROM new.visibility_level))) EXECUTE FUNCTION insert_catalog_resource_sync_event();
+
CREATE TRIGGER trigger_delete_project_namespace_on_project_delete AFTER DELETE ON projects FOR EACH ROW WHEN ((old.project_namespace_id IS NOT NULL)) EXECUTE FUNCTION delete_associated_project_namespace();
CREATE TRIGGER trigger_eaec934fe6b2 BEFORE INSERT OR UPDATE ON system_note_metadata FOR EACH ROW EXECUTE FUNCTION trigger_eaec934fe6b2();
diff --git a/doc/administration/geo/index.md b/doc/administration/geo/index.md
index 65c54d85704..27c7c73c553 100644
--- a/doc/administration/geo/index.md
+++ b/doc/administration/geo/index.md
@@ -115,15 +115,15 @@ The following are required to run Geo:
The following operating systems are known to ship with a current version of OpenSSH:
- [CentOS](https://www.centos.org) 7.4 or later
- [Ubuntu](https://ubuntu.com) 16.04 or later
-- PostgreSQL 12 or 13 with [Streaming Replication](https://wiki.postgresql.org/wiki/Streaming_Replication)
+- [Supported PostgreSQL versions](https://about.gitlab.com/handbook/engineering/development/enablement/data_stores/database/postgresql-upgrade-cadence.html) for your GitLab releases with [Streaming Replication](https://wiki.postgresql.org/wiki/Streaming_Replication).
- Note,[PostgreSQL 12 is deprecated](../../update/deprecations.md#postgresql-12-deprecated) and is removed in GitLab 16.0.
-- Git 2.9 or later
-- Git-lfs 2.4.2 or later on the user side when using LFS
-- All sites must run the same GitLab version.
- All sites must run [the same PostgreSQL versions](setup/database.md#postgresql-replication).
- If using different operating system versions between Geo sites,
[check OS locale data compatibility](replication/troubleshooting.md#check-os-locale-data-compatibility)
across Geo sites to avoid silent corruption of database indexes.
+- Git 2.9 or later
+- Git-lfs 2.4.2 or later on the user side when using LFS
+- All sites must run the same GitLab version.
- All sites must define the same [repository storages](../repository_storage_paths.md).
Additionally, check the GitLab [minimum requirements](../../install/requirements.md),
diff --git a/doc/architecture/blueprints/cells/routing-service.md b/doc/architecture/blueprints/cells/routing-service.md
index ca1f1d3c52a..9efdbdf3f91 100644
--- a/doc/architecture/blueprints/cells/routing-service.md
+++ b/doc/architecture/blueprints/cells/routing-service.md
@@ -65,7 +65,7 @@ For example:
| Security | only authorized cells can be routed to | high |
| Single domain | e.g. GitLab.com | high |
| Caching | can cache routing information for performance | high |
-| Low latency | small overhead for user requests | high |
+| [50 ms of increased latency](#low-latency) | | high |
| Path-based | can make routing decision based on path | high |
| Complexity | the routing service should be configuration-driven and small | high |
| Stateless | does not need database, Cells provide all routing information | medium |
@@ -74,6 +74,106 @@ For example:
| Self-managed | can be eventually used by [self-managed](goals.md#self-managed) | low |
| Regional | can route requests to different [regions](goals.md#regions) | low |
+### Low Latency
+
+The target latency for routing service **should be less than 50 _ms_**.
+
+Looking at the `urgency: high` request we don't have a lot of headroom on the p50.
+Adding an extra 50 _ms_ allows us to still be in or SLO on the p95 level.
+
+There is 3 primary entry points for the application; [`web`](https://gitlab.com/gitlab-com/runbooks/-/blob/5d8248314b343bef15a4c021ac33978525f809e3/services/service-catalog.yml#L492-537), [`api`](https://gitlab.com/gitlab-com/runbooks/-/blob/5d8248314b343bef15a4c021ac33978525f809e3/services/service-catalog.yml#L18-62), and [`git`](https://gitlab.com/gitlab-com/runbooks/-/blob/5d8248314b343bef15a4c021ac33978525f809e3/services/service-catalog.yml#L589-638).
+Each service is assigned a Service Level Indicator (SLI) based on latency using the [apdex](https://www.apdex.org/wp-content/uploads/2020/09/ApdexTechnicalSpecificationV11_000.pdf) standard.
+The corresponding Service Level Objectives (SLOs) for these SLIs require low latencies for large amount of requests.
+It's crucial to ensure that the addition of the routing layer in front of these services does not impact the SLIs.
+The routing layer is a proxy for these services, and we lack a comprehensive SLI monitoring system for the entire request flow (including components like the Edge network and Load Balancers) we use the SLIs for `web`, `git`, and `api` as a target.
+
+The main SLI we use is the [rails requests](../../../development/application_slis/rails_request.md).
+It has multiple `satisfied` targets (apdex) depending on the [request urgency](../../../development/application_slis/rails_request.md#how-to-adjust-the-urgency):
+
+| Urgency | Duration in ms |
+|------------|----------------|
+| `:high` | 250 _ms_ |
+| `:medium` | 500 _ms_ |
+| `:default` | 1000 _ms_ |
+| `:low` | 5000 _ms_ |
+
+#### Analysis
+
+The way we calculate the headroom we have is by using the following:
+
+```math
+\mathrm{Headroom}\ {ms} = \mathrm{Satisfied}\ {ms} - \mathrm{Duration}\ {ms}
+```
+
+**`web`**:
+
+| Target Duration | Percentile | Headroom |
+|-----------------|------------|-----------|
+| 5000 _ms_ | p99 | 4000 _ms_ |
+| 5000 _ms_ | p95 | 4500 _ms_ |
+| 5000 _ms_ | p90 | 4600 _ms_ |
+| 5000 _ms_ | p50 | 4900 _ms_ |
+| 1000 _ms_ | p99 | 500 _ms_ |
+| 1000 _ms_ | p95 | 740 _ms_ |
+| 1000 _ms_ | p90 | 840 _ms_ |
+| 1000 _ms_ | p50 | 900 _ms_ |
+| 500 _ms_ | p99 | 0 _ms_ |
+| 500 _ms_ | p95 | 60 _ms_ |
+| 500 _ms_ | p90 | 100 _ms_ |
+| 500 _ms_ | p50 | 400 _ms_ |
+| 250 _ms_ | p99 | 140 _ms_ |
+| 250 _ms_ | p95 | 170 _ms_ |
+| 250 _ms_ | p90 | 180 _ms_ |
+| 250 _ms_ | p50 | 200 _ms_ |
+
+_Analysis was done in <https://gitlab.com/gitlab-org/gitlab/-/issues/432934#note_1667993089>_
+
+**`api`**:
+
+| Target Duration | Percentile | Headroom |
+|-----------------|------------|-----------|
+| 5000 _ms_ | p99 | 3500 _ms_ |
+| 5000 _ms_ | p95 | 4300 _ms_ |
+| 5000 _ms_ | p90 | 4600 _ms_ |
+| 5000 _ms_ | p50 | 4900 _ms_ |
+| 1000 _ms_ | p99 | 440 _ms_ |
+| 1000 _ms_ | p95 | 750 _ms_ |
+| 1000 _ms_ | p90 | 830 _ms_ |
+| 1000 _ms_ | p50 | 950 _ms_ |
+| 500 _ms_ | p99 | 450 _ms_ |
+| 500 _ms_ | p95 | 480 _ms_ |
+| 500 _ms_ | p90 | 490 _ms_ |
+| 500 _ms_ | p50 | 490 _ms_ |
+| 250 _ms_ | p99 | 90 _ms_ |
+| 250 _ms_ | p95 | 170 _ms_ |
+| 250 _ms_ | p90 | 210 _ms_ |
+| 250 _ms_ | p50 | 230 _ms_ |
+
+_Analysis was done in <https://gitlab.com/gitlab-org/gitlab/-/issues/432934#note_1669995479>_
+
+**`git`**:
+
+| Target Duration | Percentile | Headroom |
+|-----------------|------------|-----------|
+| 5000 _ms_ | p99 | 3760 _ms_ |
+| 5000 _ms_ | p95 | 4280 _ms_ |
+| 5000 _ms_ | p90 | 4430 _ms_ |
+| 5000 _ms_ | p50 | 4900 _ms_ |
+| 1000 _ms_ | p99 | 500 _ms_ |
+| 1000 _ms_ | p95 | 750 _ms_ |
+| 1000 _ms_ | p90 | 800 _ms_ |
+| 1000 _ms_ | p50 | 900 _ms_ |
+| 500 _ms_ | p99 | 280 _ms_ |
+| 500 _ms_ | p95 | 370 _ms_ |
+| 500 _ms_ | p90 | 400 _ms_ |
+| 500 _ms_ | p50 | 430 _ms_ |
+| 250 _ms_ | p99 | 200 _ms_ |
+| 250 _ms_ | p95 | 230 _ms_ |
+| 250 _ms_ | p90 | 240 _ms_ |
+| 250 _ms_ | p50 | 240 _ms_ |
+
+_Analysis was done in <https://gitlab.com/gitlab-org/gitlab/-/issues/432934#note_1671385680>_
+
## Non-Goals
Not yet defined.
diff --git a/doc/integration/auth0.md b/doc/integration/auth0.md
index 4e90ade6620..bda40c0655e 100644
--- a/doc/integration/auth0.md
+++ b/doc/integration/auth0.md
@@ -80,7 +80,7 @@ application.
1. Replace `<your_auth0_client_id>` with the client ID from the Auth0 Console page.
1. Replace `<your_auth0_client_secret>` with the client secret from the Auth0 Console page.
-1. Replace `<your_auth0_client_secret>` with the domain from the Auth0 Console page.
+1. Replace `<your_auth0_domain>` with the domain from the Auth0 Console page.
1. Reconfigure or restart GitLab, depending on your installation method:
- If you installed using the Linux package,
[reconfigure GitLab](../administration/restart_gitlab.md#reconfigure-a-linux-package-installation).
diff --git a/doc/update/versions/gitlab_16_changes.md b/doc/update/versions/gitlab_16_changes.md
index 354a1596a8c..14d028818f3 100644
--- a/doc/update/versions/gitlab_16_changes.md
+++ b/doc/update/versions/gitlab_16_changes.md
@@ -16,7 +16,7 @@ For more information about upgrading GitLab Helm Chart, see [the release notes f
## Issues to be aware of when upgrading from 15.11
-- [PostgreSQL 12 is not supported from v16](../../update/deprecations.md#postgresql-12-deprecated). Upgrade your PostgreSQL to at least 13.6 before upgrading to GitLab v16.0 or higher.
+- [PostgreSQL 12 is not supported starting from GitLab 16](../../update/deprecations.md#postgresql-12-deprecated). Upgrade PostgreSQL to at least version 13.6 before upgrading to GitLab 16.0 or later.
- Some GitLab installations must upgrade to GitLab 16.0 before upgrading to any other version. For more information, see
[Long-running user type data change](#long-running-user-type-data-change).
- Other installations can skip 16.0, 16.1, and 16.2 as the first required stop on the upgrade path is 16.3. Review the notes for those intermediate
diff --git a/doc/user/custom_roles.md b/doc/user/custom_roles.md
index 7d7a1b3f8bd..69ba3a3d900 100644
--- a/doc/user/custom_roles.md
+++ b/doc/user/custom_roles.md
@@ -153,6 +153,9 @@ To add a user to your group with a custom role:
The new member with custom role and custom permissions appears on the [group's members list](group/index.md#view-group-members).
+NOTE:
+Most custom roles are considered [billable users that use a seat](#billing-and-seat-usage). When you add a user to your group with a custom role, a warning is displayed if you are about to incur additional charges for having more seats than are included in your subscription.
+
### Change a member's custom role
To change a group member's custom role:
diff --git a/doc/user/project/repository/code_suggestions/saas.md b/doc/user/project/repository/code_suggestions/saas.md
index 61036009f10..77a364f14ff 100644
--- a/doc/user/project/repository/code_suggestions/saas.md
+++ b/doc/user/project/repository/code_suggestions/saas.md
@@ -21,9 +21,9 @@ Learn about [data usage when using Code Suggestions](index.md#code-suggestions-d
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121079) in GitLab 16.1 as [Beta](../../../../policy/experiment-beta-support.md#beta).
-You must enable Code Suggestions for both your user account and your group:
+You must enable Code Suggestions for both your user account and your top-level group:
-- [Enable Code Suggestions for all group members](../../../group/manage.md#enable-code-suggestions). (You must be a group owner).
+- [Enable Code Suggestions for your top-level group](../../../group/manage.md#enable-code-suggestions) (you must be a group owner).
- [Enable Code Suggestions for your own account](../../../profile/preferences.md#enable-code-suggestions).
NOTE:
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index 901f79bb10e..01bc457c6d4 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -6404,9 +6404,6 @@ msgstr ""
msgid "Archive project"
msgstr ""
-msgid "Archive test case"
-msgstr ""
-
msgid "Archived"
msgstr ""
@@ -40181,9 +40178,6 @@ msgstr ""
msgid "Reopen milestone"
msgstr ""
-msgid "Reopen test case"
-msgstr ""
-
msgid "Reopen this %{quick_action_target}"
msgstr ""
diff --git a/package.json b/package.json
index 6bf620b2896..76c1892c608 100644
--- a/package.json
+++ b/package.json
@@ -60,7 +60,7 @@
"@gitlab/favicon-overlay": "2.0.0",
"@gitlab/fonts": "^1.3.0",
"@gitlab/svgs": "3.72.0",
- "@gitlab/ui": "^71.3.0",
+ "@gitlab/ui": "^71.6.0",
"@gitlab/visual-review-tools": "1.7.3",
"@gitlab/web-ide": "^0.0.1-dev-20231129035648",
"@mattiasbuelens/web-streams-adapter": "^0.1.0",
diff --git a/spec/controllers/projects/deploy_keys_controller_spec.rb b/spec/controllers/projects/deploy_keys_controller_spec.rb
index 96addb4b6c5..43f089cede9 100644
--- a/spec/controllers/projects/deploy_keys_controller_spec.rb
+++ b/spec/controllers/projects/deploy_keys_controller_spec.rb
@@ -2,7 +2,7 @@
require 'spec_helper'
-RSpec.describe Projects::DeployKeysController do
+RSpec.describe Projects::DeployKeysController, feature_category: :continuous_delivery do
let(:project) { create(:project, :repository) }
let(:user) { create(:user) }
let(:admin) { create(:admin) }
@@ -13,60 +13,94 @@ RSpec.describe Projects::DeployKeysController do
sign_in(user)
end
- describe 'GET index' do
- let(:params) do
- { namespace_id: project.namespace, project_id: project }
- end
+ describe 'GET actions' do
+ let_it_be(:project) { create(:project, :repository) }
+ let_it_be(:user) { create(:user) }
- context 'when html requested' do
- it 'redirects to project settings with the correct anchor' do
- get :index, params: params
+ let_it_be(:accessible_project) { create(:project, :internal).tap { |p| p.add_developer(user) } }
+ let_it_be(:inaccessible_project) { create(:project, :internal) }
+ let_it_be(:project_private) { create(:project, :private) }
- expect(response).to redirect_to(project_settings_repository_path(project, anchor: 'js-deploy-keys-settings'))
- end
+ let_it_be(:deploy_key_for_target_project) do
+ create(:deploy_keys_project, project: project, deploy_key: create(:deploy_key))
end
- context 'when json requested' do
- let(:project2) { create(:project, :internal) }
- let(:project_private) { create(:project, :private) }
+ let_it_be(:deploy_key_for_accessible_project) do
+ create(:deploy_keys_project, project: accessible_project, deploy_key: create(:deploy_key))
+ end
- let(:deploy_key_internal) { create(:deploy_key) }
- let(:deploy_key_actual) { create(:deploy_key) }
- let!(:deploy_key_public) { create(:deploy_key, public: true) }
+ let_it_be(:deploy_key_for_inaccessible_project) do
+ create(:deploy_keys_project, project: inaccessible_project, deploy_key: create(:deploy_key))
+ end
- let!(:deploy_keys_project_internal) do
- create(:deploy_keys_project, project: project2, deploy_key: deploy_key_internal)
- end
+ let_it_be(:deploy_keys_project_private) do
+ create(:deploy_keys_project, project: project_private, deploy_key: create(:another_deploy_key))
+ end
- let!(:deploy_keys_project_actual) do
- create(:deploy_keys_project, project: project, deploy_key: deploy_key_actual)
- end
+ let_it_be(:deploy_key_public) { create(:deploy_key, public: true) }
- let!(:deploy_keys_project_private) do
- create(:deploy_keys_project, project: project_private, deploy_key: create(:another_deploy_key))
+ describe 'GET index' do
+ let(:params) do
+ { namespace_id: project.namespace, project_id: project }
end
- context 'when user has access to all projects where deploy keys are used' do
- before do
- project2.add_developer(user)
+ context 'when html requested' do
+ it 'redirects to project settings with the correct anchor' do
+ get :index, params: params
+
+ expect(response).to redirect_to(project_settings_repository_path(project, anchor: 'js-deploy-keys-settings'))
end
+ end
+ context 'when json requested' do
it 'returns json in a correct format' do
get :index, params: params.merge(format: :json)
expect(json_response.keys).to match_array(%w[enabled_keys available_project_keys public_keys])
- expect(json_response['enabled_keys'].count).to eq(1)
- expect(json_response['available_project_keys'].count).to eq(1)
- expect(json_response['public_keys'].count).to eq(1)
+ expect(json_response['enabled_keys'].pluck('id')).to match_array(
+ [deploy_key_for_target_project.deploy_key_id]
+ )
+ expect(json_response['available_project_keys'].pluck('id')).to match_array(
+ [deploy_key_for_accessible_project.deploy_key_id]
+ )
+ expect(json_response['public_keys'].pluck('id')).to match_array([deploy_key_public.id])
end
end
+ end
- context 'when user has no access to all projects where deploy keys are used' do
- it 'returns json in a correct format' do
- get :index, params: params.merge(format: :json)
+ describe 'GET enabled_keys' do
+ let(:params) do
+ { namespace_id: project.namespace, project_id: project }
+ end
- expect(json_response['available_project_keys'].count).to eq(0)
- end
+ it 'returns only enabled keys' do
+ get :enabled_keys, params: params.merge(format: :json)
+
+ expect(json_response.pluck("id")).to match_array([deploy_key_for_target_project.deploy_key_id])
+ end
+ end
+
+ describe 'GET available_project_keys' do
+ let(:params) do
+ { namespace_id: project.namespace, project_id: project }
+ end
+
+ it 'returns available project keys' do
+ get :available_project_keys, params: params.merge(format: :json)
+
+ expect(json_response.pluck("id")).to match_array([deploy_key_for_accessible_project.deploy_key_id])
+ end
+ end
+
+ describe 'GET available_public_keys' do
+ let(:params) do
+ { namespace_id: project.namespace, project_id: project }
+ end
+
+ it 'returns available public keys' do
+ get :available_public_keys, params: params.merge(format: :json)
+
+ expect(json_response.pluck("id")).to match_array([deploy_key_public.id])
end
end
end
diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb
index 92835d21bb6..7e3f2a3b61e 100644
--- a/spec/db/schema_spec.rb
+++ b/spec/db/schema_spec.rb
@@ -93,6 +93,7 @@ RSpec.describe 'Database schema', feature_category: :database do
oauth_applications: %w[owner_id],
p_ci_builds: %w[erased_by_id trigger_request_id partition_id auto_canceled_by_partition_id],
p_batched_git_ref_updates_deletions: %w[project_id partition_id],
+ p_catalog_resource_sync_events: %w[catalog_resource_id project_id partition_id],
p_ci_finished_build_ch_sync_events: %w[build_id],
product_analytics_events_experimental: %w[event_id txn_id user_id],
project_build_artifacts_size_refreshes: %w[last_job_artifact_id],
diff --git a/spec/factories/ci/catalog/resources/sync_events.rb b/spec/factories/ci/catalog/resources/sync_events.rb
new file mode 100644
index 00000000000..0579cec648e
--- /dev/null
+++ b/spec/factories/ci/catalog/resources/sync_events.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+ factory :ci_catalog_resource_sync_event, class: 'Ci::Catalog::Resources::SyncEvent' do
+ catalog_resource factory: :ci_catalog_resource
+ project { catalog_resource.project }
+ end
+end
diff --git a/spec/features/explore/user_explores_projects_spec.rb b/spec/features/explore/user_explores_projects_spec.rb
index e1341824bfd..369297ff04a 100644
--- a/spec/features/explore/user_explores_projects_spec.rb
+++ b/spec/features/explore/user_explores_projects_spec.rb
@@ -25,7 +25,7 @@ RSpec.describe 'User explores projects', feature_category: :user_profile do
describe 'breadcrumbs' do
it 'has "Explore" as its root breadcrumb' do
- within '.breadcrumbs-list li:first' do
+ within '.gl-breadcrumb-list li:first' do
expect(page).to have_link('Explore', href: explore_root_path)
end
end
diff --git a/spec/features/projects/commits/user_browses_commits_spec.rb b/spec/features/projects/commits/user_browses_commits_spec.rb
index 3513e249b63..5a0b70532aa 100644
--- a/spec/features/projects/commits/user_browses_commits_spec.rb
+++ b/spec/features/projects/commits/user_browses_commits_spec.rb
@@ -42,8 +42,8 @@ RSpec.describe 'User browses commits', feature_category: :source_code_management
it 'renders breadcrumbs on specific commit path' do
visit project_commits_path(project, project.repository.root_ref + '/files/ruby/regex.rb', limit: 5)
- expect(page).to have_selector('ul.breadcrumb')
- .and have_selector('ul.breadcrumb a', count: 4)
+ expect(page).to have_selector('#content-body ul.breadcrumb')
+ .and have_selector('#content-body ul.breadcrumb a', count: 4)
end
it 'renders diff links to both the previous and current image', :js do
diff --git a/spec/finders/deploy_keys/deploy_keys_finder_spec.rb b/spec/finders/deploy_keys/deploy_keys_finder_spec.rb
new file mode 100644
index 00000000000..f0d3935cc95
--- /dev/null
+++ b/spec/finders/deploy_keys/deploy_keys_finder_spec.rb
@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe DeployKeys::DeployKeysFinder, feature_category: :continuous_delivery do
+ describe '#execute' do
+ let_it_be(:user) { create(:user) }
+ let_it_be(:project) { create(:project, :repository) }
+
+ let_it_be(:accessible_project) { create(:project, :internal).tap { |p| p.add_developer(user) } }
+ let_it_be(:inaccessible_project) { create(:project, :internal) }
+ let_it_be(:project_private) { create(:project, :private) }
+
+ let_it_be(:deploy_key_for_target_project) do
+ create(:deploy_keys_project, project: project, deploy_key: create(:deploy_key))
+ end
+
+ let_it_be(:deploy_key_for_accessible_project) do
+ create(:deploy_keys_project, project: accessible_project, deploy_key: create(:deploy_key))
+ end
+
+ let_it_be(:deploy_key_for_inaccessible_project) do
+ create(:deploy_keys_project, project: inaccessible_project, deploy_key: create(:deploy_key))
+ end
+
+ let_it_be(:deploy_keys_project_private) do
+ create(:deploy_keys_project, project: project_private, deploy_key: create(:another_deploy_key))
+ end
+
+ let_it_be(:deploy_key_public) { create(:deploy_key, public: true) }
+
+ let(:params) { {} }
+
+ subject(:result) { described_class.new(project, user, params).execute }
+
+ context 'with access' do
+ before_all do
+ project.add_maintainer(user)
+ end
+
+ context 'when filtering for enabled_keys' do
+ let(:params) { { filter: :enabled_keys } }
+
+ it 'returns the correct result' do
+ expect(result.map(&:id)).to match_array([deploy_key_for_target_project.deploy_key_id])
+ end
+ end
+
+ context 'when filtering for available project keys' do
+ let(:params) { { filter: :available_project_keys } }
+
+ it 'returns the correct result' do
+ expect(result.map(&:id)).to match_array([deploy_key_for_accessible_project.deploy_key_id])
+ end
+ end
+
+ context 'when filtering for available public keys' do
+ let(:params) { { filter: :available_public_keys } }
+
+ it 'returns the correct result' do
+ expect(result.map(&:id)).to match_array([deploy_key_public.id])
+ end
+ end
+
+ context 'when there are no set filters' do
+ it 'returns an empty collection' do
+ expect(result).to eq DeployKey.none
+ end
+ end
+ end
+
+ context 'without access' do
+ it 'returns an empty collection' do
+ expect(result).to eq DeployKey.none
+ end
+ end
+ end
+end
diff --git a/spec/frontend/vue_shared/issuable/sidebar/components/issuable_sidebar_root_spec.js b/spec/frontend/vue_shared/issuable/sidebar/components/issuable_sidebar_root_spec.js
index f2509aead77..d5c6ece8cb5 100644
--- a/spec/frontend/vue_shared/issuable/sidebar/components/issuable_sidebar_root_spec.js
+++ b/spec/frontend/vue_shared/issuable/sidebar/components/issuable_sidebar_root_spec.js
@@ -1,3 +1,4 @@
+import { GlButton, GlIcon } from '@gitlab/ui';
import { GlBreakpointInstance as bp } from '@gitlab/ui/dist/utils';
import { nextTick } from 'vue';
import Cookies from '~/lib/utils/cookies';
@@ -18,6 +19,10 @@ const createComponent = () => {
<button class="js-todo">Todo</button>
`,
},
+ stubs: {
+ GlButton,
+ GlIcon,
+ },
});
};
@@ -62,9 +67,8 @@ describe('IssuableSidebarRoot', () => {
const buttonEl = findToggleSidebarButton();
expect(buttonEl.exists()).toBe(true);
- expect(buttonEl.attributes('title')).toBe('Toggle sidebar');
- expect(buttonEl.find('span').text()).toBe('Collapse sidebar');
- expect(wrapper.findByTestId('icon-collapse').isVisible()).toBe(true);
+ expect(buttonEl.attributes('title')).toBe('Collapse sidebar');
+ expect(wrapper.findByTestId('chevron-double-lg-right-icon').isVisible()).toBe(true);
});
describe('when collapsing the sidebar', () => {
@@ -116,12 +120,12 @@ describe('IssuableSidebarRoot', () => {
assertPageLayoutClasses({ isExpanded: false });
});
- it('renders sidebar toggle button with text and icon', () => {
+ it('renders sidebar toggle button with title and icon', () => {
const buttonEl = findToggleSidebarButton();
expect(buttonEl.exists()).toBe(true);
- expect(buttonEl.attributes('title')).toBe('Toggle sidebar');
- expect(wrapper.findByTestId('icon-expand').isVisible()).toBe(true);
+ expect(buttonEl.attributes('title')).toBe('Expand sidebar');
+ expect(wrapper.findByTestId('chevron-double-lg-left-icon').isVisible()).toBe(true);
});
});
diff --git a/spec/helpers/groups_helper_spec.rb b/spec/helpers/groups_helper_spec.rb
index 2856b8004cf..fe8ed2691fb 100644
--- a/spec/helpers/groups_helper_spec.rb
+++ b/spec/helpers/groups_helper_spec.rb
@@ -93,7 +93,7 @@ RSpec.describe GroupsHelper, feature_category: :groups_and_projects do
shared_examples 'correct ancestor order' do
it 'outputs the groups in the correct order' do
expect(subject)
- .to match(%r{<li><a.*>#{deep_nested_group.name}.*</li>.*<a.*>#{very_deep_nested_group.name}</a>}m)
+ .to match(%r{<li.*><a.*>#{deep_nested_group.name}.*</li>.*<a.*>#{very_deep_nested_group.name}</a>}m)
end
end
diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml
index c91efadbf98..0361e2967e4 100644
--- a/spec/lib/gitlab/import_export/all_models.yml
+++ b/spec/lib/gitlab/import_export/all_models.yml
@@ -523,6 +523,7 @@ container_repositories:
- name
project:
- catalog_resource
+- catalog_resource_sync_events
- catalog_resource_versions
- ci_components
- external_status_checks
@@ -1054,6 +1055,7 @@ catalog_resource:
- project
- catalog_resource_components
- catalog_resource_versions
+ - catalog_resource_sync_events
catalog_resource_versions:
- project
- release
diff --git a/spec/models/ci/catalog/resource_spec.rb b/spec/models/ci/catalog/resource_spec.rb
index a9054e92204..047ba135cd5 100644
--- a/spec/models/ci/catalog/resource_spec.rb
+++ b/spec/models/ci/catalog/resource_spec.rb
@@ -3,7 +3,21 @@
require 'spec_helper'
RSpec.describe Ci::Catalog::Resource, feature_category: :pipeline_composition do
- include_context 'when there are catalog resources with versions'
+ let_it_be(:current_user) { create(:user) }
+
+ let_it_be(:project_a) { create(:project, name: 'A') }
+ let_it_be(:project_b) { create(:project, name: 'B') }
+ let_it_be(:project_c) { create(:project, name: 'C', description: 'B') }
+
+ let_it_be_with_reload(:resource_a) do
+ create(:ci_catalog_resource, project: project_a, latest_released_at: '2023-02-01T00:00:00Z')
+ end
+
+ let_it_be(:resource_b) do
+ create(:ci_catalog_resource, project: project_b, latest_released_at: '2023-01-01T00:00:00Z')
+ end
+
+ let_it_be(:resource_c) { create(:ci_catalog_resource, project: project_c) }
it { is_expected.to belong_to(:project) }
@@ -17,6 +31,11 @@ RSpec.describe Ci::Catalog::Resource, feature_category: :pipeline_composition do
have_many(:versions).class_name('Ci::Catalog::Resources::Version').with_foreign_key(:catalog_resource_id))
end
+ it do
+ is_expected.to(
+ have_many(:sync_events).class_name('Ci::Catalog::Resources::SyncEvent').with_foreign_key(:catalog_resource_id))
+ end
+
it { is_expected.to delegate_method(:avatar_path).to(:project) }
it { is_expected.to delegate_method(:star_count).to(:project) }
@@ -24,17 +43,17 @@ RSpec.describe Ci::Catalog::Resource, feature_category: :pipeline_composition do
describe '.for_projects' do
it 'returns catalog resources for the given project IDs' do
- resources_for_projects = described_class.for_projects(project1.id)
+ resources_for_projects = described_class.for_projects(project_a.id)
- expect(resources_for_projects).to contain_exactly(resource1)
+ expect(resources_for_projects).to contain_exactly(resource_a)
end
end
describe '.search' do
it 'returns catalog resources whose name or description match the search term' do
- resources = described_class.search('Z')
+ resources = described_class.search('B')
- expect(resources).to contain_exactly(resource2, resource3)
+ expect(resources).to contain_exactly(resource_b, resource_c)
end
end
@@ -42,7 +61,7 @@ RSpec.describe Ci::Catalog::Resource, feature_category: :pipeline_composition do
it 'returns catalog resources sorted by descending created at' do
ordered_resources = described_class.order_by_created_at_desc
- expect(ordered_resources.to_a).to eq([resource3, resource2, resource1])
+ expect(ordered_resources.to_a).to eq([resource_c, resource_b, resource_a])
end
end
@@ -50,7 +69,7 @@ RSpec.describe Ci::Catalog::Resource, feature_category: :pipeline_composition do
it 'returns catalog resources sorted by ascending created at' do
ordered_resources = described_class.order_by_created_at_asc
- expect(ordered_resources.to_a).to eq([resource1, resource2, resource3])
+ expect(ordered_resources.to_a).to eq([resource_a, resource_b, resource_c])
end
end
@@ -58,13 +77,13 @@ RSpec.describe Ci::Catalog::Resource, feature_category: :pipeline_composition do
subject(:ordered_resources) { described_class.order_by_name_desc }
it 'returns catalog resources sorted by descending name' do
- expect(ordered_resources.pluck(:name)).to eq(%w[Z L A])
+ expect(ordered_resources.pluck(:name)).to eq(%w[C B A])
end
it 'returns catalog resources sorted by descending name with nulls last' do
- resource1.update!(name: nil)
+ resource_a.update!(name: nil)
- expect(ordered_resources.pluck(:name)).to eq(['Z', 'L', nil])
+ expect(ordered_resources.pluck(:name)).to eq(['C', 'B', nil])
end
end
@@ -72,13 +91,13 @@ RSpec.describe Ci::Catalog::Resource, feature_category: :pipeline_composition do
subject(:ordered_resources) { described_class.order_by_name_asc }
it 'returns catalog resources sorted by ascending name' do
- expect(ordered_resources.pluck(:name)).to eq(%w[A L Z])
+ expect(ordered_resources.pluck(:name)).to eq(%w[A B C])
end
it 'returns catalog resources sorted by ascending name with nulls last' do
- resource1.update!(name: nil)
+ resource_a.update!(name: nil)
- expect(ordered_resources.pluck(:name)).to eq(['L', 'Z', nil])
+ expect(ordered_resources.pluck(:name)).to eq(['B', 'C', nil])
end
end
@@ -86,7 +105,7 @@ RSpec.describe Ci::Catalog::Resource, feature_category: :pipeline_composition do
it 'returns catalog resources sorted by latest_released_at descending with nulls last' do
ordered_resources = described_class.order_by_latest_released_at_desc
- expect(ordered_resources).to eq([resource2, resource1, resource3])
+ expect(ordered_resources).to eq([resource_a, resource_b, resource_c])
end
end
@@ -94,35 +113,35 @@ RSpec.describe Ci::Catalog::Resource, feature_category: :pipeline_composition do
it 'returns catalog resources sorted by latest_released_at ascending with nulls last' do
ordered_resources = described_class.order_by_latest_released_at_asc
- expect(ordered_resources).to eq([resource1, resource2, resource3])
+ expect(ordered_resources).to eq([resource_b, resource_a, resource_c])
end
end
describe '#state' do
it 'defaults to draft' do
- expect(resource1.state).to eq('draft')
+ expect(resource_a.state).to eq('draft')
end
end
describe '#publish!' do
context 'when the catalog resource is in draft state' do
it 'updates the state of the catalog resource to published' do
- expect(resource1.state).to eq('draft')
+ expect(resource_a.state).to eq('draft')
- resource1.publish!
+ resource_a.publish!
- expect(resource1.reload.state).to eq('published')
+ expect(resource_a.reload.state).to eq('published')
end
end
context 'when the catalog resource already has a published state' do
it 'leaves the state as published' do
- resource1.update!(state: :published)
- expect(resource1.state).to eq('published')
+ resource_a.update!(state: :published)
+ expect(resource_a.state).to eq('published')
- resource1.publish!
+ resource_a.publish!
- expect(resource1.state).to eq('published')
+ expect(resource_a.state).to eq('published')
end
end
end
@@ -130,61 +149,115 @@ RSpec.describe Ci::Catalog::Resource, feature_category: :pipeline_composition do
describe '#unpublish!' do
context 'when the catalog resource is in published state' do
it 'updates the state of the catalog resource to draft' do
- resource1.update!(state: :published)
- expect(resource1.state).to eq('published')
+ resource_a.update!(state: :published)
+ expect(resource_a.state).to eq('published')
- resource1.unpublish!
+ resource_a.unpublish!
- expect(resource1.reload.state).to eq('draft')
+ expect(resource_a.reload.state).to eq('draft')
end
end
context 'when the catalog resource is already in draft state' do
it 'leaves the state as draft' do
- expect(resource1.state).to eq('draft')
+ expect(resource_a.state).to eq('draft')
- resource1.unpublish!
+ resource_a.unpublish!
- expect(resource1.reload.state).to eq('draft')
+ expect(resource_a.reload.state).to eq('draft')
end
end
end
- describe 'synchronizing denormalized columns with `projects` table' do
- shared_examples 'denormalized columns of the catalog resource match the project' do
- it do
- resource1.reload
- project1.reload
+ describe 'synchronizing denormalized columns with `projects` table', :sidekiq_inline do
+ let_it_be_with_reload(:project) { create(:project, name: 'Test project', description: 'Test description') }
+
+ context 'when the catalog resource is created' do
+ let(:resource) { build(:ci_catalog_resource, project: project) }
- expect(resource1.name).to eq(project1.name)
- expect(resource1.description).to eq(project1.description)
- expect(resource1.visibility_level).to eq(project1.visibility_level)
+ it 'updates the catalog resource columns to match the project' do
+ resource.save!
+ resource.reload
+
+ expect(resource.name).to eq(project.name)
+ expect(resource.description).to eq(project.description)
+ expect(resource.visibility_level).to eq(project.visibility_level)
end
end
- context 'when the catalog resource is created' do
- it 'calls sync_with_project' do
- new_project = create(:project)
- new_resource = build(:ci_catalog_resource, project: new_project)
+ context 'when the project is updated' do
+ let_it_be(:resource) { create(:ci_catalog_resource, project: project) }
+
+ context 'when project name is updated' do
+ it 'updates the catalog resource name to match' do
+ project.update!(name: 'New name')
- expect(new_resource).to receive(:sync_with_project).once
+ expect(resource.reload.name).to eq(project.name)
+ end
+ end
+
+ context 'when project description is updated' do
+ it 'updates the catalog resource description to match' do
+ project.update!(description: 'New description')
- new_resource.save!
+ expect(resource.reload.description).to eq(project.description)
+ end
end
- it_behaves_like 'denormalized columns of the catalog resource match the project'
+ context 'when project visibility_level is updated' do
+ it 'updates the catalog resource visibility_level to match' do
+ project.update!(visibility_level: Gitlab::VisibilityLevel::INTERNAL)
+
+ expect(resource.reload.visibility_level).to eq(project.visibility_level)
+ end
+ end
end
- context 'when the project attributes are updated' do
- before_all do
- project1.update!(
- name: 'New name',
- description: 'New description',
- visibility_level: Gitlab::VisibilityLevel::INTERNAL
- )
+ context 'when FF `ci_process_catalog_resource_sync_events` is disabled' do
+ before do
+ stub_feature_flags(ci_process_catalog_resource_sync_events: false)
+ end
+
+ context 'when the catalog resource is created' do
+ let(:resource) { build(:ci_catalog_resource, project: project) }
+
+ it 'updates the catalog resource columns to match the project' do
+ resource.save!
+ resource.reload
+
+ expect(resource.name).to eq(project.name)
+ expect(resource.description).to eq(project.description)
+ expect(resource.visibility_level).to eq(project.visibility_level)
+ end
end
- it_behaves_like 'denormalized columns of the catalog resource match the project'
+ context 'when the project is updated' do
+ let_it_be(:resource) { create(:ci_catalog_resource, project: project) }
+
+ context 'when project name is updated' do
+ it 'updates the catalog resource name to match' do
+ project.update!(name: 'New name')
+
+ expect(resource.reload.name).to eq(project.name)
+ end
+ end
+
+ context 'when project description is updated' do
+ it 'updates the catalog resource description to match' do
+ project.update!(description: 'New description')
+
+ expect(resource.reload.description).to eq(project.description)
+ end
+ end
+
+ context 'when project visibility_level is updated' do
+ it 'updates the catalog resource visibility_level to match' do
+ project.update!(visibility_level: Gitlab::VisibilityLevel::INTERNAL)
+
+ expect(resource.reload.visibility_level).to eq(project.visibility_level)
+ end
+ end
+ end
end
end
diff --git a/spec/models/ci/catalog/resources/sync_event_spec.rb b/spec/models/ci/catalog/resources/sync_event_spec.rb
new file mode 100644
index 00000000000..5d907aae9b6
--- /dev/null
+++ b/spec/models/ci/catalog/resources/sync_event_spec.rb
@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::Catalog::Resources::SyncEvent, type: :model, feature_category: :pipeline_composition do
+ let_it_be_with_reload(:project1) { create(:project) }
+ let_it_be_with_reload(:project2) { create(:project) }
+ let_it_be(:resource1) { create(:ci_catalog_resource, project: project1) }
+
+ it { is_expected.to belong_to(:catalog_resource).class_name('Ci::Catalog::Resource') }
+ it { is_expected.to belong_to(:project) }
+
+ describe 'PG triggers' do
+ context 'when the associated project of a catalog resource is updated' do
+ context 'when project name is updated' do
+ it 'creates a sync event record' do
+ expect do
+ project1.update!(name: 'New name')
+ end.to change { described_class.count }.by(1)
+ end
+ end
+
+ context 'when project description is updated' do
+ it 'creates a sync event record' do
+ expect do
+ project1.update!(description: 'New description')
+ end.to change { described_class.count }.by(1)
+ end
+ end
+
+ context 'when project visibility_level is updated' do
+ it 'creates a sync event record' do
+ expect do
+ project1.update!(visibility_level: Gitlab::VisibilityLevel::INTERNAL)
+ end.to change { described_class.count }.by(1)
+ end
+ end
+ end
+
+ context 'when a project without an associated catalog resource is updated' do
+ it 'does not create a sync event record' do
+ expect do
+ project2.update!(name: 'New name')
+ end.not_to change { described_class.count }
+ end
+ end
+ end
+
+ describe 'when there are sync event records' do
+ let_it_be(:resource2) { create(:ci_catalog_resource, project: project2) }
+
+ before_all do
+ create(:ci_catalog_resource_sync_event, catalog_resource: resource1, status: :processed)
+ create(:ci_catalog_resource_sync_event, catalog_resource: resource1)
+ create_list(:ci_catalog_resource_sync_event, 2, catalog_resource: resource2)
+ end
+
+ describe '.unprocessed_events' do
+ it 'returns the events in pending status' do
+ # 1 pending event from resource1 + 2 pending events from resource2
+ expect(described_class.unprocessed_events.size).to eq(3)
+ end
+
+ it 'selects the partition attribute in the result' do
+ described_class.unprocessed_events.each do |event|
+ expect(event.partition).not_to be_nil
+ end
+ end
+ end
+
+ describe '.mark_records_processed' do
+ it 'updates the records to processed status' do
+ expect(described_class.status_pending.count).to eq(3)
+ expect(described_class.status_processed.count).to eq(1)
+
+ described_class.mark_records_processed(described_class.unprocessed_events)
+
+ expect(described_class.pluck(:status).uniq).to eq(['processed'])
+
+ expect(described_class.status_pending.count).to eq(0)
+ expect(described_class.status_processed.count).to eq(4)
+ end
+ end
+ end
+
+ describe '.upper_bound_count' do
+ it 'returns 0 when there are no records in the table' do
+ expect(described_class.upper_bound_count).to eq(0)
+ end
+
+ it 'returns an estimated number of unprocessed records' do
+ create_list(:ci_catalog_resource_sync_event, 5, catalog_resource: resource1)
+ described_class.order(:id).limit(2).update_all(status: :processed)
+
+ expect(described_class.upper_bound_count).to eq(3)
+ end
+ end
+
+ describe 'sliding_list partitioning' do
+ let(:partition_manager) { Gitlab::Database::Partitioning::PartitionManager.new(described_class) }
+
+ describe 'next_partition_if callback' do
+ let(:active_partition) { described_class.partitioning_strategy.active_partition }
+
+ subject(:value) { described_class.partitioning_strategy.next_partition_if.call(active_partition) }
+
+ context 'when the partition is empty' do
+ it { is_expected.to eq(false) }
+ end
+
+ context 'when the partition has records' do
+ before do
+ create(:ci_catalog_resource_sync_event, catalog_resource: resource1, status: :processed)
+ create(:ci_catalog_resource_sync_event, catalog_resource: resource1)
+ end
+
+ it { is_expected.to eq(false) }
+ end
+
+ context 'when the first record of the partition is older than PARTITION_DURATION' do
+ before do
+ create(:ci_catalog_resource_sync_event, catalog_resource: resource1)
+ described_class.first.update!(created_at: (described_class::PARTITION_DURATION + 1.day).ago)
+ end
+
+ it { is_expected.to eq(true) }
+ end
+ end
+
+ describe 'detach_partition_if callback' do
+ let(:active_partition) { described_class.partitioning_strategy.active_partition }
+
+ subject(:value) { described_class.partitioning_strategy.detach_partition_if.call(active_partition) }
+
+ before_all do
+ create(:ci_catalog_resource_sync_event, catalog_resource: resource1, status: :processed)
+ create(:ci_catalog_resource_sync_event, catalog_resource: resource1)
+ end
+
+ context 'when the partition contains unprocessed records' do
+ it { is_expected.to eq(false) }
+ end
+
+ context 'when the partition contains only processed records' do
+ before do
+ described_class.update_all(status: :processed)
+ end
+
+ it { is_expected.to eq(true) }
+ end
+ end
+
+ describe 'strategy behavior' do
+ it 'moves records to new partitions as time passes', :freeze_time do
+ # We start with partition 1
+ expect(described_class.partitioning_strategy.current_partitions.map(&:value)).to eq([1])
+
+ # Add one record so the initial partition is not empty
+ create(:ci_catalog_resource_sync_event, catalog_resource: resource1)
+
+ # It's not a day old yet so no new partitions are created
+ partition_manager.sync_partitions
+
+ expect(described_class.partitioning_strategy.current_partitions.map(&:value)).to eq([1])
+
+ # After traveling forward a day
+ travel(described_class::PARTITION_DURATION + 1.second)
+
+ # a new partition is created
+ partition_manager.sync_partitions
+
+ expect(described_class.partitioning_strategy.current_partitions.map(&:value)).to contain_exactly(1, 2)
+
+ # and we can insert to the new partition
+ create(:ci_catalog_resource_sync_event, catalog_resource: resource1)
+
+ # After processing records in partition 1
+ described_class.mark_records_processed(described_class.for_partition(1).select_with_partition)
+
+ partition_manager.sync_partitions
+
+ # partition 1 is removed
+ expect(described_class.partitioning_strategy.current_partitions.map(&:value)).to eq([2])
+
+ # and we only have the newly created partition left.
+ expect(described_class.count).to eq(1)
+ end
+ end
+ end
+end
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index dcd2e634ce3..bbff7e5ee40 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -50,6 +50,7 @@ RSpec.describe Project, factory_default: :keep, feature_category: :groups_and_pr
it { is_expected.to have_one(:catalog_resource) }
it { is_expected.to have_many(:ci_components).class_name('Ci::Catalog::Resources::Component') }
it { is_expected.to have_many(:catalog_resource_versions).class_name('Ci::Catalog::Resources::Version') }
+ it { is_expected.to have_many(:catalog_resource_sync_events).class_name('Ci::Catalog::Resources::SyncEvent') }
it { is_expected.to have_one(:microsoft_teams_integration) }
it { is_expected.to have_one(:mattermost_integration) }
it { is_expected.to have_one(:hangouts_chat_integration) }
@@ -8937,62 +8938,68 @@ RSpec.describe Project, factory_default: :keep, feature_category: :groups_and_pr
end
end
- # TODO: Remove/update this spec after background syncing is implemented. See https://gitlab.com/gitlab-org/gitlab/-/issues/429376.
- describe '#update_catalog_resource' do
- let_it_be_with_reload(:project) { create(:project, name: 'My project name', description: 'My description') }
- let_it_be_with_reload(:resource) { create(:ci_catalog_resource, project: project) }
+ describe 'catalog resource process sync events worker' do
+ let_it_be_with_reload(:project) { create(:project, name: 'Test project', description: 'Test description') }
- shared_examples 'name, description, and visibility_level of the catalog resource match the project' do
- it do
- expect(project).to receive(:update_catalog_resource).once.and_call_original
+ context 'when the project has a catalog resource' do
+ let_it_be(:resource) { create(:ci_catalog_resource, project: project) }
- project.save!
+ context 'when project name is updated' do
+ it 'enqueues Ci::Catalog::Resources::ProcessSyncEventsWorker' do
+ expect(Ci::Catalog::Resources::ProcessSyncEventsWorker).to receive(:perform_async).once
- expect(resource.name).to eq(project.name)
- expect(resource.description).to eq(project.description)
- expect(resource.visibility_level).to eq(project.visibility_level)
+ project.update!(name: 'New name')
+ end
end
- end
- context 'when the project name is updated' do
- before do
- project.name = 'My new project name'
+ context 'when project description is updated' do
+ it 'enqueues Ci::Catalog::Resources::ProcessSyncEventsWorker' do
+ expect(Ci::Catalog::Resources::ProcessSyncEventsWorker).to receive(:perform_async).once
+
+ project.update!(description: 'New description')
+ end
end
- it_behaves_like 'name, description, and visibility_level of the catalog resource match the project'
- end
+ context 'when project visibility_level is updated' do
+ it 'enqueues Ci::Catalog::Resources::ProcessSyncEventsWorker' do
+ expect(Ci::Catalog::Resources::ProcessSyncEventsWorker).to receive(:perform_async).once
- context 'when the project description is updated' do
- before do
- project.description = 'My new description'
+ project.update!(visibility_level: Gitlab::VisibilityLevel::INTERNAL)
+ end
end
- it_behaves_like 'name, description, and visibility_level of the catalog resource match the project'
- end
+ context 'when neither the project name, description, nor visibility_level are updated' do
+ it 'does not enqueue Ci::Catalog::Resources::ProcessSyncEventsWorker' do
+ expect(Ci::Catalog::Resources::ProcessSyncEventsWorker).not_to receive(:perform_async)
- context 'when the project visibility_level is updated' do
- before do
- project.visibility_level = 10
+ project.update!(path: 'path')
+ end
end
- it_behaves_like 'name, description, and visibility_level of the catalog resource match the project'
- end
+ context 'when FF `ci_process_catalog_resource_sync_events` is disabled' do
+ before do
+ stub_feature_flags(ci_process_catalog_resource_sync_events: false)
+ end
- context 'when neither the project name, description, nor visibility_level are updated' do
- it 'does not call update_catalog_resource' do
- expect(project).not_to receive(:update_catalog_resource)
+ it 'does not enqueue Ci::Catalog::Resources::ProcessSyncEventsWorker' do
+ expect(Ci::Catalog::Resources::ProcessSyncEventsWorker).not_to receive(:perform_async)
- project.update!(path: 'path')
+ project.update!(
+ name: 'New name',
+ description: 'New description',
+ visibility_level: Gitlab::VisibilityLevel::INTERNAL)
+ end
end
end
context 'when the project does not have a catalog resource' do
- let_it_be(:project2) { create(:project) }
-
- it 'does not call update_catalog_resource' do
- expect(project2).not_to receive(:update_catalog_resource)
+ it 'does not enqueue Ci::Catalog::Resources::ProcessSyncEventsWorker' do
+ expect(Ci::Catalog::Resources::ProcessSyncEventsWorker).not_to receive(:perform_async)
- project.update!(name: 'name')
+ project.update!(
+ name: 'New name',
+ description: 'New description',
+ visibility_level: Gitlab::VisibilityLevel::INTERNAL)
end
end
end
diff --git a/spec/services/ci/process_sync_events_service_spec.rb b/spec/services/ci/process_sync_events_service_spec.rb
index c58d73815b0..48b70eb38c9 100644
--- a/spec/services/ci/process_sync_events_service_spec.rb
+++ b/spec/services/ci/process_sync_events_service_spec.rb
@@ -163,5 +163,84 @@ RSpec.describe Ci::ProcessSyncEventsService, feature_category: :continuous_integ
execute
end
end
+
+ context 'for Ci::Catalog::Resources::SyncEvent' do
+ let(:sync_event_class) { Ci::Catalog::Resources::SyncEvent }
+ let(:hierarchy_class) { Ci::Catalog::Resource }
+
+ let_it_be(:project1) { create(:project) }
+ let_it_be(:project2) { create(:project) }
+ let_it_be_with_refind(:resource1) { create(:ci_catalog_resource, project: project1) }
+ let_it_be(:resource2) { create(:ci_catalog_resource, project: project2) }
+
+ before_all do
+ create(:ci_catalog_resource_sync_event, catalog_resource: resource1, status: :processed)
+ # PG trigger adds an event for each update
+ project1.update!(name: 'Name 1', description: 'Test 1')
+ project1.update!(visibility_level: Gitlab::VisibilityLevel::INTERNAL)
+ project2.update!(name: 'Name 2', description: 'Test 2')
+ project2.update!(visibility_level: Gitlab::VisibilityLevel::PUBLIC)
+ end
+
+ it 'processes the events', :aggregate_failures do
+ # 2 pending events from resource1 + 2 pending events from resource2
+ expect { execute }.to change(Ci::Catalog::Resources::SyncEvent.status_pending, :count).from(4).to(0)
+
+ expect(resource1.reload.name).to eq(project1.name)
+ expect(resource2.reload.name).to eq(project2.name)
+ expect(resource1.reload.description).to eq(project1.description)
+ expect(resource2.reload.description).to eq(project2.description)
+ expect(resource1.reload.visibility_level).to eq(project1.visibility_level)
+ expect(resource2.reload.visibility_level).to eq(project2.visibility_level)
+ end
+
+ context 'when there are no remaining unprocessed events' do
+ it 'does not enqueue Ci::Catalog::Resources::ProcessSyncEventsWorker' do
+ stub_const("#{described_class}::BATCH_SIZE", 4)
+
+ expect(Ci::Catalog::Resources::ProcessSyncEventsWorker).not_to receive(:perform_async)
+
+ execute
+ end
+ end
+
+ context 'when there are remaining unprocessed events' do
+ it 'enqueues Ci::Catalog::Resources::ProcessSyncEventsWorker' do
+ stub_const("#{described_class}::BATCH_SIZE", 1)
+
+ expect(Ci::Catalog::Resources::ProcessSyncEventsWorker).to receive(:perform_async)
+
+ execute
+ end
+
+ context 'when FF `ci_process_catalog_resource_sync_events` is disabled' do
+ before do
+ stub_feature_flags(ci_process_catalog_resource_sync_events: false)
+ end
+
+ it 'does not enqueue Ci::Catalog::Resources::ProcessSyncEventsWorker' do
+ stub_const("#{described_class}::BATCH_SIZE", 1)
+
+ expect(Ci::Catalog::Resources::ProcessSyncEventsWorker).not_to receive(:perform_async)
+
+ execute
+ end
+ end
+ end
+
+ # The `p_catalog_resource_sync_events` table does not enforce an FK on catalog_resource_id
+ context 'when there are orphaned sync events' do
+ it 'processes the events', :aggregate_failures do
+ resource1.destroy!
+
+ # 2 pending events from resource1 + 2 pending events from resource2
+ expect { execute }.to change(Ci::Catalog::Resources::SyncEvent.status_pending, :count).from(4).to(0)
+
+ expect(resource2.reload.name).to eq(project2.name)
+ expect(resource2.reload.description).to eq(project2.description)
+ expect(resource2.reload.visibility_level).to eq(project2.visibility_level)
+ end
+ end
+ end
end
end
diff --git a/spec/workers/ci/catalog/resources/process_sync_events_worker_spec.rb b/spec/workers/ci/catalog/resources/process_sync_events_worker_spec.rb
new file mode 100644
index 00000000000..3c5f7bc0bf9
--- /dev/null
+++ b/spec/workers/ci/catalog/resources/process_sync_events_worker_spec.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::Catalog::Resources::ProcessSyncEventsWorker, feature_category: :pipeline_composition do
+ subject(:worker) { described_class.new }
+
+ include_examples 'an idempotent worker'
+
+ it 'has the `until_executed` deduplicate strategy' do
+ expect(described_class.get_deduplicate_strategy).to eq(:until_executed)
+ end
+
+ it 'has the option to reschedule once if deduplicated and a TTL of 1 minute' do
+ expect(described_class.get_deduplication_options).to include({ if_deduplicated: :reschedule_once, ttl: 1.minute })
+ end
+
+ describe '#perform' do
+ let_it_be(:project) { create(:project) }
+ let_it_be(:resource) { create(:ci_catalog_resource, project: project) }
+
+ before_all do
+ create(:ci_catalog_resource_sync_event, catalog_resource: resource, status: :processed)
+ create_list(:ci_catalog_resource_sync_event, 2, catalog_resource: resource)
+ # PG trigger adds an event for this update
+ project.update!(name: 'Name', description: 'Test', visibility_level: Gitlab::VisibilityLevel::INTERNAL)
+ end
+
+ subject(:perform) { worker.perform }
+
+ it 'consumes all sync events' do
+ expect { perform }.to change { Ci::Catalog::Resources::SyncEvent.status_pending.count }
+ .from(3).to(0)
+ end
+
+ it 'syncs the denormalized columns of catalog resource with the project' do
+ perform
+
+ expect(resource.reload.name).to eq(project.name)
+ expect(resource.reload.description).to eq(project.description)
+ expect(resource.reload.visibility_level).to eq(project.visibility_level)
+ end
+
+ it 'logs the service result', :aggregate_failures do
+ expect(worker).to receive(:log_extra_metadata_on_done).with(:estimated_total_events, 3)
+ expect(worker).to receive(:log_extra_metadata_on_done).with(:consumable_events, 3)
+ expect(worker).to receive(:log_extra_metadata_on_done).with(:processed_events, 3)
+
+ perform
+ end
+ end
+end
diff --git a/yarn.lock b/yarn.lock
index dcb28a12a66..cc54dd0145d 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1274,10 +1274,10 @@
resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-3.72.0.tgz#5daaa7366913b52ea89439305067e030f967c8a5"
integrity sha512-VbSdwXxu9Y6NAXNFTROjZa83e2b8QeDAO7byqjJ0z+2Y3gGGXdw+HclAzz0Ns8B0+DMV5mV7dtmTlv/1xAXXYQ==
-"@gitlab/ui@^71.3.0":
- version "71.4.0"
- resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-71.4.0.tgz#481d594f7cdc01aac6529cc7c801221ccde13b86"
- integrity sha512-6ddhlYo5wVQJ2j0AhlrmxwBpYS7UhM6sR3XeXeMRbDqJaA/17ARwyl8JMxCqVcIcGbTmDd9FJluXzObQsyUzUQ==
+"@gitlab/ui@^71.6.0":
+ version "71.6.0"
+ resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-71.6.0.tgz#6d3b5bbe4f303e83a069e7c03e758a30a986139e"
+ integrity sha512-DAJVMoDJESmzLaRh2PWWE7wO3AqqxWVF76u11wrVg08TsicSU/rL1Sl09mbY99YPrmqoBepVsAERCAEuPaq/Ug==
dependencies:
"@floating-ui/dom" "1.2.9"
bootstrap-vue "2.23.1"