diff options
39 files changed, 967 insertions, 241 deletions
diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 5a2daa3f134..e46e3d5b462 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -534,6 +534,13 @@ rspec:undercoverage: stage: post-test needs: ["rspec:coverage"] script: + - apt install -y jq + - if [[ $(curl "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}" | jq ".labels" | grep "pipeline:skip-undercoverage") ]]; then + echo "The 'pipeline:skip-undercoverage' label is set on the MR, exiting early."; + exit 0; + else + echo "The 'pipeline:skip-undercoverage' label is not set on the MR, proceeding."; + fi - if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA" ]; then echo "HEAD is $(git rev-parse HEAD). \$CI_MERGE_REQUEST_TARGET_BRANCH_SHA is ${CI_MERGE_REQUEST_TARGET_BRANCH_SHA}"; else diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index fb8104fec11..75d4292f524 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -1694,6 +1694,7 @@ .qa:rules:code-suggestions-eval-base: rules: - !reference [".strict-ee-only-rules", rules] + - !reference [".qa:rules:package-and-test-never-run", rules] - <<: *if-fork-merge-request when: never - <<: *if-merge-request-labels-run-cs-evaluation diff --git a/app/assets/javascripts/admin/statistics_panel/components/app.vue b/app/assets/javascripts/admin/statistics_panel/components/app.vue index 87325b07144..5e7b7e959e0 100644 --- a/app/assets/javascripts/admin/statistics_panel/components/app.vue +++ b/app/assets/javascripts/admin/statistics_panel/components/app.vue @@ -29,13 +29,19 @@ export default { <template> <gl-card> - <h4>{{ __('Statistics') }}</h4> - <gl-loading-icon v-if="isLoading" size="lg" class="my-3" /> - <template v-else> - <p v-for="statistic in getStatistics(statisticsLabels)" :key="statistic.key" class="js-stats"> - {{ statistic.label }} - <span class="light float-right">{{ statistic.value }}</span> - </p> - </template> + <h4 class="gl-heading-4">{{ __('Statistics') }}</h4> + <slot name="footer"> + <gl-loading-icon v-if="isLoading" size="lg" class="my-3" /> + <template v-else> + <p + v-for="statistic in getStatistics(statisticsLabels)" + :key="statistic.key" + class="js-stats" + > + {{ statistic.label }} + <span class="light float-right">{{ statistic.value }}</span> + </p> + </template> + </slot> </gl-card> </template> diff --git a/app/assets/javascripts/graphql_shared/possible_types.json b/app/assets/javascripts/graphql_shared/possible_types.json index fe151c2b358..1e7903ffa19 100644 --- a/app/assets/javascripts/graphql_shared/possible_types.json +++ b/app/assets/javascripts/graphql_shared/possible_types.json @@ -198,6 +198,7 @@ "WorkItemWidgetMilestone", "WorkItemWidgetNotes", "WorkItemWidgetNotifications", + "WorkItemWidgetParticipants", "WorkItemWidgetProgress", "WorkItemWidgetRequirementLegacy", "WorkItemWidgetRolledupDates", diff --git a/app/assets/javascripts/vue_merge_request_widget/components/merge_checks.vue b/app/assets/javascripts/vue_merge_request_widget/components/merge_checks.vue index 750f53a29b6..89095a55a11 100644 --- a/app/assets/javascripts/vue_merge_request_widget/components/merge_checks.vue +++ b/app/assets/javascripts/vue_merge_request_widget/components/merge_checks.vue @@ -98,6 +98,17 @@ export default { checks() { return this.state.mergeabilityChecks || []; }, + sortedChecks() { + return [...this.checks] + .sort((a, b) => { + if (a.status === 'FAILED' && b.status !== 'FAILED') return -1; + if (a.status === 'SUCCESS' && b.status !== 'SUCCESS') + return b.status === 'FAILED' ? 1 : -1; + + return 0; + }) + .filter((s) => s.status !== 'INACTIVE'); + }, failedChecks() { return this.checks.filter((c) => c.status.toLowerCase() === 'failed'); }, @@ -143,14 +154,15 @@ export default { <div class="gl-px-5"> <component :is="checkComponent(check)" - v-for="(check, index) in checks" + v-for="(check, index) in sortedChecks" :key="index" :class="{ - 'gl-border-b-solid gl-border-b-1 gl-border-gray-100': index !== checks.length - 1, + 'gl-border-b-solid gl-border-b-1 gl-border-gray-100': index !== sortedChecks.length - 1, }" :check="check" :mr="mr" :service="service" + data-testid="merge-check" /> </div> </div> diff --git a/app/finders/groups_finder.rb b/app/finders/groups_finder.rb index 9cc27a3096d..2e25c97cabe 100644 --- a/app/finders/groups_finder.rb +++ b/app/finders/groups_finder.rb @@ -27,30 +27,25 @@ class GroupsFinder < UnionFinder include CustomAttributesFilter + attr_reader :current_user, :params + def initialize(current_user = nil, params = {}) @current_user = current_user @params = params end def execute - items = all_groups.map do |groups| - filter_groups(groups) - end - - find_union(items, Group).with_route.order_id_desc + # filtered_groups can contain an array of scopes, so these + # are combined into a single query using UNION. + find_union(filtered_groups, Group).with_route.order_id_desc end private - attr_reader :current_user, :params - - def filter_groups(groups) - groups = by_organization(groups) - groups = by_parent(groups) - groups = by_custom_attributes(groups) - groups = filter_group_ids(groups) - groups = exclude_group_ids(groups) - by_search(groups) + def filtered_groups + all_groups.map do |groups| + filter_groups(groups) + end end def all_groups @@ -58,20 +53,19 @@ class GroupsFinder < UnionFinder return [groups_with_min_access_level] if min_access_level? return [Group.all] if current_user&.can_read_all_resources? && all_available? - groups = [] - groups = get_groups_for_user if current_user + groups = [ + membership_groups, + authorized_groups, + public_groups + ].compact - groups << Group.unscoped.public_to_user(current_user) if include_public_groups? groups << Group.none if groups.empty? - groups - end - def groups_for_ancestors - current_user.authorized_groups + groups end - def groups_for_descendants - current_user.groups + def owned_groups + current_user&.owned_groups || Group.none end # rubocop: disable CodeReuse/ActiveRecord @@ -83,16 +77,37 @@ class GroupsFinder < UnionFinder end # rubocop: enable CodeReuse/ActiveRecord - def exclude_group_ids(groups) - return groups unless params[:exclude_group_ids] + def membership_groups + return unless current_user - groups.id_not_in(params[:exclude_group_ids]) + current_user.groups.self_and_descendants end - def filter_group_ids(groups) - return groups unless params[:filter_group_ids] + def authorized_groups + return unless current_user - groups.id_in(params[:filter_group_ids]) + if params.fetch(:include_ancestors, true) + current_user.authorized_groups.self_and_ancestors + else + current_user.authorized_groups + end + end + + def public_groups + # By default, all groups public to the user are included. This is controlled by + # the :all_available argument, which defaults to true + return unless include_public_groups? + + Group.unscoped.public_to_user(current_user) + end + + def filter_groups(groups) + groups = by_organization(groups) + groups = by_parent(groups) + groups = by_custom_attributes(groups) + groups = filter_group_ids(groups) + groups = exclude_group_ids(groups) + by_search(groups) end def by_organization(groups) @@ -106,7 +121,7 @@ class GroupsFinder < UnionFinder def by_parent(groups) return groups unless params[:parent] - if include_parent_descendants? + if params.fetch(:include_parent_descendants, false) groups.id_in(params[:parent].descendants) else groups.where(parent: params[:parent]) @@ -114,48 +129,34 @@ class GroupsFinder < UnionFinder end # rubocop: enable CodeReuse/ActiveRecord - def by_search(groups) - return groups unless params[:search].present? + def filter_group_ids(groups) + return groups unless params[:filter_group_ids] - groups.search(params[:search], include_parents: params[:parent].blank?) + groups.id_in(params[:filter_group_ids]) end - def owned_groups - current_user&.owned_groups || Group.none - end + def exclude_group_ids(groups) + return groups unless params[:exclude_group_ids] - def include_public_groups? - current_user.nil? || all_available? + groups.id_not_in(params[:exclude_group_ids]) end - def all_available? - params.fetch(:all_available, true) - end + def by_search(groups) + return groups unless params[:search].present? - def include_parent_descendants? - params.fetch(:include_parent_descendants, false) + groups.search(params[:search], include_parents: params[:parent].blank?) end def min_access_level? current_user && params[:min_access_level].present? end - def include_ancestors? - params.fetch(:include_ancestors, true) + def include_public_groups? + current_user.nil? || all_available? end - def get_groups_for_user - groups = [] - - groups << if include_ancestors? - current_user.authorized_groups.self_and_ancestors - else - current_user.authorized_groups - end - - groups << current_user.groups.self_and_descendants - - groups + def all_available? + params.fetch(:all_available, true) end end diff --git a/app/graphql/resolvers/concerns/work_items/look_ahead_preloads.rb b/app/graphql/resolvers/concerns/work_items/look_ahead_preloads.rb index 71833fbd2b9..68085088821 100644 --- a/app/graphql/resolvers/concerns/work_items/look_ahead_preloads.rb +++ b/app/graphql/resolvers/concerns/work_items/look_ahead_preloads.rb @@ -35,6 +35,7 @@ module WorkItems { last_edited_by: :last_edited_by, assignees: :assignees, + participants: WorkItem.participant_includes, parent: :work_item_parent, children: { work_item_children_by_relative_position: [:author, { project: :project_feature }] }, labels: :labels, diff --git a/app/graphql/types/work_items/widget_interface.rb b/app/graphql/types/work_items/widget_interface.rb index 9f4dbdd1038..62657f38a4b 100644 --- a/app/graphql/types/work_items/widget_interface.rb +++ b/app/graphql/types/work_items/widget_interface.rb @@ -22,7 +22,8 @@ module Types ::Types::WorkItems::Widgets::NotificationsType, ::Types::WorkItems::Widgets::CurrentUserTodosType, ::Types::WorkItems::Widgets::AwardEmojiType, - ::Types::WorkItems::Widgets::LinkedItemsType + ::Types::WorkItems::Widgets::LinkedItemsType, + ::Types::WorkItems::Widgets::ParticipantsType ].freeze def self.ce_orphan_types @@ -56,6 +57,8 @@ module Types ::Types::WorkItems::Widgets::AwardEmojiType when ::WorkItems::Widgets::LinkedItems ::Types::WorkItems::Widgets::LinkedItemsType + when ::WorkItems::Widgets::Participants + ::Types::WorkItems::Widgets::ParticipantsType else raise "Unknown GraphQL type for widget #{object}" end diff --git a/app/graphql/types/work_items/widgets/participants_type.rb b/app/graphql/types/work_items/widgets/participants_type.rb new file mode 100644 index 00000000000..0098e656eae --- /dev/null +++ b/app/graphql/types/work_items/widgets/participants_type.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Types + module WorkItems + module Widgets + # Disabling widget level authorization as we scope by `.visible_participants` + # rubocop:disable Graphql/AuthorizeTypes -- see above + class ParticipantsType < BaseObject + graphql_name 'WorkItemWidgetParticipants' + description 'Represents a participants widget' + + implements Types::WorkItems::WidgetInterface + + field :participants, Types::UserType.connection_type, + null: true, + description: 'Participants in the work item.' + + def participants + object.visible_participants(current_user) + end + end + # rubocop:enable Graphql/AuthorizeTypes + end + end +end diff --git a/app/models/concerns/participable.rb b/app/models/concerns/participable.rb index 6035cb87c9b..d9ad0e8c16b 100644 --- a/app/models/concerns/participable.rb +++ b/app/models/concerns/participable.rb @@ -162,7 +162,15 @@ module Participable when PersonalSnippet Ability.users_that_can_read_personal_snippet(participants.to_a, self) else - Ability.users_that_can_read_project(participants.to_a, project) + return Ability.users_that_can_read_project(participants.to_a, project) if project + + # handling group level work items(issues) that would have a namespace, + # We need to make sure that scenarios where some models that do not have a project set and also do not have + # a namespace are also handled and exceptions are avoided. + namespace_level_participable = respond_to?(:namespace) && namespace.present? + return Ability.users_that_can_read_group(participants.to_a, namespace) if namespace_level_participable + + [] end end @@ -171,7 +179,15 @@ module Participable when PersonalSnippet participant.can?(:read_snippet, self) else - participant.can?(:read_project, project) + return participant.can?(:read_project, project) if project + + # handling group level work items(issues) that would have a namespace, + # We need to make sure that scenarios where some models that do not have a project set and also do not have + # a namespace are also handled and exceptions are avoided. + namespace_level_participable = respond_to?(:namespace) && namespace.present? + return participant.can?(:read_group, namespace) if namespace_level_participable + + false end end diff --git a/app/models/work_items/widget_definition.rb b/app/models/work_items/widget_definition.rb index 2637a7c8185..1e00065b503 100644 --- a/app/models/work_items/widget_definition.rb +++ b/app/models/work_items/widget_definition.rb @@ -34,7 +34,8 @@ module WorkItems award_emoji: 16, linked_items: 17, color: 18, # EE-only - rolledup_dates: 19 # EE-only + rolledup_dates: 19, # EE-only + participants: 20 } def self.available_widgets diff --git a/app/models/work_items/widgets/participants.rb b/app/models/work_items/widgets/participants.rb new file mode 100644 index 00000000000..7ecbf3e0f0a --- /dev/null +++ b/app/models/work_items/widgets/participants.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module WorkItems + module Widgets + class Participants < Base + delegate :participants, :visible_participants, to: :work_item + end + end +end diff --git a/app/services/merge_requests/retarget_chain_service.rb b/app/services/merge_requests/retarget_chain_service.rb index b4b05ffb08c..b6434819914 100644 --- a/app/services/merge_requests/retarget_chain_service.rb +++ b/app/services/merge_requests/retarget_chain_service.rb @@ -29,6 +29,8 @@ module MergeRequests target_branch_was_deleted: true } ).execute(other_merge_request) + + other_merge_request.rebase_async(current_user.id) end end end diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml index bf00fbfd81d..bac237c35c7 100644 --- a/app/views/admin/dashboard/index.html.haml +++ b/app/views/admin/dashboard/index.html.haml @@ -22,29 +22,27 @@ .admin-dashboard.gl-mt-3 .h3.gl-mb-5.gl-mt-0= _('Instance overview') .row - - component_params = { body_options: { class: 'gl-display-flex gl-justify-content-space-between gl-align-items-center gl-p-6' }, - footer_options: { class: 'gl-bg-transparent'} } + - component_params = { body_options: { class: 'gl-display-flex gl-justify-content-space-between gl-align-items-baseline' } } .col-md-4.gl-mb-6 = render Pajamas::CardComponent.new(**component_params) do |c| - c.with_body do %span - .d-flex.align-items-center + .gl-display-flex.gl-align-items-center = sprite_icon('project', size: 16, css_class: 'gl-text-gray-700') - %h3.gl-m-0.gl-ml-3= approximate_count_with_delimiters(@counts, Project) - .gl-mt-3.text-uppercase= s_('AdminArea|Projects') + %h3.gl-heading-2.gl-ml-3{ class: 'gl-mb-0!' }= approximate_count_with_delimiters(@counts, Project) + .gl-heading-4{ class: 'gl-mb-0!' }= s_('AdminArea|Projects') = render Pajamas::ButtonComponent.new(href: new_project_path) do = s_('AdminArea|New project') - c.with_footer do - .d-flex.align-items-center + .gl-display-flex.gl-align-items-center = link_to(s_('AdminArea|View latest projects'), admin_projects_path(sort: 'created_desc')) - = sprite_icon('chevron-right', size: 12, css_class: 'gl-text-gray-700 gl-ml-2') .col-md-4.gl-mb-6 = render Pajamas::CardComponent.new(**component_params) do |c| - c.with_body do %span - .d-flex.align-items-center + .gl-display-flex.gl-align-items-center = sprite_icon('users', size: 16, css_class: 'gl-text-gray-700') - %h3.gl-m-0.gl-ml-3= approximate_count_with_delimiters(@counts, User) + %h3.gl-heading-2.gl-ml-3{ class: 'gl-mb-0!' }= approximate_count_with_delimiters(@counts, User) %span.gl-outline-0.gl-ml-3{ tabindex: "0", data: { container: "body", toggle: "popover", placement: "top", @@ -53,36 +51,35 @@ content: s_("AdminArea|All users created in the instance, including users who are not %{billable_users_link_start}billable users%{billable_users_link_end}.").html_safe % { billable_users_link_start: billable_users_link_start, billable_users_link_end: '</a>'.html_safe }, } } = sprite_icon('question-o', size: 16, css_class: 'gl-text-blue-600') - .gl-mt-3.text-uppercase + .gl-heading-4{ class: 'gl-mb-0!' } = s_('AdminArea|Users') - = link_to(s_('AdminArea|Users statistics'), admin_dashboard_stats_path, class: "text-capitalize gl-ml-2") = render Pajamas::ButtonComponent.new(href: new_admin_user_path) do = s_('AdminArea|New user') - c.with_footer do - .d-flex.align-items-center + .gl-display-flex.gl-flex-wrap.gl-align-items-center.gl-gap-2 = link_to(s_('AdminArea|View latest users'), admin_users_path({ sort: 'created_desc' })) - = sprite_icon('chevron-right', size: 12, css_class: 'gl-text-gray-700 gl-ml-2') + %span.gl-text-secondary= "/" + = link_to(s_('AdminArea|Users statistics'), admin_dashboard_stats_path, class: "gl-font-base gl-font-weight-normal gl-text-capitalize") .col-md-4.gl-mb-6 = render Pajamas::CardComponent.new(**component_params) do |c| - c.with_body do %span - .d-flex.align-items-center + .gl-display-flex.gl-align-items-center = sprite_icon('group', size: 16, css_class: 'gl-text-gray-700') - %h3.gl-m-0.gl-ml-3= approximate_count_with_delimiters(@counts, Group) - .gl-mt-3.text-uppercase= s_('AdminArea|Groups') + %h3.gl-heading-2.gl-ml-3{ class: 'gl-mb-0!' }= approximate_count_with_delimiters(@counts, Group) + .gl-heading-4{ class: 'gl-mb-0!' }= s_('AdminArea|Groups') = render Pajamas::ButtonComponent.new(href: new_admin_group_path) do = s_('AdminArea|New group') - c.with_footer do - .d-flex.align-items-center + .gl-display-flex.gl-align-items-center = link_to(s_('AdminArea|View latest groups'), admin_groups_path(sort: 'created_desc')) - = sprite_icon('chevron-right', size: 12, css_class: 'gl-text-gray-700 gl-ml-2') .row .col-md-4.gl-mb-6 #js-admin-statistics-container .col-md-4.gl-mb-6 = render Pajamas::CardComponent.new do |c| - c.with_body do - %h4= s_('AdminArea|Features') + %h4.gl-heading-4= s_('AdminArea|Features') = feature_entry(_('Sign up'), href: general_admin_application_settings_path(anchor: 'js-signup-settings'), enabled: allow_signup?) @@ -121,7 +118,7 @@ .col-md-4.gl-mb-6 = render Pajamas::CardComponent.new do |c| - c.with_body do - %h4 + %h4.gl-heading-4 = s_('AdminArea|Components') - if show_version_check? .float-right @@ -179,7 +176,7 @@ .col-md-4.gl-mb-6 = render Pajamas::CardComponent.new do |c| - c.with_body do - %h4= s_('AdminArea|Latest projects') + %h4.gl-heading-4= s_('AdminArea|Latest projects') - @projects.each do |project| .gl-display-flex.gl-py-3 .gl-mr-auto.gl-overflow-hidden.gl-text-overflow-ellipsis @@ -189,7 +186,7 @@ .col-md-4.gl-mb-6 = render Pajamas::CardComponent.new do |c| - c.with_body do - %h4= s_('AdminArea|Latest users') + %h4.gl-heading-4= s_('AdminArea|Latest users') - @users.each do |user| .gl-display-flex.gl-py-3 .gl-mr-auto.gl-overflow-hidden.gl-text-overflow-ellipsis @@ -200,7 +197,7 @@ .col-md-4.gl-mb-6 = render Pajamas::CardComponent.new do |c| - c.with_body do - %h4= s_('AdminArea|Latest groups') + %h4.gl-heading-4= s_('AdminArea|Latest groups') - @groups.each do |group| .gl-display-flex.gl-py-3 .gl-mr-auto.gl-overflow-hidden.gl-text-overflow-ellipsis diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 0f13af61e3d..60ad122288c 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -354,6 +354,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: cronjob:click_house_event_paths_consistency_cron + :worker_name: ClickHouse::EventPathsConsistencyCronWorker + :feature_category: :value_stream_management + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:click_house_events_sync :worker_name: ClickHouse::EventsSyncWorker :feature_category: :value_stream_management diff --git a/app/workers/click_house/concerns/consistency_worker.rb b/app/workers/click_house/concerns/consistency_worker.rb new file mode 100644 index 00000000000..5fa1608ea2f --- /dev/null +++ b/app/workers/click_house/concerns/consistency_worker.rb @@ -0,0 +1,108 @@ +# frozen_string_literal: true + +module ClickHouse + module Concerns + # This module can be used for batching over a ClickHouse database table column + # and do something with the yielded values. The module is responsible for + # correctly restoring the state (cursor) in case the processing was + # interrupted or restart the processing from the beginning of the table + # when the table was fully processed. + # + # This class acts like a "template method" pattern where the implementor classes + # need to define two methods: + # + # - init_context: Returns a memoized hash, initializing the context that controls the data processing. + # - pluck_column: which column value to take from the ClickHouse DB when iterating + # - process_collected_values: once a limit is reached or no more data, do something + # - collect_values: filter, process and store the returned values from ClickHouse + # with the collected values. + module ConsistencyWorker + extend ActiveSupport::Concern + include Gitlab::Utils::StrongMemoize + + MAX_RUNTIME = 150.seconds + MAX_TTL = 5.minutes.to_i + CLICK_HOUSE_BATCH_SIZE = 100_000 + POSTGRESQL_BATCH_SIZE = 2500 + LIMIT_STATUSES = %i[limit_reached over_time].freeze + + included do + include Gitlab::ExclusiveLeaseHelpers + end + + def perform + return unless enabled? + + init_context + runtime_limiter + click_house_each_batch do |values| + collect_values(values) + + break if limit_was_reached? + end + + process_collected_values + + context[:last_processed_id] = 0 if table_fully_processed? + ClickHouse::SyncCursor.update_cursor_for(sync_cursor, context[:last_processed_id]) + log_extra_metadata_on_done(:result, metadata) + end + + private + + attr_reader :context + + def click_house_each_batch + in_lock(self.class.to_s, ttl: MAX_TTL, retries: 0) do + iterator.each_batch(column: batch_column, of: CLICK_HOUSE_BATCH_SIZE) do |scope| + query = scope.select(Arel.sql("DISTINCT #{pluck_column}")).to_sql + ids_from_click_house = connection.select(query).pluck(pluck_column).sort # rubocop: disable CodeReuse/ActiveRecord -- limited query + + ids_from_click_house.each_slice(POSTGRESQL_BATCH_SIZE) do |values| + yield values + end + end + end + end + + def enabled? + ClickHouse::Client.database_configured?(:main) && Feature.enabled?(:event_sync_worker_for_click_house) + end + + def runtime_limiter + @runtime_limiter ||= Analytics::CycleAnalytics::RuntimeLimiter.new(MAX_RUNTIME) + end + + def iterator + builder = ClickHouse::QueryBuilder.new(table.to_s) + ClickHouse::Iterator.new(query_builder: builder, connection: connection, min_value: previous_id) + end + + def sync_cursor + "#{table}_consistency_check" + end + + def previous_id + value = ClickHouse::SyncCursor.cursor_for(sync_cursor) + value == 0 ? nil : value + end + strong_memoize_attr :previous_id + + def metadata + @metadata ||= { status: :processed, modifications: 0 } + end + + def connection + @connection ||= ClickHouse::Connection.new(:main) + end + + def table_fully_processed? + metadata[:status] == :processed + end + + def limit_was_reached? + LIMIT_STATUSES.include?(metadata[:status]) + end + end + end +end diff --git a/app/workers/click_house/event_authors_consistency_cron_worker.rb b/app/workers/click_house/event_authors_consistency_cron_worker.rb index c35aadba593..62f64f2b9ff 100644 --- a/app/workers/click_house/event_authors_consistency_cron_worker.rb +++ b/app/workers/click_house/event_authors_consistency_cron_worker.rb @@ -5,8 +5,8 @@ module ClickHouse class EventAuthorsConsistencyCronWorker include ApplicationWorker include ClickHouseWorker + include ClickHouse::Concerns::ConsistencyWorker # defines perform include Gitlab::ExclusiveLeaseHelpers - include Gitlab::Utils::StrongMemoize idempotent! queue_namespace :cronjob @@ -14,83 +14,57 @@ module ClickHouse worker_has_external_dependencies! # the worker interacts with a ClickHouse database feature_category :value_stream_management - MAX_TTL = 5.minutes.to_i - MAX_RUNTIME = 150.seconds MAX_AUTHOR_DELETIONS = 2000 - CLICK_HOUSE_BATCH_SIZE = 100_000 - POSTGRESQL_BATCH_SIZE = 2500 - def perform - return unless enabled? - - runtime_limiter = Analytics::CycleAnalytics::RuntimeLimiter.new(MAX_RUNTIME) - - in_lock(self.class.to_s, ttl: MAX_TTL, retries: 0) do - author_records_to_delete = [] - last_processed_id = 0 - iterator.each_batch(column: :author_id, of: CLICK_HOUSE_BATCH_SIZE) do |scope| - query = scope.select(Arel.sql('DISTINCT author_id')).to_sql - ids_from_click_house = connection.select(query).pluck('author_id').sort - - ids_from_click_house.each_slice(POSTGRESQL_BATCH_SIZE) do |ids| - author_records_to_delete.concat(missing_user_ids(ids)) - last_processed_id = ids.last - - to_be_deleted_size = author_records_to_delete.size - if to_be_deleted_size >= MAX_AUTHOR_DELETIONS - metadata.merge!(status: :deletion_limit_reached, deletions: to_be_deleted_size) - break - end - - if runtime_limiter.over_time? - metadata.merge!(status: :over_time, deletions: to_be_deleted_size) - break - end - end - - break if limit_was_reached? - end + private - delete_records_from_click_house(author_records_to_delete) + def collect_values(ids) + missing_user_ids_from_batch = missing_user_ids(ids) + context[:last_processed_id] = missing_user_ids_from_batch.last + context[:author_records_to_delete].concat(missing_user_ids_from_batch) - last_processed_id = 0 if table_fully_processed? - ClickHouse::SyncCursor.update_cursor_for(:event_authors_consistency_check, last_processed_id) + to_be_deleted_size = context[:author_records_to_delete].size + metadata[:modifications] = to_be_deleted_size - log_extra_metadata_on_done(:result, metadata) + if to_be_deleted_size >= MAX_AUTHOR_DELETIONS + metadata[:status] = :limit_reached + return end + + metadata[:status] = :over_time if runtime_limiter.over_time? end - private + def process_collected_values + ids = context[:author_records_to_delete] + query = ClickHouse::Client::Query.new( + raw_query: 'ALTER TABLE events DELETE WHERE author_id IN ({author_ids:Array(UInt64)})', + placeholders: { author_ids: ids.to_json } + ) - def metadata - @metadata ||= { status: :processed, deletions: 0 } - end + connection.execute(query) - def limit_was_reached? - metadata[:status] == :deletion_limit_reached || metadata[:status] == :over_time - end + query = ClickHouse::Client::Query.new( + raw_query: 'ALTER TABLE event_authors DELETE WHERE author_id IN ({author_ids:Array(UInt64)})', + placeholders: { author_ids: ids.to_json } + ) - def table_fully_processed? - metadata[:status] == :processed + connection.execute(query) end - def enabled? - ClickHouse::Client.database_configured?(:main) && Feature.enabled?(:event_sync_worker_for_click_house) + def init_context + @context = { author_records_to_delete: [], last_processed_id: 0 } end - def previous_author_id - value = ClickHouse::SyncCursor.cursor_for(:event_authors_consistency_check) - value == 0 ? nil : value + def table + 'event_authors' end - strong_memoize_attr :previous_author_id - def iterator - builder = ClickHouse::QueryBuilder.new('event_authors') - ClickHouse::Iterator.new(query_builder: builder, connection: connection, min_value: previous_author_id) + def batch_column + 'author_id' end - def connection - @connection ||= ClickHouse::Connection.new(:main) + def pluck_column + 'author_id' end def missing_user_ids(ids) @@ -100,22 +74,6 @@ module ClickHouse .where('NOT EXISTS (SELECT 1 FROM users WHERE id = user_ids.id)') .pluck(:id) end - - def delete_records_from_click_house(ids) - query = ClickHouse::Client::Query.new( - raw_query: "ALTER TABLE events DELETE WHERE author_id IN ({author_ids:Array(UInt64)})", - placeholders: { author_ids: ids.to_json } - ) - - connection.execute(query) - - query = ClickHouse::Client::Query.new( - raw_query: "ALTER TABLE event_authors DELETE WHERE author_id IN ({author_ids:Array(UInt64)})", - placeholders: { author_ids: ids.to_json } - ) - - connection.execute(query) - end end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/app/workers/click_house/event_paths_consistency_cron_worker.rb b/app/workers/click_house/event_paths_consistency_cron_worker.rb new file mode 100644 index 00000000000..9dbf55186de --- /dev/null +++ b/app/workers/click_house/event_paths_consistency_cron_worker.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +module ClickHouse + # rubocop: disable CodeReuse/ActiveRecord -- Building worker-specific ActiveRecord and ClickHouse queries + class EventPathsConsistencyCronWorker + include ApplicationWorker + include ClickHouseWorker + include ClickHouse::Concerns::ConsistencyWorker # defines perform + include Gitlab::ExclusiveLeaseHelpers + + idempotent! + queue_namespace :cronjob + data_consistency :delayed + worker_has_external_dependencies! # the worker interacts with a ClickHouse database + feature_category :value_stream_management + + MAX_RECORD_MODIFICATIONS = 500 + + private + + def collect_values(paths) + traversal_ids_by_id = paths.each_with_object({}) do |path, hash| + traversal_ids = path.split('/').map(&:to_i) + hash[traversal_ids.last] = traversal_ids + end + + namespaces_by_id = Namespace + .select(:id, :traversal_ids) + .where(id: traversal_ids_by_id.keys) + .index_by(&:id) + + traversal_ids_by_id.each do |id, old_traversal_ids| + old_path = to_path(old_traversal_ids) + if !namespaces_by_id.key?(id) + context[:paths_to_delete] << [id, old_path] + elsif namespaces_by_id[id].traversal_ids != old_traversal_ids + context[:paths_to_update] << [id, old_path, to_path(namespaces_by_id[id].traversal_ids)] + end + + context[:last_processed_id] = id + end + + modifications = context[:paths_to_update].size + context[:paths_to_delete].size + metadata[:modifications] = modifications + + if modifications >= MAX_RECORD_MODIFICATIONS + metadata[:status] = :modification_limit_reached + return + end + + return unless runtime_limiter.over_time? + + metadata[:status] = :over_time + end + + def to_path(traversal_ids) + "#{traversal_ids.join('/')}/" + end + + def process_collected_values + delete_records_from_click_house(context[:paths_to_delete]) + update_records_in_click_house(context[:paths_to_update]) + end + + def table + 'event_namespace_paths' + end + + def batch_column + 'namespace_id' + end + + def pluck_column + 'path' + end + + def init_context + @context = { paths_to_delete: [], paths_to_update: [], last_processed_id: 0 } + end + + def delete_records_from_click_house(id_paths) + return if id_paths.empty? + + paths = id_paths.map(&:second).map { |value| "'#{value}'" }.join(',') + query = ClickHouse::Client::Query.new( + raw_query: "DELETE FROM events WHERE path IN (#{paths})" + ) + + connection.execute(query) + + query = ClickHouse::Client::Query.new( + raw_query: 'DELETE FROM event_namespace_paths WHERE namespace_id IN ({namespace_ids:Array(UInt64)})', + placeholders: { namespace_ids: id_paths.map(&:first).to_json } + ) + + connection.execute(query) + end + + def update_records_in_click_house(paths_to_update) + paths_to_update.each do |id, old_path, new_path| + query = ClickHouse::Client::Query.new( + raw_query: + 'ALTER TABLE events UPDATE path={new_path:String} WHERE path = {old_path:String}', + placeholders: { new_path: new_path, old_path: old_path } + ) + connection.execute(query) + + query = ClickHouse::Client::Query.new( + raw_query: + 'ALTER TABLE event_namespace_paths UPDATE path={new_path:String} WHERE namespace_id = {namespace_id:UInt64}', + placeholders: { new_path: new_path, namespace_id: id } + ) + connection.execute(query) + end + end + end + # rubocop: enable CodeReuse/ActiveRecord +end diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index fd2a7ab6743..7007e6329e7 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -906,6 +906,9 @@ Gitlab.ee do Settings.cron_jobs['click_house_event_authors_consistency_cron_worker'] ||= {} Settings.cron_jobs['click_house_event_authors_consistency_cron_worker']['cron'] ||= "*/30 * * * *" Settings.cron_jobs['click_house_event_authors_consistency_cron_worker']['job_class'] = 'ClickHouse::EventAuthorsConsistencyCronWorker' + Settings.cron_jobs['click_house_event_namespace_paths_consistency_cron_worker'] ||= {} + Settings.cron_jobs['click_house_event_namespace_paths_consistency_cron_worker']['cron'] ||= "*/45 * * * *" + Settings.cron_jobs['click_house_event_namespace_paths_consistency_cron_worker']['job_class'] = 'ClickHouse::EventNamespacePathsConsistencyCronWorker' Settings.cron_jobs['vertex_ai_refresh_access_token_worker'] ||= {} Settings.cron_jobs['vertex_ai_refresh_access_token_worker']['cron'] ||= '*/50 * * * *' Settings.cron_jobs['vertex_ai_refresh_access_token_worker']['job_class'] = 'Llm::VertexAiAccessTokenRefreshWorker' diff --git a/config/initializers/7_gitlab_http.rb b/config/initializers/7_gitlab_http.rb index cd891f29584..23aa357c5e3 100644 --- a/config/initializers/7_gitlab_http.rb +++ b/config/initializers/7_gitlab_http.rb @@ -30,5 +30,4 @@ if Gitlab.config.gitlab['http_client'] password = Gitlab.config.gitlab['http_client']['tls_client_cert_password'] Gitlab::HTTP_V2::Client.pem(pem, password) - Gitlab::LegacyHTTP.pem(pem, password) end diff --git a/db/post_migrate/20240105121755_add_participants_widget_definition_to_work_item_types.rb b/db/post_migrate/20240105121755_add_participants_widget_definition_to_work_item_types.rb new file mode 100644 index 00000000000..7313e14c44e --- /dev/null +++ b/db/post_migrate/20240105121755_add_participants_widget_definition_to_work_item_types.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class AddParticipantsWidgetDefinitionToWorkItemTypes < Gitlab::Database::Migration[2.2] + milestone '16.9' + restrict_gitlab_migration gitlab_schema: :gitlab_main + + WIDGET_NAME = 'Participants' + WIDGET_ENUM_VALUE = 20 + + def up + widgets = [] + + work_item_types = define_batchable_model('work_item_types') + work_item_types.each_batch do |work_item_types| + work_item_types.each do |type| + widgets << { + work_item_type_id: type.id, + name: WIDGET_NAME, + widget_type: WIDGET_ENUM_VALUE + } + end + end + + return if widgets.empty? + + work_item_widget_definitions.upsert_all( + widgets, + unique_by: :index_work_item_widget_definitions_on_default_witype_and_name + ) + end + + def down + work_item_widget_definitions.where(name: WIDGET_NAME).delete_all + end + + private + + def work_item_widget_definitions + define_batchable_model('work_item_widget_definitions') + end +end diff --git a/db/schema_migrations/20240105121755 b/db/schema_migrations/20240105121755 new file mode 100644 index 00000000000..a476a0ea694 --- /dev/null +++ b/db/schema_migrations/20240105121755 @@ -0,0 +1 @@ +c4cd2bb2b69cf75a00ae68517d5464a123d7c96bde4665b5c2187cdb8465bc48
\ No newline at end of file diff --git a/doc/administration/settings/usage_statistics.md b/doc/administration/settings/usage_statistics.md index b661bcd6746..4c6af23849d 100644 --- a/doc/administration/settings/usage_statistics.md +++ b/doc/administration/settings/usage_statistics.md @@ -118,9 +118,6 @@ sequenceDiagram participant GitLab instance participant Version Application GitLab instance->>Version Application: Is there a version update? - loop Version Check - Version Application->>Version Application: Record version info - end Version Application->>GitLab instance: Response (PNG/SVG) ``` diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index f5198c54136..6b14d394e3a 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -29313,6 +29313,17 @@ Represents the notifications widget. | <a id="workitemwidgetnotificationssubscribed"></a>`subscribed` | [`Boolean!`](#boolean) | Whether the current user is subscribed to notifications on the work item. | | <a id="workitemwidgetnotificationstype"></a>`type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | +### `WorkItemWidgetParticipants` + +Represents a participants widget. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemwidgetparticipantsparticipants"></a>`participants` | [`UserCoreConnection`](#usercoreconnection) | Participants in the work item. (see [Connections](#connections)) | +| <a id="workitemwidgetparticipantstype"></a>`type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | + ### `WorkItemWidgetProgress` Represents a progress widget. @@ -32296,6 +32307,7 @@ Type of a work item widget. | <a id="workitemwidgettypemilestone"></a>`MILESTONE` | Milestone widget. | | <a id="workitemwidgettypenotes"></a>`NOTES` | Notes widget. | | <a id="workitemwidgettypenotifications"></a>`NOTIFICATIONS` | Notifications widget. | +| <a id="workitemwidgettypeparticipants"></a>`PARTICIPANTS` | Participants widget. | | <a id="workitemwidgettypeprogress"></a>`PROGRESS` | Progress widget. | | <a id="workitemwidgettyperequirement_legacy"></a>`REQUIREMENT_LEGACY` | Requirement Legacy widget. | | <a id="workitemwidgettyperolledup_dates"></a>`ROLLEDUP_DATES` | Rolledup Dates widget. | @@ -33983,6 +33995,7 @@ Implementations: - [`WorkItemWidgetMilestone`](#workitemwidgetmilestone) - [`WorkItemWidgetNotes`](#workitemwidgetnotes) - [`WorkItemWidgetNotifications`](#workitemwidgetnotifications) +- [`WorkItemWidgetParticipants`](#workitemwidgetparticipants) - [`WorkItemWidgetProgress`](#workitemwidgetprogress) - [`WorkItemWidgetRequirementLegacy`](#workitemwidgetrequirementlegacy) - [`WorkItemWidgetRolledupDates`](#workitemwidgetrolledupdates) diff --git a/doc/development/pipelines/internals.md b/doc/development/pipelines/internals.md index 16d0bfdfa30..020962e8425 100644 --- a/doc/development/pipelines/internals.md +++ b/doc/development/pipelines/internals.md @@ -46,7 +46,6 @@ Here's a list of where we're using this right now, and should try to move away from using `$FORCE_GITLAB_CI`. - [JiHu validation pipeline](https://about.gitlab.com/handbook/ceo/chief-of-staff-team/jihu-support/jihu-validation-pipelines.html) -- [Gitaly downstream GitLab pipeline](https://gitlab.com/gitlab-org/gitaly/-/issues/4615) See the next section for how we can enable pipelines without using `$FORCE_GITLAB_CI`. diff --git a/doc/install/requirements.md b/doc/install/requirements.md index bad3a2b09e7..982c00d264e 100644 --- a/doc/install/requirements.md +++ b/doc/install/requirements.md @@ -79,13 +79,22 @@ available, though the exact requirements [depend on the number of users](../admi We highly recommend using at least the minimum PostgreSQL versions (as specified in the following table) as these were used for development and testing: -| GitLab version | Minimum PostgreSQL version | -|----------------|----------------------------| -| 13.0 | 11 | -| 14.0 | 12.7 | -| 15.0 | 12.10 | -| 16.0 | 13.6 | -| 17.0 (planned) | 14.8 | +| GitLab version | Minimum PostgreSQL version<sup>1</sup> | Maximum PostgreSQL version<sup>2</sup> | +|----------------|----------------------------------------|----------------------------------------| +| 13.0 | 11 | <sup>2</sup> | +| 14.0 | 12.7 | <sup>2</sup> | +| 15.0 | 12.10 | 13.x (14.x<sup>3</sup>) | +| 16.0 | 13.6 | 15.x<sup>4</sup> | +| 17.0 (planned) | 14.9 | 15.x<sup>4</sup> | + +1. PostgreSQL minor release upgrades (for example 14.8 to 14.9) [include only bug and security fixes](https://www.postgresql.org/support/versioning/). + Patch levels in this table are not prescriptive. Always deploy the most recent patch level + to avoid [known bugs in PostgreSQL that might be triggered by GitLab](https://gitlab.com/gitlab-org/gitlab/-/issues/364763). +1. If you want to run a later major release of PostgreSQL than the specified minimum + [check if a more recent version shipped with Linux package (Omnibus) GitLab](http://gitlab-org.gitlab.io/omnibus-gitlab/licenses.html). + `postgresql-new` is a later version that's definitely supported. +1. PostgreSQL 14.x [tested against GitLab 15.11 only](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/114624). +1. [Tested against GitLab 16.1 and later](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/119344). You must also ensure the following extensions are loaded into every GitLab database. [Read more about this requirement, and troubleshooting](postgresql_extensions.md). @@ -135,7 +144,7 @@ exclusive use of GitLab. Do not make direct changes to the database, schemas, us properties except when following procedures in the GitLab documentation or following the directions of GitLab Support or other GitLab engineers. -- The main GitLab application currently uses three schemas: +- The main GitLab application uses three schemas: - The default `public` schema - `gitlab_partitions_static` (automatically created) @@ -253,7 +262,7 @@ The requirements for Redis are as follows: ## Sidekiq -Sidekiq processes the background jobs with a multithreaded process. +Sidekiq processes the background jobs with a multi-threaded process. This process starts with the entire Rails stack (200 MB+) but it can grow over time due to memory leaks. On a very active server (10,000 billable users) the Sidekiq process can use 1 GB+ of memory. diff --git a/doc/user/analytics/analytics_dashboards.md b/doc/user/analytics/analytics_dashboards.md index e647dcf170a..585512698f3 100644 --- a/doc/user/analytics/analytics_dashboards.md +++ b/doc/user/analytics/analytics_dashboards.md @@ -43,7 +43,6 @@ For more information about the development of product analytics, see the [group - Comment on issue [391970](https://gitlab.com/gitlab-org/gitlab/-/issues/391970). - Create an issue with the `group::product analytics` label. -- [Schedule a call](https://calendly.com/jheimbuck/30-minute-call) with the team. ### Value Stream Management diff --git a/lib/gitlab/database_importers/work_items/base_type_importer.rb b/lib/gitlab/database_importers/work_items/base_type_importer.rb index 6e6080a0543..50a9a0e8245 100644 --- a/lib/gitlab/database_importers/work_items/base_type_importer.rb +++ b/lib/gitlab/database_importers/work_items/base_type_importer.rb @@ -24,7 +24,8 @@ module Gitlab award_emoji: 'Award emoji', linked_items: 'Linked items', color: 'Color', - rolledup_dates: 'Rolledup dates' + rolledup_dates: 'Rolledup dates', + participants: 'Participants' }.freeze WIDGETS_FOR_TYPE = { @@ -42,7 +43,8 @@ module Gitlab :notifications, :current_user_todos, :award_emoji, - :linked_items + :linked_items, + :participants ], incident: [ :assignees, @@ -52,7 +54,8 @@ module Gitlab :notifications, :current_user_todos, :award_emoji, - :linked_items + :linked_items, + :participants ], test_case: [ :description, @@ -60,7 +63,8 @@ module Gitlab :notifications, :current_user_todos, :award_emoji, - :linked_items + :linked_items, + :participants ], requirement: [ :description, @@ -71,7 +75,8 @@ module Gitlab :notifications, :current_user_todos, :award_emoji, - :linked_items + :linked_items, + :participants ], task: [ :assignees, @@ -86,7 +91,8 @@ module Gitlab :notifications, :current_user_todos, :award_emoji, - :linked_items + :linked_items, + :participants ], objective: [ :assignees, @@ -100,7 +106,8 @@ module Gitlab :notifications, :current_user_todos, :award_emoji, - :linked_items + :linked_items, + :participants ], key_result: [ :assignees, @@ -114,7 +121,8 @@ module Gitlab :notifications, :current_user_todos, :award_emoji, - :linked_items + :linked_items, + :participants ], epic: [ :assignees, @@ -130,7 +138,8 @@ module Gitlab :award_emoji, :linked_items, :color, - :rolledup_dates + :rolledup_dates, + :participants ], ticket: [ :assignees, @@ -146,7 +155,8 @@ module Gitlab :notifications, :current_user_todos, :award_emoji, - :linked_items + :linked_items, + :participants ] }.freeze diff --git a/public/502.html b/public/502.html index 237fbff313f..5c2af3498b9 100644 --- a/public/502.html +++ b/public/502.html @@ -2,7 +2,7 @@ <html> <head> <meta content="width=device-width, initial-scale=1" name="viewport"> - <title>GitLab is not responding (502)</title> + <title>Waiting for GitLab to boot</title> <style> body { color: #666; @@ -69,24 +69,35 @@ <a href="/"> <img src='data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjAwIiBoZWlnaHQ9IjE5MiIgdmlld0JveD0iMCAwIDI1IDI0IiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxwYXRoIGQ9Im0yNC41MDcgOS41LS4wMzQtLjA5TDIxLjA4Mi41NjJhLjg5Ni44OTYgMCAwIDAtMS42OTQuMDkxbC0yLjI5IDcuMDFINy44MjVMNS41MzUuNjUzYS44OTguODk4IDAgMCAwLTEuNjk0LS4wOUwuNDUxIDkuNDExLjQxNiA5LjVhNi4yOTcgNi4yOTcgMCAwIDAgMi4wOSA3LjI3OGwuMDEyLjAxLjAzLjAyMiA1LjE2IDMuODY3IDIuNTYgMS45MzUgMS41NTQgMS4xNzZhMS4wNTEgMS4wNTEgMCAwIDAgMS4yNjggMGwxLjU1NS0xLjE3NiAyLjU2LTEuOTM1IDUuMTk3LTMuODkuMDE0LS4wMUE2LjI5NyA2LjI5NyAwIDAgMCAyNC41MDcgOS41WiIKICAgICAgICBmaWxsPSIjRTI0MzI5Ii8+CiAgPHBhdGggZD0ibTI0LjUwNyA5LjUtLjAzNC0uMDlhMTEuNDQgMTEuNDQgMCAwIDAtNC41NiAyLjA1MWwtNy40NDcgNS42MzIgNC43NDIgMy41ODQgNS4xOTctMy44OS4wMTQtLjAxQTYuMjk3IDYuMjk3IDAgMCAwIDI0LjUwNyA5LjVaIgogICAgICAgIGZpbGw9IiNGQzZEMjYiLz4KICA8cGF0aCBkPSJtNy43MDcgMjAuNjc3IDIuNTYgMS45MzUgMS41NTUgMS4xNzZhMS4wNTEgMS4wNTEgMCAwIDAgMS4yNjggMGwxLjU1NS0xLjE3NiAyLjU2LTEuOTM1LTQuNzQzLTMuNTg0LTQuNzU1IDMuNTg0WiIKICAgICAgICBmaWxsPSIjRkNBMzI2Ii8+CiAgPHBhdGggZD0iTTUuMDEgMTEuNDYxYTExLjQzIDExLjQzIDAgMCAwLTQuNTYtMi4wNUwuNDE2IDkuNWE2LjI5NyA2LjI5NyAwIDAgMCAyLjA5IDcuMjc4bC4wMTIuMDEuMDMuMDIyIDUuMTYgMy44NjcgNC43NDUtMy41ODQtNy40NDQtNS42MzJaIgogICAgICAgIGZpbGw9IiNGQzZEMjYiLz4KPC9zdmc+Cg==' alt="GitLab"/> </a> - <h1> - 502 - </h1> + + <h1>Waiting for GitLab to boot</h1> + <h2>HTTP 502</h2> + <div class="container"> - <h3>We're sorry. GitLab is taking too much time to respond.</h3> - <hr /> - <p>Try refreshing the page, or going back and attempting the action again.</p> - <p>Please contact your GitLab administrator if this problem persists.</p> + <p>It can take up to a few minutes for GitLab to boot completely.</p> + <p>This page will automatically reload every 5 seconds.</p> + <a href="javascript:history.back()" class="js-go-back go-back">Go back</a> </div> + + <hr/> + + <footer> + <p>Refreshed at <span id="refreshedAt"></span></p> + </footer> + <script> - (function () { - var goBack = document.querySelector('.js-go-back'); + const now = new Date(); + const refreshedAt = now.toLocaleTimeString(); + + document.getElementById("refreshedAt").innerText = refreshedAt; - if (history.length > 1) { - goBack.style.display = 'inline'; - } - })(); + window.setTimeout(() => location.reload(), 5000); + + if (history.length > 1) { + const goBack = document.querySelector('.js-go-back'); + goBack.style.display = 'inline'; + } </script> </body> </html> diff --git a/spec/frontend/vue_merge_request_widget/components/merge_checks_spec.js b/spec/frontend/vue_merge_request_widget/components/merge_checks_spec.js index b19095cc686..48c01e3efad 100644 --- a/spec/frontend/vue_merge_request_widget/components/merge_checks_spec.js +++ b/spec/frontend/vue_merge_request_widget/components/merge_checks_spec.js @@ -162,4 +162,26 @@ describe('Merge request merge checks component', () => { expect(wrapper.findByTestId('merge-checks-full').exists()).toBe(true); }); + + it('sorts merge checks', async () => { + mountComponent({ + mergeabilityChecks: [ + { identifier: 'discussions', status: 'SUCCESS' }, + { identifier: 'discussions', status: 'INACTIVE' }, + { identifier: 'rebase', status: 'FAILED' }, + ], + }); + + await waitForPromises(); + + await wrapper.findByTestId('widget-toggle').trigger('click'); + + const mergeChecks = wrapper.findAllByTestId('merge-check'); + + expect(mergeChecks.length).toBe(2); + expect(mergeChecks.at(0).props('check')).toEqual(expect.objectContaining({ status: 'FAILED' })); + expect(mergeChecks.at(1).props('check')).toEqual( + expect.objectContaining({ status: 'SUCCESS' }), + ); + }); }); diff --git a/spec/graphql/types/work_items/widgets/participants_type_spec.rb b/spec/graphql/types/work_items/widgets/participants_type_spec.rb new file mode 100644 index 00000000000..68d201cc52b --- /dev/null +++ b/spec/graphql/types/work_items/widgets/participants_type_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Types::WorkItems::Widgets::ParticipantsType, feature_category: :team_planning do + it 'exposes the expected fields' do + expected_fields = %i[participants type] + + expect(described_class).to have_graphql_fields(*expected_fields) + end +end diff --git a/spec/models/concerns/participable_spec.rb b/spec/models/concerns/participable_spec.rb index 58a44fec3aa..57cdf0da516 100644 --- a/spec/models/concerns/participable_spec.rb +++ b/spec/models/concerns/participable_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Participable do +RSpec.describe Participable, feature_category: :team_planning do let(:model) do Class.new do include Participable @@ -31,7 +31,7 @@ RSpec.describe Participable do expect(instance).to receive(:foo).and_return(user2) expect(instance).to receive(:bar).and_return(user3) - expect(instance).to receive(:project).thrice.and_return(project) + expect(instance).to receive(:project).exactly(4).and_return(project) participants = instance.participants(user1) @@ -66,7 +66,7 @@ RSpec.describe Participable do expect(instance).to receive(:foo).and_return(other) expect(other).to receive(:bar).and_return(user2) - expect(instance).to receive(:project).thrice.and_return(project) + expect(instance).to receive(:project).exactly(4).and_return(project) expect(instance.participants(user1)).to eq([user2]) end @@ -86,7 +86,7 @@ RSpec.describe Participable do instance = model.new - expect(instance).to receive(:project).thrice.and_return(project) + expect(instance).to receive(:project).exactly(4).and_return(project) instance.participants(user1) @@ -115,6 +115,46 @@ RSpec.describe Participable do expect(participants).to contain_exactly(user1) end end + + context 'participable is a group level object' do + it 'returns the list of participants' do + model.participant(:foo) + model.participant(:bar) + + user1 = build(:user) + user2 = build(:user) + user3 = build(:user) + group = build(:group, :public) + instance = model.new + + expect(instance).to receive(:foo).and_return(user2) + expect(instance).to receive(:bar).and_return(user3) + expect(instance).to receive(:project).exactly(3).and_return(nil) + expect(instance).to receive(:namespace).exactly(2).and_return(group) + + participants = instance.participants(user1) + + expect(participants).not_to include(user1) + expect(participants).to include(user2) + expect(participants).to include(user3) + end + end + + context 'participable is neither a project nor a group level object' do + it 'returns no participants' do + model.participant(:foo) + + user = build(:user) + instance = model.new + + expect(instance).to receive(:foo).and_return(user) + expect(instance).to receive(:project).exactly(3).and_return(nil) + + participants = instance.participants(user) + + expect(participants).to be_empty + end + end end describe '#visible_participants' do @@ -138,7 +178,7 @@ RSpec.describe Participable do allow(instance).to receive_message_chain(:model_name, :element) { 'class' } expect(instance).to receive(:foo).and_return(user2) expect(instance).to receive(:bar).and_return(user3) - expect(instance).to receive(:project).thrice.and_return(project) + expect(instance).to receive(:project).exactly(4).and_return(project) participants = instance.visible_participants(user1) @@ -158,17 +198,68 @@ RSpec.describe Participable do allow(instance).to receive_message_chain(:model_name, :element) { 'class' } allow(instance).to receive(:bar).and_return(user2) - expect(instance).to receive(:project).thrice.and_return(project) + expect(instance).to receive(:project).exactly(4).and_return(project) expect(instance.visible_participants(user1)).to be_empty end end + context 'when participable is a group level object' do + let(:group) { create(:group, :private) } + + it 'returns the list of participants' do + model.participant(:foo) + model.participant(:bar) + + user1 = create(:user) + user2 = create(:user) + user3 = create(:user) + instance = model.new + + group.add_reporter(user1) + group.add_reporter(user3) + + allow(instance).to receive_message_chain(:model_name, :element) { 'class' } + expect(instance).to receive(:foo).and_return(user2) + expect(instance).to receive(:bar).and_return(user3) + expect(instance).to receive(:project).exactly(3).and_return(nil) + expect(instance).to receive(:namespace).exactly(2).and_return(group) + + participants = instance.visible_participants(user1) + + expect(participants).not_to include(user1) # not returned by participant attr + expect(participants).not_to include(user2) # not a member of group + expect(participants).to include(user3) # member of group + end + end + + context 'when participable is neither project nor group level object' do + let(:group) { create(:group, :private) } + + it 'returns no participants' do + model.participant(:foo) + + user = create(:user) + instance = model.new + + group.add_reporter(user) + + allow(instance).to receive_message_chain(:model_name, :element) { 'class' } + expect(instance).to receive(:foo).and_return(user) + expect(instance).to receive(:project).exactly(3).and_return(nil) + + # user is returned by participant attr and is a member of the group, + # but participable model is neither a group or project object + participants = instance.visible_participants(user) + expect(participants).to be_empty + end + end + context 'with multiple system notes from the same author and mentioned_users' do let!(:user1) { create(:user) } let!(:user2) { create(:user) } - it 'skips expensive checks if the author is aleady in participants list' do + it 'skips expensive checks if the author is already in participants list' do model.participant(:notes) instance = model.new @@ -215,7 +306,7 @@ RSpec.describe Participable do it 'caches the list of raw participants' do expect(instance).to receive(:raw_participants).once.and_return([]) - expect(instance).to receive(:project).twice.and_return(project) + expect(instance).to receive(:project).exactly(4).and_return(project) instance.participant?(user1) instance.participant?(user1) @@ -234,5 +325,41 @@ RSpec.describe Participable do expect(instance.participant?(user3)).to be false end end + + context 'when participable is a group level object' do + let(:group) { create(:group, :private) } + + before do + # we need users to be created to add them as members to the group + user1.save! + user2.save! + user3.save! + + group.add_reporter(user1) + group.add_reporter(user2) + end + + it 'returns whether the user is a participant' do + allow(instance).to receive(:foo).and_return(user1) + allow(instance).to receive(:bar).and_return(user3) + allow(instance).to receive(:project).and_return(nil) + allow(instance).to receive(:namespace).and_return(group) + + expect(instance.participant?(user1)).to be true # returned by participant attr and a member of group + expect(instance.participant?(user2)).to be false # returned by participant attr + expect(instance.participant?(user3)).to be false # not a member of group + end + + context 'when participable is neither project nor group level object' do + it 'returns whether the user is a participant' do + allow(instance).to receive(:foo).and_return(user1) + allow(instance).to receive(:project).and_return(nil) + + # user1 is returned by participant attr and is a member of group, + # but participable model is neither a group or project object + expect(instance.participant?(user1)).to be false + end + end + end end end diff --git a/spec/models/work_item_spec.rb b/spec/models/work_item_spec.rb index 843e3d40dc2..eeb1e3699d8 100644 --- a/spec/models/work_item_spec.rb +++ b/spec/models/work_item_spec.rb @@ -6,6 +6,7 @@ RSpec.describe WorkItem, feature_category: :portfolio_management do using RSpec::Parameterized::TableSyntax let_it_be(:reusable_project) { create(:project) } + let_it_be(:reusable_group) { create(:group) } describe 'associations' do it { is_expected.to belong_to(:namespace) } @@ -725,4 +726,22 @@ RSpec.describe WorkItem, feature_category: :portfolio_management do expect(item4.linked_items_count).to eq(0) end end + + context 'work item participants' do + context 'project level work item' do + let_it_be(:work_item) { create(:work_item, project: reusable_project) } + + it 'has participants' do + expect(work_item.participants).to match_array([work_item.author]) + end + end + + context 'group level work item' do + let_it_be(:work_item) { create(:work_item, namespace: reusable_group) } + + it 'has participants' do + expect(work_item.participants).to match_array([work_item.author]) + end + end + end end diff --git a/spec/models/work_items/widget_definition_spec.rb b/spec/models/work_items/widget_definition_spec.rb index 1540ee57ff4..f1f498cef88 100644 --- a/spec/models/work_items/widget_definition_spec.rb +++ b/spec/models/work_items/widget_definition_spec.rb @@ -15,7 +15,8 @@ RSpec.describe WorkItems::WidgetDefinition, feature_category: :team_planning do ::WorkItems::Widgets::Notifications, ::WorkItems::Widgets::CurrentUserTodos, ::WorkItems::Widgets::AwardEmoji, - ::WorkItems::Widgets::LinkedItems + ::WorkItems::Widgets::LinkedItems, + ::WorkItems::Widgets::Participants ] if Gitlab.ee? diff --git a/spec/requests/api/graphql/project/work_items_spec.rb b/spec/requests/api/graphql/project/work_items_spec.rb index d0f80bcfebe..982696593bb 100644 --- a/spec/requests/api/graphql/project/work_items_spec.rb +++ b/spec/requests/api/graphql/project/work_items_spec.rb @@ -279,14 +279,14 @@ RSpec.describe 'getting a work item list for a project', feature_category: :team context 'when fetching work item notifications widget' do let(:fields) do <<~GRAPHQL - nodes { - widgets { - type - ... on WorkItemWidgetNotifications { - subscribed - } + nodes { + widgets { + type + ... on WorkItemWidgetNotifications { + subscribed } } + } GRAPHQL end @@ -307,22 +307,22 @@ RSpec.describe 'getting a work item list for a project', feature_category: :team context 'when fetching work item award emoji widget' do let(:fields) do <<~GRAPHQL - nodes { - widgets { - type - ... on WorkItemWidgetAwardEmoji { - awardEmoji { - nodes { - name - emoji - user { id } - } + nodes { + widgets { + type + ... on WorkItemWidgetAwardEmoji { + awardEmoji { + nodes { + name + emoji + user { id } } - upvotes - downvotes } + upvotes + downvotes } } + } GRAPHQL end @@ -407,6 +407,54 @@ RSpec.describe 'getting a work item list for a project', feature_category: :team end end + context 'when fetching work item participants widget' do + let_it_be(:other_project) { create(:project, group: group) } + let_it_be(:project) { other_project } + let_it_be(:users) { create_list(:user, 3) } + let_it_be(:work_items) { create_list(:work_item, 3, project: project, assignees: users) } + + let(:fields) do + <<~GRAPHQL + nodes { + id + widgets { + type + ... on WorkItemWidgetParticipants { + participants { + nodes { + id + username + } + } + } + } + } + GRAPHQL + end + + before do + project.add_guest(current_user) + end + + it 'returns participants' do + post_graphql(query, current_user: current_user) + + participants_usernames = graphql_dig_at(items_data, 'widgets', 'participants', 'nodes', 'username') + expect(participants_usernames).to match_array(work_items.flat_map(&:participants).map(&:username)) + end + + it 'executes limited number of N+1 queries', :use_sql_query_cache do + control = ActiveRecord::QueryRecorder.new(skip_cached: false) do + post_graphql(query, current_user: current_user) + end + + create_list(:work_item, 2, project: project, assignees: users) + + expect_graphql_errors_to_be_empty + expect { post_graphql(query, current_user: current_user) }.not_to exceed_all_query_limit(control) + end + end + def item_ids graphql_dig_at(items_data, :id) end diff --git a/spec/requests/api/graphql/work_item_spec.rb b/spec/requests/api/graphql/work_item_spec.rb index c6d44b057a7..e5d8131fc7e 100644 --- a/spec/requests/api/graphql/work_item_spec.rb +++ b/spec/requests/api/graphql/work_item_spec.rb @@ -650,7 +650,7 @@ RSpec.describe 'Query.work_item(id)', feature_category: :team_planning do let(:first_param) { 1 } let(:all_records) { [link1, link2] } - let(:data_path) { ['workItem', 'widgets', "linkedItems", -1] } + let(:data_path) { ['workItem', 'widgets', 'linkedItems', -2] } def widget_fields(args) query_graphql_field( diff --git a/spec/services/merge_requests/retarget_chain_service_spec.rb b/spec/services/merge_requests/retarget_chain_service_spec.rb index ef8cd0a861e..a048c21d39a 100644 --- a/spec/services/merge_requests/retarget_chain_service_spec.rb +++ b/spec/services/merge_requests/retarget_chain_service_spec.rb @@ -41,9 +41,13 @@ RSpec.describe MergeRequests::RetargetChainService, feature_category: :code_revi 'target', 'delete', merge_request.source_branch, merge_request.target_branch) + expect(another_merge_request.rebase_jid).to be_blank + expect { subject }.to change { another_merge_request.reload.target_branch } .from(merge_request.source_branch) .to(merge_request.target_branch) + + expect(another_merge_request.rebase_jid).to be_present end end @@ -132,9 +136,17 @@ RSpec.describe MergeRequests::RetargetChainService, feature_category: :code_revi merge_request.mark_as_merged end - it 'retargets only 4 of them' do + it 'retargets and rebases only 4 of them' do + expect(many_merge_requests.any? { |mr| mr.reload.rebase_jid.present? }).to be_falsey + subject + first_four = many_merge_requests.first(4) + others = many_merge_requests - first_four + + expect(others.any? { |mr| mr.reload.rebase_jid.present? }).to be_falsey + expect(first_four.all? { |mr| mr.reload.rebase_jid.present? }).to be_truthy + expect(many_merge_requests.each(&:reload).pluck(:target_branch).tally) .to eq( merge_request.source_branch => 6, diff --git a/spec/workers/click_house/event_authors_consistency_cron_worker_spec.rb b/spec/workers/click_house/event_authors_consistency_cron_worker_spec.rb index d4fa35b9b82..4d7e0e138e9 100644 --- a/spec/workers/click_house/event_authors_consistency_cron_worker_spec.rb +++ b/spec/workers/click_house/event_authors_consistency_cron_worker_spec.rb @@ -73,10 +73,10 @@ RSpec.describe ClickHouse::EventAuthorsConsistencyCronWorker, feature_category: User.where(id: [user1.id, user2.id]).delete_all stub_const("#{described_class}::MAX_AUTHOR_DELETIONS", 2) - stub_const("#{described_class}::POSTGRESQL_BATCH_SIZE", 1) + stub_const("#{ClickHouse::Concerns::ConsistencyWorker}::POSTGRESQL_BATCH_SIZE", 1) expect(worker).to receive(:log_extra_metadata_on_done).with(:result, - { status: :deletion_limit_reached, deletions: 2 }) + { status: :limit_reached, modifications: 2 }) worker.perform @@ -87,13 +87,13 @@ RSpec.describe ClickHouse::EventAuthorsConsistencyCronWorker, feature_category: context 'when time limit is reached' do it 'stops the processing earlier' do - stub_const("#{described_class}::POSTGRESQL_BATCH_SIZE", 1) + stub_const("#{ClickHouse::Concerns::ConsistencyWorker}::POSTGRESQL_BATCH_SIZE", 1) # stop at the third author_id allow_next_instance_of(Analytics::CycleAnalytics::RuntimeLimiter) do |runtime_limiter| allow(runtime_limiter).to receive(:over_time?).and_return(false, false, true) end - expect(worker).to receive(:log_extra_metadata_on_done).with(:result, { status: :over_time, deletions: 1 }) + expect(worker).to receive(:log_extra_metadata_on_done).with(:result, { status: :over_time, modifications: 1 }) worker.perform diff --git a/spec/workers/click_house/event_paths_consistency_cron_worker_spec.rb b/spec/workers/click_house/event_paths_consistency_cron_worker_spec.rb new file mode 100644 index 00000000000..76ce63ed2e4 --- /dev/null +++ b/spec/workers/click_house/event_paths_consistency_cron_worker_spec.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ClickHouse::EventPathsConsistencyCronWorker, feature_category: :value_stream_management do + let(:worker) { described_class.new } + + context 'when ClickHouse is disabled' do + it 'does nothing' do + allow(ClickHouse::Client).to receive(:database_configured?).and_return(false) + + expect(worker).not_to receive(:log_extra_metadata_on_done) + + worker.perform + end + end + + context 'when the event_sync_worker_for_click_house feature flag is off' do + before do + stub_feature_flags(event_sync_worker_for_click_house: false) + end + + it 'does nothing' do + allow(ClickHouse::Client).to receive(:database_configured?).and_return(true) + + expect(worker).not_to receive(:log_extra_metadata_on_done) + + worker.perform + end + end + + context 'when ClickHouse is available', :click_house do + let_it_be(:connection) { ClickHouse::Connection.new(:main) } + let_it_be_with_reload(:namespace1) { create(:group) } + let_it_be_with_reload(:namespace2) { create(:project).project_namespace } + let_it_be_with_reload(:namespace_with_updated_parent) { create(:group, parent: create(:group)) } + + let(:leftover_paths) { connection.select('SELECT DISTINCT path FROM events FINAL').pluck('path') } + let(:deleted_namespace_id) { namespace_with_updated_parent.id + 1 } + + before do + insert_query = <<~SQL + INSERT INTO events (id, path) VALUES + (1, '#{namespace1.id}/'), + (2, '#{namespace2.traversal_ids.join('/')}/'), + (3, '#{namespace1.id}/#{namespace_with_updated_parent.id}/'), + (4, '#{deleted_namespace_id}/'), + (5, '#{deleted_namespace_id}/') + SQL + + connection.execute(insert_query) + end + + it 'fixes all inconsistent records in ClickHouse' do + worker.perform + + paths = [ + "#{namespace1.id}/", + "#{namespace2.traversal_ids.join('/')}/", + "#{namespace_with_updated_parent.traversal_ids.join('/')}/" + ] + + expect(leftover_paths).to match_array(paths) + + # the next job starts from the beginning of the table + expect(ClickHouse::SyncCursor.cursor_for(:event_namespace_paths_consistency_check)).to eq(0) + end + + context 'when the table is empty' do + it 'does not do anything' do + connection.execute('TRUNCATE TABLE event_namespace_paths') + + expect { worker.perform }.not_to change { connection.select('SELECT * FROM events FINAL ORDER BY id') } + end + end + + context 'when the previous job was not finished' do + it 'continues the processing from the cursor' do + ClickHouse::SyncCursor.update_cursor_for(:event_namespace_paths_consistency_check, deleted_namespace_id) + + worker.perform + + paths = [ + "#{namespace1.id}/", + "#{namespace2.traversal_ids.join('/')}/", + "#{namespace1.id}/#{namespace_with_updated_parent.id}/" + ] + # the previous records should remain + expect(leftover_paths).to match_array(paths) + end + end + + context 'when processing stops due to the record clean up limit' do + it 'stores the last processed id value' do + stub_const("#{described_class}::MAX_RECORD_MODIFICATIONS", 1) + stub_const("#{ClickHouse::Concerns::ConsistencyWorker}::POSTGRESQL_BATCH_SIZE", 1) + + expect(worker).to receive(:log_extra_metadata_on_done).with(:result, + { status: :modification_limit_reached, modifications: 2 }) + + worker.perform + + paths = [ + "#{namespace1.id}/", + "#{namespace2.traversal_ids.join('/')}/", + "#{namespace_with_updated_parent.traversal_ids.join('/')}/" + ] + + expect(leftover_paths).to match_array(paths) + expect(ClickHouse::SyncCursor.cursor_for(:event_namespace_paths_consistency_check)).to eq(deleted_namespace_id) + end + end + + context 'when the processing stops due to time limit' do + it 'returns over_time status' do + stub_const("#{ClickHouse::Concerns::ConsistencyWorker}::POSTGRESQL_BATCH_SIZE", 1) + + allow_next_instance_of(Analytics::CycleAnalytics::RuntimeLimiter) do |runtime_limiter| + allow(runtime_limiter).to receive(:over_time?).and_return(false, true) + end + + expect(worker).to receive(:log_extra_metadata_on_done).with(:result, + { status: :over_time, modifications: 1 }) + + worker.perform + end + end + end +end |