Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/app
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2023-11-14 21:09:21 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2023-11-14 21:09:21 +0300
commitc3ccd2fdf136f7b3962ef5243ed6ce01de47d8ef (patch)
tree4d8b1df9d6b4bc09edcf8ae610e014b36a2d4269 /app
parent15c1cc886c5785d49f2a6dae064a1e8290f59f46 (diff)
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'app')
-rw-r--r--app/assets/javascripts/ci/catalog/components/list/ci_resources_list_item.vue6
-rw-r--r--app/assets/javascripts/diffs/index.js2
-rw-r--r--app/assets/javascripts/environments/folder/environments_folder_view.vue3
-rw-r--r--app/assets/javascripts/issues/constants.js1
-rw-r--r--app/assets/javascripts/projects/settings/components/default_branch_selector.vue6
-rw-r--r--app/assets/javascripts/projects/settings/mount_default_branch_selector.js4
-rw-r--r--app/assets/javascripts/ref/components/ref_selector.vue12
-rw-r--r--app/assets/javascripts/vue_merge_request_widget/components/mr_widget_status_icon.vue6
-rw-r--r--app/assets/javascripts/vue_merge_request_widget/components/states/new_ready_to_merge.vue2
-rw-r--r--app/assets/javascripts/vue_merge_request_widget/components/states/nothing_to_merge.vue60
-rw-r--r--app/mailers/emails/service_desk.rb11
-rw-r--r--app/models/project_setting.rb2
-rw-r--r--app/models/vulnerability.rb2
-rw-r--r--app/services/bulk_imports/process_service.rb4
-rw-r--r--app/views/projects/branch_defaults/_default_branch_fields.html.haml14
-rw-r--r--app/workers/bulk_imports/entity_worker.rb16
-rw-r--r--app/workers/bulk_imports/export_request_worker.rb4
-rw-r--r--app/workers/bulk_imports/finish_batched_pipeline_worker.rb4
-rw-r--r--app/workers/bulk_imports/pipeline_batch_worker.rb19
-rw-r--r--app/workers/bulk_imports/pipeline_worker.rb18
-rw-r--r--app/workers/click_house/events_sync_worker.rb34
21 files changed, 156 insertions, 74 deletions
diff --git a/app/assets/javascripts/ci/catalog/components/list/ci_resources_list_item.vue b/app/assets/javascripts/ci/catalog/components/list/ci_resources_list_item.vue
index 080955b4322..3544ff448b7 100644
--- a/app/assets/javascripts/ci/catalog/components/list/ci_resources_list_item.vue
+++ b/app/assets/javascripts/ci/catalog/components/list/ci_resources_list_item.vue
@@ -112,11 +112,9 @@ export default {
</div>
</div>
<div
- class="gl-display-flex gl-flex-direction-column gl-md-flex-direction-row gl-justify-content-space-between"
+ class="gl-display-flex gl-sm-flex-direction-column gl-justify-content-space-between gl-font-sm"
>
- <span class="gl-display-flex gl-flex-basis-two-thirds gl-font-sm">{{
- resource.description
- }}</span>
+ <span class="gl-display-flex gl-flex-basis-two-thirds">{{ resource.description }}</span>
<div class="gl-display-flex gl-justify-content-end">
<span v-if="hasReleasedVersion">
<gl-sprintf :message="$options.i18n.releasedMessage">
diff --git a/app/assets/javascripts/diffs/index.js b/app/assets/javascripts/diffs/index.js
index 034dd4cf6d2..18e90fad45f 100644
--- a/app/assets/javascripts/diffs/index.js
+++ b/app/assets/javascripts/diffs/index.js
@@ -36,7 +36,7 @@ export default function initDiffsApp(store = notesStore) {
iid: dataset.iid || '',
endpointCoverage: dataset.endpointCoverage || '',
endpointCodequality: dataset.endpointCodequality || '',
- sastReportAvailable: dataset.endpointSast,
+ sastReportAvailable: parseBoolean(dataset.sastReportAvailable),
helpPagePath: dataset.helpPagePath,
currentUser: JSON.parse(dataset.currentUserData) || {},
changesEmptyStateIllustration: dataset.changesEmptyStateIllustration,
diff --git a/app/assets/javascripts/environments/folder/environments_folder_view.vue b/app/assets/javascripts/environments/folder/environments_folder_view.vue
index adb14ce3d6f..35a754c757b 100644
--- a/app/assets/javascripts/environments/folder/environments_folder_view.vue
+++ b/app/assets/javascripts/environments/folder/environments_folder_view.vue
@@ -4,6 +4,7 @@ import DeleteEnvironmentModal from '../components/delete_environment_modal.vue';
import StopEnvironmentModal from '../components/stop_environment_modal.vue';
import environmentsMixin from '../mixins/environments_mixin';
import EnvironmentsPaginationApiMixin from '../mixins/environments_pagination_api_mixin';
+import ConfirmRollbackModal from '../components/confirm_rollback_modal.vue';
export default {
components: {
@@ -12,6 +13,7 @@ export default {
GlTab,
GlTabs,
StopEnvironmentModal,
+ ConfirmRollbackModal,
},
mixins: [environmentsMixin, EnvironmentsPaginationApiMixin],
@@ -42,6 +44,7 @@ export default {
<div :class="cssContainerClass">
<stop-environment-modal :environment="environmentInStopModal" />
<delete-environment-modal :environment="environmentInDeleteModal" />
+ <confirm-rollback-modal :environment="environmentInRollbackModal" />
<h4 class="gl-font-weight-normal" data-testid="folder-name">
{{ s__('Environments|Environments') }} /
diff --git a/app/assets/javascripts/issues/constants.js b/app/assets/javascripts/issues/constants.js
index 3d8017e6e07..0a762b161ef 100644
--- a/app/assets/javascripts/issues/constants.js
+++ b/app/assets/javascripts/issues/constants.js
@@ -6,6 +6,7 @@ export const STATUS_MERGED = 'merged';
export const STATUS_OPEN = 'opened';
export const STATUS_REOPENED = 'reopened';
export const STATUS_LOCKED = 'locked';
+export const STATUS_EMPTY = 'empty';
export const TITLE_LENGTH_MAX = 255;
diff --git a/app/assets/javascripts/projects/settings/components/default_branch_selector.vue b/app/assets/javascripts/projects/settings/components/default_branch_selector.vue
index f5fb72e84bc..d1c143b96f7 100644
--- a/app/assets/javascripts/projects/settings/components/default_branch_selector.vue
+++ b/app/assets/javascripts/projects/settings/components/default_branch_selector.vue
@@ -8,6 +8,11 @@ export default {
RefSelector,
},
props: {
+ disabled: {
+ type: Boolean,
+ required: false,
+ default: false,
+ },
persistedDefaultBranch: {
type: String,
required: true,
@@ -26,6 +31,7 @@ export default {
</script>
<template>
<ref-selector
+ :disabled="disabled"
:value="persistedDefaultBranch"
class="gl-w-full"
:project-id="projectId"
diff --git a/app/assets/javascripts/projects/settings/mount_default_branch_selector.js b/app/assets/javascripts/projects/settings/mount_default_branch_selector.js
index 611561e38f2..8e64d29e947 100644
--- a/app/assets/javascripts/projects/settings/mount_default_branch_selector.js
+++ b/app/assets/javascripts/projects/settings/mount_default_branch_selector.js
@@ -1,4 +1,5 @@
import Vue from 'vue';
+import { parseBoolean } from '~/lib/utils/common_utils';
import DefaultBranchSelector from './components/default_branch_selector.vue';
export default (el) => {
@@ -6,13 +7,14 @@ export default (el) => {
return null;
}
- const { projectId, defaultBranch } = el.dataset;
+ const { projectId, defaultBranch, disabled } = el.dataset;
return new Vue({
el,
render(createElement) {
return createElement(DefaultBranchSelector, {
props: {
+ disabled: parseBoolean(disabled),
persistedDefaultBranch: defaultBranch,
projectId,
},
diff --git a/app/assets/javascripts/ref/components/ref_selector.vue b/app/assets/javascripts/ref/components/ref_selector.vue
index ed9fd521e67..0d6b19829f2 100644
--- a/app/assets/javascripts/ref/components/ref_selector.vue
+++ b/app/assets/javascripts/ref/components/ref_selector.vue
@@ -28,12 +28,17 @@ export default {
},
inheritAttrs: false,
props: {
+ disabled: {
+ type: Boolean,
+ required: false,
+ default: false,
+ },
enabledRefTypes: {
type: Array,
required: false,
default: () => ALL_REF_TYPES,
validator: (val) =>
- // It has to be an arrray
+ // It has to be an array
isArray(val) &&
// with at least one item
val.length > 0 &&
@@ -234,6 +239,10 @@ export default {
this.debouncedSearch();
},
selectRef(ref) {
+ if (this.disabled) {
+ return;
+ }
+
this.setSelectedRef(ref);
this.$emit('input', this.selectedRef);
},
@@ -262,6 +271,7 @@ export default {
:toggle-class="extendedToggleButtonClass"
:toggle-text="buttonText"
:icon="dropdownIcon"
+ :disabled="disabled"
v-bind="$attrs"
v-on="$listeners"
@hidden="$emit('hide')"
diff --git a/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_status_icon.vue b/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_status_icon.vue
index 370e07b397c..d0771a79a5d 100644
--- a/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_status_icon.vue
+++ b/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_status_icon.vue
@@ -1,6 +1,6 @@
<script>
import { GlIcon } from '@gitlab/ui';
-import { STATUS_CLOSED, STATUS_MERGED } from '~/issues/constants';
+import { STATUS_CLOSED, STATUS_MERGED, STATUS_EMPTY } from '~/issues/constants';
import StatusIcon from './extensions/status_icon.vue';
export default {
@@ -24,6 +24,9 @@ export default {
isMerged() {
return this.status === STATUS_MERGED;
},
+ isEmpty() {
+ return this.status === STATUS_EMPTY;
+ },
},
};
</script>
@@ -33,6 +36,7 @@ export default {
<gl-icon v-if="isMerged" name="merge" :size="16" class="gl-text-blue-500" />
<gl-icon v-else-if="isClosed" name="merge-request-close" :size="16" class="gl-text-red-500" />
<gl-icon v-else-if="status === 'approval'" name="approval" :size="16" />
+ <status-icon v-else-if="isEmpty" icon-name="neutral" :level="1" class="gl-m-0!" />
<status-icon v-else :is-loading="isLoading" :icon-name="status" :level="1" class="gl-m-0!" />
</div>
</div>
diff --git a/app/assets/javascripts/vue_merge_request_widget/components/states/new_ready_to_merge.vue b/app/assets/javascripts/vue_merge_request_widget/components/states/new_ready_to_merge.vue
index 2db5c71be82..16f1bac73ab 100644
--- a/app/assets/javascripts/vue_merge_request_widget/components/states/new_ready_to_merge.vue
+++ b/app/assets/javascripts/vue_merge_request_widget/components/states/new_ready_to_merge.vue
@@ -37,7 +37,7 @@ export default {
<template>
<div class="mr-widget-body media">
<status-icon status="success" />
- <p class="media-body gl-m-0! gl-font-weight-bold gl-text-gray-900!">
+ <p class="media-body gl-mt-1 gl-mb-0! gl-font-weight-bold gl-text-gray-900!">
<template v-if="canMerge">
{{ __('Ready to merge!') }}
</template>
diff --git a/app/assets/javascripts/vue_merge_request_widget/components/states/nothing_to_merge.vue b/app/assets/javascripts/vue_merge_request_widget/components/states/nothing_to_merge.vue
index e1c54a8827c..b80b5246e24 100644
--- a/app/assets/javascripts/vue_merge_request_widget/components/states/nothing_to_merge.vue
+++ b/app/assets/javascripts/vue_merge_request_widget/components/states/nothing_to_merge.vue
@@ -1,48 +1,48 @@
<script>
import { GlSprintf, GlLink } from '@gitlab/ui';
-import EMPTY_STATE_SVG_URL from '@gitlab/svgs/dist/illustrations/empty-state/empty-merge-requests-md.svg?url';
+import { STATUS_EMPTY } from '~/issues/constants';
import { helpPagePath } from '~/helpers/help_page_helper';
+import StatusIcon from '../mr_widget_status_icon.vue';
export default {
name: 'MRWidgetNothingToMerge',
components: {
GlSprintf,
GlLink,
+ StatusIcon,
+ },
+ computed: {
+ statusEmpty() {
+ return STATUS_EMPTY;
+ },
},
ciHelpPage: helpPagePath('ci/quick_start/index.html'),
- EMPTY_STATE_SVG_URL,
};
</script>
<template>
- <div class="mr-widget-body mr-widget-empty-state">
- <div class="row">
- <div
- class="col-md-3 col-12 text-center d-flex justify-content-center align-items-center svg-content svg-130 pb-0 pt-0"
- >
- <img :src="$options.EMPTY_STATE_SVG_URL" :alt="''" />
- </div>
- <div class="text col-md-9 col-12">
- <p class="highlight mt-3">
- {{ s__('mrWidgetNothingToMerge|Merge request contains no changes') }}
- </p>
- <p data-testid="nothing-to-merge-body">
- <gl-sprintf
- :message="
- s__(
- 'mrWidgetNothingToMerge|Use merge requests to propose changes to your project and discuss them with your team. To make changes, use the %{boldStart}Code%{boldEnd} dropdown list above, then test them with %{linkStart}CI/CD%{linkEnd} before merging.',
- )
- "
- >
- <template #bold="{ content }">
- <b>{{ content }}</b>
- </template>
- <template #link="{ content }">
- <gl-link :href="$options.ciHelpPage" target="_blank">{{ content }}</gl-link>
- </template>
- </gl-sprintf>
- </p>
- </div>
+ <div class="mr-widget-body media">
+ <status-icon :status="statusEmpty" />
+ <div>
+ <p class="media-body gl-mt-1 gl-mb-1 gl-font-weight-bold gl-text-gray-900!">
+ {{ s__('mrWidgetNothingToMerge|Merge request contains no changes') }}
+ </p>
+ <p class="gl-m-0! gl-text-secondary" data-testid="nothing-to-merge-body">
+ <gl-sprintf
+ :message="
+ s__(
+ 'mrWidgetNothingToMerge|Use merge requests to propose changes to your project and discuss them with your team. To make changes, use the %{boldStart}Code%{boldEnd} dropdown list above, then test them with %{linkStart}CI/CD%{linkEnd} before merging.',
+ )
+ "
+ >
+ <template #bold="{ content }">
+ <b>{{ content }}</b>
+ </template>
+ <template #link="{ content }">
+ <gl-link :href="$options.ciHelpPage" target="_blank">{{ content }}</gl-link>
+ </template>
+ </gl-sprintf>
+ </p>
</div>
</div>
</template>
diff --git a/app/mailers/emails/service_desk.rb b/app/mailers/emails/service_desk.rb
index f67c2636fc6..a421e76e0de 100644
--- a/app/mailers/emails/service_desk.rb
+++ b/app/mailers/emails/service_desk.rb
@@ -7,6 +7,7 @@ module Emails
include ::ServiceDesk::CustomEmails::Logger
EMAIL_ATTACHMENTS_SIZE_LIMIT = 10.megabytes.freeze
+ VERIFICATION_EMAIL_TIMEOUT = 7
included do
layout 'service_desk', only: [:service_desk_thank_you_email, :service_desk_new_note_email]
@@ -146,7 +147,15 @@ module Emails
log_info(project: @project)
- mail.delivery_method(::Mail::SMTP, @service_desk_setting.custom_email_credential.delivery_options)
+ delivery_options = @service_desk_setting.custom_email_credential.delivery_options
+ # We force the use of custom email settings when sending out the verification email.
+ # If the credentials aren't correct some servers tend to take a while to answer
+ # which leads to some Net::ReadTimeout errors which disguises the
+ # real configuration issue.
+ # We increase the timeout for verification emails only.
+ delivery_options[:read_timeout] = VERIFICATION_EMAIL_TIMEOUT if force
+
+ mail.delivery_method(::Mail::SMTP, delivery_options)
end
def service_desk_custom_email_enabled?
diff --git a/app/models/project_setting.rb b/app/models/project_setting.rb
index 349625a90f6..f684feacace 100644
--- a/app/models/project_setting.rb
+++ b/app/models/project_setting.rb
@@ -11,8 +11,6 @@ class ProjectSetting < ApplicationRecord
scope :for_projects, ->(projects) { where(project_id: projects) }
- ignore_column :jitsu_key, remove_with: '16.7', remove_after: '2023-11-17'
-
attr_encrypted :cube_api_key,
mode: :per_attribute_iv,
key: Settings.attr_encrypted_db_key_base_32,
diff --git a/app/models/vulnerability.rb b/app/models/vulnerability.rb
index 0e3fe2cc8ac..abdf585af81 100644
--- a/app/models/vulnerability.rb
+++ b/app/models/vulnerability.rb
@@ -5,6 +5,8 @@ class Vulnerability < ApplicationRecord
include EachBatch
include IgnorableColumns
+ ignore_column :milestone_id, remove_with: '16.9', remove_after: '2023-01-13'
+
alias_attribute :vulnerability_id, :id
scope :with_projects, -> { includes(:project) }
diff --git a/app/services/bulk_imports/process_service.rb b/app/services/bulk_imports/process_service.rb
index 7a6a883f1a9..2d9a0d6a6c9 100644
--- a/app/services/bulk_imports/process_service.rb
+++ b/app/services/bulk_imports/process_service.rb
@@ -32,7 +32,9 @@ module BulkImports
entity.start!
- BulkImports::ExportRequestWorker.perform_async(entity.id)
+ Gitlab::ApplicationContext.with_context(bulk_import_entity_id: entity.id) do
+ BulkImports::ExportRequestWorker.perform_async(entity.id)
+ end
end
end
diff --git a/app/views/projects/branch_defaults/_default_branch_fields.html.haml b/app/views/projects/branch_defaults/_default_branch_fields.html.haml
index 78ce43ca8c9..f3a7e93a5f7 100644
--- a/app/views/projects/branch_defaults/_default_branch_fields.html.haml
+++ b/app/views/projects/branch_defaults/_default_branch_fields.html.haml
@@ -1,3 +1,13 @@
+- change_default_disabled = @default_branch_blocked_by_security_policy
+- popover_data = {}
+
+- if change_default_disabled
+ - tag_pair_security_policies_page = tag_pair(link_to('', namespace_project_security_policies_path, target: '_blank', rel: 'noopener noreferrer'), :security_policies_link_start, :security_policies_link_end)
+ - tag_pair_security_policies_docs = tag_pair(link_to('', help_page_path('user/application_security/policies/scan-result-policies'), target: '_blank', rel: 'noopener noreferrer'), :learn_more_link_start, :learn_more_link_end)
+ - popover_content = safe_format(s_("SecurityOrchestration|You can't change the default branch because its protection is enforced by one or more %{security_policies_link_start}security policies%{security_policies_link_end}. %{learn_more_link_start}Learn more%{learn_more_link_end}."), tag_pair_security_policies_docs, tag_pair_security_policies_page)
+ - popover_title = s_("SecurityOrchestration|Security policy overwrites this setting")
+ - popover_data = { container: 'body', toggle: 'popover', html: 'true', triggers: 'hover', title: popover_title, content: popover_content }
+
%fieldset#default-branch-settings
- if @project.empty_repo?
.text-secondary
@@ -6,8 +16,8 @@
.form-group
= f.label :default_branch, _("Default branch"), class: 'label-bold'
%p= s_('ProjectSettings|All merge requests and commits are made against this branch unless you specify a different one.')
- .gl-form-input-xl
- .js-select-default-branch{ data: { default_branch: @project.default_branch, project_id: @project.id } }
+ .gl-form-input-xl{ data: { **popover_data } }
+ .js-select-default-branch{ data: { default_branch: @project.default_branch, project_id: @project.id, disabled: change_default_disabled.to_s } }
.form-group
- help_text = _("When merge requests and commits in the default branch close, any issues they reference also close.")
diff --git a/app/workers/bulk_imports/entity_worker.rb b/app/workers/bulk_imports/entity_worker.rb
index e510a8c0d06..caee292a504 100644
--- a/app/workers/bulk_imports/entity_worker.rb
+++ b/app/workers/bulk_imports/entity_worker.rb
@@ -49,7 +49,9 @@ module BulkImports
attr_reader :entity
def re_enqueue
- BulkImports::EntityWorker.perform_in(PERFORM_DELAY, entity.id)
+ with_context(bulk_import_entity_id: entity.id) do
+ BulkImports::EntityWorker.perform_in(PERFORM_DELAY, entity.id)
+ end
end
def running_tracker
@@ -66,11 +68,13 @@ module BulkImports
next_pipeline_trackers.each_with_index do |pipeline_tracker, index|
log_info(message: 'Stage starting', entity_stage: pipeline_tracker.stage) if index == 0
- BulkImports::PipelineWorker.perform_async(
- pipeline_tracker.id,
- pipeline_tracker.stage,
- entity.id
- )
+ with_context(bulk_import_entity_id: entity.id) do
+ BulkImports::PipelineWorker.perform_async(
+ pipeline_tracker.id,
+ pipeline_tracker.stage,
+ entity.id
+ )
+ end
end
end
diff --git a/app/workers/bulk_imports/export_request_worker.rb b/app/workers/bulk_imports/export_request_worker.rb
index f7456ddccb1..54815c05c67 100644
--- a/app/workers/bulk_imports/export_request_worker.rb
+++ b/app/workers/bulk_imports/export_request_worker.rb
@@ -20,7 +20,9 @@ module BulkImports
set_source_xid
request_export
- BulkImports::EntityWorker.perform_async(entity_id)
+ with_context(bulk_import_entity_id: entity_id) do
+ BulkImports::EntityWorker.perform_async(entity_id)
+ end
end
def perform_failure(exception, entity_id)
diff --git a/app/workers/bulk_imports/finish_batched_pipeline_worker.rb b/app/workers/bulk_imports/finish_batched_pipeline_worker.rb
index 40d26e14dc1..73ae2188ac4 100644
--- a/app/workers/bulk_imports/finish_batched_pipeline_worker.rb
+++ b/app/workers/bulk_imports/finish_batched_pipeline_worker.rb
@@ -38,7 +38,9 @@ module BulkImports
attr_reader :tracker
def re_enqueue
- self.class.perform_in(REQUEUE_DELAY, tracker.id)
+ with_context(bulk_import_entity_id: tracker.entity.id) do
+ self.class.perform_in(REQUEUE_DELAY, tracker.id)
+ end
end
def import_in_progress?
diff --git a/app/workers/bulk_imports/pipeline_batch_worker.rb b/app/workers/bulk_imports/pipeline_batch_worker.rb
index 1485275e616..957669d4a66 100644
--- a/app/workers/bulk_imports/pipeline_batch_worker.rb
+++ b/app/workers/bulk_imports/pipeline_batch_worker.rb
@@ -42,6 +42,7 @@ module BulkImports
@batch = ::BulkImports::BatchTracker.find(batch_id)
@tracker = @batch.tracker
+ @entity = @tracker.entity
@pending_retry = false
return unless process_batch?
@@ -50,7 +51,11 @@ module BulkImports
try_obtain_lease { run }
ensure
- ::BulkImports::FinishBatchedPipelineWorker.perform_async(tracker.id) unless pending_retry
+ unless pending_retry
+ with_context(bulk_import_entity_id: entity.id) do
+ ::BulkImports::FinishBatchedPipelineWorker.perform_async(tracker.id)
+ end
+ end
end
def perform_failure(batch_id, exception)
@@ -62,7 +67,7 @@ module BulkImports
private
- attr_reader :batch, :tracker, :pending_retry
+ attr_reader :batch, :tracker, :pending_retry, :entity
def run
return batch.skip! if tracker.failed? || tracker.finished?
@@ -83,7 +88,7 @@ module BulkImports
Gitlab::ErrorTracking.track_exception(exception, log_attributes(message: 'Batch tracker failed'))
BulkImports::Failure.create(
- bulk_import_entity_id: batch.tracker.entity.id,
+ bulk_import_entity_id: tracker.entity.id,
pipeline_class: tracker.pipeline_name,
pipeline_step: 'pipeline_batch_worker_run',
exception_class: exception.class.to_s,
@@ -91,7 +96,9 @@ module BulkImports
correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id
)
- ::BulkImports::FinishBatchedPipelineWorker.perform_async(tracker.id)
+ with_context(bulk_import_entity_id: tracker.entity.id) do
+ ::BulkImports::FinishBatchedPipelineWorker.perform_async(tracker.id)
+ end
end
def context
@@ -115,7 +122,9 @@ module BulkImports
def re_enqueue(delay = FILE_EXTRACTION_PIPELINE_PERFORM_DELAY)
log_extra_metadata_on_done(:re_enqueue, true)
- self.class.perform_in(delay, batch.id)
+ with_context(bulk_import_entity_id: entity.id) do
+ self.class.perform_in(delay, batch.id)
+ end
end
def process_batch?
diff --git a/app/workers/bulk_imports/pipeline_worker.rb b/app/workers/bulk_imports/pipeline_worker.rb
index 2c1d28b33c5..4bf6da73fdf 100644
--- a/app/workers/bulk_imports/pipeline_worker.rb
+++ b/app/workers/bulk_imports/pipeline_worker.rb
@@ -124,12 +124,14 @@ module BulkImports
def re_enqueue(delay = FILE_EXTRACTION_PIPELINE_PERFORM_DELAY)
log_extra_metadata_on_done(:re_enqueue, true)
- self.class.perform_in(
- delay,
- pipeline_tracker.id,
- pipeline_tracker.stage,
- entity.id
- )
+ with_context(bulk_import_entity_id: entity.id) do
+ self.class.perform_in(
+ delay,
+ pipeline_tracker.id,
+ pipeline_tracker.stage,
+ entity.id
+ )
+ end
end
def context
@@ -218,7 +220,9 @@ module BulkImports
1.upto(export_status.batches_count) do |batch_number|
batch = pipeline_tracker.batches.find_or_create_by!(batch_number: batch_number) # rubocop:disable CodeReuse/ActiveRecord
- ::BulkImports::PipelineBatchWorker.perform_async(batch.id)
+ with_context(bulk_import_entity_id: entity.id) do
+ ::BulkImports::PipelineBatchWorker.perform_async(batch.id)
+ end
end
end
end
diff --git a/app/workers/click_house/events_sync_worker.rb b/app/workers/click_house/events_sync_worker.rb
index e884a43b1e3..5936d30b8b2 100644
--- a/app/workers/click_house/events_sync_worker.rb
+++ b/app/workers/click_house/events_sync_worker.rb
@@ -4,6 +4,7 @@ module ClickHouse
class EventsSyncWorker
include ApplicationWorker
include Gitlab::ExclusiveLeaseHelpers
+ include Gitlab::Utils::StrongMemoize
idempotent!
queue_namespace :cronjob
@@ -91,6 +92,11 @@ module ClickHouse
)
end
+ def last_event_id_in_postgresql
+ Event.maximum(:id)
+ end
+ strong_memoize_attr :last_event_id_in_postgresql
+
def enabled?
ClickHouse::Client.configuration.databases[:main].present? && Feature.enabled?(:event_sync_worker_for_click_house)
end
@@ -110,24 +116,34 @@ module ClickHouse
def process_batch(context)
Enumerator.new do |yielder|
- has_data = false
- # rubocop: disable CodeReuse/ActiveRecord
- Event.where(Event.arel_table[:id].gt(context.last_record_id)).each_batch(of: BATCH_SIZE) do |relation|
- has_data = true
-
- relation.select(*EVENT_PROJECTIONS).each do |row|
+ has_more_data = false
+ batching_scope.each_batch(of: BATCH_SIZE) do |relation|
+ records = relation.select(*EVENT_PROJECTIONS).to_a
+ has_more_data = records.size == BATCH_SIZE
+ records.each do |row|
yielder << row
context.last_processed_id = row.id
break if context.record_limit_reached?
end
- break if context.over_time? || context.record_limit_reached?
+ break if context.over_time? || context.record_limit_reached? || !has_more_data
end
- context.no_more_records! if has_data == false
- # rubocop: enable CodeReuse/ActiveRecord
+ context.no_more_records! unless has_more_data
end
end
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ def batching_scope
+ return Event.none unless last_event_id_in_postgresql
+
+ table = Event.arel_table
+
+ Event
+ .where(table[:id].gt(context.last_record_id))
+ .where(table[:id].lteq(last_event_id_in_postgresql))
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
end
end