diff options
Diffstat (limited to 'app')
70 files changed, 647 insertions, 59 deletions
diff --git a/app/assets/javascripts/monitoring/components/charts/anomaly.vue b/app/assets/javascripts/monitoring/components/charts/anomaly.vue new file mode 100644 index 00000000000..8eeac737a11 --- /dev/null +++ b/app/assets/javascripts/monitoring/components/charts/anomaly.vue @@ -0,0 +1,227 @@ +<script> +import { flatten, isNumber } from 'underscore'; +import { GlLineChart, GlChartSeriesLabel } from '@gitlab/ui/dist/charts'; +import { roundOffFloat } from '~/lib/utils/common_utils'; +import { hexToRgb } from '~/lib/utils/color_utils'; +import { areaOpacityValues, symbolSizes, colorValues } from '../../constants'; +import { graphDataValidatorForAnomalyValues } from '../../utils'; +import MonitorTimeSeriesChart from './time_series.vue'; + +/** + * Series indexes + */ +const METRIC = 0; +const UPPER = 1; +const LOWER = 2; + +/** + * Boundary area appearance + */ +const AREA_COLOR = colorValues.anomalyAreaColor; +const AREA_OPACITY = areaOpacityValues.default; +const AREA_COLOR_RGBA = `rgba(${hexToRgb(AREA_COLOR).join(',')},${AREA_OPACITY})`; + +/** + * The anomaly component highlights when a metric shows + * some anomalous behavior. + * + * It shows both a metric line and a boundary band in a + * time series chart, the boundary band shows the normal + * range of values the metric should take. + * + * This component accepts 3 queries, which contain the + * "metric", "upper" limit and "lower" limit. + * + * The upper and lower series are "stacked areas" visually + * to create the boundary band, and if any "metric" value + * is outside this band, it is highlighted to warn users. + * + * The boundary band stack must be painted above the 0 line + * so the area is shown correctly. If any of the values of + * the data are negative, the chart data is shifted to be + * above 0 line. + * + * The data passed to the time series is will always be + * positive, but reformatted to show the original values of + * data. + * + */ +export default { + components: { + GlLineChart, + GlChartSeriesLabel, + MonitorTimeSeriesChart, + }, + inheritAttrs: false, + props: { + graphData: { + type: Object, + required: true, + validator: graphDataValidatorForAnomalyValues, + }, + }, + computed: { + series() { + return this.graphData.queries.map(query => { + const values = query.result[0] ? query.result[0].values : []; + return { + label: query.label, + data: values.filter(([, value]) => !Number.isNaN(value)), + }; + }); + }, + /** + * If any of the values of the data is negative, the + * chart data is shifted to the lowest value + * + * This offset is the lowest value. + */ + yOffset() { + const values = flatten(this.series.map(ser => ser.data.map(([, y]) => y))); + const min = values.length ? Math.floor(Math.min(...values)) : 0; + return min < 0 ? -min : 0; + }, + metricData() { + const originalMetricQuery = this.graphData.queries[0]; + + const metricQuery = { ...originalMetricQuery }; + metricQuery.result[0].values = metricQuery.result[0].values.map(([x, y]) => [ + x, + y + this.yOffset, + ]); + return { + ...this.graphData, + type: 'line-chart', + queries: [metricQuery], + }; + }, + metricSeriesConfig() { + return { + type: 'line', + symbol: 'circle', + symbolSize: (val, params) => { + if (this.isDatapointAnomaly(params.dataIndex)) { + return symbolSizes.anomaly; + } + // 0 causes echarts to throw an error, use small number instead + // see https://gitlab.com/gitlab-org/gitlab-ui/issues/423 + return 0.001; + }, + showSymbol: true, + itemStyle: { + color: params => { + if (this.isDatapointAnomaly(params.dataIndex)) { + return colorValues.anomalySymbol; + } + return colorValues.primaryColor; + }, + }, + }; + }, + chartOptions() { + const [, upperSeries, lowerSeries] = this.series; + const calcOffsetY = (data, offsetCallback) => + data.map((value, dataIndex) => { + const [x, y] = value; + return [x, y + offsetCallback(dataIndex)]; + }); + + const yAxisWithOffset = { + name: this.yAxisLabel, + axisLabel: { + formatter: num => roundOffFloat(num - this.yOffset, 3).toString(), + }, + }; + + /** + * Boundary is rendered by 2 series: An invisible + * series (opacity: 0) stacked on a visible one. + * + * Order is important, lower boundary is stacked + * *below* the upper boundary. + */ + const boundarySeries = []; + + if (upperSeries.data.length && lowerSeries.data.length) { + // Lower boundary, plus the offset if negative values + boundarySeries.push( + this.makeBoundarySeries({ + name: this.formatLegendLabel(lowerSeries), + data: calcOffsetY(lowerSeries.data, () => this.yOffset), + }), + ); + // Upper boundary, minus the lower boundary + boundarySeries.push( + this.makeBoundarySeries({ + name: this.formatLegendLabel(upperSeries), + data: calcOffsetY(upperSeries.data, i => -this.yValue(LOWER, i)), + areaStyle: { + color: AREA_COLOR, + opacity: AREA_OPACITY, + }, + }), + ); + } + return { yAxis: yAxisWithOffset, series: boundarySeries }; + }, + }, + methods: { + formatLegendLabel(query) { + return query.label; + }, + yValue(seriesIndex, dataIndex) { + const d = this.series[seriesIndex].data[dataIndex]; + return d && d[1]; + }, + yValueFormatted(seriesIndex, dataIndex) { + const y = this.yValue(seriesIndex, dataIndex); + return isNumber(y) ? y.toFixed(3) : ''; + }, + isDatapointAnomaly(dataIndex) { + const yVal = this.yValue(METRIC, dataIndex); + const yUpper = this.yValue(UPPER, dataIndex); + const yLower = this.yValue(LOWER, dataIndex); + return (isNumber(yUpper) && yVal > yUpper) || (isNumber(yLower) && yVal < yLower); + }, + makeBoundarySeries(series) { + const stackKey = 'anomaly-boundary-series-stack'; + return { + type: 'line', + stack: stackKey, + lineStyle: { + width: 0, + color: AREA_COLOR_RGBA, // legend color + }, + color: AREA_COLOR_RGBA, // tooltip color + symbol: 'none', + ...series, + }; + }, + }, +}; +</script> + +<template> + <monitor-time-series-chart + v-bind="$attrs" + :graph-data="metricData" + :option="chartOptions" + :series-config="metricSeriesConfig" + > + <slot></slot> + <template v-slot:tooltipContent="slotProps"> + <div + v-for="(content, seriesIndex) in slotProps.tooltip.content" + :key="seriesIndex" + class="d-flex justify-content-between" + > + <gl-chart-series-label :color="content.color"> + {{ content.name }} + </gl-chart-series-label> + <div class="prepend-left-32"> + {{ yValueFormatted(seriesIndex, content.dataIndex) }} + </div> + </div> + </template> + </monitor-time-series-chart> +</template> diff --git a/app/assets/javascripts/monitoring/components/charts/time_series.vue b/app/assets/javascripts/monitoring/components/charts/time_series.vue index 434debb67f5..6a88c8a5ee3 100644 --- a/app/assets/javascripts/monitoring/components/charts/time_series.vue +++ b/app/assets/javascripts/monitoring/components/charts/time_series.vue @@ -1,12 +1,20 @@ <script> import { s__, __ } from '~/locale'; +import _ from 'underscore'; import { GlLink, GlButton, GlTooltip, GlResizeObserverDirective } from '@gitlab/ui'; import { GlAreaChart, GlLineChart, GlChartSeriesLabel } from '@gitlab/ui/dist/charts'; import dateFormat from 'dateformat'; import { roundOffFloat } from '~/lib/utils/common_utils'; import { getSvgIconPathContent } from '~/lib/utils/icon_utils'; import Icon from '~/vue_shared/components/icon.vue'; -import { chartHeight, graphTypes, lineTypes, symbolSizes, dateFormats } from '../../constants'; +import { + chartHeight, + graphTypes, + lineTypes, + lineWidths, + symbolSizes, + dateFormats, +} from '../../constants'; import { makeDataSeries } from '~/helpers/monitor_helper'; import { graphDataValidatorForValues } from '../../utils'; @@ -30,6 +38,16 @@ export default { required: true, validator: graphDataValidatorForValues.bind(null, false), }, + option: { + type: Object, + required: false, + default: () => ({}), + }, + seriesConfig: { + type: Object, + required: false, + default: () => ({}), + }, deploymentData: { type: Array, required: false, @@ -96,29 +114,35 @@ export default { const lineWidth = appearance && appearance.line && appearance.line.width ? appearance.line.width - : undefined; + : lineWidths.default; const areaStyle = { opacity: appearance && appearance.area && typeof appearance.area.opacity === 'number' ? appearance.area.opacity : undefined, }; - const series = makeDataSeries(query.result, { name: this.formatLegendLabel(query), lineStyle: { type: lineType, width: lineWidth, + color: this.primaryColor, }, showSymbol: false, areaStyle: this.graphData.type === 'area-chart' ? areaStyle : undefined, + ...this.seriesConfig, }); return acc.concat(series); }, []); }, + chartOptionSeries() { + return (this.option.series || []).concat(this.scatterSeries ? [this.scatterSeries] : []); + }, chartOptions() { + const option = _.omit(this.option, 'series'); return { + series: this.chartOptionSeries, xAxis: { name: __('Time'), type: 'time', @@ -135,8 +159,8 @@ export default { formatter: num => roundOffFloat(num, 3).toString(), }, }, - series: this.scatterSeries, dataZoom: [this.dataZoomConfig], + ...option, }; }, dataZoomConfig() { @@ -144,6 +168,14 @@ export default { return handleIcon ? { handleIcon } : {}; }, + /** + * This method returns the earliest time value in all series of a chart. + * Takes a chart data with data to populate a timeseries. + * data should be an array of data points [t, y] where t is a ISO formatted date, + * and is sorted by t (time). + * @returns {(String|null)} earliest x value from all series, or null when the + * chart series data is empty. + */ earliestDatapoint() { return this.chartData.reduce((acc, series) => { const { data } = series; @@ -230,10 +262,11 @@ export default { this.tooltip.sha = deploy.sha.substring(0, 8); this.tooltip.commitUrl = deploy.commitUrl; } else { - const { seriesName, color } = dataPoint; + const { seriesName, color, dataIndex } = dataPoint; const value = yVal.toFixed(3); this.tooltip.content.push({ name: seriesName, + dataIndex, value, color, }); @@ -306,23 +339,27 @@ export default { </template> <template v-else> <template slot="tooltipTitle"> - <div class="text-nowrap"> - {{ tooltip.title }} - </div> + <slot name="tooltipTitle"> + <div class="text-nowrap"> + {{ tooltip.title }} + </div> + </slot> </template> <template slot="tooltipContent"> - <div - v-for="(content, key) in tooltip.content" - :key="key" - class="d-flex justify-content-between" - > - <gl-chart-series-label :color="isMultiSeries ? content.color : ''"> - {{ content.name }} - </gl-chart-series-label> - <div class="prepend-left-32"> - {{ content.value }} + <slot name="tooltipContent" :tooltip="tooltip"> + <div + v-for="(content, key) in tooltip.content" + :key="key" + class="d-flex justify-content-between" + > + <gl-chart-series-label :color="isMultiSeries ? content.color : ''"> + {{ content.name }} + </gl-chart-series-label> + <div class="prepend-left-32"> + {{ content.value }} + </div> </div> - </div> + </slot> </template> </template> </component> diff --git a/app/assets/javascripts/monitoring/components/panel_type.vue b/app/assets/javascripts/monitoring/components/panel_type.vue index 2c56966f120..e3f99dbda9a 100644 --- a/app/assets/javascripts/monitoring/components/panel_type.vue +++ b/app/assets/javascripts/monitoring/components/panel_type.vue @@ -11,6 +11,7 @@ import { } from '@gitlab/ui'; import Icon from '~/vue_shared/components/icon.vue'; import MonitorTimeSeriesChart from './charts/time_series.vue'; +import MonitorAnomalyChart from './charts/anomaly.vue'; import MonitorSingleStatChart from './charts/single_stat.vue'; import MonitorEmptyChart from './charts/empty_chart.vue'; import TrackEventDirective from '~/vue_shared/directives/track_event'; @@ -19,7 +20,6 @@ import { downloadCSVOptions, generateLinkToChartOptions } from '../utils'; export default { components: { MonitorSingleStatChart, - MonitorTimeSeriesChart, MonitorEmptyChart, Icon, GlDropdown, @@ -67,6 +67,12 @@ export default { const data = new Blob([this.csvText], { type: 'text/plain' }); return window.URL.createObjectURL(data); }, + monitorChartComponent() { + if (this.isPanelType('anomaly-chart')) { + return MonitorAnomalyChart; + } + return MonitorTimeSeriesChart; + }, }, methods: { getGraphAlerts(queries) { @@ -93,13 +99,14 @@ export default { v-if="isPanelType('single-stat') && graphDataHasMetrics" :graph-data="graphData" /> - <monitor-time-series-chart + <component + :is="monitorChartComponent" v-else-if="graphDataHasMetrics" :graph-data="graphData" :deployment-data="deploymentData" :project-path="projectPath" :thresholds="getGraphAlertValues(graphData.queries)" - group-id="monitor-area-chart" + group-id="panel-type-chart" > <div class="d-flex align-items-center"> <alert-widget @@ -141,6 +148,6 @@ export default { </gl-dropdown-item> </gl-dropdown> </div> - </monitor-time-series-chart> + </component> <monitor-empty-chart v-else :graph-title="graphData.title" /> </template> diff --git a/app/assets/javascripts/monitoring/constants.js b/app/assets/javascripts/monitoring/constants.js index 2836fe4fc26..1a1fcdd0e66 100644 --- a/app/assets/javascripts/monitoring/constants.js +++ b/app/assets/javascripts/monitoring/constants.js @@ -14,13 +14,28 @@ export const graphTypes = { }; export const symbolSizes = { + anomaly: 8, default: 14, }; +export const areaOpacityValues = { + default: 0.2, +}; + +export const colorValues = { + primaryColor: '#1f78d1', // $blue-500 (see variables.scss) + anomalySymbol: '#db3b21', + anomalyAreaColor: '#1f78d1', +}; + export const lineTypes = { default: 'solid', }; +export const lineWidths = { + default: 2, +}; + export const timeWindows = { thirtyMinutes: __('30 minutes'), threeHours: __('3 hours'), diff --git a/app/assets/javascripts/monitoring/utils.js b/app/assets/javascripts/monitoring/utils.js index 00f188c1d5a..6747306a6d9 100644 --- a/app/assets/javascripts/monitoring/utils.js +++ b/app/assets/javascripts/monitoring/utils.js @@ -131,4 +131,20 @@ export const downloadCSVOptions = title => { return { category, action, label: 'Chart title', property: title }; }; +/** + * This function validates the graph data contains exactly 3 queries plus + * value validations from graphDataValidatorForValues. + * @param {Object} isValues + * @param {Object} graphData the graph data response from a prometheus request + * @returns {boolean} true if the data is valid + */ +export const graphDataValidatorForAnomalyValues = graphData => { + const anomalySeriesCount = 3; // metric, upper, lower + return ( + graphData.queries && + graphData.queries.length === anomalySeriesCount && + graphDataValidatorForValues(false, graphData) + ); +}; + export default {}; diff --git a/app/assets/javascripts/pages/projects/shared/permissions/components/settings_panel.vue b/app/assets/javascripts/pages/projects/shared/permissions/components/settings_panel.vue index fcc57da0649..4802cc2ad25 100644 --- a/app/assets/javascripts/pages/projects/shared/permissions/components/settings_panel.vue +++ b/app/assets/javascripts/pages/projects/shared/permissions/components/settings_panel.vue @@ -1,7 +1,6 @@ <script> -/* eslint-disable @gitlab/vue-i18n/no-bare-strings */ import settingsMixin from 'ee_else_ce/pages/projects/shared/permissions/mixins/settings_pannel_mixin'; -import { __ } from '~/locale'; +import { s__ } from '~/locale'; import projectFeatureSetting from './project_feature_setting.vue'; import projectFeatureToggle from '~/vue_shared/components/toggle_button.vue'; import projectSettingRow from './project_setting_row.vue'; @@ -13,7 +12,7 @@ import { } from '../constants'; import { toggleHiddenClassBySelector } from '../external'; -const PAGE_FEATURE_ACCESS_LEVEL = __('Everyone'); +const PAGE_FEATURE_ACCESS_LEVEL = s__('ProjectSettings|Everyone'); export default { components: { @@ -207,7 +206,10 @@ export default { <template> <div> <div class="project-visibility-setting"> - <project-setting-row :help-path="visibilityHelpPath" label="Project visibility"> + <project-setting-row + :help-path="visibilityHelpPath" + :label="s__('ProjectSettings|Project visibility')" + > <div class="project-feature-controls"> <div class="select-wrapper"> <select @@ -220,17 +222,17 @@ export default { <option :value="visibilityOptions.PRIVATE" :disabled="!visibilityAllowed(visibilityOptions.PRIVATE)" - >{{ __('Private') }}</option + >{{ s__('ProjectSettings|Private') }}</option > <option :value="visibilityOptions.INTERNAL" :disabled="!visibilityAllowed(visibilityOptions.INTERNAL)" - >{{ __('Internal') }}</option + >{{ s__('ProjectSettings|Internal') }}</option > <option :value="visibilityOptions.PUBLIC" :disabled="!visibilityAllowed(visibilityOptions.PUBLIC)" - >{{ __('Public') }}</option + >{{ s__('ProjectSettings|Public') }}</option > </select> <i aria-hidden="true" data-hidden="true" class="fa fa-chevron-down"></i> @@ -243,14 +245,15 @@ export default { type="hidden" name="project[request_access_enabled]" /> - <input v-model="requestAccessEnabled" type="checkbox" /> Allow users to request access + <input v-model="requestAccessEnabled" type="checkbox" /> + {{ s__('ProjectSettings|Allow users to request access') }} </label> </project-setting-row> </div> <div :class="{ 'highlight-changes': highlightChangesClass }" class="project-feature-settings"> <project-setting-row - label="Issues" - help-text="Lightweight issue tracking system for this project" + :label="s__('ProjectSettings|Issues')" + :help-text="s__('ProjectSettings|Lightweight issue tracking system for this project')" > <project-feature-setting v-model="issuesAccessLevel" @@ -258,7 +261,10 @@ export default { name="project[project_feature_attributes][issues_access_level]" /> </project-setting-row> - <project-setting-row label="Repository" help-text="View and edit files in this project"> + <project-setting-row + :label="s__('ProjectSettings|Repository')" + :help-text="s__('ProjectSettings|View and edit files in this project')" + > <project-feature-setting v-model="repositoryAccessLevel" :options="featureAccessLevelOptions" @@ -267,8 +273,8 @@ export default { </project-setting-row> <div class="project-feature-setting-group"> <project-setting-row - label="Merge requests" - help-text="Submit changes to be merged upstream" + :label="s__('ProjectSettings|Merge requests')" + :help-text="s__('ProjectSettings|Submit changes to be merged upstream')" > <project-feature-setting v-model="mergeRequestsAccessLevel" @@ -277,7 +283,10 @@ export default { name="project[project_feature_attributes][merge_requests_access_level]" /> </project-setting-row> - <project-setting-row label="Pipelines" help-text="Build, test, and deploy your changes"> + <project-setting-row + :label="s__('ProjectSettings|Pipelines')" + :help-text="s__('ProjectSettings|Build, test, and deploy your changes')" + > <project-feature-setting v-model="buildsAccessLevel" :options="repoFeatureAccessLevelOptions" @@ -288,11 +297,17 @@ export default { <project-setting-row v-if="registryAvailable" :help-path="registryHelpPath" - label="Container registry" - help-text="Every project can have its own space to store its Docker images" + :label="s__('ProjectSettings|Container registry')" + :help-text=" + s__('ProjectSettings|Every project can have its own space to store its Docker images') + " > <div v-if="showContainerRegistryPublicNote" class="text-muted"> - {{ __('Note: the container registry is always visible when a project is public') }} + {{ + s__( + 'ProjectSettings|Note: the container registry is always visible when a project is public', + ) + }} </div> <project-feature-toggle v-model="containerRegistryEnabled" @@ -303,8 +318,10 @@ export default { <project-setting-row v-if="lfsAvailable" :help-path="lfsHelpPath" - label="Git Large File Storage" - help-text="Manages large files such as audio, video, and graphics files" + :label="s__('ProjectSettings|Git Large File Storage')" + :help-text=" + s__('ProjectSettings|Manages large files such as audio, video, and graphics files') + " > <project-feature-toggle v-model="lfsEnabled" @@ -315,8 +332,10 @@ export default { <project-setting-row v-if="packagesAvailable" :help-path="packagesHelpPath" - label="Packages" - help-text="Every project can have its own space to store its packages" + :label="s__('ProjectSettings|Packages')" + :help-text=" + s__('ProjectSettings|Every project can have its own space to store its packages') + " > <project-feature-toggle v-model="packagesEnabled" @@ -325,7 +344,10 @@ export default { /> </project-setting-row> </div> - <project-setting-row label="Wiki" help-text="Pages for project documentation"> + <project-setting-row + :label="s__('ProjectSettings|Wiki')" + :help-text="s__('ProjectSettings|Pages for project documentation')" + > <project-feature-setting v-model="wikiAccessLevel" :options="featureAccessLevelOptions" @@ -333,8 +355,8 @@ export default { /> </project-setting-row> <project-setting-row - label="Snippets" - help-text="Share code pastes with others out of Git repository" + :label="s__('ProjectSettings|Snippets')" + :help-text="s__('ProjectSettings|Share code pastes with others out of Git repository')" > <project-feature-setting v-model="snippetsAccessLevel" @@ -346,7 +368,9 @@ export default { v-if="pagesAvailable && pagesAccessControlEnabled" :help-path="pagesHelpPath" :label="s__('ProjectSettings|Pages')" - :help-text="__('With GitLab Pages you can host your static websites on GitLab')" + :help-text=" + s__('ProjectSettings|With GitLab Pages you can host your static websites on GitLab') + " > <project-feature-setting v-model="pagesAccessLevel" @@ -358,10 +382,13 @@ export default { <project-setting-row v-if="canDisableEmails" class="mb-3"> <label class="js-emails-disabled"> <input :value="emailsDisabled" type="hidden" name="project[emails_disabled]" /> - <input v-model="emailsDisabled" type="checkbox" /> {{ __('Disable email notifications') }} + <input v-model="emailsDisabled" type="checkbox" /> + {{ s__('ProjectSettings|Disable email notifications') }} </label> <span class="form-text text-muted">{{ - __('This setting will override user notification preferences for all project members.') + s__( + 'ProjectSettings|This setting will override user notification preferences for all project members.', + ) }}</span> </project-setting-row> </div> diff --git a/app/assets/javascripts/repository/components/table/row.vue b/app/assets/javascripts/repository/components/table/row.vue index 171841178a3..81ae5143082 100644 --- a/app/assets/javascripts/repository/components/table/row.vue +++ b/app/assets/javascripts/repository/components/table/row.vue @@ -97,11 +97,13 @@ export default { }, }, methods: { - openRow() { - if (this.isFolder) { + openRow(e) { + if (e.target.tagName === 'A') return; + + if (this.isFolder && !e.metaKey) { this.$router.push(this.routerLinkTo); } else { - visitUrl(this.url); + visitUrl(this.url, e.metaKey); } }, }, diff --git a/app/controllers/groups/group_links_controller.rb b/app/controllers/groups/group_links_controller.rb new file mode 100644 index 00000000000..7965311c5f1 --- /dev/null +++ b/app/controllers/groups/group_links_controller.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class Groups::GroupLinksController < Groups::ApplicationController + before_action :check_feature_flag! + before_action :authorize_admin_group! + + def create + shared_with_group = Group.find(params[:shared_with_group_id]) if params[:shared_with_group_id].present? + + if shared_with_group + result = Groups::GroupLinks::CreateService + .new(shared_with_group, current_user, group_link_create_params) + .execute(group) + + return render_404 if result[:http_status] == 404 + + flash[:alert] = result[:message] if result[:status] == :error + else + flash[:alert] = _('Please select a group.') + end + + redirect_to group_group_members_path(group) + end + + private + + def group_link_create_params + params.permit(:shared_group_access, :expires_at) + end + + def check_feature_flag! + render_404 unless Feature.enabled?(:share_group_with_group) + end +end diff --git a/app/models/concerns/worker_attributes.rb b/app/models/concerns/worker_attributes.rb index af40e9e3b19..506215ca9ed 100644 --- a/app/models/concerns/worker_attributes.rb +++ b/app/models/concerns/worker_attributes.rb @@ -3,6 +3,10 @@ module WorkerAttributes extend ActiveSupport::Concern + # Resource boundaries that workers can declare through the + # `worker_resource_boundary` attribute + VALID_RESOURCE_BOUNDARIES = [:memory, :cpu, :unknown].freeze + class_methods do def feature_category(value) raise "Invalid category. Use `feature_category_not_owned!` to mark a worker as not owned" if value == :not_owned @@ -24,6 +28,48 @@ module WorkerAttributes get_worker_attribute(:feature_category) == :not_owned end + # This should be set for jobs that need to be run immediately, or, if + # they are delayed, risk creating inconsistencies in the application + # that could being perceived by the user as incorrect behavior + # (ie, a bug) + # See doc/development/sidekiq_style_guide.md#Latency-Sensitive-Jobs + # for details + def latency_sensitive_worker! + worker_attributes[:latency_sensitive] = true + end + + # Returns a truthy value if the worker is latency sensitive. + # See doc/development/sidekiq_style_guide.md#Latency-Sensitive-Jobs + # for details + def latency_sensitive_worker? + worker_attributes[:latency_sensitive] + end + + # Set this attribute on a job when it will call to services outside of the + # application, such as 3rd party applications, other k8s clusters etc See + # doc/development/sidekiq_style_guide.md#Jobs-with-External-Dependencies for + # details + def worker_has_external_dependencies! + worker_attributes[:external_dependencies] = true + end + + # Returns a truthy value if the worker has external dependencies. + # See doc/development/sidekiq_style_guide.md#Jobs-with-External-Dependencies + # for details + def worker_has_external_dependencies? + worker_attributes[:external_dependencies] + end + + def worker_resource_boundary(boundary) + raise "Invalid boundary" unless VALID_RESOURCE_BOUNDARIES.include? boundary + + worker_attributes[:resource_boundary] = boundary + end + + def get_worker_resource_boundary + worker_attributes[:resource_boundary] || :unknown + end + protected # Returns a worker attribute declared on this class or its parent class. diff --git a/app/models/group.rb b/app/models/group.rb index 042201ffa14..27e4d709823 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -30,6 +30,10 @@ class Group < Namespace has_many :members_and_requesters, as: :source, class_name: 'GroupMember' has_many :milestones + has_many :shared_group_links, foreign_key: :shared_with_group_id, class_name: 'GroupGroupLink' + has_many :shared_with_group_links, foreign_key: :shared_group_id, class_name: 'GroupGroupLink' + has_many :shared_groups, through: :shared_group_links, source: :shared_group + has_many :shared_with_groups, through: :shared_with_group_links, source: :shared_with_group has_many :project_group_links, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :shared_projects, through: :project_group_links, source: :project @@ -376,11 +380,12 @@ class Group < Namespace return GroupMember::OWNER if user.admin? - members_with_parents - .where(user_id: user) - .reorder(access_level: :desc) - .first&. - access_level || GroupMember::NO_ACCESS + max_member_access = members_with_parents.where(user_id: user) + .reorder(access_level: :desc) + .first + &.access_level + + max_member_access || max_member_access_for_user_from_shared_groups(user) || GroupMember::NO_ACCESS end def mattermost_team_params @@ -474,6 +479,26 @@ class Group < Namespace errors.add(:visibility_level, "#{visibility} is not allowed since there are sub-groups with higher visibility.") end + def max_member_access_for_user_from_shared_groups(user) + return unless Feature.enabled?(:share_group_with_group) + + group_group_link_table = GroupGroupLink.arel_table + group_member_table = GroupMember.arel_table + + group_group_links_query = GroupGroupLink.where(shared_group_id: self_and_ancestors_ids) + cte = Gitlab::SQL::CTE.new(:group_group_links_cte, group_group_links_query) + + link = GroupGroupLink + .with(cte.to_arel) + .from([group_member_table, cte.alias_to(group_group_link_table)]) + .where(group_member_table[:user_id].eq(user.id)) + .where(group_member_table[:source_id].eq(group_group_link_table[:shared_with_group_id])) + .reorder(Arel::Nodes::Descending.new(group_group_link_table[:group_access])) + .first + + link&.group_access + end + def self.groups_including_descendants_by(group_ids) Gitlab::ObjectHierarchy .new(Group.where(id: group_ids)) diff --git a/app/models/group_group_link.rb b/app/models/group_group_link.rb new file mode 100644 index 00000000000..4b279b7af5b --- /dev/null +++ b/app/models/group_group_link.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class GroupGroupLink < ApplicationRecord + include Expirable + + belongs_to :shared_group, class_name: 'Group', foreign_key: :shared_group_id + belongs_to :shared_with_group, class_name: 'Group', foreign_key: :shared_with_group_id + + validates :shared_group, presence: true + validates :shared_group_id, uniqueness: { scope: [:shared_with_group_id], + message: _('The group has already been shared with this group') } + validates :shared_with_group, presence: true + validates :group_access, inclusion: { in: Gitlab::Access.values }, + presence: true + + def self.access_options + Gitlab::Access.options + end + + def self.default_access + Gitlab::Access::DEVELOPER + end +end diff --git a/app/services/groups/group_links/create_service.rb b/app/services/groups/group_links/create_service.rb new file mode 100644 index 00000000000..2ce53fcfe4a --- /dev/null +++ b/app/services/groups/group_links/create_service.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Groups + module GroupLinks + class CreateService < BaseService + def execute(shared_group) + unless group && shared_group && + can?(current_user, :admin_group, shared_group) && + can?(current_user, :read_group, group) + return error('Not Found', 404) + end + + link = GroupGroupLink.new( + shared_group: shared_group, + shared_with_group: group, + group_access: params[:shared_group_access], + expires_at: params[:expires_at] + ) + + if link.save + group.refresh_members_authorized_projects + success(link: link) + else + error(link.errors.full_messages.to_sentence, 409) + end + end + end + end +end diff --git a/app/views/ci/variables/_header.html.haml b/app/views/ci/variables/_header.html.haml index dbfa0a9e5a1..ce4dd5a4877 100644 --- a/app/views/ci/variables/_header.html.haml +++ b/app/views/ci/variables/_header.html.haml @@ -7,5 +7,5 @@ %button.btn.btn-default.js-settings-toggle{ type: 'button' } = expanded ? _('Collapse') : _('Expand') -%p.append-bottom-0 +%p = render "ci/variables/content" diff --git a/app/workers/authorized_projects_worker.rb b/app/workers/authorized_projects_worker.rb index 577c439f4a2..9492cfe217c 100644 --- a/app/workers/authorized_projects_worker.rb +++ b/app/workers/authorized_projects_worker.rb @@ -5,6 +5,7 @@ class AuthorizedProjectsWorker prepend WaitableWorker feature_category :authentication_and_authorization + latency_sensitive_worker! # This is a workaround for a Ruby 2.3.7 bug. rspec-mocks cannot restore the # visibility of prepended modules. See https://github.com/rspec/rspec-mocks/issues/1231 diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb index e95b6b38d28..e61f37ddce1 100644 --- a/app/workers/build_finished_worker.rb +++ b/app/workers/build_finished_worker.rb @@ -5,6 +5,8 @@ class BuildFinishedWorker include PipelineQueue queue_namespace :pipeline_processing + latency_sensitive_worker! + worker_resource_boundary :cpu # rubocop: disable CodeReuse/ActiveRecord def perform(build_id) diff --git a/app/workers/build_hooks_worker.rb b/app/workers/build_hooks_worker.rb index 15b31acf3e5..fa55769e486 100644 --- a/app/workers/build_hooks_worker.rb +++ b/app/workers/build_hooks_worker.rb @@ -6,6 +6,7 @@ class BuildHooksWorker queue_namespace :pipeline_hooks feature_category :continuous_integration + latency_sensitive_worker! # rubocop: disable CodeReuse/ActiveRecord def perform(build_id) diff --git a/app/workers/build_queue_worker.rb b/app/workers/build_queue_worker.rb index 6584fba4c65..6f75f403e6e 100644 --- a/app/workers/build_queue_worker.rb +++ b/app/workers/build_queue_worker.rb @@ -6,6 +6,8 @@ class BuildQueueWorker queue_namespace :pipeline_processing feature_category :continuous_integration + latency_sensitive_worker! + worker_resource_boundary :cpu # rubocop: disable CodeReuse/ActiveRecord def perform(build_id) diff --git a/app/workers/build_success_worker.rb b/app/workers/build_success_worker.rb index ac947f3cf38..b7dbd367fee 100644 --- a/app/workers/build_success_worker.rb +++ b/app/workers/build_success_worker.rb @@ -5,6 +5,7 @@ class BuildSuccessWorker include PipelineQueue queue_namespace :pipeline_processing + latency_sensitive_worker! # rubocop: disable CodeReuse/ActiveRecord def perform(build_id) diff --git a/app/workers/chat_notification_worker.rb b/app/workers/chat_notification_worker.rb index 3bc2edad62c..42a23cd472a 100644 --- a/app/workers/chat_notification_worker.rb +++ b/app/workers/chat_notification_worker.rb @@ -4,6 +4,11 @@ class ChatNotificationWorker include ApplicationWorker feature_category :chatops + latency_sensitive_worker! + # TODO: break this into multiple jobs + # as the `responder` uses external dependencies + # See https://gitlab.com/gitlab-com/gl-infra/scalability/issues/34 + # worker_has_external_dependencies! RESCHEDULE_INTERVAL = 2.seconds diff --git a/app/workers/ci/build_schedule_worker.rb b/app/workers/ci/build_schedule_worker.rb index f22ec4c7810..e34f16f46c2 100644 --- a/app/workers/ci/build_schedule_worker.rb +++ b/app/workers/ci/build_schedule_worker.rb @@ -7,6 +7,7 @@ module Ci queue_namespace :pipeline_processing feature_category :continuous_integration + worker_resource_boundary :cpu def perform(build_id) ::Ci::Build.find_by_id(build_id).try do |build| diff --git a/app/workers/cluster_install_app_worker.rb b/app/workers/cluster_install_app_worker.rb index 32e2ea7996c..0e075b295dd 100644 --- a/app/workers/cluster_install_app_worker.rb +++ b/app/workers/cluster_install_app_worker.rb @@ -5,6 +5,8 @@ class ClusterInstallAppWorker include ClusterQueue include ClusterApplications + worker_has_external_dependencies! + def perform(app_name, app_id) find_application(app_name, app_id) do |app| Clusters::Applications::InstallService.new(app).execute diff --git a/app/workers/cluster_patch_app_worker.rb b/app/workers/cluster_patch_app_worker.rb index 0549e81ed05..3f95a764567 100644 --- a/app/workers/cluster_patch_app_worker.rb +++ b/app/workers/cluster_patch_app_worker.rb @@ -5,6 +5,8 @@ class ClusterPatchAppWorker include ClusterQueue include ClusterApplications + worker_has_external_dependencies! + def perform(app_name, app_id) find_application(app_name, app_id) do |app| Clusters::Applications::PatchService.new(app).execute diff --git a/app/workers/cluster_project_configure_worker.rb b/app/workers/cluster_project_configure_worker.rb index ad2437a77e9..614029c2b5c 100644 --- a/app/workers/cluster_project_configure_worker.rb +++ b/app/workers/cluster_project_configure_worker.rb @@ -4,6 +4,8 @@ class ClusterProjectConfigureWorker include ApplicationWorker include ClusterQueue + worker_has_external_dependencies! + def perform(project_id) # Scheduled for removal in https://gitlab.com/gitlab-org/gitlab-foss/issues/59319 end diff --git a/app/workers/cluster_provision_worker.rb b/app/workers/cluster_provision_worker.rb index 59de7903c1c..375bcff4d3d 100644 --- a/app/workers/cluster_provision_worker.rb +++ b/app/workers/cluster_provision_worker.rb @@ -4,6 +4,8 @@ class ClusterProvisionWorker include ApplicationWorker include ClusterQueue + worker_has_external_dependencies! + def perform(cluster_id) Clusters::Cluster.find_by_id(cluster_id).try do |cluster| cluster.provider.try do |provider| diff --git a/app/workers/cluster_upgrade_app_worker.rb b/app/workers/cluster_upgrade_app_worker.rb index d1a538859b4..cd06f0a2224 100644 --- a/app/workers/cluster_upgrade_app_worker.rb +++ b/app/workers/cluster_upgrade_app_worker.rb @@ -5,6 +5,8 @@ class ClusterUpgradeAppWorker include ClusterQueue include ClusterApplications + worker_has_external_dependencies! + def perform(app_name, app_id) find_application(app_name, app_id) do |app| Clusters::Applications::UpgradeService.new(app).execute diff --git a/app/workers/cluster_wait_for_app_installation_worker.rb b/app/workers/cluster_wait_for_app_installation_worker.rb index e8d7e52f70f..7155dc6f835 100644 --- a/app/workers/cluster_wait_for_app_installation_worker.rb +++ b/app/workers/cluster_wait_for_app_installation_worker.rb @@ -8,6 +8,9 @@ class ClusterWaitForAppInstallationWorker INTERVAL = 10.seconds TIMEOUT = 20.minutes + worker_has_external_dependencies! + worker_resource_boundary :cpu + def perform(app_name, app_id) find_application(app_name, app_id) do |app| Clusters::Applications::CheckInstallationProgressService.new(app).execute diff --git a/app/workers/cluster_wait_for_ingress_ip_address_worker.rb b/app/workers/cluster_wait_for_ingress_ip_address_worker.rb index 6865384df44..14b1651cc72 100644 --- a/app/workers/cluster_wait_for_ingress_ip_address_worker.rb +++ b/app/workers/cluster_wait_for_ingress_ip_address_worker.rb @@ -5,6 +5,8 @@ class ClusterWaitForIngressIpAddressWorker include ClusterQueue include ClusterApplications + worker_has_external_dependencies! + def perform(app_name, app_id) find_application(app_name, app_id) do |app| Clusters::Applications::CheckIngressIpAddressService.new(app).execute diff --git a/app/workers/clusters/applications/uninstall_worker.rb b/app/workers/clusters/applications/uninstall_worker.rb index 85e8ecc4ad5..6180998c8d9 100644 --- a/app/workers/clusters/applications/uninstall_worker.rb +++ b/app/workers/clusters/applications/uninstall_worker.rb @@ -7,6 +7,8 @@ module Clusters include ClusterQueue include ClusterApplications + worker_has_external_dependencies! + def perform(app_name, app_id) find_application(app_name, app_id) do |app| Clusters::Applications::UninstallService.new(app).execute diff --git a/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb b/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb index 163c99d3c3c..7907aa8dfff 100644 --- a/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb +++ b/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb @@ -10,6 +10,9 @@ module Clusters INTERVAL = 10.seconds TIMEOUT = 20.minutes + worker_has_external_dependencies! + worker_resource_boundary :cpu + def perform(app_name, app_id) find_application(app_name, app_id) do |app| Clusters::Applications::CheckUninstallProgressService.new(app).execute diff --git a/app/workers/concerns/gitlab/github_import/object_importer.rb b/app/workers/concerns/gitlab/github_import/object_importer.rb index b856a9329dd..bd0b566658e 100644 --- a/app/workers/concerns/gitlab/github_import/object_importer.rb +++ b/app/workers/concerns/gitlab/github_import/object_importer.rb @@ -14,6 +14,7 @@ module Gitlab include NotifyUponDeath feature_category :importers + worker_has_external_dependencies! end # project - An instance of `Project` to import the data into. diff --git a/app/workers/create_pipeline_worker.rb b/app/workers/create_pipeline_worker.rb index 70412ffd095..a75cc643038 100644 --- a/app/workers/create_pipeline_worker.rb +++ b/app/workers/create_pipeline_worker.rb @@ -6,6 +6,8 @@ class CreatePipelineWorker queue_namespace :pipeline_creation feature_category :continuous_integration + latency_sensitive_worker! + worker_resource_boundary :cpu def perform(project_id, user_id, ref, source, params = {}) project = Project.find(project_id) diff --git a/app/workers/deployments/finished_worker.rb b/app/workers/deployments/finished_worker.rb index 79a1caccc92..90bbc193651 100644 --- a/app/workers/deployments/finished_worker.rb +++ b/app/workers/deployments/finished_worker.rb @@ -6,6 +6,7 @@ module Deployments queue_namespace :deployment feature_category :continuous_delivery + worker_resource_boundary :cpu def perform(deployment_id) Deployment.find_by_id(deployment_id).try(:execute_hooks) diff --git a/app/workers/deployments/success_worker.rb b/app/workers/deployments/success_worker.rb index f6520307186..4a29f1aef52 100644 --- a/app/workers/deployments/success_worker.rb +++ b/app/workers/deployments/success_worker.rb @@ -6,6 +6,7 @@ module Deployments queue_namespace :deployment feature_category :continuous_delivery + worker_resource_boundary :cpu def perform(deployment_id) Deployment.find_by_id(deployment_id).try do |deployment| diff --git a/app/workers/email_receiver_worker.rb b/app/workers/email_receiver_worker.rb index c82728be329..b56bf4ed833 100644 --- a/app/workers/email_receiver_worker.rb +++ b/app/workers/email_receiver_worker.rb @@ -4,6 +4,7 @@ class EmailReceiverWorker include ApplicationWorker feature_category :issue_tracking + latency_sensitive_worker! def perform(raw) return unless Gitlab::IncomingEmail.enabled? diff --git a/app/workers/emails_on_push_worker.rb b/app/workers/emails_on_push_worker.rb index 2231c91a720..f523f5953e1 100644 --- a/app/workers/emails_on_push_worker.rb +++ b/app/workers/emails_on_push_worker.rb @@ -6,6 +6,8 @@ class EmailsOnPushWorker attr_reader :email, :skip_premailer feature_category :source_code_management + latency_sensitive_worker! + worker_resource_boundary :cpu def perform(project_id, recipients, push_data, options = {}) options.symbolize_keys! diff --git a/app/workers/expire_job_cache_worker.rb b/app/workers/expire_job_cache_worker.rb index b09d0a5d121..0363429587e 100644 --- a/app/workers/expire_job_cache_worker.rb +++ b/app/workers/expire_job_cache_worker.rb @@ -5,6 +5,7 @@ class ExpireJobCacheWorker include PipelineQueue queue_namespace :pipeline_cache + latency_sensitive_worker! # rubocop: disable CodeReuse/ActiveRecord def perform(job_id) diff --git a/app/workers/expire_pipeline_cache_worker.rb b/app/workers/expire_pipeline_cache_worker.rb index 78e68d7bf46..ab57c59ffda 100644 --- a/app/workers/expire_pipeline_cache_worker.rb +++ b/app/workers/expire_pipeline_cache_worker.rb @@ -5,6 +5,8 @@ class ExpirePipelineCacheWorker include PipelineQueue queue_namespace :pipeline_cache + latency_sensitive_worker! + worker_resource_boundary :cpu # rubocop: disable CodeReuse/ActiveRecord def perform(pipeline_id) diff --git a/app/workers/gitlab_shell_worker.rb b/app/workers/gitlab_shell_worker.rb index 9766331cf4b..5dcf901e041 100644 --- a/app/workers/gitlab_shell_worker.rb +++ b/app/workers/gitlab_shell_worker.rb @@ -5,6 +5,7 @@ class GitlabShellWorker include Gitlab::ShellAdapter feature_category :source_code_management + latency_sensitive_worker! def perform(action, *arg) gitlab_shell.__send__(action, *arg) # rubocop:disable GitlabSecurity/PublicSend diff --git a/app/workers/import_issues_csv_worker.rb b/app/workers/import_issues_csv_worker.rb index d9834320318..d2733dc5f56 100644 --- a/app/workers/import_issues_csv_worker.rb +++ b/app/workers/import_issues_csv_worker.rb @@ -4,6 +4,7 @@ class ImportIssuesCsvWorker include ApplicationWorker feature_category :issue_tracking + worker_resource_boundary :cpu sidekiq_retries_exhausted do |job| Upload.find(job['args'][2]).destroy diff --git a/app/workers/mail_scheduler/notification_service_worker.rb b/app/workers/mail_scheduler/notification_service_worker.rb index 0d06dab3b2e..4130ce25878 100644 --- a/app/workers/mail_scheduler/notification_service_worker.rb +++ b/app/workers/mail_scheduler/notification_service_worker.rb @@ -8,6 +8,7 @@ module MailScheduler include MailSchedulerQueue feature_category :issue_tracking + worker_resource_boundary :cpu def perform(meth, *args) check_arguments!(args) diff --git a/app/workers/merge_worker.rb b/app/workers/merge_worker.rb index 70b909afea8..ed88c57e8d4 100644 --- a/app/workers/merge_worker.rb +++ b/app/workers/merge_worker.rb @@ -4,6 +4,7 @@ class MergeWorker include ApplicationWorker feature_category :source_code_management + latency_sensitive_worker! def perform(merge_request_id, current_user_id, params) params = params.with_indifferent_access diff --git a/app/workers/namespaces/prune_aggregation_schedules_worker.rb b/app/workers/namespaces/prune_aggregation_schedules_worker.rb index 16259ffbfa6..9a5f533fe9a 100644 --- a/app/workers/namespaces/prune_aggregation_schedules_worker.rb +++ b/app/workers/namespaces/prune_aggregation_schedules_worker.rb @@ -6,6 +6,7 @@ module Namespaces include CronjobQueue feature_category :source_code_management + worker_resource_boundary :cpu # Worker to prune pending rows on Namespace::AggregationSchedule # It's scheduled to run once a day at 1:05am. diff --git a/app/workers/new_issue_worker.rb b/app/workers/new_issue_worker.rb index 1b0fec597e7..af9ca332d3c 100644 --- a/app/workers/new_issue_worker.rb +++ b/app/workers/new_issue_worker.rb @@ -5,6 +5,8 @@ class NewIssueWorker include NewIssuable feature_category :issue_tracking + latency_sensitive_worker! + worker_resource_boundary :cpu def perform(issue_id, user_id) return unless objects_found?(issue_id, user_id) diff --git a/app/workers/new_merge_request_worker.rb b/app/workers/new_merge_request_worker.rb index 0a5b2f86331..aa3f85c157b 100644 --- a/app/workers/new_merge_request_worker.rb +++ b/app/workers/new_merge_request_worker.rb @@ -5,6 +5,8 @@ class NewMergeRequestWorker include NewIssuable feature_category :source_code_management + latency_sensitive_worker! + worker_resource_boundary :cpu def perform(merge_request_id, user_id) return unless objects_found?(merge_request_id, user_id) diff --git a/app/workers/new_note_worker.rb b/app/workers/new_note_worker.rb index d0d2a563738..2a5988a7e32 100644 --- a/app/workers/new_note_worker.rb +++ b/app/workers/new_note_worker.rb @@ -4,6 +4,8 @@ class NewNoteWorker include ApplicationWorker feature_category :issue_tracking + latency_sensitive_worker! + worker_resource_boundary :cpu # Keep extra parameter to preserve backwards compatibility with # old `NewNoteWorker` jobs (can remove later) diff --git a/app/workers/object_pool/join_worker.rb b/app/workers/object_pool/join_worker.rb index 9c5161fd55a..ddd002eabb8 100644 --- a/app/workers/object_pool/join_worker.rb +++ b/app/workers/object_pool/join_worker.rb @@ -5,6 +5,8 @@ module ObjectPool include ApplicationWorker include ObjectPoolQueue + worker_resource_boundary :cpu + # The use of pool id is deprecated. Keeping the argument allows old jobs to # still be performed. def perform(_pool_id, project_id) diff --git a/app/workers/pages_domain_removal_cron_worker.rb b/app/workers/pages_domain_removal_cron_worker.rb index 25e747c78d0..b1506831056 100644 --- a/app/workers/pages_domain_removal_cron_worker.rb +++ b/app/workers/pages_domain_removal_cron_worker.rb @@ -5,6 +5,7 @@ class PagesDomainRemovalCronWorker include CronjobQueue feature_category :pages + worker_resource_boundary :cpu def perform PagesDomain.for_removal.find_each do |domain| diff --git a/app/workers/pipeline_hooks_worker.rb b/app/workers/pipeline_hooks_worker.rb index eae1115e60c..04abc9c88fd 100644 --- a/app/workers/pipeline_hooks_worker.rb +++ b/app/workers/pipeline_hooks_worker.rb @@ -5,6 +5,8 @@ class PipelineHooksWorker include PipelineQueue queue_namespace :pipeline_hooks + latency_sensitive_worker! + worker_resource_boundary :cpu # rubocop: disable CodeReuse/ActiveRecord def perform(pipeline_id) diff --git a/app/workers/pipeline_metrics_worker.rb b/app/workers/pipeline_metrics_worker.rb index 0ddad43b8d5..3830522aaa1 100644 --- a/app/workers/pipeline_metrics_worker.rb +++ b/app/workers/pipeline_metrics_worker.rb @@ -4,6 +4,8 @@ class PipelineMetricsWorker include ApplicationWorker include PipelineQueue + latency_sensitive_worker! + # rubocop: disable CodeReuse/ActiveRecord def perform(pipeline_id) Ci::Pipeline.find_by(id: pipeline_id).try do |pipeline| diff --git a/app/workers/pipeline_notification_worker.rb b/app/workers/pipeline_notification_worker.rb index e4a18573d20..62ecbc8a047 100644 --- a/app/workers/pipeline_notification_worker.rb +++ b/app/workers/pipeline_notification_worker.rb @@ -4,6 +4,9 @@ class PipelineNotificationWorker include ApplicationWorker include PipelineQueue + latency_sensitive_worker! + worker_resource_boundary :cpu + # rubocop: disable CodeReuse/ActiveRecord def perform(pipeline_id, recipients = nil) pipeline = Ci::Pipeline.find_by(id: pipeline_id) diff --git a/app/workers/pipeline_process_worker.rb b/app/workers/pipeline_process_worker.rb index 96f3725dbbe..2a36ab992e9 100644 --- a/app/workers/pipeline_process_worker.rb +++ b/app/workers/pipeline_process_worker.rb @@ -6,6 +6,7 @@ class PipelineProcessWorker queue_namespace :pipeline_processing feature_category :continuous_integration + latency_sensitive_worker! # rubocop: disable CodeReuse/ActiveRecord def perform(pipeline_id, build_ids = nil) diff --git a/app/workers/pipeline_schedule_worker.rb b/app/workers/pipeline_schedule_worker.rb index f500ea08353..19c3c5fcc2f 100644 --- a/app/workers/pipeline_schedule_worker.rb +++ b/app/workers/pipeline_schedule_worker.rb @@ -5,6 +5,7 @@ class PipelineScheduleWorker include CronjobQueue feature_category :continuous_integration + worker_resource_boundary :cpu def perform Ci::PipelineSchedule.runnable_schedules.preloaded.find_in_batches do |schedules| diff --git a/app/workers/pipeline_success_worker.rb b/app/workers/pipeline_success_worker.rb index 666331e6cd4..5c24f00e0c3 100644 --- a/app/workers/pipeline_success_worker.rb +++ b/app/workers/pipeline_success_worker.rb @@ -5,6 +5,7 @@ class PipelineSuccessWorker include PipelineQueue queue_namespace :pipeline_processing + latency_sensitive_worker! def perform(pipeline_id) # no-op diff --git a/app/workers/pipeline_update_worker.rb b/app/workers/pipeline_update_worker.rb index 13a748e1551..5b742461f7a 100644 --- a/app/workers/pipeline_update_worker.rb +++ b/app/workers/pipeline_update_worker.rb @@ -5,6 +5,7 @@ class PipelineUpdateWorker include PipelineQueue queue_namespace :pipeline_processing + latency_sensitive_worker! # rubocop: disable CodeReuse/ActiveRecord def perform(pipeline_id) diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index a3bc7e5b9c9..334a98a0017 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -4,6 +4,8 @@ class PostReceive include ApplicationWorker feature_category :source_code_management + latency_sensitive_worker! + worker_resource_boundary :cpu def perform(gl_repository, identifier, changes, push_options = {}) project, repo_type = Gitlab::GlRepository.parse(gl_repository) diff --git a/app/workers/process_commit_worker.rb b/app/workers/process_commit_worker.rb index 1e4561fc6ea..8b4d66ae493 100644 --- a/app/workers/process_commit_worker.rb +++ b/app/workers/process_commit_worker.rb @@ -11,6 +11,7 @@ class ProcessCommitWorker include ApplicationWorker feature_category :source_code_management + latency_sensitive_worker! # project_id - The ID of the project this commit belongs to. # user_id - The ID of the user that pushed the commit. diff --git a/app/workers/project_cache_worker.rb b/app/workers/project_cache_worker.rb index 57a01c0dd8e..ae1d57aa124 100644 --- a/app/workers/project_cache_worker.rb +++ b/app/workers/project_cache_worker.rb @@ -3,6 +3,9 @@ # Worker for updating any project specific caches. class ProjectCacheWorker include ApplicationWorker + + latency_sensitive_worker! + LEASE_TIMEOUT = 15.minutes.to_i feature_category :source_code_management diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb index bbcf3b72718..11f3fed82cd 100644 --- a/app/workers/project_export_worker.rb +++ b/app/workers/project_export_worker.rb @@ -6,6 +6,7 @@ class ProjectExportWorker sidekiq_options retry: 3 feature_category :source_code_management + worker_resource_boundary :memory def perform(current_user_id, project_id, after_export_strategy = {}, params = {}) current_user = User.find(current_user_id) diff --git a/app/workers/project_service_worker.rb b/app/workers/project_service_worker.rb index 8041404fc71..38a2a7414a5 100644 --- a/app/workers/project_service_worker.rb +++ b/app/workers/project_service_worker.rb @@ -5,6 +5,7 @@ class ProjectServiceWorker sidekiq_options dead: false feature_category :integrations + worker_has_external_dependencies! def perform(hook_id, data) data = data.with_indifferent_access diff --git a/app/workers/reactive_caching_worker.rb b/app/workers/reactive_caching_worker.rb index af4a3def062..f3a83e0e8d4 100644 --- a/app/workers/reactive_caching_worker.rb +++ b/app/workers/reactive_caching_worker.rb @@ -5,6 +5,14 @@ class ReactiveCachingWorker feature_category_not_owned! + # TODO: The reactive caching worker should be split into + # two different workers, one for latency_sensitive jobs without external dependencies + # and another worker without latency_sensitivity, but with external dependencies + # https://gitlab.com/gitlab-com/gl-infra/scalability/issues/34 + # This worker should also have `worker_has_external_dependencies!` enabled + latency_sensitive_worker! + worker_resource_boundary :cpu + def perform(class_name, id, *args) klass = begin class_name.constantize diff --git a/app/workers/remove_expired_members_worker.rb b/app/workers/remove_expired_members_worker.rb index 75f06fd9f6b..bf209fcec9f 100644 --- a/app/workers/remove_expired_members_worker.rb +++ b/app/workers/remove_expired_members_worker.rb @@ -5,6 +5,7 @@ class RemoveExpiredMembersWorker include CronjobQueue feature_category :authentication_and_authorization + worker_resource_boundary :cpu def perform Member.expired.find_each do |member| diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index bc2d0366fdd..15677fb0a95 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -7,6 +7,7 @@ class RepositoryImportWorker include ProjectImportOptions feature_category :importers + worker_has_external_dependencies! # technical debt: https://gitlab.com/gitlab-org/gitlab/issues/33991 sidekiq_options memory_killer_memory_growth_kb: ENV.fetch('MEMORY_KILLER_REPOSITORY_IMPORT_WORKER_MEMORY_GROWTH_KB', 50).to_i diff --git a/app/workers/repository_update_remote_mirror_worker.rb b/app/workers/repository_update_remote_mirror_worker.rb index b4d96546fa4..d1dec4cb732 100644 --- a/app/workers/repository_update_remote_mirror_worker.rb +++ b/app/workers/repository_update_remote_mirror_worker.rb @@ -6,6 +6,8 @@ class RepositoryUpdateRemoteMirrorWorker include ApplicationWorker include Gitlab::ExclusiveLeaseHelpers + worker_has_external_dependencies! + sidekiq_options retry: 3, dead: false feature_category :source_code_management diff --git a/app/workers/stage_update_worker.rb b/app/workers/stage_update_worker.rb index ea587789d03..de2454128f6 100644 --- a/app/workers/stage_update_worker.rb +++ b/app/workers/stage_update_worker.rb @@ -5,6 +5,7 @@ class StageUpdateWorker include PipelineQueue queue_namespace :pipeline_processing + latency_sensitive_worker! # rubocop: disable CodeReuse/ActiveRecord def perform(stage_id) diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index 971edb1f14f..5a248ab1137 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -5,6 +5,7 @@ class StuckCiJobsWorker include CronjobQueue feature_category :continuous_integration + worker_resource_boundary :cpu EXCLUSIVE_LEASE_KEY = 'stuck_ci_builds_worker_lease' diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb index 4993cd1220c..d9a9a613ca9 100644 --- a/app/workers/stuck_import_jobs_worker.rb +++ b/app/workers/stuck_import_jobs_worker.rb @@ -5,6 +5,7 @@ class StuckImportJobsWorker include CronjobQueue feature_category :importers + worker_resource_boundary :cpu IMPORT_JOBS_EXPIRATION = 15.hours.to_i diff --git a/app/workers/update_head_pipeline_for_merge_request_worker.rb b/app/workers/update_head_pipeline_for_merge_request_worker.rb index 77859abfea4..e069b16eb90 100644 --- a/app/workers/update_head_pipeline_for_merge_request_worker.rb +++ b/app/workers/update_head_pipeline_for_merge_request_worker.rb @@ -6,6 +6,8 @@ class UpdateHeadPipelineForMergeRequestWorker queue_namespace :pipeline_processing feature_category :continuous_integration + latency_sensitive_worker! + worker_resource_boundary :cpu def perform(merge_request_id) MergeRequest.find_by_id(merge_request_id).try do |merge_request| diff --git a/app/workers/update_merge_requests_worker.rb b/app/workers/update_merge_requests_worker.rb index 8e1703cdd0b..acb95353983 100644 --- a/app/workers/update_merge_requests_worker.rb +++ b/app/workers/update_merge_requests_worker.rb @@ -4,6 +4,8 @@ class UpdateMergeRequestsWorker include ApplicationWorker feature_category :source_code_management + latency_sensitive_worker! + worker_resource_boundary :cpu LOG_TIME_THRESHOLD = 90 # seconds diff --git a/app/workers/wait_for_cluster_creation_worker.rb b/app/workers/wait_for_cluster_creation_worker.rb index 8aa1d9290fd..770417398cb 100644 --- a/app/workers/wait_for_cluster_creation_worker.rb +++ b/app/workers/wait_for_cluster_creation_worker.rb @@ -4,6 +4,8 @@ class WaitForClusterCreationWorker include ApplicationWorker include ClusterQueue + worker_has_external_dependencies! + def perform(cluster_id) Clusters::Cluster.find_by_id(cluster_id).try do |cluster| cluster.provider.try do |provider| diff --git a/app/workers/web_hook_worker.rb b/app/workers/web_hook_worker.rb index fd7ca93683e..c3fa3162c14 100644 --- a/app/workers/web_hook_worker.rb +++ b/app/workers/web_hook_worker.rb @@ -4,6 +4,8 @@ class WebHookWorker include ApplicationWorker feature_category :integrations + worker_has_external_dependencies! + sidekiq_options retry: 4, dead: false def perform(hook_id, data, hook_name) |