diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-02-07 15:09:13 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-02-07 15:09:13 +0300 |
commit | 211a8c3361ccf4eb92f36edbdcf15c98fcdcc8b7 (patch) | |
tree | 0ad37172721a39b0d57240bb1b4e70f200a0d93e /app | |
parent | 456a7247f9e88fc2518b69a1a00e905c6db6d775 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'app')
34 files changed, 340 insertions, 158 deletions
diff --git a/app/assets/javascripts/monitoring/components/charts/time_series.vue b/app/assets/javascripts/monitoring/components/charts/time_series.vue index e58d9f23201..8b2c5e44bb5 100644 --- a/app/assets/javascripts/monitoring/components/charts/time_series.vue +++ b/app/assets/javascripts/monitoring/components/charts/time_series.vue @@ -1,5 +1,5 @@ <script> -import { omit } from 'lodash'; +import { omit, throttle } from 'lodash'; import { GlLink, GlButton, GlTooltip, GlResizeObserverDirective } from '@gitlab/ui'; import { GlAreaChart, GlLineChart, GlChartSeriesLabel } from '@gitlab/ui/dist/charts'; import dateFormat from 'dateformat'; @@ -18,6 +18,13 @@ import { import { makeDataSeries } from '~/helpers/monitor_helper'; import { graphDataValidatorForValues } from '../../utils'; +const THROTTLED_DATAZOOM_WAIT = 1000; // miliseconds +const timestampToISODate = timestamp => new Date(timestamp).toISOString(); + +const events = { + datazoom: 'datazoom', +}; + export default { components: { GlAreaChart, @@ -98,6 +105,7 @@ export default { height: chartHeight, svgs: {}, primaryColor: null, + throttledDatazoom: null, }; }, computed: { @@ -245,6 +253,11 @@ export default { this.setSvg('rocket'); this.setSvg('scroll-handle'); }, + destroyed() { + if (this.throttledDatazoom) { + this.throttledDatazoom.cancel(); + } + }, methods: { formatLegendLabel(query) { return `${query.label}`; @@ -287,8 +300,39 @@ export default { console.error('SVG could not be rendered correctly: ', e); }); }, - onChartUpdated(chart) { - [this.primaryColor] = chart.getOption().color; + onChartUpdated(eChart) { + [this.primaryColor] = eChart.getOption().color; + }, + + onChartCreated(eChart) { + // Emit a datazoom event that corresponds to the eChart + // `datazoom` event. + + if (this.throttledDatazoom) { + // Chart can be created multiple times in this component's + // lifetime, remove previous handlers every time + // chart is created. + this.throttledDatazoom.cancel(); + } + + // Emitting is throttled to avoid flurries of calls when + // the user changes or scrolls the zoom bar. + this.throttledDatazoom = throttle( + () => { + const { startValue, endValue } = eChart.getOption().dataZoom[0]; + this.$emit(events.datazoom, { + start: timestampToISODate(startValue), + end: timestampToISODate(endValue), + }); + }, + THROTTLED_DATAZOOM_WAIT, + { + leading: false, + }, + ); + + eChart.off('datazoom'); + eChart.on('datazoom', this.throttledDatazoom); }, onResize() { if (!this.$refs.chart) return; @@ -331,6 +375,7 @@ export default { :height="height" :average-text="legendAverageText" :max-text="legendMaxText" + @created="onChartCreated" @updated="onChartUpdated" > <template v-if="tooltip.isDeployment"> diff --git a/app/assets/javascripts/monitoring/components/dashboard.vue b/app/assets/javascripts/monitoring/components/dashboard.vue index 391cd6dd15e..79f32b357fc 100644 --- a/app/assets/javascripts/monitoring/components/dashboard.vue +++ b/app/assets/javascripts/monitoring/components/dashboard.vue @@ -21,7 +21,6 @@ import createFlash from '~/flash'; import glFeatureFlagsMixin from '~/vue_shared/mixins/gl_feature_flags_mixin'; import { mergeUrlParams, redirectTo } from '~/lib/utils/url_utility'; import invalidUrl from '~/lib/utils/invalid_url'; -import { convertToFixedRange } from '~/lib/utils/datetime_range'; import Icon from '~/vue_shared/components/icon.vue'; import DateTimePicker from '~/vue_shared/components/date_time_picker/date_time_picker.vue'; @@ -102,6 +101,11 @@ export default { type: String, required: true, }, + logsPath: { + type: String, + required: false, + default: invalidUrl, + }, defaultBranch: { type: String, required: true, @@ -247,22 +251,20 @@ export default { dashboardsEndpoint: this.dashboardsEndpoint, currentDashboard: this.currentDashboard, projectPath: this.projectPath, + logsPath: this.logsPath, }); }, mounted() { if (!this.hasMetrics) { this.setGettingStartedEmptyState(); } else { - const { start, end } = convertToFixedRange(this.selectedTimeRange); - - this.fetchData({ - start, - end, - }); + this.setTimeRange(this.selectedTimeRange); + this.fetchData(); } }, methods: { ...mapActions('monitoringDashboard', [ + 'setTimeRange', 'fetchData', 'setGettingStartedEmptyState', 'setEndpoints', diff --git a/app/assets/javascripts/monitoring/components/embed.vue b/app/assets/javascripts/monitoring/components/embed.vue index e55de1c0105..49188a7af8f 100644 --- a/app/assets/javascripts/monitoring/components/embed.vue +++ b/app/assets/javascripts/monitoring/components/embed.vue @@ -19,14 +19,8 @@ export default { }, data() { const timeRange = timeRangeFromUrl(this.dashboardUrl) || defaultTimeRange; - const { start, end } = convertToFixedRange(timeRange); - const params = { - start, - end, - }; - return { - params, + timeRange: convertToFixedRange(timeRange), elWidth: 0, }; }, @@ -49,7 +43,9 @@ export default { }, mounted() { this.setInitialState(); - this.fetchMetricsData(this.params); + this.setTimeRange(this.timeRange); + this.fetchDashboard(); + sidebarMutationObserver = new MutationObserver(this.onSidebarMutation); sidebarMutationObserver.observe(document.querySelector('.layout-page'), { attributes: true, @@ -64,7 +60,8 @@ export default { }, methods: { ...mapActions('monitoringDashboard', [ - 'fetchMetricsData', + 'setTimeRange', + 'fetchDashboard', 'setEndpoints', 'setFeatureFlags', 'setShowErrorBanner', diff --git a/app/assets/javascripts/monitoring/components/panel_type.vue b/app/assets/javascripts/monitoring/components/panel_type.vue index 6751f3d31e8..0258a62b390 100644 --- a/app/assets/javascripts/monitoring/components/panel_type.vue +++ b/app/assets/javascripts/monitoring/components/panel_type.vue @@ -1,6 +1,7 @@ <script> import { mapState } from 'vuex'; import { pickBy } from 'lodash'; +import invalidUrl from '~/lib/utils/invalid_url'; import { GlDropdown, GlDropdownItem, @@ -18,7 +19,7 @@ import MonitorColumnChart from './charts/column.vue'; import MonitorStackedColumnChart from './charts/stacked_column.vue'; import MonitorEmptyChart from './charts/empty_chart.vue'; import TrackEventDirective from '~/vue_shared/directives/track_event'; -import { downloadCSVOptions, generateLinkToChartOptions } from '../utils'; +import { timeRangeToUrl, downloadCSVOptions, generateLinkToChartOptions } from '../utils'; export default { components: { @@ -58,8 +59,13 @@ export default { default: 'panel-type-chart', }, }, + data() { + return { + zoomedTimeRange: null, + }; + }, computed: { - ...mapState('monitoringDashboard', ['deploymentData', 'projectPath']), + ...mapState('monitoringDashboard', ['deploymentData', 'projectPath', 'logsPath', 'timeRange']), alertWidgetAvailable() { return IS_EE && this.prometheusAlertsAvailable && this.alertsEndpoint && this.graphData; }, @@ -70,6 +76,14 @@ export default { this.graphData.metrics[0].result.length > 0 ); }, + logsPathWithTimeRange() { + const timeRange = this.zoomedTimeRange || this.timeRange; + + if (this.logsPath && this.logsPath !== invalidUrl && timeRange) { + return timeRangeToUrl(timeRange, this.logsPath); + } + return null; + }, csvText() { const chartData = this.graphData.metrics[0].result[0].values; const yLabel = this.graphData.y_label; @@ -107,6 +121,10 @@ export default { }, downloadCSVOptions, generateLinkToChartOptions, + + onDatazoom({ start, end }) { + this.zoomedTimeRange = { start, end }; + }, }, }; </script> @@ -130,11 +148,13 @@ export default { <component :is="monitorChartComponent" v-else-if="graphDataHasMetrics" + ref="timeChart" :graph-data="graphData" :deployment-data="deploymentData" :project-path="projectPath" :thresholds="getGraphAlertValues(graphData.metrics)" :group-id="groupId" + @datazoom="onDatazoom" > <div class="d-flex align-items-center"> <alert-widget @@ -157,6 +177,15 @@ export default { <template slot="button-content"> <icon name="ellipsis_v" class="text-secondary" /> </template> + + <gl-dropdown-item + v-if="logsPathWithTimeRange" + ref="viewLogsLink" + :href="logsPathWithTimeRange" + > + {{ s__('Metrics|View logs') }} + </gl-dropdown-item> + <gl-dropdown-item v-track-event="downloadCSVOptions(graphData.title)" :href="downloadCsv" diff --git a/app/assets/javascripts/monitoring/stores/actions.js b/app/assets/javascripts/monitoring/stores/actions.js index 29000475bd4..3a052200ab9 100644 --- a/app/assets/javascripts/monitoring/stores/actions.js +++ b/app/assets/javascripts/monitoring/stores/actions.js @@ -1,6 +1,7 @@ import * as types from './mutation_types'; import axios from '~/lib/utils/axios_utils'; import createFlash from '~/flash'; +import { convertToFixedRange } from '~/lib/utils/datetime_range'; import { gqClient, parseEnvironmentsResponse, removeLeadingSlash } from './utils'; import trackDashboardLoad from '../monitoring_tracking_helper'; import getEnvironments from '../queries/getEnvironments.query.graphql'; @@ -32,6 +33,10 @@ export const setEndpoints = ({ commit }, endpoints) => { commit(types.SET_ENDPOINTS, endpoints); }; +export const setTimeRange = ({ commit }, timeRange) => { + commit(types.SET_TIME_RANGE, timeRange); +}; + export const filterEnvironments = ({ commit, dispatch }, searchTerm) => { commit(types.SET_ENVIRONMENTS_FILTER, searchTerm); dispatch('fetchEnvironmentsData'); @@ -63,19 +68,24 @@ export const receiveEnvironmentsDataSuccess = ({ commit }, data) => export const receiveEnvironmentsDataFailure = ({ commit }) => commit(types.RECEIVE_ENVIRONMENTS_DATA_FAILURE); -export const fetchData = ({ dispatch }, params) => { - dispatch('fetchMetricsData', params); +export const fetchData = ({ dispatch }) => { + dispatch('fetchDashboard'); dispatch('fetchDeploymentsData'); dispatch('fetchEnvironmentsData'); }; -export const fetchMetricsData = ({ dispatch }, params) => dispatch('fetchDashboard', params); - -export const fetchDashboard = ({ state, dispatch }, params) => { +export const fetchDashboard = ({ state, dispatch }) => { dispatch('requestMetricsDashboard'); + const params = {}; + + if (state.timeRange) { + const { start, end } = convertToFixedRange(state.timeRange); + params.start = start; + params.end = end; + } + if (state.currentDashboard) { - // eslint-disable-next-line no-param-reassign params.dashboard = state.currentDashboard; } diff --git a/app/assets/javascripts/monitoring/stores/mutation_types.js b/app/assets/javascripts/monitoring/stores/mutation_types.js index bdfaf42b35c..8873142accc 100644 --- a/app/assets/javascripts/monitoring/stores/mutation_types.js +++ b/app/assets/javascripts/monitoring/stores/mutation_types.js @@ -14,7 +14,7 @@ export const REQUEST_METRIC_RESULT = 'REQUEST_METRIC_RESULT'; export const RECEIVE_METRIC_RESULT_SUCCESS = 'RECEIVE_METRIC_RESULT_SUCCESS'; export const RECEIVE_METRIC_RESULT_FAILURE = 'RECEIVE_METRIC_RESULT_FAILURE'; -export const SET_TIME_WINDOW = 'SET_TIME_WINDOW'; +export const SET_TIME_RANGE = 'SET_TIME_RANGE'; export const SET_ALL_DASHBOARDS = 'SET_ALL_DASHBOARDS'; export const SET_ENDPOINTS = 'SET_ENDPOINTS'; export const SET_GETTING_STARTED_EMPTY_STATE = 'SET_GETTING_STARTED_EMPTY_STATE'; diff --git a/app/assets/javascripts/monitoring/stores/mutations.js b/app/assets/javascripts/monitoring/stores/mutations.js index 2a86a6a26d8..5f559290ff7 100644 --- a/app/assets/javascripts/monitoring/stores/mutations.js +++ b/app/assets/javascripts/monitoring/stores/mutations.js @@ -182,6 +182,10 @@ export default { state.dashboardsEndpoint = endpoints.dashboardsEndpoint; state.currentDashboard = endpoints.currentDashboard; state.projectPath = endpoints.projectPath; + state.logsPath = endpoints.logsPath || state.logsPath; + }, + [types.SET_TIME_RANGE](state, timeRange) { + state.timeRange = timeRange; }, [types.SET_GETTING_STARTED_EMPTY_STATE](state) { state.emptyState = 'gettingStarted'; diff --git a/app/assets/javascripts/monitoring/stores/state.js b/app/assets/javascripts/monitoring/stores/state.js index 9d3227e8aae..a2050f8e893 100644 --- a/app/assets/javascripts/monitoring/stores/state.js +++ b/app/assets/javascripts/monitoring/stores/state.js @@ -1,22 +1,31 @@ import invalidUrl from '~/lib/utils/invalid_url'; export default () => ({ + // API endpoints metricsEndpoint: null, deploymentsEndpoint: null, dashboardEndpoint: invalidUrl, + + // Dashboard request parameters + timeRange: null, + currentDashboard: null, + + // Dashboard data emptyState: 'gettingStarted', showEmptyState: true, showErrorBanner: true, - dashboard: { panel_groups: [], }, + allDashboards: [], + // Other project data deploymentData: [], environments: [], environmentsSearchTerm: '', environmentsLoading: false, - allDashboards: [], - currentDashboard: null, + + // GitLab paths to other pages projectPath: null, + logsPath: invalidUrl, }); diff --git a/app/assets/javascripts/monitoring/utils.js b/app/assets/javascripts/monitoring/utils.js index 915812596c6..b2fa44835e6 100644 --- a/app/assets/javascripts/monitoring/utils.js +++ b/app/assets/javascripts/monitoring/utils.js @@ -103,8 +103,9 @@ export const graphDataValidatorForAnomalyValues = graphData => { /** * Returns a time range from the current URL params * - * @returns {Object} The time range defined by the - * current URL, reading from `window.location.search` + * @returns {Object|null} The time range defined by the + * current URL, reading from search query or `window.location.search`. + * Returns `null` if no parameters form a time range. */ export const timeRangeFromUrl = (search = window.location.search) => { const params = queryToObject(search); diff --git a/app/assets/stylesheets/framework/typography.scss b/app/assets/stylesheets/framework/typography.scss index a1bfa03a5ac..6cd69fe75ce 100644 --- a/app/assets/stylesheets/framework/typography.scss +++ b/app/assets/stylesheets/framework/typography.scss @@ -102,6 +102,7 @@ padding-bottom: 0.3em; border-bottom: 1px solid $white-dark; color: $gl-text-color; + overflow: hidden; &:first-child { margin-top: 0; @@ -115,6 +116,7 @@ padding-bottom: 0.3em; border-bottom: 1px solid $white-dark; color: $gl-text-color; + overflow: hidden; } h3 { diff --git a/app/controllers/admin/services_controller.rb b/app/controllers/admin/services_controller.rb index e31e0e09978..9cea25cc7fe 100644 --- a/app/controllers/admin/services_controller.rb +++ b/app/controllers/admin/services_controller.rb @@ -7,7 +7,7 @@ class Admin::ServicesController < Admin::ApplicationController before_action :service, only: [:edit, :update] def index - @services = services_templates + @services = instance_level_services end def edit @@ -19,7 +19,7 @@ class Admin::ServicesController < Admin::ApplicationController def update if service.update(service_params[:service]) - PropagateServiceTemplateWorker.perform_async(service.id) if service.active? + PropagateInstanceLevelServiceWorker.perform_async(service.id) if service.active? redirect_to admin_application_settings_services_path, notice: 'Application settings saved successfully' @@ -31,17 +31,17 @@ class Admin::ServicesController < Admin::ApplicationController private # rubocop: disable CodeReuse/ActiveRecord - def services_templates + def instance_level_services Service.available_services_names.map do |service_name| - service_template = "#{service_name}_service".camelize.constantize - service_template.where(template: true).first_or_create + service = "#{service_name}_service".camelize.constantize + service.where(instance: true).first_or_create end end # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord def service - @service ||= Service.where(id: params[:id], template: true).first + @service ||= Service.where(id: params[:id], instance: true).first end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/controllers/concerns/lfs_request.rb b/app/controllers/concerns/lfs_request.rb index 61072eec535..3152d959ae4 100644 --- a/app/controllers/concerns/lfs_request.rb +++ b/app/controllers/concerns/lfs_request.rb @@ -112,10 +112,6 @@ module LfsRequest has_authentication_ability?(:build_download_code) && can?(user, :build_download_code, project) end - def storage_project - @storage_project ||= project.lfs_storage_project - end - def objects @objects ||= (params[:objects] || []).to_a end diff --git a/app/controllers/repositories/lfs_storage_controller.rb b/app/controllers/repositories/lfs_storage_controller.rb index 58f496e16d3..ec5ca5bbeec 100644 --- a/app/controllers/repositories/lfs_storage_controller.rb +++ b/app/controllers/repositories/lfs_storage_controller.rb @@ -80,12 +80,13 @@ module Repositories LfsObject.create!(oid: oid, size: size, file: uploaded_file) end - # rubocop: disable CodeReuse/ActiveRecord def link_to_project!(object) - if object && !object.projects.exists?(storage_project.id) - object.lfs_objects_projects.create!(project: storage_project) - end + return unless object + + LfsObjectsProject.safe_find_or_create_by!( + project: project, + lfs_object: object + ) end - # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/app/models/concerns/sortable.rb b/app/models/concerns/sortable.rb index c4af1b1fab2..4fe2a0e1827 100644 --- a/app/models/concerns/sortable.rb +++ b/app/models/concerns/sortable.rb @@ -8,13 +8,13 @@ module Sortable extend ActiveSupport::Concern included do - scope :with_order_id_desc, -> { order(id: :desc) } - scope :order_id_desc, -> { reorder(id: :desc) } - scope :order_id_asc, -> { reorder(id: :asc) } - scope :order_created_desc, -> { reorder(created_at: :desc) } - scope :order_created_asc, -> { reorder(created_at: :asc) } - scope :order_updated_desc, -> { reorder(updated_at: :desc) } - scope :order_updated_asc, -> { reorder(updated_at: :asc) } + scope :with_order_id_desc, -> { order(self.arel_table['id'].desc) } + scope :order_id_desc, -> { reorder(self.arel_table['id'].desc) } + scope :order_id_asc, -> { reorder(self.arel_table['id'].asc) } + scope :order_created_desc, -> { reorder(self.arel_table['created_at'].desc) } + scope :order_created_asc, -> { reorder(self.arel_table['created_at'].asc) } + scope :order_updated_desc, -> { reorder(self.arel_table['updated_at'].desc) } + scope :order_updated_asc, -> { reorder(self.arel_table['updated_at'].asc) } scope :order_name_asc, -> { reorder(Arel::Nodes::Ascending.new(arel_table[:name].lower)) } scope :order_name_desc, -> { reorder(Arel::Nodes::Descending.new(arel_table[:name].lower)) } end diff --git a/app/models/project.rb b/app/models/project.rb index 816d964519d..717075161aa 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -358,6 +358,8 @@ class Project < ApplicationRecord project_path: true, length: { maximum: 255 } + validates :project_feature, presence: true + validates :namespace, presence: true validates :name, uniqueness: { scope: :namespace_id } validates :import_url, public_url: { schemes: ->(project) { project.persisted? ? VALID_MIRROR_PROTOCOLS : VALID_IMPORT_PROTOCOLS }, @@ -395,11 +397,11 @@ class Project < ApplicationRecord # last_activity_at is throttled every minute, but last_repository_updated_at is updated with every push scope :sorted_by_activity, -> { reorder(Arel.sql("GREATEST(COALESCE(last_activity_at, '1970-01-01'), COALESCE(last_repository_updated_at, '1970-01-01')) DESC")) } - scope :sorted_by_stars_desc, -> { reorder(star_count: :desc) } - scope :sorted_by_stars_asc, -> { reorder(star_count: :asc) } + scope :sorted_by_stars_desc, -> { reorder(self.arel_table['star_count'].desc) } + scope :sorted_by_stars_asc, -> { reorder(self.arel_table['star_count'].asc) } scope :sorted_by_name_asc_limited, ->(limit) { reorder(name: :asc).limit(limit) } # Sometimes queries (e.g. using CTEs) require explicit disambiguation with table name - scope :projects_order_id_desc, -> { reorder("#{table_name}.id DESC") } + scope :projects_order_id_desc, -> { reorder(self.arel_table['id'].desc) } scope :in_namespace, ->(namespace_ids) { where(namespace_id: namespace_ids) } scope :personal, ->(user) { where(namespace_id: user.namespace_id) } @@ -595,9 +597,9 @@ class Project < ApplicationRecord # pass a string to avoid AR adding the table name reorder('project_statistics.storage_size DESC, projects.id DESC') when 'latest_activity_desc' - reorder(last_activity_at: :desc) + reorder(self.arel_table['last_activity_at'].desc) when 'latest_activity_asc' - reorder(last_activity_at: :asc) + reorder(self.arel_table['last_activity_at'].asc) when 'stars_desc' sorted_by_stars_desc when 'stars_asc' @@ -1219,13 +1221,13 @@ class Project < ApplicationRecord service = find_service(services, name) return service if service - # We should check if template for the service exists - template = find_service(services_templates, name) + # We should check if an instance-level service exists + instance_level_service = find_service(instance_level_services, name) - if template - Service.build_from_template(id, template) + if instance_level_service + Service.build_from_instance(id, instance_level_service) else - # If no template, we should create an instance. Ex `build_gitlab_ci_service` + # If no instance-level service exists, we should create a new service. Ex `build_gitlab_ci_service` public_send("build_#{name}_service") # rubocop:disable GitlabSecurity/PublicSend end end @@ -1357,6 +1359,10 @@ class Project < ApplicationRecord forked_from_project || fork_network&.root_project end + # TODO: Remove this method once all LfsObjectsProject records are backfilled + # for forks. + # + # See https://gitlab.com/gitlab-org/gitlab/issues/122002 for more info. def lfs_storage_project @lfs_storage_project ||= begin result = self @@ -1369,14 +1375,27 @@ class Project < ApplicationRecord end end - # This will return all `lfs_objects` that are accessible to the project. - # So this might be `self.lfs_objects` if the project is not part of a fork - # network, or it is the base of the fork network. + # This will return all `lfs_objects` that are accessible to the project and + # the fork source. This is needed since older forks won't have access to some + # LFS objects directly and have to get it from the fork source. + # + # TODO: Remove this method once all LfsObjectsProject records are backfilled + # for forks. At that point, projects can look at their own `lfs_objects`. # - # TODO: refactor this to get the correct lfs objects when implementing - # https://gitlab.com/gitlab-org/gitlab-foss/issues/39769 + # See https://gitlab.com/gitlab-org/gitlab/issues/122002 for more info. def all_lfs_objects - lfs_storage_project.lfs_objects + LfsObject + .distinct + .joins(:lfs_objects_projects) + .where(lfs_objects_projects: { project_id: [self, lfs_storage_project] }) + end + + # TODO: Call `#lfs_objects` instead once all LfsObjectsProject records are + # backfilled. At that point, projects can look at their own `lfs_objects`. + # + # See https://gitlab.com/gitlab-org/gitlab/issues/122002 for more info. + def lfs_objects_oids + all_lfs_objects.pluck(:oid) end def personal? @@ -2438,8 +2457,8 @@ class Project < ApplicationRecord end end - def services_templates - @services_templates ||= Service.where(template: true) + def instance_level_services + @instance_level_services ||= Service.where(instance: true) end def ensure_pages_metadatum diff --git a/app/models/project_services/issue_tracker_service.rb b/app/models/project_services/issue_tracker_service.rb index 9e1393196ff..2ccf8e094e6 100644 --- a/app/models/project_services/issue_tracker_service.rb +++ b/app/models/project_services/issue_tracker_service.rb @@ -164,7 +164,7 @@ class IssueTrackerService < Service end def one_issue_tracker - return if template? + return if instance? return if project.blank? if project.services.external_issue_trackers.where.not(id: id).any? diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb index 00b06ae2595..72dabe2578a 100644 --- a/app/models/project_services/prometheus_service.rb +++ b/app/models/project_services/prometheus_service.rb @@ -85,7 +85,7 @@ class PrometheusService < MonitoringService end def prometheus_available? - return false if template? + return false if instance? return false unless project project.all_clusters.enabled.any? { |cluster| cluster.application_prometheus_available? } diff --git a/app/models/service.rb b/app/models/service.rb index 95b7c6927cf..6d0e375e757 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -32,7 +32,7 @@ class Service < ApplicationRecord belongs_to :project, inverse_of: :services has_one :service_hook - validates :project_id, presence: true, unless: proc { |service| service.template? } + validates :project_id, presence: true, unless: proc { |service| service.instance? } validates :type, presence: true scope :visible, -> { where.not(type: 'GitlabIssueTrackerService') } @@ -70,8 +70,8 @@ class Service < ApplicationRecord true end - def template? - template + def instance? + instance end def category @@ -299,15 +299,15 @@ class Service < ApplicationRecord service_names.sort_by(&:downcase) end - def self.build_from_template(project_id, template) - service = template.dup + def self.build_from_instance(project_id, instance_level_service) + service = instance_level_service.dup - if template.supports_data_fields? - data_fields = template.data_fields.dup + if instance_level_service.supports_data_fields? + data_fields = instance_level_service.data_fields.dup data_fields.service = service end - service.template = false + service.instance = false service.project_id = project_id service.active = false if service.active? && !service.valid? service @@ -321,10 +321,6 @@ class Service < ApplicationRecord nil end - def self.find_by_template - find_by(template: true) - end - # override if needed def supports_data_fields? false diff --git a/app/presenters/project_presenter.rb b/app/presenters/project_presenter.rb index 42c707908e6..3af6be26843 100644 --- a/app/presenters/project_presenter.rb +++ b/app/presenters/project_presenter.rb @@ -302,7 +302,7 @@ class ProjectPresenter < Gitlab::View::Presenter::Delegated cluster_link = clusters.count == 1 ? project_cluster_path(project, clusters.first) : project_clusters_path(project) AnchorData.new(false, - _('Kubernetes configured'), + _('Kubernetes'), cluster_link, 'default') end diff --git a/app/services/merge_requests/create_service.rb b/app/services/merge_requests/create_service.rb index 9a37a0330fc..4a05d1fd7ef 100644 --- a/app/services/merge_requests/create_service.rb +++ b/app/services/merge_requests/create_service.rb @@ -27,6 +27,7 @@ module MergeRequests create_pipeline_for(issuable, current_user) issuable.update_head_pipeline Gitlab::UsageDataCounters::MergeRequestCounter.count(:create) + link_lfs_objects(issuable) super end @@ -64,6 +65,10 @@ module MergeRequests raise Gitlab::Access::AccessDeniedError end end + + def link_lfs_objects(issuable) + LinkLfsObjectsService.new(issuable.target_project).execute(issuable) + end end end diff --git a/app/services/merge_requests/link_lfs_objects_service.rb b/app/services/merge_requests/link_lfs_objects_service.rb new file mode 100644 index 00000000000..191da594095 --- /dev/null +++ b/app/services/merge_requests/link_lfs_objects_service.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module MergeRequests + class LinkLfsObjectsService < ::BaseService + def execute(merge_request, oldrev: merge_request.diff_base_sha, newrev: merge_request.diff_head_sha) + return if merge_request.source_project == project + return if no_changes?(oldrev, newrev) + + new_lfs_oids = lfs_oids(merge_request.source_project.repository, oldrev, newrev) + + return if new_lfs_oids.empty? + + Projects::LfsPointers::LfsLinkService + .new(project) + .execute(new_lfs_oids) + end + + private + + def no_changes?(oldrev, newrev) + oldrev == newrev + end + + def lfs_oids(source_repository, oldrev, newrev) + Gitlab::Git::LfsChanges + .new(source_repository, newrev) + .new_pointers(not_in: [oldrev]) + .map(&:lfs_oid) + end + end +end diff --git a/app/services/merge_requests/refresh_service.rb b/app/services/merge_requests/refresh_service.rb index 396ddec6383..c6e1651fa26 100644 --- a/app/services/merge_requests/refresh_service.rb +++ b/app/services/merge_requests/refresh_service.rb @@ -21,6 +21,7 @@ module MergeRequests # empty diff during a manual merge close_upon_missing_source_branch_ref post_merge_manually_merged + link_forks_lfs_objects reload_merge_requests outdate_suggestions refresh_pipelines_on_merge_requests @@ -91,17 +92,25 @@ module MergeRequests end # rubocop: enable CodeReuse/ActiveRecord + # Link LFS objects that exists in forks but does not exists in merge requests + # target project + def link_forks_lfs_objects + return unless @push.branch_updated? + + merge_requests_for_forks.find_each do |mr| + LinkLfsObjectsService + .new(mr.target_project) + .execute(mr, oldrev: @push.oldrev, newrev: @push.newrev) + end + end + # Refresh merge request diff if we push to source or target branch of merge request # Note: we should update merge requests from forks too - # rubocop: disable CodeReuse/ActiveRecord def reload_merge_requests merge_requests = @project.merge_requests.opened .by_source_or_target_branch(@push.branch_name).to_a - # Fork merge requests - merge_requests += MergeRequest.opened - .where(source_branch: @push.branch_name, source_project: @project) - .where.not(target_project: @project).to_a + merge_requests += merge_requests_for_forks.to_a filter_merge_requests(merge_requests).each do |merge_request| if branch_and_project_match?(merge_request) || @push.force_push? @@ -117,7 +126,6 @@ module MergeRequests # @source_merge_requests diffs (for MergeRequest#commit_shas for instance). merge_requests_for_source_branch(reload: true) end - # rubocop: enable CodeReuse/ActiveRecord def push_commit_ids @push_commit_ids ||= @commits.map(&:id) @@ -282,6 +290,15 @@ module MergeRequests @source_merge_requests = nil if reload @source_merge_requests ||= merge_requests_for(@push.branch_name) end + + # rubocop: disable CodeReuse/ActiveRecord + def merge_requests_for_forks + @merge_requests_for_forks ||= + MergeRequest.opened + .where(source_branch: @push.branch_name, source_project: @project) + .where.not(target_project: @project) + end + # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index ef06545b27d..f3666f100a3 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -134,7 +134,7 @@ module Projects if @project.save unless @project.gitlab_project_import? - create_services_from_active_templates(@project) + create_services_from_active_instance_level_services(@project) @project.create_labels end @@ -160,9 +160,9 @@ module Projects end # rubocop: disable CodeReuse/ActiveRecord - def create_services_from_active_templates(project) - Service.where(template: true, active: true).each do |template| - service = Service.build_from_template(project.id, template) + def create_services_from_active_instance_level_services(project) + Service.where(instance: true, active: true).each do |template| + service = Service.build_from_instance(project.id, template) service.save! end end diff --git a/app/services/projects/fork_service.rb b/app/services/projects/fork_service.rb index e66a0ed181a..fcfea567885 100644 --- a/app/services/projects/fork_service.rb +++ b/app/services/projects/fork_service.rb @@ -26,17 +26,7 @@ module Projects build_fork_network_member(fork_to_project) - if link_fork_network(fork_to_project) - # A forked project stores its LFS objects in the `forked_from_project`. - # So the LFS objects become inaccessible, and therefore delete them from - # the database so they'll get cleaned up. - # - # TODO: refactor this to get the correct lfs objects when implementing - # https://gitlab.com/gitlab-org/gitlab-foss/issues/39769 - fork_to_project.lfs_objects_projects.delete_all - - fork_to_project - end + fork_to_project if link_fork_network(fork_to_project) end def fork_new_project diff --git a/app/services/projects/lfs_pointers/lfs_download_service.rb b/app/services/projects/lfs_pointers/lfs_download_service.rb index a009f479d5d..bd70012c76c 100644 --- a/app/services/projects/lfs_pointers/lfs_download_service.rb +++ b/app/services/projects/lfs_pointers/lfs_download_service.rb @@ -39,9 +39,9 @@ module Projects def download_lfs_file! with_tmp_file do |tmp_file| download_and_save_file!(tmp_file) - project.all_lfs_objects << LfsObject.new(oid: lfs_oid, - size: lfs_size, - file: tmp_file) + project.lfs_objects << LfsObject.new(oid: lfs_oid, + size: lfs_size, + file: tmp_file) success end diff --git a/app/services/projects/move_lfs_objects_projects_service.rb b/app/services/projects/move_lfs_objects_projects_service.rb index 10e19014db4..8cc420d7ba7 100644 --- a/app/services/projects/move_lfs_objects_projects_service.rb +++ b/app/services/projects/move_lfs_objects_projects_service.rb @@ -16,7 +16,7 @@ module Projects private def move_lfs_objects_projects - non_existent_lfs_objects_projects.update_all(project_id: @project.lfs_storage_project.id) + non_existent_lfs_objects_projects.update_all(project_id: @project.id) end def remove_remaining_lfs_objects_project diff --git a/app/services/projects/propagate_service_template.rb b/app/services/projects/propagate_instance_level_service.rb index 6013b00b8c6..dc75977ba0f 100644 --- a/app/services/projects/propagate_service_template.rb +++ b/app/services/projects/propagate_instance_level_service.rb @@ -1,38 +1,38 @@ # frozen_string_literal: true module Projects - class PropagateServiceTemplate + class PropagateInstanceLevelService BATCH_SIZE = 100 def self.propagate(*args) new(*args).propagate end - def initialize(template) - @template = template + def initialize(instance_level_service) + @instance_level_service = instance_level_service end def propagate - return unless @template.active? + return unless @instance_level_service.active? - Rails.logger.info("Propagating services for template #{@template.id}") # rubocop:disable Gitlab/RailsLogger + Rails.logger.info("Propagating services for instance_level_service #{@instance_level_service.id}") # rubocop:disable Gitlab/RailsLogger - propagate_projects_with_template + propagate_projects_with_instance_level_service end private - def propagate_projects_with_template + def propagate_projects_with_instance_level_service loop do batch = Project.uncached { project_ids_batch } - bulk_create_from_template(batch) unless batch.empty? + bulk_create_from_instance_level_service(batch) unless batch.empty? break if batch.size < BATCH_SIZE end end - def bulk_create_from_template(batch) + def bulk_create_from_instance_level_service(batch) service_list = batch.map do |project_id| service_hash.values << project_id end @@ -52,7 +52,7 @@ module Projects SELECT true FROM services WHERE services.project_id = projects.id - AND services.type = '#{@template.type}' + AND services.type = '#{@instance_level_service.type}' ) AND projects.pending_delete = false AND projects.archived = false @@ -73,9 +73,9 @@ module Projects def service_hash @service_hash ||= begin - template_hash = @template.as_json(methods: :type).except('id', 'template', 'project_id') + instance_hash = @instance_level_service.as_json(methods: :type).except('id', 'instance', 'project_id') - template_hash.each_with_object({}) do |(key, value), service_hash| + instance_hash.each_with_object({}) do |(key, value), service_hash| value = value.is_a?(Hash) ? value.to_json : value service_hash[ActiveRecord::Base.connection.quote_column_name(key)] = @@ -97,11 +97,11 @@ module Projects # rubocop: enable CodeReuse/ActiveRecord def active_external_issue_tracker? - @template.issue_tracker? && !@template.default + @instance_level_service.issue_tracker? && !@instance_level_service.default end def active_external_wiki? - @template.type == 'ExternalWikiService' + @instance_level_service.type == 'ExternalWikiService' end end end diff --git a/app/services/projects/unlink_fork_service.rb b/app/services/projects/unlink_fork_service.rb index e7e0141099e..b3cf27373cd 100644 --- a/app/services/projects/unlink_fork_service.rb +++ b/app/services/projects/unlink_fork_service.rb @@ -52,6 +52,10 @@ module Projects Projects::ForksCountService.new(project).refresh_cache end + # TODO: Remove this method once all LfsObjectsProject records are backfilled + # for forks. + # + # See https://gitlab.com/gitlab-org/gitlab/issues/122002 for more info. def save_lfs_objects return unless @project.forked? diff --git a/app/views/projects/services/mattermost_slash_commands/_help.html.haml b/app/views/projects/services/mattermost_slash_commands/_help.html.haml index cc005dd69b7..7f14128a0cb 100644 --- a/app/views/projects/services/mattermost_slash_commands/_help.html.haml +++ b/app/views/projects/services/mattermost_slash_commands/_help.html.haml @@ -10,8 +10,8 @@ %p.inline = s_("MattermostService|See list of available commands in Mattermost after setting up this service, by entering") %kbd.inline /<trigger> help - - unless enabled || @service.template? + - unless enabled || @service.instance? = render 'projects/services/mattermost_slash_commands/detailed_help', subject: @service -- if enabled && !@service.template? +- if enabled && !@service.instance? = render 'projects/services/mattermost_slash_commands/installation_info', subject: @service diff --git a/app/views/projects/services/slack_slash_commands/_help.html.haml b/app/views/projects/services/slack_slash_commands/_help.html.haml index 7f6717e298c..447f7f074a8 100644 --- a/app/views/projects/services/slack_slash_commands/_help.html.haml +++ b/app/views/projects/services/slack_slash_commands/_help.html.haml @@ -11,7 +11,7 @@ %p.inline = s_("SlackService|See list of available commands in Slack after setting up this service, by entering") %kbd.inline /<command> help - - unless @service.template? + - unless @service.instance? %p= _("To set up this service:") %ul.list-unstyled.indent-list %li diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 35852742b0d..b344e1e36b8 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -951,7 +951,7 @@ :latency_sensitive: :resource_boundary: :unknown :weight: 1 -- :name: propagate_service_template +- :name: propagate_instance_level_service :feature_category: :source_code_management :has_external_dependencies: :latency_sensitive: diff --git a/app/workers/propagate_instance_level_service_worker.rb b/app/workers/propagate_instance_level_service_worker.rb new file mode 100644 index 00000000000..64ea61cabfa --- /dev/null +++ b/app/workers/propagate_instance_level_service_worker.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# Worker for updating any project specific caches. +class PropagateInstanceLevelServiceWorker + include ApplicationWorker + + feature_category :source_code_management + + LEASE_TIMEOUT = 4.hours.to_i + + # rubocop: disable CodeReuse/ActiveRecord + def perform(instance_level_service_id) + return unless try_obtain_lease_for(instance_level_service_id) + + Projects::PropagateInstanceLevelService.propagate(Service.find_by(id: instance_level_service_id)) + end + # rubocop: enable CodeReuse/ActiveRecord + + private + + def try_obtain_lease_for(instance_level_service_id) + Gitlab::ExclusiveLease + .new("propagate_instance_level_service_worker:#{instance_level_service_id}", timeout: LEASE_TIMEOUT) + .try_obtain + end +end diff --git a/app/workers/propagate_service_template_worker.rb b/app/workers/propagate_service_template_worker.rb deleted file mode 100644 index 73a2b453207..00000000000 --- a/app/workers/propagate_service_template_worker.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -# Worker for updating any project specific caches. -class PropagateServiceTemplateWorker - include ApplicationWorker - - feature_category :source_code_management - - LEASE_TIMEOUT = 4.hours.to_i - - # rubocop: disable CodeReuse/ActiveRecord - def perform(template_id) - return unless try_obtain_lease_for(template_id) - - Projects::PropagateServiceTemplate.propagate(Service.find_by(id: template_id)) - end - # rubocop: enable CodeReuse/ActiveRecord - - private - - def try_obtain_lease_for(template_id) - Gitlab::ExclusiveLease - .new("propagate_service_template_worker:#{template_id}", timeout: LEASE_TIMEOUT) - .try_obtain - end -end diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb index 0adf745c7ac..ba141f808a7 100644 --- a/app/workers/repository_fork_worker.rb +++ b/app/workers/repository_fork_worker.rb @@ -29,7 +29,15 @@ class RepositoryForkWorker result = gitlab_shell.fork_repository(source_project, target_project) - raise "Unable to fork project #{target_project.id} for repository #{source_project.disk_path} -> #{target_project.disk_path}" unless result + if result + link_lfs_objects(source_project, target_project) + else + raise_fork_failure( + source_project, + target_project, + 'Failed to create fork repository' + ) + end target_project.after_import end @@ -40,4 +48,20 @@ class RepositoryForkWorker Rails.logger.info("Project #{project.full_path} was in inconsistent state (#{project.import_status}) while forking.") # rubocop:disable Gitlab/RailsLogger false end + + def link_lfs_objects(source_project, target_project) + Projects::LfsPointers::LfsLinkService + .new(target_project) + .execute(source_project.lfs_objects_oids) + rescue Projects::LfsPointers::LfsLinkService::TooManyOidsError + raise_fork_failure( + source_project, + target_project, + 'Source project has too many LFS objects' + ) + end + + def raise_fork_failure(source_project, target_project, reason) + raise "Unable to fork project #{target_project.id} for repository #{source_project.disk_path} -> #{target_project.disk_path}: #{reason}" + end end |