diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-03-24 15:09:32 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-03-24 15:09:32 +0300 |
| commit | a46b489e175708cc64fc5198f458f927558f11ba (patch) | |
| tree | 96bb3be8fa37d729d1417b4e91df0bf718039911 | |
| parent | bd818d0618b0c1316c23f47f9ae41449e5d41fa8 (diff) | |
Add latest changes from gitlab-org/gitlab@master
85 files changed, 1183 insertions, 178 deletions
diff --git a/app/assets/javascripts/error_tracking_settings/components/error_tracking_form.vue b/app/assets/javascripts/error_tracking_settings/components/error_tracking_form.vue index dbcda0877b4..4df324b396c 100644 --- a/app/assets/javascripts/error_tracking_settings/components/error_tracking_form.vue +++ b/app/assets/javascripts/error_tracking_settings/components/error_tracking_form.vue @@ -33,7 +33,7 @@ export default { <p class="form-text text-muted"> {{ s__( - "ErrorTracking|If you self-host Sentry, enter the full URL of your Sentry instance. If you're using Sentry's hosted solution, enter https://sentry.io", + "ErrorTracking|If you self-host Sentry, enter your Sentry instance's full URL. If you use Sentry's hosted solution, enter https://sentry.io", ) }} </p> @@ -75,12 +75,12 @@ export default { </div> </div> <p v-if="connectError" class="gl-field-error"> - {{ s__('ErrorTracking|Connection has failed. Re-check Auth Token and try again.') }} + {{ s__('ErrorTracking|Connection failed. Check Auth Token and try again.') }} </p> <p v-else class="form-text text-muted"> {{ s__( - "ErrorTracking|After adding your Auth Token, use the 'Connect' button to load projects", + 'ErrorTracking|After adding your Auth Token, select the Connect button to load projects.', ) }} </p> diff --git a/app/assets/javascripts/error_tracking_settings/store/getters.js b/app/assets/javascripts/error_tracking_settings/store/getters.js index 30828778574..f203a259b16 100644 --- a/app/assets/javascripts/error_tracking_settings/store/getters.js +++ b/app/assets/javascripts/error_tracking_settings/store/getters.js @@ -34,8 +34,8 @@ export const invalidProjectLabel = (state) => { export const projectSelectionLabel = (state) => { if (state.token) { return s__( - "ErrorTracking|Click 'Connect' to re-establish the connection to Sentry and activate the dropdown.", + 'ErrorTracking|Click Connect to reestablish the connection to Sentry and activate the dropdown.', ); } - return s__('ErrorTracking|To enable project selection, enter a valid Auth Token'); + return s__('ErrorTracking|To enable project selection, enter a valid Auth Token.'); }; diff --git a/app/assets/javascripts/mr_notes/init_notes.js b/app/assets/javascripts/mr_notes/init_notes.js index ff194d1a171..d85fd10be45 100644 --- a/app/assets/javascripts/mr_notes/init_notes.js +++ b/app/assets/javascripts/mr_notes/init_notes.js @@ -58,6 +58,8 @@ export default () => { created() { this.setActiveTab(window.mrTabs.getCurrentAction()); this.setEndpoints(this.endpoints); + + this.fetchMrMetadata(); }, mounted() { this.notesCountBadge = $('.issuable-details').find('.notes-tab .badge'); @@ -69,7 +71,7 @@ export default () => { window.mrTabs.eventHub.$off('MergeRequestTabChange', this.setActiveTab); }, methods: { - ...mapActions(['setActiveTab', 'setEndpoints']), + ...mapActions(['setActiveTab', 'setEndpoints', 'fetchMrMetadata']), updateDiscussionTabCounter() { this.notesCountBadge.text(this.discussionTabCounter); }, diff --git a/app/assets/javascripts/mr_notes/stores/actions.js b/app/assets/javascripts/mr_notes/stores/actions.js index d1874dcb214..bc66d1dd68f 100644 --- a/app/assets/javascripts/mr_notes/stores/actions.js +++ b/app/assets/javascripts/mr_notes/stores/actions.js @@ -1,3 +1,5 @@ +import axios from '~/lib/utils/axios_utils'; + import types from './mutation_types'; export function setActiveTab({ commit }, tab) { @@ -7,3 +9,24 @@ export function setActiveTab({ commit }, tab) { export function setEndpoints({ commit }, endpoints) { commit(types.SET_ENDPOINTS, endpoints); } + +export function setMrMetadata({ commit }, metadata) { + commit(types.SET_MR_METADATA, metadata); +} + +export function fetchMrMetadata({ dispatch, state }) { + if (state.endpoints?.metadata) { + axios + .get(state.endpoints.metadata) + .then((response) => { + dispatch('setMrMetadata', response.data); + }) + .catch(() => { + // https://gitlab.com/gitlab-org/gitlab/-/issues/324740 + // We can't even do a simple console warning here because + // the pipeline will fail. However, the issue above will + // eventually handle errors appropriately. + // console.warn('Failed to load MR Metadata for the Overview tab.'); + }); + } +} diff --git a/app/assets/javascripts/mr_notes/stores/modules/index.js b/app/assets/javascripts/mr_notes/stores/modules/index.js index 6e228c62a72..52e12ba664c 100644 --- a/app/assets/javascripts/mr_notes/stores/modules/index.js +++ b/app/assets/javascripts/mr_notes/stores/modules/index.js @@ -6,6 +6,7 @@ export default () => ({ state: { endpoints: {}, activeTab: null, + mrMetadata: {}, }, actions, getters, diff --git a/app/assets/javascripts/mr_notes/stores/mutation_types.js b/app/assets/javascripts/mr_notes/stores/mutation_types.js index 67fa63f882d..88cf6e48988 100644 --- a/app/assets/javascripts/mr_notes/stores/mutation_types.js +++ b/app/assets/javascripts/mr_notes/stores/mutation_types.js @@ -1,4 +1,5 @@ export default { SET_ACTIVE_TAB: 'SET_ACTIVE_TAB', SET_ENDPOINTS: 'SET_ENDPOINTS', + SET_MR_METADATA: 'SET_MR_METADATA', }; diff --git a/app/assets/javascripts/mr_notes/stores/mutations.js b/app/assets/javascripts/mr_notes/stores/mutations.js index 3843103f4d0..6af6adb4e18 100644 --- a/app/assets/javascripts/mr_notes/stores/mutations.js +++ b/app/assets/javascripts/mr_notes/stores/mutations.js @@ -7,4 +7,7 @@ export default { [types.SET_ENDPOINTS](state, endpoints) { Object.assign(state, { endpoints }); }, + [types.SET_MR_METADATA](state, metadata) { + Object.assign(state, { mrMetadata: metadata }); + }, }; diff --git a/app/assets/javascripts/pipeline_editor/index.js b/app/assets/javascripts/pipeline_editor/index.js index c20eeac8f41..377905c54bb 100644 --- a/app/assets/javascripts/pipeline_editor/index.js +++ b/app/assets/javascripts/pipeline_editor/index.js @@ -3,7 +3,8 @@ import Vue from 'vue'; import VueApollo from 'vue-apollo'; import createDefaultClient from '~/lib/graphql'; import { resetServiceWorkersPublicPath } from '../lib/utils/webpack'; -import { EDITOR_APP_STATUS_LOADING } from './constants'; +import getCommitSha from './graphql/queries/client/commit_sha.graphql'; +import getCurrentBranch from './graphql/queries/client/current_branch.graphql'; import { resolvers } from './graphql/resolvers'; import typeDefs from './graphql/typedefs.graphql'; import PipelineEditorApp from './pipeline_editor_app.vue'; @@ -41,12 +42,19 @@ export const initPipelineEditor = (selector = '#js-pipeline-editor') => { const apolloProvider = new VueApollo({ defaultClient: createDefaultClient(resolvers, { typeDefs }), }); + const { cache } = apolloProvider.clients.defaultClient; - apolloProvider.clients.defaultClient.cache.writeData({ + cache.writeQuery({ + query: getCurrentBranch, data: { currentBranch: initialBranchName || defaultBranch, + }, + }); + + cache.writeQuery({ + query: getCommitSha, + data: { commitSha, - status: EDITOR_APP_STATUS_LOADING, }, }); diff --git a/app/assets/javascripts/vue_shared/alert_details/index.js b/app/assets/javascripts/vue_shared/alert_details/index.js index 50f2e63702b..8c2aceb0336 100644 --- a/app/assets/javascripts/vue_shared/alert_details/index.js +++ b/app/assets/javascripts/vue_shared/alert_details/index.js @@ -42,7 +42,8 @@ export default (selector) => { }), }); - apolloProvider.clients.defaultClient.cache.writeData({ + apolloProvider.clients.defaultClient.cache.writeQuery({ + query: sidebarStatusQuery, data: { sidebarStatus: false, }, diff --git a/app/controllers/projects/commits_controller.rb b/app/controllers/projects/commits_controller.rb index c48e5f40d99..2ced72aab7e 100644 --- a/app/controllers/projects/commits_controller.rb +++ b/app/controllers/projects/commits_controller.rb @@ -8,7 +8,6 @@ class Projects::CommitsController < Projects::ApplicationController prepend_before_action(only: [:show]) { authenticate_sessionless_user!(:rss) } around_action :allow_gitaly_ref_name_caching - before_action :disable_query_limiting, except: :commits_root before_action :require_non_empty_project before_action :assign_ref_vars, except: :commits_root before_action :authorize_download_code! @@ -82,8 +81,4 @@ class Projects::CommitsController < Projects::ApplicationController @commits = @commits.with_latest_pipeline(@ref) @commits = set_commits_for_rendering(@commits) end - - def disable_query_limiting - Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42330') - end end diff --git a/app/finders/packages/go/package_finder.rb b/app/finders/packages/go/package_finder.rb new file mode 100644 index 00000000000..4573417d11f --- /dev/null +++ b/app/finders/packages/go/package_finder.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Packages + module Go + class PackageFinder + delegate :exists?, to: :candidates + + def initialize(project, module_name, module_version) + @project = project + @module_name = module_name + @module_version = module_version + end + + def execute + candidates.first + end + + private + + def candidates + @project + .packages + .golang + .with_name(@module_name) + .with_version(@module_version) + end + end + end +end diff --git a/app/finders/packages/go/version_finder.rb b/app/finders/packages/go/version_finder.rb index 8e2fab8ba35..6ee02b8c6f6 100644 --- a/app/finders/packages/go/version_finder.rb +++ b/app/finders/packages/go/version_finder.rb @@ -23,7 +23,8 @@ module Packages when String if pseudo_version? target semver = parse_semver(target) - commit = pseudo_version_commit(@mod.project, semver) + version = parse_pseudo_version(semver) + commit = validate_pseudo_version(@mod.project, version) Packages::Go::ModuleVersion.new(@mod, :pseudo, commit, name: target, semver: semver) else @mod.version_by(ref: target) diff --git a/app/models/issue.rb b/app/models/issue.rb index 4c46f779893..16b30f193c7 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -24,6 +24,8 @@ class Issue < ApplicationRecord include Todoable include FromUnion + extend ::Gitlab::Utils::Override + DueDateStruct = Struct.new(:title, :name).freeze NoDueDate = DueDateStruct.new('No Due Date', '0').freeze AnyDueDate = DueDateStruct.new('Any Due Date', '').freeze @@ -448,8 +450,14 @@ class Issue < ApplicationRecord private + # Ensure that the metrics association is safely created and respecting the unique constraint on issue_id + override :ensure_metrics def ensure_metrics - super + if !association(:metrics).loaded? || metrics.blank? + metrics_record = Issue::Metrics.safe_find_or_create_by(issue: self) + self.metrics = metrics_record + end + metrics.record! end diff --git a/app/models/packages/go/module_version.rb b/app/models/packages/go/module_version.rb index a50c78f8e69..fd575e6c96c 100644 --- a/app/models/packages/go/module_version.rb +++ b/app/models/packages/go/module_version.rb @@ -4,6 +4,7 @@ module Packages module Go class ModuleVersion include Gitlab::Utils::StrongMemoize + include Gitlab::Golang VALID_TYPES = %i[ref commit pseudo].freeze @@ -81,6 +82,9 @@ module Packages end def valid? + # assume the module version is valid if a corresponding Package exists + return true if ::Packages::Go::PackageFinder.new(mod.project, mod.name, name).exists? + @mod.path_valid?(major) && @mod.gomod_valid?(gomod) end diff --git a/app/services/issue_rebalancing_service.rb b/app/services/issue_rebalancing_service.rb index db5c5ddfb84..f9c3388204f 100644 --- a/app/services/issue_rebalancing_service.rb +++ b/app/services/issue_rebalancing_service.rb @@ -62,7 +62,7 @@ class IssueRebalancingService def run_update_query(values, query_name) Issue.connection.exec_query(<<~SQL, query_name) - WITH cte(cte_id, new_pos) AS ( + WITH cte(cte_id, new_pos) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT * FROM (VALUES #{values}) as t (id, pos) ) diff --git a/app/services/notes/create_service.rb b/app/services/notes/create_service.rb index 488c847dcbb..e63099a0820 100644 --- a/app/services/notes/create_service.rb +++ b/app/services/notes/create_service.rb @@ -75,16 +75,9 @@ module Notes increment_usage_counter(note) track_event(note, current_user) - if Feature.enabled?(:notes_create_service_tracking, project) - Gitlab::Tracking.event('Notes::CreateService', 'execute', **tracking_data_for(note)) - end - if note.for_merge_request? && note.diff_note? && note.start_of_discussion? Discussions::CaptureDiffNotePositionService.new(note.noteable, note.diff_file&.paths).execute(note.discussion) end - - track_note_creation_usage_for_issues(note) if note.for_issue? - track_note_creation_usage_for_merge_requests(note) if note.for_merge_request? end def do_commands(note, update_params, message, only_commands) @@ -111,6 +104,16 @@ module Notes } end + def track_event(note, user) + track_note_creation_usage_for_issues(note) if note.for_issue? + track_note_creation_usage_for_merge_requests(note) if note.for_merge_request? + track_usage_event(:incident_management_incident_comment, user.id) if note.for_issue? && note.noteable.incident? + + if Feature.enabled?(:notes_create_service_tracking, project) + Gitlab::Tracking.event('Notes::CreateService', 'execute', **tracking_data_for(note)) + end + end + def tracking_data_for(note) label = Gitlab.ee? && note.author == User.visual_review_bot ? 'anonymous_visual_review_note' : 'note' @@ -120,12 +123,6 @@ module Notes } end - def track_event(note, user) - return unless note.noteable.is_a?(Issue) && note.noteable.incident? - - track_usage_event(:incident_management_incident_comment, user.id) - end - def track_note_creation_usage_for_issues(note) Gitlab::UsageDataCounters::IssueActivityUniqueCounter.track_issue_comment_added_action(author: note.author) end @@ -135,3 +132,5 @@ module Notes end end end + +Notes::CreateService.prepend_if_ee('EE::Notes::CreateService') diff --git a/app/services/packages/go/create_package_service.rb b/app/services/packages/go/create_package_service.rb new file mode 100644 index 00000000000..4e8b8ef8d6b --- /dev/null +++ b/app/services/packages/go/create_package_service.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Packages + module Go + class CreatePackageService < BaseService + GoZipSizeError = Class.new(StandardError) + + attr_accessor :version + + def initialize(project, user = nil, version:) + super(project, user) + + @version = version + end + + def execute + # check for existing package to avoid SQL errors due to the index + package = ::Packages::Go::PackageFinder.new(version.mod.project, version.mod.name, version.name).execute + return package if package + + # this can be expensive, so do it outside the transaction + files = {} + files[:mod] = prepare_file(version, :mod, version.gomod) + files[:zip] = prepare_file(version, :zip, version.archive.string) + + ActiveRecord::Base.transaction do + # create new package and files + package = create_package + files.each { |type, (file, digests)| create_file(package, type, file, digests) } + package + end + end + + private + + def prepare_file(version, type, content) + file = CarrierWaveStringFile.new(content) + raise GoZipSizeError, "#{version.mod.name}@#{version.name}.#{type} exceeds size limit" if file.size > project.actual_limits.golang_max_file_size + + digests = { + md5: Digest::MD5.hexdigest(content), + sha1: Digest::SHA1.hexdigest(content), + sha256: Digest::SHA256.hexdigest(content) + } + + [file, digests] + end + + def create_package + version.mod.project.packages.create!( + name: version.mod.name, + version: version.name, + package_type: :golang, + created_at: version.commit.committed_date + ) + end + + def create_file(package, type, file, digests) + CreatePackageFileService.new(package, + file: file, + size: file.size, + file_name: "#{version.name}.#{type}", + file_md5: digests[:md5], + file_sha1: digests[:sha1], + file_sha256: digests[:sha256] + ).execute + end + end + end +end diff --git a/app/services/packages/go/sync_packages_service.rb b/app/services/packages/go/sync_packages_service.rb new file mode 100644 index 00000000000..c35d3600388 --- /dev/null +++ b/app/services/packages/go/sync_packages_service.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Packages + module Go + class SyncPackagesService < BaseService + include Gitlab::Golang + + def initialize(project, ref, path = '') + super(project) + + @ref = ref + @path = path + + raise ArgumentError, 'project is required' unless project + raise ArgumentError, 'ref is required' unless ref + raise ArgumentError, "ref #{ref} not found" unless project.repository.find_tag(ref) || project.repository.find_branch(ref) + end + + def execute_async + Packages::Go::SyncPackagesWorker.perform_async(project.id, @ref, @path) + end + end + end +end diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index d405b00dc32..3dbb087d8d8 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -1083,6 +1083,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: package_repositories:packages_go_sync_packages + :feature_category: :package_registry + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: package_repositories:packages_maven_metadata_sync :feature_category: :package_registry :has_external_dependencies: diff --git a/app/workers/packages/go/sync_packages_worker.rb b/app/workers/packages/go/sync_packages_worker.rb new file mode 100644 index 00000000000..e41f27f2252 --- /dev/null +++ b/app/workers/packages/go/sync_packages_worker.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Packages + module Go + class SyncPackagesWorker + include ApplicationWorker + include Gitlab::Golang + + queue_namespace :package_repositories + feature_category :package_registry + + deduplicate :until_executing + idempotent! + + def perform(project_id, ref_name, path) + project = Project.find_by_id(project_id) + return unless project && project.repository.find_tag(ref_name) + + module_name = go_path(project, path) + mod = Packages::Go::ModuleFinder.new(project, module_name).execute + return unless mod + + ver = Packages::Go::VersionFinder.new(mod).find(ref_name) + return unless ver + + Packages::Go::CreatePackageService.new(project, nil, version: ver).execute + + rescue ::Packages::Go::CreatePackageService::GoZipSizeError => ex + Gitlab::ErrorTracking.log_exception(ex) + end + end + end +end diff --git a/changelogs/unreleased/214456-dedup-issues-metrics.yml b/changelogs/unreleased/214456-dedup-issues-metrics.yml new file mode 100644 index 00000000000..6fe71bb23cf --- /dev/null +++ b/changelogs/unreleased/214456-dedup-issues-metrics.yml @@ -0,0 +1,5 @@ +--- +title: Deduplicate issue_metrics table +merge_request: 55285 +author: +type: other diff --git a/changelogs/unreleased/220628-go-proxy-packages.yml b/changelogs/unreleased/220628-go-proxy-packages.yml new file mode 100644 index 00000000000..9847b5b9c14 --- /dev/null +++ b/changelogs/unreleased/220628-go-proxy-packages.yml @@ -0,0 +1,5 @@ +--- +title: Add Go Packages as a cache for the Go proxy +merge_request: 34558 +author: Ethan Reesor (@firelizzard) +type: added diff --git a/changelogs/unreleased/245323-arel-support-for-materialized-cte.yml b/changelogs/unreleased/245323-arel-support-for-materialized-cte.yml new file mode 100644 index 00000000000..b822c2810f2 --- /dev/null +++ b/changelogs/unreleased/245323-arel-support-for-materialized-cte.yml @@ -0,0 +1,5 @@ +--- +title: Add support for the MATERIALIZED keyword when using WITH (CTE) queries in PostgreSQL 12 +merge_request: 56976 +author: +type: other diff --git a/changelogs/unreleased/292253-track-epic-note-created.yml b/changelogs/unreleased/292253-track-epic-note-created.yml new file mode 100644 index 00000000000..b050784589c --- /dev/null +++ b/changelogs/unreleased/292253-track-epic-note-created.yml @@ -0,0 +1,5 @@ +--- +title: Track epic note created via usage ping +merge_request: 56609 +author: +type: other diff --git a/changelogs/unreleased/id-n-1-for-deploy-keys.yml b/changelogs/unreleased/id-n-1-for-deploy-keys.yml new file mode 100644 index 00000000000..34db7d37751 --- /dev/null +++ b/changelogs/unreleased/id-n-1-for-deploy-keys.yml @@ -0,0 +1,5 @@ +--- +title: Remove N+1 for API :id/deploy_keys +merge_request: 57295 +author: +type: performance diff --git a/changelogs/unreleased/ui-text-error-tracking.yml b/changelogs/unreleased/ui-text-error-tracking.yml new file mode 100644 index 00000000000..f31d7f3338a --- /dev/null +++ b/changelogs/unreleased/ui-text-error-tracking.yml @@ -0,0 +1,5 @@ +--- +title: Updated UI text to match style guidelines +merge_request: 57276 +author: +type: other diff --git a/config/initializers/postgres_cte_as_materialized.rb b/config/initializers/postgres_cte_as_materialized.rb new file mode 100644 index 00000000000..85b3361e25e --- /dev/null +++ b/config/initializers/postgres_cte_as_materialized.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# This patch adds support for AS MATERIALIZED in Arel, see Gitlab::Database::AsWithMaterialized for more info +module Arel + module Visitors + class Arel::Visitors::PostgreSQL + def visit_Gitlab_Database_AsWithMaterialized(obj, collector) # rubocop:disable Naming/MethodName + collector = visit obj.left, collector + collector << " AS#{obj.expr} " + visit obj.right, collector + end + end + end +end diff --git a/config/initializers/postgresql_cte.rb b/config/initializers/postgresql_cte.rb index 1ea0b4cfb58..6a9af7b4868 100644 --- a/config/initializers/postgresql_cte.rb +++ b/config/initializers/postgresql_cte.rb @@ -121,6 +121,8 @@ module ActiveRecord end when Arel::Nodes::As with_value + when Gitlab::Database::AsWithMaterialized + with_value end end diff --git a/config/metrics/counts_all/20210216175446_network_policy_forwards.yml b/config/metrics/counts_all/20210216175446_network_policy_forwards.yml index f1330d775a8..36605f8cd92 100644 --- a/config/metrics/counts_all/20210216175446_network_policy_forwards.yml +++ b/config/metrics/counts_all/20210216175446_network_policy_forwards.yml @@ -9,7 +9,7 @@ product_category: container_network_security value_type: number status: data_available time_frame: all -data_source: database +data_source: redis distribution: - ce - ee @@ -17,4 +17,3 @@ tier: - free - premium - ultimate -skip_validation: true diff --git a/config/metrics/counts_all/20210216175448_network_policy_drops.yml b/config/metrics/counts_all/20210216175448_network_policy_drops.yml index e76c53f7b95..d254fedced4 100644 --- a/config/metrics/counts_all/20210216175448_network_policy_drops.yml +++ b/config/metrics/counts_all/20210216175448_network_policy_drops.yml @@ -9,7 +9,7 @@ product_category: container_network_security value_type: number status: data_available time_frame: all -data_source: database +data_source: redis distribution: - ce - ee @@ -17,4 +17,3 @@ tier: - free - premium - ultimate -skip_validation: true diff --git a/danger/changelog/Dangerfile b/danger/changelog/Dangerfile index 26a844a3757..4507f41b798 100644 --- a/danger/changelog/Dangerfile +++ b/danger/changelog/Dangerfile @@ -21,7 +21,7 @@ def check_changelog_yaml(path) fail "`type` should be set, in #{helper.html_link(path)}! #{SEE_DOC}" if yaml["type"].nil? return if helper.security_mr? - return if helper.mr_iid.empty? + return if helper.mr_iid.to_s.empty? cherry_pick_against_stable_branch = helper.cherry_pick_mr? && helper.stable_branch? diff --git a/db/migrate/20200609212701_add_incident_settings_to_all_existing_projects.rb b/db/migrate/20200609212701_add_incident_settings_to_all_existing_projects.rb index 60286e0dca6..a2931824ef5 100644 --- a/db/migrate/20200609212701_add_incident_settings_to_all_existing_projects.rb +++ b/db/migrate/20200609212701_add_incident_settings_to_all_existing_projects.rb @@ -8,7 +8,7 @@ class AddIncidentSettingsToAllExistingProjects < ActiveRecord::Migration[6.0] # to preserve behavior for existing projects that # are using the create issue functionality with the default setting of true query = <<-SQL - WITH project_ids AS ( + WITH project_ids AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported}( SELECT DISTINCT issues.project_id AS id FROM issues LEFT OUTER JOIN project_incident_management_settings diff --git a/db/migrate/20210106061254_add_unique_index_for_golang_packages.rb b/db/migrate/20210106061254_add_unique_index_for_golang_packages.rb new file mode 100644 index 00000000000..44237699fda --- /dev/null +++ b/db/migrate/20210106061254_add_unique_index_for_golang_packages.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddUniqueIndexForGolangPackages < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + INDEX_NAME = 'index_packages_on_project_id_name_version_unique_when_golang' + PACKAGE_TYPE_GOLANG = 8 + + disable_ddl_transaction! + + def up + add_concurrent_index :packages_packages, [:project_id, :name, :version], unique: true, where: "package_type = #{PACKAGE_TYPE_GOLANG}", name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name(:packages_packages, INDEX_NAME) + end +end diff --git a/db/post_migrate/20191105094625_set_report_type_for_vulnerabilities.rb b/db/post_migrate/20191105094625_set_report_type_for_vulnerabilities.rb index 5a8529c24d7..40e9e3bddc8 100644 --- a/db/post_migrate/20191105094625_set_report_type_for_vulnerabilities.rb +++ b/db/post_migrate/20191105094625_set_report_type_for_vulnerabilities.rb @@ -7,7 +7,7 @@ class SetReportTypeForVulnerabilities < ActiveRecord::Migration[5.2] # set report_type based on vulnerability_occurrences from which the vulnerabilities were promoted, # that is, first vulnerability_occurrences among those having the same vulnerability_id execute <<~SQL - WITH first_findings_for_vulnerabilities AS ( + WITH first_findings_for_vulnerabilities AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT MIN(id) AS id, vulnerability_id FROM vulnerability_occurrences WHERE vulnerability_id IS NOT NULL diff --git a/db/post_migrate/20191114173624_set_resolved_state_on_vulnerabilities.rb b/db/post_migrate/20191114173624_set_resolved_state_on_vulnerabilities.rb index b28aecdc0a3..2900ef852a5 100644 --- a/db/post_migrate/20191114173624_set_resolved_state_on_vulnerabilities.rb +++ b/db/post_migrate/20191114173624_set_resolved_state_on_vulnerabilities.rb @@ -6,7 +6,7 @@ class SetResolvedStateOnVulnerabilities < ActiveRecord::Migration[5.2] def up execute <<~SQL -- selecting IDs for all non-orphan Findings that either have no feedback or it's a non-dismissal feedback - WITH resolved_vulnerability_ids AS ( + WITH resolved_vulnerability_ids AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT DISTINCT vulnerability_id AS id FROM vulnerability_occurrences LEFT JOIN vulnerability_feedback ON vulnerability_feedback.project_fingerprint = ENCODE(vulnerability_occurrences.project_fingerprint::bytea, 'HEX') diff --git a/db/post_migrate/20200305082754_remove_duplicate_labels_from_project.rb b/db/post_migrate/20200305082754_remove_duplicate_labels_from_project.rb index 33f8118534d..4bb43da43bb 100644 --- a/db/post_migrate/20200305082754_remove_duplicate_labels_from_project.rb +++ b/db/post_migrate/20200305082754_remove_duplicate_labels_from_project.rb @@ -55,7 +55,7 @@ class RemoveDuplicateLabelsFromProject < ActiveRecord::Migration[6.0] # project_id title template description type color duplicate_labels = ApplicationRecord.connection.execute(<<-SQL.squish) -WITH data AS ( +WITH data AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT labels.*, row_number() OVER (PARTITION BY labels.project_id, labels.title, labels.template, labels.description, labels.type, labels.color ORDER BY labels.id) AS row_number, #{CREATE} AS restore_action @@ -83,7 +83,7 @@ WITH data AS ( # then add `_duplicate#{ID}` soft_duplicates = ApplicationRecord.connection.execute(<<-SQL.squish) -WITH data AS ( +WITH data AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT *, substring(title from 1 for 245 - length(id::text)) || '_duplicate' || id::text as new_title, @@ -108,7 +108,7 @@ WHERE labels.id IN (#{soft_duplicates.map { |dup| dup["id"] }.join(", ")}); def restore_renamed_labels(start_id, stop_id) # the backup label IDs are not incremental, they are copied directly from the Labels table ApplicationRecord.connection.execute(<<-SQL.squish) -WITH backups AS ( +WITH backups AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT id, title FROM backup_labels WHERE project_id BETWEEN #{start_id} AND #{stop_id} AND diff --git a/db/post_migrate/20200716234259_remove_duplicate_labels_from_group.rb b/db/post_migrate/20200716234259_remove_duplicate_labels_from_group.rb index f19a209092b..114276df875 100644 --- a/db/post_migrate/20200716234259_remove_duplicate_labels_from_group.rb +++ b/db/post_migrate/20200716234259_remove_duplicate_labels_from_group.rb @@ -59,7 +59,7 @@ class RemoveDuplicateLabelsFromGroup < ActiveRecord::Migration[6.0] # group_id title template description type color duplicate_labels = ApplicationRecord.connection.execute(<<-SQL.squish) -WITH data AS ( +WITH data AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT labels.*, row_number() OVER (PARTITION BY labels.group_id, labels.title, labels.template, labels.description, labels.type, labels.color ORDER BY labels.id) AS row_number, #{CREATE} AS restore_action @@ -87,7 +87,7 @@ WITH data AS ( # then add `_duplicate#{ID}` soft_duplicates = ApplicationRecord.connection.execute(<<-SQL.squish) -WITH data AS ( +WITH data AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT *, substring(title from 1 for 245 - length(id::text)) || '_duplicate' || id::text as new_title, @@ -112,7 +112,7 @@ WHERE labels.id IN (#{soft_duplicates.map { |dup| dup["id"] }.join(", ")}); def restore_renamed_labels(start_id, stop_id) # the backup label IDs are not incremental, they are copied directly from the Labels table ApplicationRecord.connection.execute(<<-SQL.squish) -WITH backups AS ( +WITH backups AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT id, title FROM backup_labels WHERE id BETWEEN #{start_id} AND #{stop_id} diff --git a/db/post_migrate/20200809221641_migrate_license_management_artifacts_to_license_scanning.rb b/db/post_migrate/20200809221641_migrate_license_management_artifacts_to_license_scanning.rb index 0a5dfd72392..66ef4b35dfa 100644 --- a/db/post_migrate/20200809221641_migrate_license_management_artifacts_to_license_scanning.rb +++ b/db/post_migrate/20200809221641_migrate_license_management_artifacts_to_license_scanning.rb @@ -26,7 +26,7 @@ class MigrateLicenseManagementArtifactsToLicenseScanning < ActiveRecord::Migrati min, max = relation.pluck('MIN(job_id)', 'MAX(job_id)').flatten ActiveRecord::Base.connection.execute <<~SQL - WITH ci_job_artifacts_with_row_number as ( + WITH ci_job_artifacts_with_row_number as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT job_id, id, ROW_NUMBER() OVER (PARTITION BY job_id ORDER BY id ASC) as row_number FROM ci_job_artifacts WHERE (file_type = #{LICENSE_SCANNING_FILE_TYPE} OR file_type = #{LICENSE_MANAGEMENT_FILE_TYPE}) diff --git a/db/post_migrate/20200831065705_ensure_target_project_id_is_filled.rb b/db/post_migrate/20200831065705_ensure_target_project_id_is_filled.rb index 9b267933b04..8693dca0000 100644 --- a/db/post_migrate/20200831065705_ensure_target_project_id_is_filled.rb +++ b/db/post_migrate/20200831065705_ensure_target_project_id_is_filled.rb @@ -32,7 +32,7 @@ class EnsureTargetProjectIdIsFilled < ActiveRecord::Migration[6.0] ) MergeRequestMetrics.connection.execute <<-SQL - WITH target_project_id_and_metrics_id as ( + WITH target_project_id_and_metrics_id as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( #{query_for_cte.to_sql} ) UPDATE #{MergeRequestMetrics.connection.quote_table_name(MergeRequestMetrics.table_name)} diff --git a/db/post_migrate/20210105030125_cleanup_projects_with_bad_has_external_wiki_data.rb b/db/post_migrate/20210105030125_cleanup_projects_with_bad_has_external_wiki_data.rb index 73725062bb3..faaa3f47e57 100644 --- a/db/post_migrate/20210105030125_cleanup_projects_with_bad_has_external_wiki_data.rb +++ b/db/post_migrate/20210105030125_cleanup_projects_with_bad_has_external_wiki_data.rb @@ -45,7 +45,7 @@ class CleanupProjectsWithBadHasExternalWikiData < ActiveRecord::Migration[6.0] .merge(Project.where(has_external_wiki: false).where(pending_delete: false).where(archived: false)) execute(<<~SQL) - WITH project_ids_to_update (id) AS ( + WITH project_ids_to_update (id) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( #{scope_with_projects.to_sql} ) UPDATE projects SET has_external_wiki = true WHERE id IN (SELECT id FROM project_ids_to_update) @@ -75,7 +75,7 @@ class CleanupProjectsWithBadHasExternalWikiData < ActiveRecord::Migration[6.0] Project.where(index_where).each_batch(of: BATCH_SIZE) do |relation| relation_with_exists_query = relation.where('NOT EXISTS (?)', services_sub_query) execute(<<~SQL) - WITH project_ids_to_update (id) AS ( + WITH project_ids_to_update (id) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( #{relation_with_exists_query.select(:id).to_sql} ) UPDATE projects SET has_external_wiki = false WHERE id IN (SELECT id FROM project_ids_to_update) diff --git a/db/post_migrate/20210210221006_cleanup_projects_with_bad_has_external_issue_tracker_data.rb b/db/post_migrate/20210210221006_cleanup_projects_with_bad_has_external_issue_tracker_data.rb index 4b8bf014066..25ef93e692b 100644 --- a/db/post_migrate/20210210221006_cleanup_projects_with_bad_has_external_issue_tracker_data.rb +++ b/db/post_migrate/20210210221006_cleanup_projects_with_bad_has_external_issue_tracker_data.rb @@ -44,7 +44,7 @@ class CleanupProjectsWithBadHasExternalIssueTrackerData < ActiveRecord::Migratio .merge(Project.where(has_external_issue_tracker: false).where(pending_delete: false)) execute(<<~SQL) - WITH project_ids_to_update (id) AS ( + WITH project_ids_to_update (id) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( #{scope_with_projects.to_sql} ) UPDATE projects SET has_external_issue_tracker = true WHERE id IN (SELECT id FROM project_ids_to_update) @@ -71,7 +71,7 @@ class CleanupProjectsWithBadHasExternalIssueTrackerData < ActiveRecord::Migratio Project.where(index_where).each_batch(of: BATCH_SIZE) do |relation| relation_with_exists_query = relation.where('NOT EXISTS (?)', services_sub_query) execute(<<~SQL) - WITH project_ids_to_update (id) AS ( + WITH project_ids_to_update (id) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( #{relation_with_exists_query.select(:id).to_sql} ) UPDATE projects SET has_external_issue_tracker = false WHERE id IN (SELECT id FROM project_ids_to_update) diff --git a/db/post_migrate/20210226141517_dedup_issue_metrics.rb b/db/post_migrate/20210226141517_dedup_issue_metrics.rb new file mode 100644 index 00000000000..8228d509e07 --- /dev/null +++ b/db/post_migrate/20210226141517_dedup_issue_metrics.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +class DedupIssueMetrics < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + TMP_INDEX_NAME = 'tmp_unique_issue_metrics_by_issue_id' + OLD_INDEX_NAME = 'index_issue_metrics' + INDEX_NAME = 'index_unique_issue_metrics_issue_id' + BATCH_SIZE = 1_000 + + disable_ddl_transaction! + + class IssueMetrics < ActiveRecord::Base + self.table_name = 'issue_metrics' + + include EachBatch + end + + def up + IssueMetrics.reset_column_information + + last_metrics_record_id = IssueMetrics.maximum(:id) || 0 + + # This index will disallow further duplicates while we're deduplicating the data. + add_concurrent_index(:issue_metrics, :issue_id, where: "id > #{Integer(last_metrics_record_id)}", unique: true, name: TMP_INDEX_NAME) + + IssueMetrics.each_batch(of: BATCH_SIZE) do |relation| + duplicated_issue_ids = IssueMetrics + .where(issue_id: relation.select(:issue_id)) + .select(:issue_id) + .group(:issue_id) + .having('COUNT(issue_metrics.issue_id) > 1') + .pluck(:issue_id) + + duplicated_issue_ids.each do |issue_id| + deduplicate_item(issue_id) + end + end + + add_concurrent_index(:issue_metrics, :issue_id, unique: true, name: INDEX_NAME) + remove_concurrent_index_by_name(:issue_metrics, TMP_INDEX_NAME) + remove_concurrent_index_by_name(:issue_metrics, OLD_INDEX_NAME) + end + + def down + add_concurrent_index(:issue_metrics, :issue_id, name: OLD_INDEX_NAME) + remove_concurrent_index_by_name(:issue_metrics, TMP_INDEX_NAME) + remove_concurrent_index_by_name(:issue_metrics, INDEX_NAME) + end + + private + + def deduplicate_item(issue_id) + issue_metrics_records = IssueMetrics.where(issue_id: issue_id).order(updated_at: :asc).to_a + + attributes = {} + issue_metrics_records.each do |issue_metrics_record| + params = issue_metrics_record.attributes.except('id') + attributes.merge!(params.compact) + end + + ActiveRecord::Base.transaction do + record_to_keep = issue_metrics_records.pop + records_to_delete = issue_metrics_records + + IssueMetrics.where(id: records_to_delete.map(&:id)).delete_all + record_to_keep.update!(attributes) + end + end +end diff --git a/db/schema_migrations/20210106061254 b/db/schema_migrations/20210106061254 new file mode 100644 index 00000000000..3780e444cd3 --- /dev/null +++ b/db/schema_migrations/20210106061254 @@ -0,0 +1 @@ +f4c81be1168dc8dc3eaadbc9b0d46cfd5aefa0b9e4d61fa8276bbc4f59216da8
\ No newline at end of file diff --git a/db/schema_migrations/20210226141517 b/db/schema_migrations/20210226141517 new file mode 100644 index 00000000000..00c57cbe827 --- /dev/null +++ b/db/schema_migrations/20210226141517 @@ -0,0 +1 @@ +400dd521f5c462afdcb3c556815f840e916df7576a6d6dd301fe5a49a1fe6011
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index a51e47a9317..bede4f8fddb 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -22816,8 +22816,6 @@ CREATE UNIQUE INDEX index_issue_links_on_source_id_and_target_id ON issue_links CREATE INDEX index_issue_links_on_target_id ON issue_links USING btree (target_id); -CREATE INDEX index_issue_metrics ON issue_metrics USING btree (issue_id); - CREATE INDEX index_issue_metrics_on_issue_id_and_timestamps ON issue_metrics USING btree (issue_id, first_mentioned_in_commit_at, first_associated_with_milestone_at, first_added_to_board_at); CREATE INDEX index_issue_on_project_id_state_id_and_blocking_issues_count ON issues USING btree (project_id, state_id, blocking_issues_count); @@ -23296,6 +23294,8 @@ CREATE INDEX index_packages_nuget_dl_metadata_on_dependency_link_id ON packages_ CREATE UNIQUE INDEX index_packages_on_project_id_name_version_unique_when_generic ON packages_packages USING btree (project_id, name, version) WHERE (package_type = 7); +CREATE UNIQUE INDEX index_packages_on_project_id_name_version_unique_when_golang ON packages_packages USING btree (project_id, name, version) WHERE (package_type = 8); + CREATE INDEX index_packages_package_file_build_infos_on_package_file_id ON packages_package_file_build_infos USING btree (package_file_id); CREATE INDEX index_packages_package_file_build_infos_on_pipeline_id ON packages_package_file_build_infos USING btree (pipeline_id); @@ -23966,6 +23966,8 @@ CREATE INDEX index_u2f_registrations_on_key_handle ON u2f_registrations USING bt CREATE INDEX index_u2f_registrations_on_user_id ON u2f_registrations USING btree (user_id); +CREATE UNIQUE INDEX index_unique_issue_metrics_issue_id ON issue_metrics USING btree (issue_id); + CREATE INDEX index_unit_test_failures_failed_at ON ci_unit_test_failures USING btree (failed_at DESC); CREATE UNIQUE INDEX index_unit_test_failures_unique_columns ON ci_unit_test_failures USING btree (unit_test_id, failed_at DESC, build_id); diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 331d3be2b8c..d6bebfb5280 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -7485,10 +7485,14 @@ Roadmap sort values. | Value | Description | | ----- | ----------- | -| `end_date_asc` | End date at ascending order. | -| `end_date_desc` | End date at descending order. | -| `start_date_asc` | Start date at ascending order. | -| `start_date_desc` | Start date at descending order. | +| `END_DATE_ASC` | Sort by end date in ascending order. | +| `END_DATE_DESC` | Sort by end date in descending order. | +| `START_DATE_ASC` | Sort by start date in ascending order. | +| `START_DATE_DESC` | Sort by start date in descending order. | +| `end_date_asc` **{warning-solid}** | **Deprecated:** Use END_DATE_ASC. Deprecated in 13.11. | +| `end_date_desc` **{warning-solid}** | **Deprecated:** Use END_DATE_DESC. Deprecated in 13.11. | +| `start_date_asc` **{warning-solid}** | **Deprecated:** Use START_DATE_ASC. Deprecated in 13.11. | +| `start_date_desc` **{warning-solid}** | **Deprecated:** Use START_DATE_DESC. Deprecated in 13.11. | ### `EpicState` diff --git a/doc/development/changelog.md b/doc/development/changelog.md index 2cabb447781..5d2833902a4 100644 --- a/doc/development/changelog.md +++ b/doc/development/changelog.md @@ -57,7 +57,7 @@ the `author` field. GitLab team members **should not**. - Any change behind a feature flag that is **enabled** by default **should** have a changelog entry. - Any change that adds new usage data metrics and changes that needs to be documented in Product Intelligence [Event Dictionary](https://about.gitlab.com/handbook/product/product-intelligence-guide/#event-dictionary) **should** have a changelog entry. - A change that adds snowplow events **should** have a changelog entry - -- A change that [removes a feature flag](feature_flags/index.md) **must** have a changelog entry. +- A change that [removes a feature flag, or removes a feature and its feature flag](feature_flags/index.md) **must** have a changelog entry. - A fix for a regression introduced and then fixed in the same release (i.e., fixing a bug introduced during a monthly release candidate) **should not** have a changelog entry. diff --git a/doc/development/usage_ping/dictionary.md b/doc/development/usage_ping/dictionary.md index 887ecd858ff..9cd3ac3dee8 100644 --- a/doc/development/usage_ping/dictionary.md +++ b/doc/development/usage_ping/dictionary.md @@ -9908,6 +9908,30 @@ Status: `implemented` Tiers: `premium`, `ultimate` +### `redis_hll_counters.epics_usage.g_project_management_users_creating_epic_notes_monthly` + +Counts of MAU adding epic notes + +[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/config/metrics/counts_28d/20210314215451_g_project_management_users_creating_epic_notes_monthly.yml) + +Group: `group:product planning` + +Status: `implemented` + +Tiers: `premium`, `ultimate` + +### `redis_hll_counters.epics_usage.g_project_management_users_creating_epic_notes_weekly` + +Counts of WAU adding epic notes + +[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/config/metrics/counts_7d/20210314231518_g_project_management_users_creating_epic_notes_weekly.yml) + +Group: `group:product planning` + +Status: `implemented` + +Tiers: `premium`, `ultimate` + ### `redis_hll_counters.epics_usage.g_project_management_users_destroying_epic_notes_monthly` Counts of MAU destroying epic notes @@ -15538,7 +15562,7 @@ Tiers: `free` ### `usage_activity_by_stage.secure.user_container_scanning_jobs` -no idea, Count of Container Scanning jobs run, it implies user but AFAIK we don't track per user +Distinct count per user of Container Scanning jobs run [YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/config/metrics/counts_all/20210216175501_user_container_scanning_jobs.yml) @@ -17374,7 +17398,7 @@ Tiers: `free` ### `usage_activity_by_stage_monthly.secure.container_scanning_pipeline` -no idea, what is this when did it get added? guess pipelines containing a CS job +Pipelines containing a Container Scanning job [YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/config/metrics/counts_28d/20210216175507_container_scanning_pipeline.yml) @@ -17542,7 +17566,7 @@ Tiers: `free` ### `usage_activity_by_stage_monthly.secure.user_container_scanning_jobs` -no idea, Count of Container Scanning jobs run, it implies user and monthly, but AFAIK we don't track per user +Distinct count per user of Container Scanning jobs run monthly [YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/config/metrics/counts_28d/20210216175505_user_container_scanning_jobs.yml) diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 0a541620c3a..9f0f569b711 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -44,7 +44,7 @@ module API end # rubocop: disable CodeReuse/ActiveRecord get ":id/deploy_keys" do - keys = user_project.deploy_keys_projects.preload(:deploy_key) + keys = user_project.deploy_keys_projects.preload(deploy_key: :user) present paginate(keys), with: Entities::DeployKeysProject end diff --git a/lib/gitlab/background_migration/backfill_design_internal_ids.rb b/lib/gitlab/background_migration/backfill_design_internal_ids.rb index 553571d5d00..6d1df95c66d 100644 --- a/lib/gitlab/background_migration/backfill_design_internal_ids.rb +++ b/lib/gitlab/background_migration/backfill_design_internal_ids.rb @@ -97,13 +97,13 @@ module Gitlab ActiveRecord::Base.connection.execute <<~SQL WITH - starting_iids(project_id, iid) as ( + starting_iids(project_id, iid) as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported}( SELECT project_id, MAX(COALESCE(iid, 0)) FROM #{table} WHERE project_id BETWEEN #{start_id} AND #{end_id} GROUP BY project_id ), - with_calculated_iid(id, iid) as ( + with_calculated_iid(id, iid) as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported}( SELECT design.id, init.iid + ROW_NUMBER() OVER (PARTITION BY design.project_id ORDER BY design.id ASC) FROM #{table} as design, starting_iids as init diff --git a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb index 7484027a0fa..030dfd2d99b 100644 --- a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb +++ b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb @@ -8,7 +8,7 @@ module Gitlab updated_repository_storages = Projects::RepositoryStorageMove.select("project_id, MAX(updated_at) as updated_at").where(project_id: project_ids).group(:project_id) Project.connection.execute <<-SQL - WITH repository_storage_cte as ( + WITH repository_storage_cte as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( #{updated_repository_storages.to_sql} ) UPDATE projects diff --git a/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb b/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb index 6014ccc12eb..691bdb457d7 100644 --- a/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb +++ b/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb @@ -8,7 +8,7 @@ module Gitlab def perform(start_id, stop_id) ActiveRecord::Base.connection.execute <<~SQL - WITH merge_requests_batch AS ( + WITH merge_requests_batch AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT id, target_project_id FROM merge_requests WHERE id BETWEEN #{Integer(start_id)} AND #{Integer(stop_id)} ) diff --git a/lib/gitlab/background_migration/fix_projects_without_project_feature.rb b/lib/gitlab/background_migration/fix_projects_without_project_feature.rb index 68665db522e..83c01afa432 100644 --- a/lib/gitlab/background_migration/fix_projects_without_project_feature.rb +++ b/lib/gitlab/background_migration/fix_projects_without_project_feature.rb @@ -22,7 +22,7 @@ module Gitlab def sql(from_id, to_id) <<~SQL - WITH created_records AS ( + WITH created_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( INSERT INTO project_features ( project_id, merge_requests_access_level, diff --git a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb index e750b8ca374..b8e4562b3bf 100644 --- a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb +++ b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb @@ -136,7 +136,7 @@ module Gitlab # there is no uniq constraint on project_id and type pair, which prevents us from using ON CONFLICT def create_sql(from_id, to_id) <<~SQL - WITH created_records AS ( + WITH created_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( INSERT INTO services (project_id, #{DEFAULTS.keys.map { |key| %("#{key}")}.join(',')}, created_at, updated_at) #{select_insert_values_sql(from_id, to_id)} RETURNING * @@ -149,7 +149,7 @@ module Gitlab # there is no uniq constraint on project_id and type pair, which prevents us from using ON CONFLICT def update_sql(from_id, to_id) <<~SQL - WITH updated_records AS ( + WITH updated_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( UPDATE services SET active = TRUE WHERE services.project_id BETWEEN #{Integer(from_id)} AND #{Integer(to_id)} AND services.properties = '{}' AND services.type = '#{Migratable::PrometheusService.type}' AND #{group_cluster_condition(from_id, to_id)} AND services.active = FALSE diff --git a/lib/gitlab/background_migration/fix_user_namespace_names.rb b/lib/gitlab/background_migration/fix_user_namespace_names.rb index d767cbfd8f5..cd5b4ab103d 100644 --- a/lib/gitlab/background_migration/fix_user_namespace_names.rb +++ b/lib/gitlab/background_migration/fix_user_namespace_names.rb @@ -14,7 +14,7 @@ module Gitlab def fix_namespace_names(from_id, to_id) ActiveRecord::Base.connection.execute <<~UPDATE_NAMESPACES - WITH namespaces_to_update AS ( + WITH namespaces_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT namespaces.id, users.name AS correct_name @@ -39,7 +39,7 @@ module Gitlab def fix_namespace_route_names(from_id, to_id) ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE - WITH routes_to_update AS ( + WITH routes_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT routes.id, users.name AS correct_name diff --git a/lib/gitlab/background_migration/fix_user_project_route_names.rb b/lib/gitlab/background_migration/fix_user_project_route_names.rb index 6b99685fd68..e534f2449aa 100644 --- a/lib/gitlab/background_migration/fix_user_project_route_names.rb +++ b/lib/gitlab/background_migration/fix_user_project_route_names.rb @@ -8,7 +8,7 @@ module Gitlab class FixUserProjectRouteNames def perform(from_id, to_id) ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE - WITH routes_to_update AS ( + WITH routes_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT routes.id, users.name || ' / ' || projects.name AS correct_name diff --git a/lib/gitlab/background_migration/populate_has_vulnerabilities.rb b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb index 78140b768fc..28ff2070209 100644 --- a/lib/gitlab/background_migration/populate_has_vulnerabilities.rb +++ b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb @@ -8,21 +8,23 @@ module Gitlab class ProjectSetting < ActiveRecord::Base # rubocop:disable Style/Documentation self.table_name = 'project_settings' - UPSERT_SQL = <<~SQL - WITH upsert_data (project_id, has_vulnerabilities, created_at, updated_at) AS ( - SELECT projects.id, true, current_timestamp, current_timestamp FROM projects WHERE projects.id IN (%{project_ids}) - ) - INSERT INTO project_settings - (project_id, has_vulnerabilities, created_at, updated_at) - (SELECT * FROM upsert_data) - ON CONFLICT (project_id) - DO UPDATE SET - has_vulnerabilities = true, - updated_at = EXCLUDED.updated_at - SQL - def self.upsert_for(project_ids) - connection.execute(UPSERT_SQL % { project_ids: project_ids.join(', ') }) + connection.execute(upsert_sql % { project_ids: project_ids.join(', ') }) + end + + def self.upsert_sql + <<~SQL + WITH upsert_data (project_id, has_vulnerabilities, created_at, updated_at) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + SELECT projects.id, true, current_timestamp, current_timestamp FROM projects WHERE projects.id IN (%{project_ids}) + ) + INSERT INTO project_settings + (project_id, has_vulnerabilities, created_at, updated_at) + (SELECT * FROM upsert_data) + ON CONFLICT (project_id) + DO UPDATE SET + has_vulnerabilities = true, + updated_at = EXCLUDED.updated_at + SQL end end diff --git a/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb b/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb index baacc912df3..5b01141d8c1 100644 --- a/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb +++ b/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb @@ -57,7 +57,7 @@ module Gitlab def update_email_records(start_id, stop_id) EmailModel.connection.execute <<-SQL - WITH md5_strings as ( + WITH md5_strings as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( #{email_query_for_update(start_id, stop_id).to_sql} ) UPDATE #{EmailModel.connection.quote_table_name(EmailModel.table_name)} diff --git a/lib/gitlab/database/as_with_materialized.rb b/lib/gitlab/database/as_with_materialized.rb new file mode 100644 index 00000000000..7c45f416638 --- /dev/null +++ b/lib/gitlab/database/as_with_materialized.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # This class is a special Arel node which allows optionally define the `MATERIALIZED` keyword for CTE and Recursive CTE queries. + class AsWithMaterialized < Arel::Nodes::Binary + extend Gitlab::Utils::StrongMemoize + + MATERIALIZED = Arel.sql(' MATERIALIZED') + EMPTY_STRING = Arel.sql('') + attr_reader :expr + + def initialize(left, right, materialized: true) + @expr = if materialized && self.class.materialized_supported? + MATERIALIZED + else + EMPTY_STRING + end + + super(left, right) + end + + # Note: to be deleted after the minimum PG version is set to 12.0 + def self.materialized_supported? + strong_memoize(:materialized_supported) do + Gitlab::Database.version.match?(/^1[2-9]\./) # version 12.x and above + end + end + + # Note: to be deleted after the minimum PG version is set to 12.0 + def self.materialized_if_supported + materialized_supported? ? 'MATERIALIZED' : '' + end + end + end +end diff --git a/lib/gitlab/database/bulk_update.rb b/lib/gitlab/database/bulk_update.rb index 1403d561890..b1f9da30585 100644 --- a/lib/gitlab/database/bulk_update.rb +++ b/lib/gitlab/database/bulk_update.rb @@ -130,7 +130,7 @@ module Gitlab def sql <<~SQL - WITH cte(#{list_of(cte_columns)}) AS (VALUES #{list_of(values)}) + WITH cte(#{list_of(cte_columns)}) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (VALUES #{list_of(values)}) UPDATE #{table_name} SET #{list_of(updates)} FROM cte WHERE cte_id = id SQL end diff --git a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb index 62dfaeeaae3..e8b49c7f62c 100644 --- a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb +++ b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb @@ -41,19 +41,6 @@ module Gitlab BUCKET_ID_MASK = (Buckets::TOTAL_BUCKETS - ZERO_OFFSET).to_s(2) BIT_31_MASK = "B'0#{'1' * 31}'" BIT_32_NORMALIZED_BUCKET_ID_MASK = "B'#{'0' * (32 - BUCKET_ID_MASK.size)}#{BUCKET_ID_MASK}'" - # @example source_query - # SELECT CAST(('X' || md5(CAST(%{column} as text))) as bit(32)) attr_hash_32_bits - # FROM %{relation} - # WHERE %{pkey} >= %{batch_start} - # AND %{pkey} < %{batch_end} - # AND %{column} IS NOT NULL - BUCKETED_DATA_SQL = <<~SQL - WITH hashed_attributes AS (%{source_query}) - SELECT (attr_hash_32_bits & #{BIT_32_NORMALIZED_BUCKET_ID_MASK})::int AS bucket_num, - (31 - floor(log(2, min((attr_hash_32_bits & #{BIT_31_MASK})::int))))::int as bucket_hash - FROM hashed_attributes - GROUP BY 1 - SQL WRONG_CONFIGURATION_ERROR = Class.new(ActiveRecord::StatementInvalid) @@ -103,7 +90,7 @@ module Gitlab def hll_buckets_for_batch(start, finish) @relation .connection - .execute(BUCKETED_DATA_SQL % { source_query: source_query(start, finish) }) + .execute(bucketed_data_sql % { source_query: source_query(start, finish) }) .map(&:values) .to_h end @@ -139,6 +126,22 @@ module Gitlab def actual_finish(finish) finish || @relation.unscope(:group, :having).maximum(@relation.primary_key) || 0 end + + # @example source_query + # SELECT CAST(('X' || md5(CAST(%{column} as text))) as bit(32)) attr_hash_32_bits + # FROM %{relation} + # WHERE %{pkey} >= %{batch_start} + # AND %{pkey} < %{batch_end} + # AND %{column} IS NOT NULL + def bucketed_data_sql + <<~SQL + WITH hashed_attributes AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (%{source_query}) + SELECT (attr_hash_32_bits & #{BIT_32_NORMALIZED_BUCKET_ID_MASK})::int AS bucket_num, + (31 - floor(log(2, min((attr_hash_32_bits & #{BIT_31_MASK})::int))))::int as bucket_hash + FROM hashed_attributes + GROUP BY 1 + SQL + end end end end diff --git a/lib/gitlab/golang.rb b/lib/gitlab/golang.rb index f2dc668c482..097967cbcf5 100644 --- a/lib/gitlab/golang.rb +++ b/lib/gitlab/golang.rb @@ -2,6 +2,8 @@ module Gitlab module Golang + PseudoVersion = Struct.new(:semver, :timestamp, :commit_id) + extend self def local_module_prefix @@ -37,11 +39,11 @@ module Gitlab end # This pattern is intentionally more forgiving than the patterns - # above. Correctness is verified by #pseudo_version_commit. + # above. Correctness is verified by #validate_pseudo_version. /\A\d{14}-\h+\z/.freeze.match? pre end - def pseudo_version_commit(project, semver) + def parse_pseudo_version(semver) # Per Go's implementation of pseudo-versions, a tag should be # considered a pseudo-version if it matches one of the patterns # listed in #pseudo_version?, regardless of the content of the @@ -55,9 +57,14 @@ module Gitlab # - [Pseudo-version request processing](https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/coderepo.go) # Go ignores anything before '.' or after the second '-', so we will do the same - timestamp, sha = semver.prerelease.split('-').last 2 + timestamp, commit_id = semver.prerelease.split('-').last 2 timestamp = timestamp.split('.').last - commit = project.repository.commit_by(oid: sha) + + PseudoVersion.new(semver, timestamp, commit_id) + end + + def validate_pseudo_version(project, version, commit = nil) + commit ||= project.repository.commit_by(oid: version.commit_id) # Error messages are based on the responses of proxy.golang.org @@ -65,10 +72,10 @@ module Gitlab raise ArgumentError.new 'invalid pseudo-version: unknown commit' unless commit # Require the SHA fragment to be 12 characters long - raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless sha.length == 12 + raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless version.commit_id.length == 12 # Require the timestamp to match that of the commit - raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == timestamp + raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == version.timestamp commit end @@ -77,6 +84,14 @@ module Gitlab Packages::SemVer.parse(str, prefixed: true) end + def go_path(project, path = nil) + if path.blank? + "#{local_module_prefix}/#{project.full_path}" + else + "#{local_module_prefix}/#{project.full_path}/#{path}" + end + end + def pkg_go_dev_url(name, version = nil) if version "https://pkg.go.dev/#{name}@#{version}" diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index dad629f5074..9a74266693b 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -74,9 +74,16 @@ module Gitlab read_only(model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)).order(depth: hierarchy_order)) else recursive_query = base_and_ancestors_cte(upto).apply_to(model.all) - recursive_query = recursive_query.reselect(*recursive_query.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct - recursive_query = model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)) - read_only(remove_depth_and_maintain_order(recursive_query, hierarchy_order: hierarchy_order)) + + if skip_ordering? + recursive_query = recursive_query.distinct + else + recursive_query = recursive_query.reselect(*recursive_query.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct + recursive_query = model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)) + recursive_query = remove_depth_and_maintain_order(recursive_query, hierarchy_order: hierarchy_order) + end + + read_only(recursive_query) end else recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all) @@ -100,9 +107,16 @@ module Gitlab read_only(model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)).order(depth: :asc)) else base_cte = base_and_descendants_cte.apply_to(model.all) - base_cte = base_cte.reselect(*base_cte.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct - base_cte = model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)) - read_only(remove_depth_and_maintain_order(base_cte, hierarchy_order: :asc)) + + if skip_ordering? + base_cte = base_cte.distinct + else + base_cte = base_cte.reselect(*base_cte.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct + base_cte = model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)) + base_cte = remove_depth_and_maintain_order(base_cte, hierarchy_order: :asc) + end + + read_only(base_cte) end else read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(model.all)) @@ -173,6 +187,13 @@ module Gitlab false end + # Skips the extra ordering when using distinct on the namespace queries + def skip_ordering? + return options[:skip_ordering] if options.key?(:skip_ordering) + + false + end + # Remove the extra `depth` field using an INNER JOIN to avoid breaking UNION queries # and ordering the rows based on the `depth` column to maintain the row order. # diff --git a/lib/gitlab/sql/cte.rb b/lib/gitlab/sql/cte.rb index 7817a2a1ce2..8f37602aeaa 100644 --- a/lib/gitlab/sql/cte.rb +++ b/lib/gitlab/sql/cte.rb @@ -15,20 +15,27 @@ module Gitlab # Namespace # with(cte.to_arel). # from(cte.alias_to(ns)) + # + # To skip materialization of the CTE query by passing materialized: false + # More context: https://www.postgresql.org/docs/12/queries-with.html + # + # cte = CTE.new(:my_cte_name, materialized: false) + # class CTE attr_reader :table, :query # name - The name of the CTE as a String or Symbol. - def initialize(name, query) + def initialize(name, query, materialized: true) @table = Arel::Table.new(name) @query = query + @materialized = materialized end # Returns the Arel relation for this CTE. def to_arel sql = Arel::Nodes::SqlLiteral.new("(#{query.to_sql})") - Arel::Nodes::As.new(table, sql) + Gitlab::Database::AsWithMaterialized.new(table, sql, materialized: @materialized) end # Returns an "AS" statement that aliases the CTE name as the given table diff --git a/lib/gitlab/usage_data_counters/known_events/epic_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_events.yml index 3defa3e932e..836621511f1 100644 --- a/lib/gitlab/usage_data_counters/known_events/epic_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/epic_events.yml @@ -9,6 +9,12 @@ aggregation: daily feature_flag: track_epics_activity +- name: g_project_management_users_creating_epic_notes + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + - name: g_project_management_users_updating_epic_notes category: epics_usage redis_slot: project_management diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 118b971b35f..28c074290f5 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -12320,19 +12320,19 @@ msgstr "" msgid "ErrorTracking|Active" msgstr "" -msgid "ErrorTracking|After adding your Auth Token, use the 'Connect' button to load projects" +msgid "ErrorTracking|After adding your Auth Token, select the Connect button to load projects." msgstr "" msgid "ErrorTracking|Auth Token" msgstr "" -msgid "ErrorTracking|Click 'Connect' to re-establish the connection to Sentry and activate the dropdown." +msgid "ErrorTracking|Click Connect to reestablish the connection to Sentry and activate the dropdown." msgstr "" -msgid "ErrorTracking|Connection has failed. Re-check Auth Token and try again." +msgid "ErrorTracking|Connection failed. Check Auth Token and try again." msgstr "" -msgid "ErrorTracking|If you self-host Sentry, enter the full URL of your Sentry instance. If you're using Sentry's hosted solution, enter https://sentry.io" +msgid "ErrorTracking|If you self-host Sentry, enter your Sentry instance's full URL. If you use Sentry's hosted solution, enter https://sentry.io" msgstr "" msgid "ErrorTracking|No projects available" @@ -12341,7 +12341,7 @@ msgstr "" msgid "ErrorTracking|Select project" msgstr "" -msgid "ErrorTracking|To enable project selection, enter a valid Auth Token" +msgid "ErrorTracking|To enable project selection, enter a valid Auth Token." msgstr "" msgid "Errors" diff --git a/spec/features/projects/settings/operations_settings_spec.rb b/spec/features/projects/settings/operations_settings_spec.rb index fe0ee52e4fa..ca976997142 100644 --- a/spec/features/projects/settings/operations_settings_spec.rb +++ b/spec/features/projects/settings/operations_settings_spec.rb @@ -146,7 +146,7 @@ RSpec.describe 'Projects > Settings > For a forked project', :js do click_button('Connect') - assert_text('Connection has failed. Re-check Auth Token and try again.') + assert_text('Connection failed. Check Auth Token and try again.') end end end diff --git a/spec/finders/packages/go/package_finder_spec.rb b/spec/finders/packages/go/package_finder_spec.rb new file mode 100644 index 00000000000..b6fad1e7061 --- /dev/null +++ b/spec/finders/packages/go/package_finder_spec.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Packages::Go::PackageFinder do + include_context 'basic Go module' + + let_it_be(:mod) { create :go_module, project: project } + let_it_be(:version) { create :go_module_version, :tagged, mod: mod, name: 'v1.0.1' } + let_it_be(:package) { create :golang_package, project: project, name: mod.name, version: 'v1.0.1' } + + let(:finder) { described_class.new(project, mod_name, version_name) } + + describe '#exists?' do + subject { finder.exists? } + + context 'with a valid name and version' do + let(:mod_name) { mod.name } + let(:version_name) { version.name } + + it 'executes SELECT 1' do + expect { subject }.to exceed_query_limit(0).for_query(/^SELECT 1/) + end + + it { is_expected.to eq(true) } + end + + context 'with an invalid name' do + let(:mod_name) { 'foo/bar' } + let(:version_name) { 'baz' } + + it { is_expected.to eq(false) } + end + + context 'with an invalid version' do + let(:mod_name) { mod.name } + let(:version_name) { 'baz' } + + it { is_expected.to eq(false) } + end + end + + describe '#execute' do + subject { finder.execute } + + context 'with a valid name and version' do + let(:mod_name) { mod.name } + let(:version_name) { version.name } + + it 'executes a single query' do + expect { subject }.not_to exceed_query_limit(1) + end + + it { is_expected.to eq(package) } + end + + context 'with an invalid name' do + let(:mod_name) { 'foo/bar' } + let(:version_name) { 'baz' } + + it { is_expected.to eq(nil) } + end + + context 'with an invalid version' do + let(:mod_name) { mod.name } + let(:version_name) { 'baz' } + + it { is_expected.to eq(nil) } + end + end +end diff --git a/spec/frontend/error_tracking_settings/components/error_tracking_form_spec.js b/spec/frontend/error_tracking_settings/components/error_tracking_form_spec.js index 7ebaf0c3f2a..f02a261f323 100644 --- a/spec/frontend/error_tracking_settings/components/error_tracking_form_spec.js +++ b/spec/frontend/error_tracking_settings/components/error_tracking_form_spec.js @@ -44,13 +44,13 @@ describe('error tracking settings form', () => { const pageText = wrapper.text(); expect(pageText).toContain( - "If you self-host Sentry, enter the full URL of your Sentry instance. If you're using Sentry's hosted solution, enter https://sentry.io", + "If you self-host Sentry, enter your Sentry instance's full URL. If you use Sentry's hosted solution, enter https://sentry.io", ); expect(pageText).toContain( - "After adding your Auth Token, use the 'Connect' button to load projects", + 'After adding your Auth Token, select the Connect button to load projects.', ); - expect(pageText).not.toContain('Connection has failed. Re-check Auth Token and try again'); + expect(pageText).not.toContain('Connection failed. Check Auth Token and try again.'); expect(wrapper.findAll(GlFormInput).at(0).attributes('placeholder')).toContain( 'https://mysentryserver.com', ); @@ -80,9 +80,7 @@ describe('error tracking settings form', () => { }); it('does not show an error', () => { - expect(wrapper.text()).not.toContain( - 'Connection has failed. Re-check Auth Token and try again', - ); + expect(wrapper.text()).not.toContain('Connection failed. Check Auth Token and try again.'); }); }); @@ -96,7 +94,7 @@ describe('error tracking settings form', () => { }); it('shows an error', () => { - expect(wrapper.text()).toContain('Connection has failed. Re-check Auth Token and try again'); + expect(wrapper.text()).toContain('Connection failed. Check Auth Token and try again.'); }); }); }); diff --git a/spec/frontend/error_tracking_settings/store/getters_spec.js b/spec/frontend/error_tracking_settings/store/getters_spec.js index b135fdee40b..4bb8d38e294 100644 --- a/spec/frontend/error_tracking_settings/store/getters_spec.js +++ b/spec/frontend/error_tracking_settings/store/getters_spec.js @@ -78,7 +78,7 @@ describe('Error Tracking Settings - Getters', () => { describe('projectSelectionLabel', () => { it('should show the correct message when the token is empty', () => { expect(getters.projectSelectionLabel(state)).toEqual( - 'To enable project selection, enter a valid Auth Token', + 'To enable project selection, enter a valid Auth Token.', ); }); @@ -86,7 +86,7 @@ describe('Error Tracking Settings - Getters', () => { state.token = 'test-token'; expect(getters.projectSelectionLabel(state)).toEqual( - "Click 'Connect' to re-establish the connection to Sentry and activate the dropdown.", + 'Click Connect to reestablish the connection to Sentry and activate the dropdown.', ); }); }); diff --git a/spec/frontend/mr_notes/stores/actions_spec.js b/spec/frontend/mr_notes/stores/actions_spec.js index dbceedface1..c6578453d85 100644 --- a/spec/frontend/mr_notes/stores/actions_spec.js +++ b/spec/frontend/mr_notes/stores/actions_spec.js @@ -1,5 +1,9 @@ +import MockAdapter from 'axios-mock-adapter'; + import testAction from 'helpers/vuex_action_helper'; -import { setEndpoints } from '~/mr_notes/stores/actions'; +import axios from '~/lib/utils/axios_utils'; + +import { setEndpoints, setMrMetadata, fetchMrMetadata } from '~/mr_notes/stores/actions'; import mutationTypes from '~/mr_notes/stores/mutation_types'; describe('MR Notes Mutator Actions', () => { @@ -22,4 +26,67 @@ describe('MR Notes Mutator Actions', () => { ); }); }); + + describe('setMrMetadata', () => { + it('should trigger the SET_MR_METADATA state mutation', async () => { + const mrMetadata = { propA: 'a', propB: 'b' }; + + await testAction( + setMrMetadata, + mrMetadata, + {}, + [ + { + type: mutationTypes.SET_MR_METADATA, + payload: mrMetadata, + }, + ], + [], + ); + }); + }); + + describe('fetchMrMetadata', () => { + const mrMetadata = { meta: true, data: 'foo' }; + const state = { + endpoints: { + metadata: 'metadata', + }, + }; + let mock; + + beforeEach(() => { + mock = new MockAdapter(axios); + + mock.onGet(state.endpoints.metadata).reply(200, mrMetadata); + }); + + afterEach(() => { + mock.restore(); + }); + + it('should fetch the data from the API', async () => { + await fetchMrMetadata({ state, dispatch: () => {} }); + + await axios.waitForAll(); + + expect(mock.history.get).toHaveLength(1); + expect(mock.history.get[0].url).toBe(state.endpoints.metadata); + }); + + it('should set the fetched data into state', () => { + return testAction( + fetchMrMetadata, + {}, + state, + [], + [ + { + type: 'setMrMetadata', + payload: mrMetadata, + }, + ], + ); + }); + }); }); diff --git a/spec/frontend/mr_notes/stores/mutations_spec.js b/spec/frontend/mr_notes/stores/mutations_spec.js index 422db3d5a38..35b8a2e4be2 100644 --- a/spec/frontend/mr_notes/stores/mutations_spec.js +++ b/spec/frontend/mr_notes/stores/mutations_spec.js @@ -12,4 +12,16 @@ describe('MR Notes Mutations', () => { expect(state.endpoints).toEqual(endpoints); }); }); + + describe(mutationTypes.SET_MR_METADATA, () => { + it('store the provided MR Metadata in the state', () => { + const state = {}; + const metadata = { propA: 'A', propB: 'B' }; + + mutations[mutationTypes.SET_MR_METADATA](state, metadata); + + expect(state.mrMetadata.propA).toBe('A'); + expect(state.mrMetadata.propB).toBe('B'); + }); + }); }); diff --git a/spec/lib/gitlab/object_hierarchy_spec.rb b/spec/lib/gitlab/object_hierarchy_spec.rb index 98a194ac73c..c48a7b8303f 100644 --- a/spec/lib/gitlab/object_hierarchy_spec.rb +++ b/spec/lib/gitlab/object_hierarchy_spec.rb @@ -3,14 +3,16 @@ require 'spec_helper' RSpec.describe Gitlab::ObjectHierarchy do - let!(:parent) { create(:group) } - let!(:child1) { create(:group, parent: parent) } - let!(:child2) { create(:group, parent: child1) } + let_it_be(:parent) { create(:group) } + let_it_be(:child1) { create(:group, parent: parent) } + let_it_be(:child2) { create(:group, parent: child1) } + + let(:options) { {} } shared_context 'Gitlab::ObjectHierarchy test cases' do describe '#base_and_ancestors' do let(:relation) do - described_class.new(Group.where(id: child2.id)).base_and_ancestors + described_class.new(Group.where(id: child2.id), options: options).base_and_ancestors end it 'includes the base rows' do @@ -22,13 +24,13 @@ RSpec.describe Gitlab::ObjectHierarchy do end it 'can find ancestors upto a certain level' do - relation = described_class.new(Group.where(id: child2)).base_and_ancestors(upto: child1) + relation = described_class.new(Group.where(id: child2), options: options).base_and_ancestors(upto: child1) expect(relation).to contain_exactly(child2) end it 'uses ancestors_base #initialize argument' do - relation = described_class.new(Group.where(id: child2.id), Group.none).base_and_ancestors + relation = described_class.new(Group.where(id: child2.id), Group.none, options: options).base_and_ancestors expect(relation).to include(parent, child1, child2) end @@ -40,7 +42,7 @@ RSpec.describe Gitlab::ObjectHierarchy do describe 'hierarchy_order option' do let(:relation) do - described_class.new(Group.where(id: child2.id)).base_and_ancestors(hierarchy_order: hierarchy_order) + described_class.new(Group.where(id: child2.id), options: options).base_and_ancestors(hierarchy_order: hierarchy_order) end context ':asc' do @@ -63,7 +65,7 @@ RSpec.describe Gitlab::ObjectHierarchy do describe '#base_and_descendants' do let(:relation) do - described_class.new(Group.where(id: parent.id)).base_and_descendants + described_class.new(Group.where(id: parent.id), options: options).base_and_descendants end it 'includes the base rows' do @@ -75,7 +77,7 @@ RSpec.describe Gitlab::ObjectHierarchy do end it 'uses descendants_base #initialize argument' do - relation = described_class.new(Group.none, Group.where(id: parent.id)).base_and_descendants + relation = described_class.new(Group.none, Group.where(id: parent.id), options: options).base_and_descendants expect(relation).to include(parent, child1, child2) end @@ -87,7 +89,7 @@ RSpec.describe Gitlab::ObjectHierarchy do context 'when with_depth is true' do let(:relation) do - described_class.new(Group.where(id: parent.id)).base_and_descendants(with_depth: true) + described_class.new(Group.where(id: parent.id), options: options).base_and_descendants(with_depth: true) end it 'includes depth in the results' do @@ -106,14 +108,14 @@ RSpec.describe Gitlab::ObjectHierarchy do describe '#descendants' do it 'includes only the descendants' do - relation = described_class.new(Group.where(id: parent)).descendants + relation = described_class.new(Group.where(id: parent), options: options).descendants expect(relation).to contain_exactly(child1, child2) end end describe '#max_descendants_depth' do - subject { described_class.new(base_relation).max_descendants_depth } + subject { described_class.new(base_relation, options: options).max_descendants_depth } context 'when base relation is empty' do let(:base_relation) { Group.where(id: nil) } @@ -136,13 +138,13 @@ RSpec.describe Gitlab::ObjectHierarchy do describe '#ancestors' do it 'includes only the ancestors' do - relation = described_class.new(Group.where(id: child2)).ancestors + relation = described_class.new(Group.where(id: child2), options: options).ancestors expect(relation).to contain_exactly(child1, parent) end it 'can find ancestors upto a certain level' do - relation = described_class.new(Group.where(id: child2)).ancestors(upto: child1) + relation = described_class.new(Group.where(id: child2), options: options).ancestors(upto: child1) expect(relation).to be_empty end @@ -150,7 +152,7 @@ RSpec.describe Gitlab::ObjectHierarchy do describe '#all_objects' do let(:relation) do - described_class.new(Group.where(id: child1.id)).all_objects + described_class.new(Group.where(id: child1.id), options: options).all_objects end it 'includes the base rows' do @@ -166,13 +168,13 @@ RSpec.describe Gitlab::ObjectHierarchy do end it 'uses ancestors_base #initialize argument for ancestors' do - relation = described_class.new(Group.where(id: child1.id), Group.where(id: non_existing_record_id)).all_objects + relation = described_class.new(Group.where(id: child1.id), Group.where(id: non_existing_record_id), options: options).all_objects expect(relation).to include(parent) end it 'uses descendants_base #initialize argument for descendants' do - relation = described_class.new(Group.where(id: non_existing_record_id), Group.where(id: child1.id)).all_objects + relation = described_class.new(Group.where(id: non_existing_record_id), Group.where(id: child1.id), options: options).all_objects expect(relation).to include(child2) end @@ -210,6 +212,19 @@ RSpec.describe Gitlab::ObjectHierarchy do expect(parent.self_and_descendants.to_sql).to include("DISTINCT") expect(child2.self_and_ancestors.to_sql).to include("DISTINCT") end + + context 'when the skip_ordering option is set' do + let(:options) { { skip_ordering: true } } + + it_behaves_like 'Gitlab::ObjectHierarchy test cases' + + it 'does not include ROW_NUMBER()' do + query = described_class.new(Group.where(id: parent.id), options: options).base_and_descendants.to_sql + + expect(query).to include("DISTINCT") + expect(query).not_to include("ROW_NUMBER()") + end + end end context 'when the use_distinct_in_object_hierarchy feature flag is disabled' do diff --git a/spec/lib/gitlab/sql/cte_spec.rb b/spec/lib/gitlab/sql/cte_spec.rb index fdc150cd4b9..33f382cde26 100644 --- a/spec/lib/gitlab/sql/cte_spec.rb +++ b/spec/lib/gitlab/sql/cte_spec.rb @@ -41,4 +41,15 @@ RSpec.describe Gitlab::SQL::CTE do expect(relation.to_a).to eq(User.where(id: user.id).to_a) end end + + it_behaves_like 'CTE with MATERIALIZED keyword examples' do + let(:expected_query_block_with_materialized) { 'WITH "some_cte" AS MATERIALIZED (' } + let(:expected_query_block_without_materialized) { 'WITH "some_cte" AS (' } + + let(:query) do + cte = described_class.new(:some_cte, User.active, **options) + + User.with(cte.to_arel).to_sql + end + end end diff --git a/spec/lib/gitlab/sql/recursive_cte_spec.rb b/spec/lib/gitlab/sql/recursive_cte_spec.rb index 02611620989..edcacd404c2 100644 --- a/spec/lib/gitlab/sql/recursive_cte_spec.rb +++ b/spec/lib/gitlab/sql/recursive_cte_spec.rb @@ -57,4 +57,17 @@ RSpec.describe Gitlab::SQL::RecursiveCTE do expect(relation.to_a).to eq(User.where(id: user.id).to_a) end end + + it_behaves_like 'CTE with MATERIALIZED keyword examples' do + # MATERIALIZED keyword is not needed for recursive queries + let(:expected_query_block_with_materialized) { 'WITH RECURSIVE "some_cte" AS (' } + let(:expected_query_block_without_materialized) { 'WITH RECURSIVE "some_cte" AS (' } + + let(:query) do + recursive_cte = described_class.new(:some_cte) + recursive_cte << User.active + + User.with.recursive(recursive_cte.to_arel).to_sql + end + end end diff --git a/spec/migrations/20210226141517_dedup_issue_metrics_spec.rb b/spec/migrations/20210226141517_dedup_issue_metrics_spec.rb new file mode 100644 index 00000000000..043884eb7b2 --- /dev/null +++ b/spec/migrations/20210226141517_dedup_issue_metrics_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20210226141517_dedup_issue_metrics.rb') + +RSpec.describe DedupIssueMetrics, :migration, schema: 20210205104425 do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:issues) { table(:issues) } + let(:metrics) { table(:issue_metrics) } + let(:issue_params) { { title: 'title', project_id: project.id } } + + let!(:namespace) { namespaces.create!(name: 'foo', path: 'foo') } + let!(:project) { projects.create!(namespace_id: namespace.id) } + let!(:issue_1) { issues.create!(issue_params) } + let!(:issue_2) { issues.create!(issue_params) } + let!(:issue_3) { issues.create!(issue_params) } + + let!(:duplicated_metrics_1) { metrics.create!(issue_id: issue_1.id, first_mentioned_in_commit_at: 1.day.ago, first_added_to_board_at: 5.days.ago, updated_at: 2.months.ago) } + let!(:duplicated_metrics_2) { metrics.create!(issue_id: issue_1.id, first_mentioned_in_commit_at: Time.now, first_associated_with_milestone_at: Time.now, updated_at: 1.month.ago) } + + let!(:duplicated_metrics_3) { metrics.create!(issue_id: issue_3.id, first_mentioned_in_commit_at: 1.day.ago, updated_at: 2.months.ago) } + let!(:duplicated_metrics_4) { metrics.create!(issue_id: issue_3.id, first_added_to_board_at: 1.day.ago, updated_at: 1.month.ago) } + + let!(:non_duplicated_metrics) { metrics.create!(issue_id: issue_2.id, first_added_to_board_at: 2.days.ago) } + + it 'deduplicates issue_metrics table' do + expect { migrate! }.to change { metrics.count }.from(5).to(3) + end + + it 'merges `duplicated_metrics_1` with `duplicated_metrics_2`' do + migrate! + + expect(metrics.where(id: duplicated_metrics_1.id)).not_to exist + + merged_metrics = metrics.find_by(id: duplicated_metrics_2.id) + + expect(merged_metrics).to be_present + expect(merged_metrics.first_mentioned_in_commit_at).to be_like_time(duplicated_metrics_2.first_mentioned_in_commit_at) + expect(merged_metrics.first_added_to_board_at).to be_like_time(duplicated_metrics_1.first_added_to_board_at) + end + + it 'merges `duplicated_metrics_3` with `duplicated_metrics_4`' do + migrate! + + expect(metrics.where(id: duplicated_metrics_3.id)).not_to exist + + merged_metrics = metrics.find_by(id: duplicated_metrics_4.id) + + expect(merged_metrics).to be_present + expect(merged_metrics.first_mentioned_in_commit_at).to be_like_time(duplicated_metrics_3.first_mentioned_in_commit_at) + expect(merged_metrics.first_added_to_board_at).to be_like_time(duplicated_metrics_4.first_added_to_board_at) + end + + it 'does not change non duplicated records' do + expect { migrate! }.not_to change { non_duplicated_metrics.reload.attributes } + end + + it 'does nothing when there are no metrics' do + metrics.delete_all + + migrate! + + expect(metrics.count).to eq(0) + end +end diff --git a/spec/models/issue_spec.rb b/spec/models/issue_spec.rb index a3e245f4def..ac98e7b047f 100644 --- a/spec/models/issue_spec.rb +++ b/spec/models/issue_spec.rb @@ -85,18 +85,14 @@ RSpec.describe Issue do describe 'callbacks' do describe '#ensure_metrics' do it 'creates metrics after saving' do - issue = create(:issue, project: reusable_project) - - expect(issue.metrics).to be_persisted + expect(subject.metrics).to be_persisted expect(Issue::Metrics.count).to eq(1) end it 'does not create duplicate metrics for an issue' do - issue = create(:issue, project: reusable_project) + subject.close! - issue.close! - - expect(issue.metrics).to be_persisted + expect(subject.metrics).to be_persisted expect(Issue::Metrics.count).to eq(1) end @@ -105,6 +101,20 @@ RSpec.describe Issue do create(:issue, project: reusable_project) end + + context 'when metrics record is missing' do + before do + subject.metrics.delete + subject.reload + subject.metrics # make sure metrics association is cached (currently nil) + end + + it 'creates the metrics record' do + subject.update!(title: 'title') + + expect(subject.metrics).to be_present + end + end end describe '#record_create_action' do diff --git a/spec/models/packages/go/module_version_spec.rb b/spec/models/packages/go/module_version_spec.rb index c4c6a07d9e9..7fa416d8537 100644 --- a/spec/models/packages/go/module_version_spec.rb +++ b/spec/models/packages/go/module_version_spec.rb @@ -3,19 +3,9 @@ require 'spec_helper' RSpec.describe Packages::Go::ModuleVersion, type: :model do - let_it_be(:user) { create :user } - let_it_be(:project) { create :project_empty_repo, creator: user, path: 'my-go-lib' } - let_it_be(:mod) { create :go_module, project: project } + include_context 'basic Go module' - before :all do - create :go_module_commit, :files, project: project, tag: 'v1.0.0', files: { 'README.md' => 'Hi' } - create :go_module_commit, :module, project: project, tag: 'v1.0.1' - create :go_module_commit, :package, project: project, tag: 'v1.0.2', path: 'pkg' - create :go_module_commit, :module, project: project, tag: 'v1.0.3', name: 'mod' - create :go_module_commit, :files, project: project, files: { 'y.go' => "package a\n" } - create :go_module_commit, :module, project: project, name: 'v2' - create :go_module_commit, :files, project: project, tag: 'v2.0.0', files: { 'v2/x.go' => "package a\n" } - end + let_it_be(:mod) { create :go_module, project: project } shared_examples '#files' do |desc, *entries| it "returns #{desc}" do diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index b59f9b7fed1..e4295d2ab40 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -4109,7 +4109,7 @@ RSpec.describe Project, factory_default: :keep do subject { described_class.wrap_with_cte(projects) } it 'wrapped query matches original' do - expect(subject.to_sql).to match(/^WITH "projects_cte" AS/) + expect(subject.to_sql).to match(/^WITH "projects_cte" AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported}/) expect(subject).to match_array(projects) end end diff --git a/spec/requests/api/deploy_keys_spec.rb b/spec/requests/api/deploy_keys_spec.rb index 591d994fec9..a01c66a311c 100644 --- a/spec/requests/api/deploy_keys_spec.rb +++ b/spec/requests/api/deploy_keys_spec.rb @@ -3,12 +3,13 @@ require 'spec_helper' RSpec.describe API::DeployKeys do - let(:user) { create(:user) } - let(:maintainer) { create(:user) } - let(:admin) { create(:admin) } - let(:project) { create(:project, creator_id: user.id) } - let(:project2) { create(:project, creator_id: user.id) } - let(:deploy_key) { create(:deploy_key, public: true) } + let_it_be(:user) { create(:user) } + let_it_be(:maintainer) { create(:user) } + let_it_be(:admin) { create(:admin) } + let_it_be(:project) { create(:project, creator_id: user.id) } + let_it_be(:project2) { create(:project, creator_id: user.id) } + + let(:deploy_key) { create(:deploy_key, public: true) } let!(:deploy_keys_project) do create(:deploy_keys_project, project: project, deploy_key: deploy_key) @@ -44,18 +45,30 @@ RSpec.describe API::DeployKeys do end describe 'GET /projects/:id/deploy_keys' do - before do - deploy_key + let(:deploy_key) { create(:deploy_key, public: true, user: admin) } + + def perform_request + get api("/projects/#{project.id}/deploy_keys", admin) end it 'returns array of ssh keys' do - get api("/projects/#{project.id}/deploy_keys", admin) + perform_request expect(response).to have_gitlab_http_status(:ok) expect(response).to include_pagination_headers expect(json_response).to be_an Array expect(json_response.first['title']).to eq(deploy_key.title) end + + it 'returns multiple deploy keys without N + 1' do + perform_request + + control_count = ActiveRecord::QueryRecorder.new { perform_request }.count + + create(:deploy_key, public: true, projects: [project], user: maintainer) + + expect { perform_request }.not_to exceed_query_limit(control_count) + end end describe 'GET /projects/:id/deploy_keys/:key_id' do diff --git a/spec/services/packages/go/create_package_service_spec.rb b/spec/services/packages/go/create_package_service_spec.rb new file mode 100644 index 00000000000..5c5fec0aa3a --- /dev/null +++ b/spec/services/packages/go/create_package_service_spec.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Packages::Go::CreatePackageService do + let_it_be(:project) { create :project_empty_repo, path: 'my-go-lib' } + let_it_be(:mod) { create :go_module, project: project } + + before :all do + create :go_module_commit, :module, project: project, tag: 'v1.0.0' + end + + shared_examples 'creates a package' do |files:| + it "returns a valid package with #{files ? files.to_s : 'no'} file(s)" do + expect(subject).to be_valid + expect(subject.name).to eq(version.mod.name) + expect(subject.version).to eq(version.name) + expect(subject.package_type).to eq('golang') + expect(subject.created_at).to eq(version.commit.committed_date) + expect(subject.package_files.count).to eq(files) + end + end + + shared_examples 'creates a package file' do |type| + it "returns a package with a #{type} file" do + file_name = "#{version.name}.#{type}" + expect(subject.package_files.map { |f| f.file_name }).to include(file_name) + + file = subject.package_files.with_file_name(file_name).first + expect(file).not_to be_nil + expect(file.file).not_to be_nil + expect(file.size).to eq(file.file.size) + expect(file.file_name).to eq(file_name) + expect(file.file_md5).not_to be_nil + expect(file.file_sha1).not_to be_nil + expect(file.file_sha256).not_to be_nil + end + end + + describe '#execute' do + subject { described_class.new(project, nil, version: version).execute } + + let(:version) { create :go_module_version, :tagged, mod: mod, name: 'v1.0.0' } + + context 'with no existing package' do + it_behaves_like 'creates a package', files: 2 + it_behaves_like 'creates a package file', :mod + it_behaves_like 'creates a package file', :zip + + it 'creates a new package' do + expect { subject } + .to change { project.packages.count }.by(1) + .and change { Packages::PackageFile.count }.by(2) + end + end + + context 'with an existing package' do + before do + described_class.new(project, version: version).execute + end + + it_behaves_like 'creates a package', files: 2 + it_behaves_like 'creates a package file', :mod + it_behaves_like 'creates a package file', :zip + + it 'does not create a package or files' do + expect { subject } + .to not_change { project.packages.count } + .and not_change { Packages::PackageFile.count } + end + end + end +end diff --git a/spec/services/packages/go/sync_packages_service_spec.rb b/spec/services/packages/go/sync_packages_service_spec.rb new file mode 100644 index 00000000000..565b0f252ce --- /dev/null +++ b/spec/services/packages/go/sync_packages_service_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Packages::Go::SyncPackagesService do + include_context 'basic Go module' + + let(:params) { { info: true, mod: true, zip: true } } + + describe '#execute_async' do + it 'schedules a package refresh' do + expect(::Packages::Go::SyncPackagesWorker).to receive(:perform_async).once + + described_class.new(project, 'master').execute_async + end + end + + describe '#initialize' do + context 'without a project' do + it 'raises an error' do + expect { described_class.new(nil, 'master') } + .to raise_error(ArgumentError, 'project is required') + end + end + + context 'without a ref' do + it 'raises an error' do + expect { described_class.new(project, nil) } + .to raise_error(ArgumentError, 'ref is required') + end + end + + context 'with an invalid ref' do + it 'raises an error' do + expect { described_class.new(project, 'not-a-ref') } + .to raise_error(ArgumentError) + end + end + end +end diff --git a/spec/support/shared_contexts/requests/api/go_modules_shared_context.rb b/spec/support/shared_contexts/requests/api/go_modules_shared_context.rb new file mode 100644 index 00000000000..5a90c3076b1 --- /dev/null +++ b/spec/support/shared_contexts/requests/api/go_modules_shared_context.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +RSpec.shared_context 'basic Go module' do + let_it_be(:user) { create :user } + let_it_be(:project) { create :project_empty_repo, creator: user, path: 'my-go-lib' } + + let_it_be(:commit_v1_0_0) { create :go_module_commit, :files, project: project, tag: 'v1.0.0', files: { 'README.md' => 'Hi' } } + let_it_be(:commit_v1_0_1) { create :go_module_commit, :module, project: project, tag: 'v1.0.1' } + let_it_be(:commit_v1_0_2) { create :go_module_commit, :package, project: project, tag: 'v1.0.2', path: 'pkg' } + let_it_be(:commit_v1_0_3) { create :go_module_commit, :module, project: project, tag: 'v1.0.3', name: 'mod' } + let_it_be(:commit_file_y) { create :go_module_commit, :files, project: project, files: { 'y.go' => "package a\n" } } + let_it_be(:commit_mod_v2) { create :go_module_commit, :module, project: project, name: 'v2' } + let_it_be(:commit_v2_0_0) { create :go_module_commit, :files, project: project, tag: 'v2.0.0', files: { 'v2/x.go' => "package a\n" } } +end diff --git a/spec/support/shared_examples/lib/gitlab/database/cte_materialized_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/database/cte_materialized_shared_examples.rb new file mode 100644 index 00000000000..88e6ffd15a8 --- /dev/null +++ b/spec/support/shared_examples/lib/gitlab/database/cte_materialized_shared_examples.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +RSpec.shared_examples 'CTE with MATERIALIZED keyword examples' do + describe 'adding MATERIALIZE to the CTE' do + let(:options) { {} } + + before do + # Clear the cached value before the test + Gitlab::Database::AsWithMaterialized.clear_memoization(:materialized_supported) + end + + context 'when PG version is <12' do + it 'does not add MATERIALIZE keyword' do + allow(Gitlab::Database).to receive(:version).and_return('11.1') + + expect(query).to include(expected_query_block_without_materialized) + end + end + + context 'when PG version is >=12' do + it 'adds MATERIALIZE keyword' do + allow(Gitlab::Database).to receive(:version).and_return('12.1') + + expect(query).to include(expected_query_block_with_materialized) + end + + context 'when version is higher than 12' do + it 'adds MATERIALIZE keyword' do + allow(Gitlab::Database).to receive(:version).and_return('15.1') + + expect(query).to include(expected_query_block_with_materialized) + end + end + + context 'when materialized is disabled' do + let(:options) { { materialized: false } } + + it 'does not add MATERIALIZE keyword' do + expect(query).to include(expected_query_block_without_materialized) + end + end + end + end +end diff --git a/spec/workers/packages/go/sync_packages_worker_spec.rb b/spec/workers/packages/go/sync_packages_worker_spec.rb new file mode 100644 index 00000000000..ad1a85b26e4 --- /dev/null +++ b/spec/workers/packages/go/sync_packages_worker_spec.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Packages::Go::SyncPackagesWorker, type: :worker do + include_context 'basic Go module' + + before do + project.update!(visibility_level: Gitlab::VisibilityLevel::PUBLIC) + end + + def perform(ref_name, path) + described_class.new.perform(project.id, ref_name, path) + end + + def validate_package(package, mod, ver) + expect(package).not_to be_nil + expect(package.name).to eq(mod.name) + expect(package.version).to eq(ver.name) + expect(package.package_type).to eq('golang') + expect(package.created_at).to eq(ver.commit.committed_date) + expect(package.package_files.count).to eq(2) + end + + shared_examples 'it creates a package' do |path, version, exists: false| + subject { perform(version, path) } + + it "returns a package for example.com/project#{path.empty? ? '' : '/' + path}@#{version}" do + expect { subject } + .to change { project.packages.count }.by(exists ? 0 : 1) + .and change { Packages::PackageFile.count }.by(exists ? 0 : 2) + + mod = create :go_module, project: project, path: path + ver = create :go_module_version, :tagged, mod: mod, name: version + validate_package(subject, mod, ver) + end + end + + describe '#perform' do + context 'with no existing packages' do + it_behaves_like 'it creates a package', '', 'v1.0.1' + it_behaves_like 'it creates a package', '', 'v1.0.2' + it_behaves_like 'it creates a package', '', 'v1.0.3' + it_behaves_like 'it creates a package', 'mod', 'v1.0.3' + it_behaves_like 'it creates a package', 'v2', 'v2.0.0' + end + + context 'with existing packages' do + before do + mod = create :go_module, project: project + ver = create :go_module_version, :tagged, mod: mod, name: 'v1.0.1' + Packages::Go::CreatePackageService.new(project, nil, version: ver).execute + end + + it_behaves_like 'it creates a package', '', 'v1.0.1', exists: true + it_behaves_like 'it creates a package', '', 'v1.0.2' + it_behaves_like 'it creates a package', '', 'v1.0.3' + it_behaves_like 'it creates a package', 'mod', 'v1.0.3' + it_behaves_like 'it creates a package', 'v2', 'v2.0.0' + end + + context 'with a package that exceeds project limits' do + before do + Plan.default.actual_limits.update!({ 'golang_max_file_size': 1 }) + end + + it 'logs an exception' do + expect(Gitlab::ErrorTracking).to receive(:log_exception).with(instance_of(::Packages::Go::CreatePackageService::GoZipSizeError)) + + perform('v2.0.0', 'v2') + end + end + + where(:path, :version) do + [ + ['', 'v1.0.1'], + ['', 'v1.0.2'], + ['', 'v1.0.3'], + ['mod', 'v1.0.3'], + ['v2', 'v2.0.0'] + ] + end + + with_them do + it_behaves_like 'an idempotent worker' do + let(:job_args) { [project.id, version, path] } + + it 'creates a package' do + expect { subject } + .to change { project.packages.count }.by(1) + .and change { Packages::PackageFile.count }.by(2) + + mod = create :go_module, project: project, path: path + ver = create :go_module_version, :tagged, mod: mod, name: version + package = ::Packages::Go::PackageFinder.new(project, mod.name, ver.name).execute + validate_package(package, mod, ver) + end + end + end + end +end |