diff options
Diffstat (limited to 'app')
| -rw-r--r-- | app/assets/javascripts/behaviors/toggler_behavior.js | 16 | ||||
| -rw-r--r-- | app/models/group.rb | 7 | ||||
| -rw-r--r-- | app/models/packages/dependency_link.rb | 28 | ||||
| -rw-r--r-- | app/models/project_authorization.rb | 9 | ||||
| -rw-r--r-- | app/models/project_authorizations/changes.rb | 4 | ||||
| -rw-r--r-- | app/services/packages/npm/generate_metadata_service.rb | 70 | ||||
| -rw-r--r-- | app/views/admin/application_settings/_gitpod.html.haml | 2 | ||||
| -rw-r--r-- | app/views/projects/merge_requests/_page.html.haml | 1 | ||||
| -rw-r--r-- | app/workers/all_queues.yml | 27 | ||||
| -rw-r--r-- | app/workers/concerns/gitlab/bitbucket_import/stage_methods.rb | 76 | ||||
| -rw-r--r-- | app/workers/gitlab/bitbucket_import/advance_stage_worker.rb | 37 | ||||
| -rw-r--r-- | app/workers/gitlab/bitbucket_import/stage/finish_import_worker.rb | 19 | ||||
| -rw-r--r-- | app/workers/gitlab/bitbucket_import/stage/import_repository_worker.rb | 29 |
13 files changed, 308 insertions, 17 deletions
diff --git a/app/assets/javascripts/behaviors/toggler_behavior.js b/app/assets/javascripts/behaviors/toggler_behavior.js index 30424fee46a..4e643f71c4b 100644 --- a/app/assets/javascripts/behaviors/toggler_behavior.js +++ b/app/assets/javascripts/behaviors/toggler_behavior.js @@ -1,4 +1,5 @@ import $ from 'jquery'; +import { fixTitle } from '~/tooltips'; import { getLocationHash } from '../lib/utils/url_utility'; // Toggle button. Show/hide content inside parent container. @@ -29,9 +30,22 @@ $(() => { $container.find('.js-toggle-content').toggle(toggleState); } + function updateTitle(el, container) { + const $container = $(container); + const isExpanded = $container.data('is-expanded'); + + el.setAttribute('title', isExpanded ? el.dataset.collapseTitle : el.dataset.expandTitle); + + fixTitle(el); + } + $('body').on('click', '.js-toggle-button', function toggleButton(e) { e.currentTarget.classList.toggle(e.currentTarget.dataset.toggleOpenClass || 'selected'); - toggleContainer($(this).closest('.js-toggle-container')); + + const containerEl = this.closest('.js-toggle-container'); + + toggleContainer(containerEl); + updateTitle(this, containerEl); const targetTag = e.currentTarget.tagName.toLowerCase(); if (targetTag === 'a' || targetTag === 'button') { diff --git a/app/models/group.rb b/app/models/group.rb index 6fc050c0c5f..7baf86b878b 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -35,7 +35,8 @@ class Group < Namespace foreign_key: :member_namespace_id, inverse_of: :group, class_name: 'GroupMember' alias_method :members, :group_members - has_many :users, through: :group_members + has_many :users, -> { allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/422405") }, + through: :group_members has_many :owners, -> { where(members: { access_level: Gitlab::Access::OWNER }) .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/422405") @@ -341,10 +342,6 @@ class Group < Namespace end end - def users - super.loaded? ? super : super.allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/422405") - end - # Overrides notification_settings has_many association # This allows to apply notification settings from parent groups # to child groups and projects. diff --git a/app/models/packages/dependency_link.rb b/app/models/packages/dependency_link.rb index 51018602bdc..400b4cce208 100644 --- a/app/models/packages/dependency_link.rb +++ b/app/models/packages/dependency_link.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true class Packages::DependencyLink < ApplicationRecord + include EachBatch + belongs_to :package, inverse_of: :dependency_links belongs_to :dependency, inverse_of: :dependency_links, class_name: 'Packages::Dependency' has_one :nuget_metadatum, inverse_of: :dependency_link, class_name: 'Packages::Nuget::DependencyLinkMetadatum' @@ -14,6 +16,32 @@ class Packages::DependencyLink < ApplicationRecord scope :with_dependency_type, ->(dependency_type) { where(dependency_type: dependency_type) } scope :includes_dependency, -> { includes(:dependency) } scope :for_package, ->(package) { where(package_id: package.id) } + scope :for_packages, ->(packages) { where(package: packages) } scope :preload_dependency, -> { preload(:dependency) } scope :preload_nuget_metadatum, -> { preload(:nuget_metadatum) } + scope :select_dependency_id, -> { select(:dependency_id) } + + def self.dependency_ids_grouped_by_type(packages) + inner_query = where(package_id: packages) + .select(' + package_id, + dependency_type, + ARRAY_AGG(dependency_id) as dependency_ids + ') + .group(:package_id, :dependency_type) + + cte = Gitlab::SQL::CTE.new(:dependency_links_cte, inner_query) + cte_alias = cte.table.alias(table_name) + + with(cte.to_arel) + .select(' + package_id, + JSON_OBJECT_AGG( + dependency_type, + dependency_ids + ) AS dependency_ids_by_type + ') + .from(cte_alias) + .group(:package_id) + end end diff --git a/app/models/project_authorization.rb b/app/models/project_authorization.rb index cc3b38a7703..3161bdaacd8 100644 --- a/app/models/project_authorization.rb +++ b/app/models/project_authorization.rb @@ -13,6 +13,9 @@ class ProjectAuthorization < ApplicationRecord scope :non_guests, -> { where('access_level > ?', ::Gitlab::Access::GUEST) } + # TODO: To be removed after https://gitlab.com/gitlab-org/gitlab/-/issues/418205 + before_create :assign_is_unique + def self.select_from_union(relations) from_union(relations) .select(['project_id', 'MAX(access_level) AS access_level']) @@ -27,6 +30,12 @@ class ProjectAuthorization < ApplicationRecord def self.insert_all(attributes) super(attributes, unique_by: connection.schema_cache.primary_keys(table_name)) end + + private + + def assign_is_unique + self.is_unique = true if Feature.enabled?(:write_project_authorizations_is_unique) + end end ProjectAuthorization.prepend_mod_with('ProjectAuthorization') diff --git a/app/models/project_authorizations/changes.rb b/app/models/project_authorizations/changes.rb index 1d717950c1c..d0175b4662d 100644 --- a/app/models/project_authorizations/changes.rb +++ b/app/models/project_authorizations/changes.rb @@ -89,7 +89,11 @@ module ProjectAuthorizations add_delay = add_delay_between_batches?(entire_size: attributes.size, batch_size: BATCH_SIZE) log_details(entire_size: attributes.size, batch_size: BATCH_SIZE) if add_delay + write_is_unique = Feature.enabled?(:write_project_authorizations_is_unique) + attributes.each_slice(BATCH_SIZE) do |attributes_batch| + attributes_batch.each { |attrs| attrs[:is_unique] = true } if write_is_unique + ProjectAuthorization.insert_all(attributes_batch) perform_delay if add_delay end diff --git a/app/services/packages/npm/generate_metadata_service.rb b/app/services/packages/npm/generate_metadata_service.rb index e1795079513..cae086d74c5 100644 --- a/app/services/packages/npm/generate_metadata_service.rb +++ b/app/services/packages/npm/generate_metadata_service.rb @@ -4,6 +4,7 @@ module Packages module Npm class GenerateMetadataService include API::Helpers::RelatedResourcesHelpers + include Gitlab::Utils::StrongMemoize # Allowed fields are those defined in the abbreviated form # defined here: https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object @@ -13,6 +14,8 @@ module Packages def initialize(name, packages) @name = name @packages = packages + @dependencies = {} + @dependency_ids = Hash.new { |h, key| h[key] = {} } end def execute(only_dist_tags: false) @@ -21,7 +24,7 @@ module Packages private - attr_reader :name, :packages + attr_reader :name, :packages, :dependencies, :dependency_ids def metadata(only_dist_tags) result = { dist_tags: dist_tags } @@ -38,9 +41,17 @@ module Packages package_versions = {} packages.each_batch do |relation| - batched_packages = relation.including_dependency_links - .preload_files - .preload_npm_metadatum + batched_packages = if optimization_enabled? + load_dependencies(relation) + load_dependency_ids(relation) + + relation.preload_files + .preload_npm_metadatum + else + relation.including_dependency_links + .preload_files + .preload_npm_metadatum + end batched_packages.each do |package| package_file = package.installable_package_files.last @@ -82,14 +93,23 @@ module Packages end def build_package_dependencies(package) - dependencies = Hash.new { |h, key| h[key] = {} } + if optimization_enabled? + inverted_dependency_types = Packages::DependencyLink.dependency_types.invert.stringify_keys + dependency_ids[package.id].each_with_object(Hash.new { |h, key| h[key] = {} }) do |(type, ids), memo| + ids.each do |id| + memo[inverted_dependency_types[type]].merge!(dependencies[id]) + end + end + else + dependencies = Hash.new { |h, key| h[key] = {} } - package.dependency_links.each do |dependency_link| - dependency = dependency_link.dependency - dependencies[dependency_link.dependency_type][dependency.name] = dependency.version_pattern - end + package.dependency_links.each do |dependency_link| + dependency = dependency_link.dependency + dependencies[dependency_link.dependency_type][dependency.name] = dependency.version_pattern + end - dependencies + dependencies + end end def sorted_versions @@ -106,6 +126,36 @@ module Packages json = package.npm_metadatum&.package_json || {} json.slice(*PACKAGE_JSON_ALLOWED_FIELDS) end + + def load_dependencies(packages) + Packages::Dependency + .id_in( + Packages::DependencyLink + .for_packages(packages) + .select_dependency_id + ) + .id_not_in(dependencies.keys) + .each_batch do |relation| + relation.each do |dependency| + dependencies[dependency.id] = { dependency.name => dependency.version_pattern } + end + end + end + + def load_dependency_ids(packages) + Packages::DependencyLink + .dependency_ids_grouped_by_type(packages) + .each_batch(column: :package_id) do |relation| + relation.each do |dependency_link| + dependency_ids[dependency_link.package_id] = dependency_link.dependency_ids_by_type + end + end + end + + def optimization_enabled? + Feature.enabled?(:npm_optimize_metadata_generation) + end + strong_memoize_attr :optimization_enabled? end end end diff --git a/app/views/admin/application_settings/_gitpod.html.haml b/app/views/admin/application_settings/_gitpod.html.haml index 988153d45a4..1f56487cea4 100644 --- a/app/views/admin/application_settings/_gitpod.html.haml +++ b/app/views/admin/application_settings/_gitpod.html.haml @@ -6,7 +6,7 @@ = _('Gitpod') = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded ? _('Collapse') : _('Expand') - .gl-text-secondary + .gl-text-secondary.gl-mb-5 #js-gitpod-settings-help-text{ data: {"message" => gitpod_enable_description, "message-url" => "https://gitpod.io/" } } = link_to sprite_icon('question-o'), help_page_path('integration/gitpod.md'), target: '_blank', class: 'has-tooltip', title: _('More information') diff --git a/app/views/projects/merge_requests/_page.html.haml b/app/views/projects/merge_requests/_page.html.haml index dc97aa62c26..dfb18b52021 100644 --- a/app/views/projects/merge_requests/_page.html.haml +++ b/app/views/projects/merge_requests/_page.html.haml @@ -61,6 +61,7 @@ = render "projects/merge_requests/tabs/pane", id: "notes", class: "notes voting_notes" do %div{ class: "#{'merge-request-overview' if moved_mr_sidebar_enabled?}" } %section + = render_if_exists "projects/merge_requests/diff_summary" .issuable-discussion.js-vue-notes-event - if @merge_request.description.present? .detail-page-description.gl-pb-0 diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 94b9acbc48b..606a0cfeee4 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -2334,6 +2334,33 @@ :weight: 1 :idempotent: true :tags: [] +- :name: bitbucket_import_advance_stage + :worker_name: Gitlab::BitbucketImport::AdvanceStageWorker + :feature_category: :importers + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: false + :tags: [] +- :name: bitbucket_import_stage_finish_import + :worker_name: Gitlab::BitbucketImport::Stage::FinishImportWorker + :feature_category: :importers + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: false + :tags: [] +- :name: bitbucket_import_stage_import_repository + :worker_name: Gitlab::BitbucketImport::Stage::ImportRepositoryWorker + :feature_category: :importers + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: false + :tags: [] - :name: bitbucket_server_import_advance_stage :worker_name: Gitlab::BitbucketServerImport::AdvanceStageWorker :feature_category: :importers diff --git a/app/workers/concerns/gitlab/bitbucket_import/stage_methods.rb b/app/workers/concerns/gitlab/bitbucket_import/stage_methods.rb new file mode 100644 index 00000000000..2885cc29532 --- /dev/null +++ b/app/workers/concerns/gitlab/bitbucket_import/stage_methods.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module StageMethods + extend ActiveSupport::Concern + + included do + include ApplicationWorker + + worker_has_external_dependencies! + + feature_category :importers + + data_consistency :always + + sidekiq_options dead: false, retry: 3 + + sidekiq_retries_exhausted do |msg, e| + Gitlab::Import::ImportFailureService.track( + project_id: msg['args'][0], + exception: e, + fail_import: true + ) + end + end + + # project_id - The ID of the GitLab project to import the data into. + def perform(project_id) + info(project_id, message: 'starting stage') + + project = find_project(project_id) + + return unless project + + import(project) + + info(project_id, message: 'stage finished') + rescue StandardError => e + Gitlab::Import::ImportFailureService.track( + project_id: project_id, + exception: e, + error_source: self.class.name, + fail_import: abort_on_failure + ) + + raise(e) + end + + def find_project(id) + # If the project has been marked as failed we want to bail out + # automatically. + # rubocop: disable CodeReuse/ActiveRecord + Project.joins_import_state.where(import_state: { status: :started }).find_by_id(id) + # rubocop: enable CodeReuse/ActiveRecord + end + + def abort_on_failure + false + end + + private + + def info(project_id, extra = {}) + Logger.info(log_attributes(project_id, extra)) + end + + def log_attributes(project_id, extra = {}) + extra.merge( + project_id: project_id, + import_stage: self.class.name + ) + end + end + end +end diff --git a/app/workers/gitlab/bitbucket_import/advance_stage_worker.rb b/app/workers/gitlab/bitbucket_import/advance_stage_worker.rb new file mode 100644 index 00000000000..7f281352a1b --- /dev/null +++ b/app/workers/gitlab/bitbucket_import/advance_stage_worker.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + # AdvanceStageWorker is a worker used by the BitBucket Importer to wait for a + # number of jobs to complete, without blocking a thread. Once all jobs have + # been completed this worker will advance the import process to the next + # stage. + class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + include ::Gitlab::Import::AdvanceStage + + data_consistency :delayed + + sidekiq_options dead: false, retry: 3 + + feature_category :importers + + loggable_arguments 1, 2 + + # The known importer stages and their corresponding Sidekiq workers. + STAGES = { + finish: Stage::FinishImportWorker + }.freeze + + def find_import_state(project_id) + ProjectImportState.jid_by(project_id: project_id, status: :started) + end + + private + + def next_stage_worker(next_stage) + STAGES.fetch(next_stage.to_sym) + end + end + end +end diff --git a/app/workers/gitlab/bitbucket_import/stage/finish_import_worker.rb b/app/workers/gitlab/bitbucket_import/stage/finish_import_worker.rb new file mode 100644 index 00000000000..a1c5f5787be --- /dev/null +++ b/app/workers/gitlab/bitbucket_import/stage/finish_import_worker.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Stage + class FinishImportWorker # rubocop:disable Scalability/IdempotentWorker + include StageMethods + + private + + def import(project) + project.after_import + + Gitlab::Import::Metrics.new(:bitbucket_importer, project).track_finished_import + end + end + end + end +end diff --git a/app/workers/gitlab/bitbucket_import/stage/import_repository_worker.rb b/app/workers/gitlab/bitbucket_import/stage/import_repository_worker.rb new file mode 100644 index 00000000000..e311b837311 --- /dev/null +++ b/app/workers/gitlab/bitbucket_import/stage/import_repository_worker.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Stage + class ImportRepositoryWorker # rubocop:disable Scalability/IdempotentWorker + include StageMethods + + private + + def import(project) + importer = importer_class.new(project) + + importer.execute + + FinishImportWorker.perform_async(project.id) + end + + def importer_class + Importers::RepositoryImporter + end + + def abort_on_failure + true + end + end + end + end +end |