From ac382b5682dc2d5eea750313c59fb2581af13326 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 24 Apr 2017 17:19:22 +0200 Subject: Use CTEs for nested groups and authorizations This commit introduces the usage of Common Table Expressions (CTEs) to efficiently retrieve nested group hierarchies, without having to rely on the "routes" table (which is an _incredibly_ inefficient way of getting the data). This requires a patch to ActiveRecord (found in the added initializer) to work properly as ActiveRecord doesn't support WITH statements properly out of the box. Unfortunately MySQL provides no efficient way of getting nested groups. For example, the old routes setup could easily take 5-10 seconds depending on the amount of "routes" in a database. Providing vastly different logic for both MySQL and PostgreSQL will negatively impact the development process. Because of this the various nested groups related methods return empty relations when used in combination with MySQL. For project authorizations the logic is split up into two classes: * Gitlab::ProjectAuthorizations::WithNestedGroups * Gitlab::ProjectAuthorizations::WithoutNestedGroups Both classes get the fresh project authorizations (= as they should be in the "project_authorizations" table), including nested groups if PostgreSQL is used. The logic of these two classes is quite different apart from their public interface. This complicates development a bit, but unfortunately there is no way around this. This commit also introduces Gitlab::GroupHierarchy. This class can be used to get the ancestors and descendants of a base relation, or both by using a UNION. This in turn is used by methods such as: * Namespace#ancestors * Namespace#descendants * User#all_expanded_groups Again this class relies on CTEs and thus only works on PostgreSQL. The Namespace methods will return an empty relation when MySQL is used, while User#all_expanded_groups will return only the groups a user is a direct member of. Performance wise the impact is quite large. For example, on GitLab.com Namespace#descendants used to take around 580 ms to retrieve data for a particular user. Using CTEs we are able to reduce this down to roughly 1 millisecond, returning the exact same data. == On The Fly Refreshing Refreshing of authorizations on the fly (= when users.authorized_projects_populated was not set) is removed with this commit. This simplifies the code, and ensures any queries used for authorizations are not mutated because they are executed in a Rails scope (e.g. Project.visible_to_user). This commit includes a migration to schedule refreshing authorizations for all users, ensuring all of them have their authorizations in place. Said migration schedules users in batches of 5000, with 5 minutes between every batch to smear the load around a bit. == Spec Changes This commit also introduces some changes to various specs. For example, some specs for ProjectTeam assumed that creating a personal project would _not_ lead to the owner having access, which is incorrect. Because we also no longer refresh authorizations on the fly for new users some code had to be added to the "empty_project" factory. This chunk of code ensures that the owner's permissions are refreshed after creating the project, something that is normally done in Projects::CreateService. --- app/models/concerns/routable.rb | 83 ---------------------- .../concerns/select_for_project_authorization.rb | 6 +- app/models/group.rb | 6 +- app/models/namespace.rb | 26 +++---- app/models/project_authorization.rb | 6 ++ app/models/user.rb | 36 ++++------ .../users/refresh_authorized_projects_service.rb | 40 ++--------- 7 files changed, 49 insertions(+), 154 deletions(-) (limited to 'app') diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index c4463abdfe6..63d02b76f6b 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -84,89 +84,6 @@ module Routable joins(:route).where(wheres.join(' OR ')) end end - - # Builds a relation to find multiple objects that are nested under user membership - # - # Usage: - # - # Klass.member_descendants(1) - # - # Returns an ActiveRecord::Relation. - def member_descendants(user_id) - joins(:route). - joins("INNER JOIN routes r2 ON routes.path LIKE CONCAT(r2.path, '/%') - INNER JOIN members ON members.source_id = r2.source_id - AND members.source_type = r2.source_type"). - where('members.user_id = ?', user_id) - end - - # Builds a relation to find multiple objects that are nested under user - # membership. Includes the parent, as opposed to `#member_descendants` - # which only includes the descendants. - # - # Usage: - # - # Klass.member_self_and_descendants(1) - # - # Returns an ActiveRecord::Relation. - def member_self_and_descendants(user_id) - joins(:route). - joins("INNER JOIN routes r2 ON routes.path LIKE CONCAT(r2.path, '/%') - OR routes.path = r2.path - INNER JOIN members ON members.source_id = r2.source_id - AND members.source_type = r2.source_type"). - where('members.user_id = ?', user_id) - end - - # Returns all objects in a hierarchy, where any node in the hierarchy is - # under the user membership. - # - # Usage: - # - # Klass.member_hierarchy(1) - # - # Examples: - # - # Given the following group tree... - # - # _______group_1_______ - # | | - # | | - # nested_group_1 nested_group_2 - # | | - # | | - # nested_group_1_1 nested_group_2_1 - # - # - # ... the following results are returned: - # - # * the user is a member of group 1 - # => 'group_1', - # 'nested_group_1', nested_group_1_1', - # 'nested_group_2', 'nested_group_2_1' - # - # * the user is a member of nested_group_2 - # => 'group1', - # 'nested_group_2', 'nested_group_2_1' - # - # * the user is a member of nested_group_2_1 - # => 'group1', - # 'nested_group_2', 'nested_group_2_1' - # - # Returns an ActiveRecord::Relation. - def member_hierarchy(user_id) - paths = member_self_and_descendants(user_id).pluck('routes.path') - - return none if paths.empty? - - wheres = paths.map do |path| - "#{connection.quote(path)} = routes.path - OR - #{connection.quote(path)} LIKE CONCAT(routes.path, '/%')" - end - - joins(:route).where(wheres.join(' OR ')) - end end def full_name diff --git a/app/models/concerns/select_for_project_authorization.rb b/app/models/concerns/select_for_project_authorization.rb index 50a1d7fc3e1..58194b0ea13 100644 --- a/app/models/concerns/select_for_project_authorization.rb +++ b/app/models/concerns/select_for_project_authorization.rb @@ -3,7 +3,11 @@ module SelectForProjectAuthorization module ClassMethods def select_for_project_authorization - select("members.user_id, projects.id AS project_id, members.access_level") + select("projects.id AS project_id, members.access_level") + end + + def select_as_master_for_project_authorization + select(["projects.id AS project_id", "#{Gitlab::Access::MASTER} AS access_level"]) end end end diff --git a/app/models/group.rb b/app/models/group.rb index 6aab477f431..be944da5a67 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -38,6 +38,10 @@ class Group < Namespace after_save :update_two_factor_requirement class << self + def supports_nested_groups? + Gitlab::Database.postgresql? + end + # Searches for groups matching the given query. # # This method uses ILIKE on PostgreSQL and LIKE on MySQL. @@ -78,7 +82,7 @@ class Group < Namespace if current_scope.joins_values.include?(:shared_projects) joins('INNER JOIN namespaces project_namespace ON project_namespace.id = projects.namespace_id') .where('project_namespace.share_with_group_lock = ?', false) - .select("members.user_id, projects.id AS project_id, LEAST(project_group_links.group_access, members.access_level) AS access_level") + .select("projects.id AS project_id, LEAST(project_group_links.group_access, members.access_level) AS access_level") else super end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index a7ede5e3b9e..985395a6fe2 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -176,26 +176,22 @@ class Namespace < ActiveRecord::Base projects.with_shared_runners.any? end - # Scopes the model on ancestors of the record + # Returns all the ancestors of the current namespaces. def ancestors - if parent_id - path = route ? route.path : full_path - paths = [] + return self.class.none if !Group.supports_nested_groups? || !parent_id - until path.blank? - path = path.rpartition('/').first - paths << path - end - - self.class.joins(:route).where('routes.path IN (?)', paths).reorder('routes.path ASC') - else - self.class.none - end + Gitlab::GroupHierarchy. + new(self.class.where(id: parent_id)). + base_and_ancestors end - # Scopes the model on direct and indirect children of the record + # Returns all the descendants of the current namespace. def descendants - self.class.joins(:route).merge(Route.inside_path(route.path)).reorder('routes.path ASC') + return self.class.none unless Group.supports_nested_groups? + + Gitlab::GroupHierarchy. + new(self.class.where(parent_id: id)). + base_and_descendants end def user_ids_for_project_authorizations diff --git a/app/models/project_authorization.rb b/app/models/project_authorization.rb index 4c7f4f5a429..def09675253 100644 --- a/app/models/project_authorization.rb +++ b/app/models/project_authorization.rb @@ -6,6 +6,12 @@ class ProjectAuthorization < ActiveRecord::Base validates :access_level, inclusion: { in: Gitlab::Access.all_values }, presence: true validates :user, uniqueness: { scope: [:project, :access_level] }, presence: true + def self.select_from_union(union) + select(['project_id', 'MAX(access_level) AS access_level']). + from("(#{union.to_sql}) #{ProjectAuthorization.table_name}"). + group(:project_id) + end + def self.insert_authorizations(rows, per_batch = 1000) rows.each_slice(per_batch) do |slice| tuples = slice.map do |tuple| diff --git a/app/models/user.rb b/app/models/user.rb index c7160a6af14..2cf995f8c31 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -10,9 +10,12 @@ class User < ActiveRecord::Base include Sortable include CaseSensitivity include TokenAuthenticatable + include IgnorableColumn DEFAULT_NOTIFICATION_LEVEL = :participating + ignore_column :authorized_projects_populated + add_authentication_token_field :authentication_token add_authentication_token_field :incoming_email_token @@ -212,7 +215,6 @@ class User < ActiveRecord::Base scope :blocked, -> { with_states(:blocked, :ldap_blocked) } scope :external, -> { where(external: true) } scope :active, -> { with_state(:active).non_internal } - scope :not_in_project, ->(project) { project.users.present? ? where("id not in (:ids)", ids: project.users.map(&:id) ) : all } scope :without_projects, -> { where('id NOT IN (SELECT DISTINCT(user_id) FROM members WHERE user_id IS NOT NULL AND requested_at IS NULL)') } scope :todo_authors, ->(user_id, state) { where(id: Todo.where(user_id: user_id, state: state).select(:author_id)) } scope :order_recent_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('last_sign_in_at', 'DESC')) } @@ -504,23 +506,18 @@ class User < ActiveRecord::Base Group.where("namespaces.id IN (#{union.to_sql})") end - def nested_groups - Group.member_descendants(id) - end - + # Returns a relation of groups the user has access to, including their parent + # and child groups (recursively). def all_expanded_groups - Group.member_hierarchy(id) + return groups unless Group.supports_nested_groups? + + Gitlab::GroupHierarchy.new(groups).all_groups end def expanded_groups_requiring_two_factor_authentication all_expanded_groups.where(require_two_factor_authentication: true) end - def nested_groups_projects - Project.joins(:namespace).where('namespaces.parent_id IS NOT NULL'). - member_descendants(id) - end - def refresh_authorized_projects Users::RefreshAuthorizedProjectsService.new(self).execute end @@ -529,18 +526,15 @@ class User < ActiveRecord::Base project_authorizations.where(project_id: project_ids).delete_all end - def set_authorized_projects_column - unless authorized_projects_populated - update_column(:authorized_projects_populated, true) - end - end - def authorized_projects(min_access_level = nil) - refresh_authorized_projects unless authorized_projects_populated - - # We're overriding an association, so explicitly call super with no arguments or it would be passed as `force_reload` to the association + # We're overriding an association, so explicitly call super with no + # arguments or it would be passed as `force_reload` to the association projects = super() - projects = projects.where('project_authorizations.access_level >= ?', min_access_level) if min_access_level + + if min_access_level + projects = projects. + where('project_authorizations.access_level >= ?', min_access_level) + end projects end diff --git a/app/services/users/refresh_authorized_projects_service.rb b/app/services/users/refresh_authorized_projects_service.rb index 8f6f5b937c4..3e07b811027 100644 --- a/app/services/users/refresh_authorized_projects_service.rb +++ b/app/services/users/refresh_authorized_projects_service.rb @@ -73,12 +73,11 @@ module Users # remove - The IDs of the authorization rows to remove. # add - Rows to insert in the form `[user id, project id, access level]` def update_authorizations(remove = [], add = []) - return if remove.empty? && add.empty? && user.authorized_projects_populated + return if remove.empty? && add.empty? User.transaction do user.remove_project_authorizations(remove) unless remove.empty? ProjectAuthorization.insert_authorizations(add) unless add.empty? - user.set_authorized_projects_column end # Since we batch insert authorization rows, Rails' associations may get @@ -101,38 +100,13 @@ module Users end def fresh_authorizations - ProjectAuthorization. - unscoped. - select('project_id, MAX(access_level) AS access_level'). - from("(#{project_authorizations_union.to_sql}) #{ProjectAuthorization.table_name}"). - group(:project_id) - end - - private - - # Returns a union query of projects that the user is authorized to access - def project_authorizations_union - relations = [ - # Personal projects - user.personal_projects.select("#{user.id} AS user_id, projects.id AS project_id, #{Gitlab::Access::MASTER} AS access_level"), - - # Projects the user is a member of - user.projects.select_for_project_authorization, - - # Projects of groups the user is a member of - user.groups_projects.select_for_project_authorization, - - # Projects of subgroups of groups the user is a member of - user.nested_groups_projects.select_for_project_authorization, - - # Projects shared with groups the user is a member of - user.groups.joins(:shared_projects).select_for_project_authorization, - - # Projects shared with subgroups of groups the user is a member of - user.nested_groups.joins(:shared_projects).select_for_project_authorization - ] + klass = if Group.supports_nested_groups? + Gitlab::ProjectAuthorizations::WithNestedGroups + else + Gitlab::ProjectAuthorizations::WithoutNestedGroups + end - Gitlab::SQL::Union.new(relations) + klass.new(user).calculate end end end -- cgit v1.2.3 From 34974258bc3f745c86512319231bad47232fe007 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 3 May 2017 14:49:37 +0200 Subject: Hide nested group UI/API support for MySQL This hides/disables some UI elements and API parameters related to nested groups when MySQL is used, since nested groups are not supported for MySQL. --- app/controllers/groups_controller.rb | 2 ++ app/models/namespace.rb | 4 +--- app/models/user.rb | 2 -- app/views/groups/_show_nav.html.haml | 7 ++++--- 4 files changed, 7 insertions(+), 8 deletions(-) (limited to 'app') diff --git a/app/controllers/groups_controller.rb b/app/controllers/groups_controller.rb index 1515173d0ac..26a4b884c3a 100644 --- a/app/controllers/groups_controller.rb +++ b/app/controllers/groups_controller.rb @@ -64,6 +64,8 @@ class GroupsController < Groups::ApplicationController end def subgroups + return not_found unless Group.supports_nested_groups? + @nested_groups = GroupsFinder.new(current_user, parent: group).execute @nested_groups = @nested_groups.search(params[:filter_groups]) if params[:filter_groups].present? end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 985395a6fe2..5ceb3d0aee6 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -178,7 +178,7 @@ class Namespace < ActiveRecord::Base # Returns all the ancestors of the current namespaces. def ancestors - return self.class.none if !Group.supports_nested_groups? || !parent_id + return self.class.none unless parent_id Gitlab::GroupHierarchy. new(self.class.where(id: parent_id)). @@ -187,8 +187,6 @@ class Namespace < ActiveRecord::Base # Returns all the descendants of the current namespace. def descendants - return self.class.none unless Group.supports_nested_groups? - Gitlab::GroupHierarchy. new(self.class.where(parent_id: id)). base_and_descendants diff --git a/app/models/user.rb b/app/models/user.rb index 2cf995f8c31..149a80b6083 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -509,8 +509,6 @@ class User < ActiveRecord::Base # Returns a relation of groups the user has access to, including their parent # and child groups (recursively). def all_expanded_groups - return groups unless Group.supports_nested_groups? - Gitlab::GroupHierarchy.new(groups).all_groups end diff --git a/app/views/groups/_show_nav.html.haml b/app/views/groups/_show_nav.html.haml index b2097e88741..35b75bc0923 100644 --- a/app/views/groups/_show_nav.html.haml +++ b/app/views/groups/_show_nav.html.haml @@ -2,6 +2,7 @@ = nav_link(page: group_path(@group)) do = link_to group_path(@group) do Projects - = nav_link(page: subgroups_group_path(@group)) do - = link_to subgroups_group_path(@group) do - Subgroups + - if Group.supports_nested_groups? + = nav_link(page: subgroups_group_path(@group)) do + = link_to subgroups_group_path(@group) do + Subgroups -- cgit v1.2.3 From 893b1eb1d3290a662a01188d2055798778bc442a Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Mon, 22 May 2017 19:51:09 +0300 Subject: Fix: Wiki is not searchable with Guest permissions --- app/services/search_service.rb | 2 +- app/views/search/_category.html.haml | 77 ++++++++++++++++++++---------------- 2 files changed, 43 insertions(+), 36 deletions(-) (limited to 'app') diff --git a/app/services/search_service.rb b/app/services/search_service.rb index 22736c71725..1d4d03a8b7d 100644 --- a/app/services/search_service.rb +++ b/app/services/search_service.rb @@ -12,7 +12,7 @@ class SearchService @project = if params[:project_id].present? the_project = Project.find_by(id: params[:project_id]) - can?(current_user, :download_code, the_project) ? the_project : nil + can?(current_user, :read_project, the_project) ? the_project : nil else nil end diff --git a/app/views/search/_category.html.haml b/app/views/search/_category.html.haml index 059a0d1ac78..7ec4aa9998f 100644 --- a/app/views/search/_category.html.haml +++ b/app/views/search/_category.html.haml @@ -3,41 +3,48 @@ .fade-right= icon('angle-right') %ul.nav-links.search-filter.scrolling-tabs - if @project - %li{ class: active_when(@scope == 'blobs') } - = link_to search_filter_path(scope: 'blobs') do - Code - %span.badge - = @search_results.blobs_count - %li{ class: active_when(@scope == 'issues') } - = link_to search_filter_path(scope: 'issues') do - Issues - %span.badge - = @search_results.issues_count - %li{ class: active_when(@scope == 'merge_requests') } - = link_to search_filter_path(scope: 'merge_requests') do - Merge requests - %span.badge - = @search_results.merge_requests_count - %li{ class: active_when(@scope == 'milestones') } - = link_to search_filter_path(scope: 'milestones') do - Milestones - %span.badge - = @search_results.milestones_count - %li{ class: active_when(@scope == 'notes') } - = link_to search_filter_path(scope: 'notes') do - Comments - %span.badge - = @search_results.notes_count - %li{ class: active_when(@scope == 'wiki_blobs') } - = link_to search_filter_path(scope: 'wiki_blobs') do - Wiki - %span.badge - = @search_results.wiki_blobs_count - %li{ class: active_when(@scope == 'commits') } - = link_to search_filter_path(scope: 'commits') do - Commits - %span.badge - = @search_results.commits_count + - if can?(current_user, :download_code, @project) + %li{ class: active_when(@scope == 'blobs') } + = link_to search_filter_path(scope: 'blobs') do + Code + %span.badge + = @search_results.blobs_count + - if can?(current_user, :read_issue, @project) + %li{ class: active_when(@scope == 'issues') } + = link_to search_filter_path(scope: 'issues') do + Issues + %span.badge + = @search_results.issues_count + - if can?(current_user, :read_merge_request, @project) + %li{ class: active_when(@scope == 'merge_requests') } + = link_to search_filter_path(scope: 'merge_requests') do + Merge requests + %span.badge + = @search_results.merge_requests_count + - if can?(current_user, :read_milestone, @project) + %li{ class: active_when(@scope == 'milestones') } + = link_to search_filter_path(scope: 'milestones') do + Milestones + %span.badge + = @search_results.milestones_count + - if can?(current_user, :read_merge_request, @project) || can?(current_user, :read_issue, @project) + %li{ class: active_when(@scope == 'notes') } + = link_to search_filter_path(scope: 'notes') do + Comments + %span.badge + = @search_results.notes_count + - if can?(current_user, :read_wiki, @project) + %li{ class: active_when(@scope == 'wiki_blobs') } + = link_to search_filter_path(scope: 'wiki_blobs') do + Wiki + %span.badge + = @search_results.wiki_blobs_count + - if can?(current_user, :download_code, @project) + %li{ class: active_when(@scope == 'commits') } + = link_to search_filter_path(scope: 'commits') do + Commits + %span.badge + = @search_results.commits_count - elsif @show_snippets %li{ class: active_when(@scope == 'snippet_blobs') } -- cgit v1.2.3 From a25fb529167f773a634f6034533c24513b15157f Mon Sep 17 00:00:00 2001 From: Bryce Johnson Date: Fri, 19 May 2017 13:33:03 -0400 Subject: Upgrade Remove Source Branch checkbox UX. --- .../components/states/mr_widget_ready_to_merge.js | 8 ++++++-- .../vue_merge_request_widget/stores/mr_widget_store.js | 4 +++- app/views/shared/issuable/form/_merge_params.html.haml | 10 ++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) (limited to 'app') diff --git a/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_ready_to_merge.js b/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_ready_to_merge.js index 74613a1089e..f30c831fd99 100644 --- a/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_ready_to_merge.js +++ b/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_ready_to_merge.js @@ -13,7 +13,7 @@ export default { }, data() { return { - removeSourceBranch: true, + removeSourceBranch: this.mr.shouldRemoveSourceBranch, mergeWhenBuildSucceeds: false, useCommitMessageWithDescription: false, setToMergeWhenPipelineSucceeds: false, @@ -69,6 +69,9 @@ export default { || this.isMakingRequest || this.mr.preventMerge); }, + isRemoveSourceBranchButtonDisabled() { + return this.isMergeButtonDisabled || !this.mr.canRemoveSourceBranch; + }, shouldShowSquashBeforeMerge() { const { commitsCount, enableSquashBeforeMerge } = this.mr; return enableSquashBeforeMerge && commitsCount > 1; @@ -252,8 +255,9 @@ export default {