diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-10-23 18:08:42 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-10-23 18:08:42 +0300 |
commit | 9086e66ee72527839053ec6db19ed321a3b3a61b (patch) | |
tree | f2904493d8539228823f15cf4126eb8c4ffa79e3 /lib | |
parent | b17c74a7e2cf516ed189e525291cb096411b7ac5 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
17 files changed, 514 insertions, 20 deletions
diff --git a/lib/bulk_imports/clients/graphql.rb b/lib/bulk_imports/clients/graphql.rb new file mode 100644 index 00000000000..89698cb53ef --- /dev/null +++ b/lib/bulk_imports/clients/graphql.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module BulkImports + module Clients + class Graphql + attr_reader :client + + delegate :query, :parse, :execute, to: :client + + def initialize(url: Gitlab::COM_URL, token: nil) + @url = Gitlab::Utils.append_path(url, '/api/graphql') + @token = token + @client = Graphlient::Client.new( + @url, + request_headers + ) + end + + def request_headers + return {} unless @token + + { + headers: { + 'Content-Type' => 'application/json', + 'Authorization' => "Bearer #{@token}" + } + } + end + end + end +end diff --git a/lib/gitlab/bulk_import/client.rb b/lib/bulk_imports/clients/http.rb index c6e77a158cd..39f56fcc114 100644 --- a/lib/gitlab/bulk_import/client.rb +++ b/lib/bulk_imports/clients/http.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -module Gitlab - module BulkImport - class Client +module BulkImports + module Clients + class Http API_VERSION = 'v4'.freeze DEFAULT_PAGE = 1.freeze DEFAULT_PER_PAGE = 30.freeze diff --git a/lib/bulk_imports/common/extractors/graphql_extractor.rb b/lib/bulk_imports/common/extractors/graphql_extractor.rb new file mode 100644 index 00000000000..571be747dca --- /dev/null +++ b/lib/bulk_imports/common/extractors/graphql_extractor.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Extractors + class GraphqlExtractor + def initialize(query) + @query = query[:query] + @query_string = @query.to_s + @variables = @query.variables + end + + def extract(context) + @context = context + + Enumerator.new do |yielder| + context.entities.each do |entity| + result = graphql_client.execute(parsed_query, query_variables(entity)) + + yielder << result.original_hash.deep_dup + end + end + end + + private + + def graphql_client + @graphql_client ||= BulkImports::Clients::Graphql.new( + url: @context.configuration.url, + token: @context.configuration.access_token + ) + end + + def parsed_query + @parsed_query ||= graphql_client.parse(@query.to_s) + end + + def query_variables(entity) + return unless @variables + + @variables.transform_values do |entity_attribute| + entity.public_send(entity_attribute) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + end + end +end diff --git a/lib/bulk_imports/common/transformers/graphql_cleaner_transformer.rb b/lib/bulk_imports/common/transformers/graphql_cleaner_transformer.rb new file mode 100644 index 00000000000..dce0fac6999 --- /dev/null +++ b/lib/bulk_imports/common/transformers/graphql_cleaner_transformer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# Cleanup GraphQL original response hash from unnecessary nesting +# 1. Remove ['data']['group'] or ['data']['project'] hash nesting +# 2. Remove ['edges'] & ['nodes'] array wrappings +# 3. Remove ['node'] hash wrapping +# +# @example +# data = {"data"=>{"group"=> { +# "name"=>"test", +# "fullName"=>"test", +# "description"=>"test", +# "labels"=>{"edges"=>[{"node"=>{"title"=>"label1"}}, {"node"=>{"title"=>"label2"}}, {"node"=>{"title"=>"label3"}}]}}}} +# +# BulkImports::Common::Transformers::GraphqlCleanerTransformer.new.transform(nil, data) +# +# {"name"=>"test", "fullName"=>"test", "description"=>"test", "labels"=>[{"title"=>"label1"}, {"title"=>"label2"}, {"title"=>"label3"}]} +module BulkImports + module Common + module Transformers + class GraphqlCleanerTransformer + EDGES = 'edges' + NODE = 'node' + + def initialize(options = {}) + @options = options + end + + def transform(_, data) + return data unless data.is_a?(Hash) + + data = data.dig('data', 'group') || data.dig('data', 'project') || data + + clean_edges_and_nodes(data) + end + + def clean_edges_and_nodes(data) + case data + when Array + data.map(&method(:clean_edges_and_nodes)) + when Hash + if data.key?(NODE) + clean_edges_and_nodes(data[NODE]) + else + data.transform_values { |value| clean_edges_and_nodes(value.try(:fetch, EDGES, value) || value) } + end + else + data + end + end + end + end + end +end diff --git a/lib/bulk_imports/common/transformers/underscorify_keys_transformer.rb b/lib/bulk_imports/common/transformers/underscorify_keys_transformer.rb new file mode 100644 index 00000000000..b32ab28fdbb --- /dev/null +++ b/lib/bulk_imports/common/transformers/underscorify_keys_transformer.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Transformers + class UnderscorifyKeysTransformer + def initialize(options = {}) + @options = options + end + + def transform(_, data) + data.deep_transform_keys do |key| + key.to_s.underscore + end + end + end + end + end +end diff --git a/lib/bulk_imports/groups/graphql/get_group_query.rb b/lib/bulk_imports/groups/graphql/get_group_query.rb new file mode 100644 index 00000000000..c50b99aae4e --- /dev/null +++ b/lib/bulk_imports/groups/graphql/get_group_query.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Graphql + module GetGroupQuery + extend self + + def to_s + <<-'GRAPHQL' + query($full_path: ID!) { + group(fullPath: $full_path) { + name + path + fullPath + description + visibility + emailsDisabled + lfsEnabled + mentionsDisabled + projectCreationLevel + requestAccessEnabled + requireTwoFactorAuthentication + shareWithGroupLock + subgroupCreationLevel + twoFactorGracePeriod + } + } + GRAPHQL + end + + def variables + { full_path: :source_full_path } + end + end + end + end +end diff --git a/lib/bulk_imports/groups/loaders/group_loader.rb b/lib/bulk_imports/groups/loaders/group_loader.rb new file mode 100644 index 00000000000..394f0ee10ec --- /dev/null +++ b/lib/bulk_imports/groups/loaders/group_loader.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Loaders + class GroupLoader + def initialize(options = {}) + @options = options + end + + def load(context, data) + return unless user_can_create_group?(context.current_user, data) + + ::Groups::CreateService.new(context.current_user, data).execute + end + + private + + def user_can_create_group?(current_user, data) + if data['parent_id'] + parent = Namespace.find_by_id(data['parent_id']) + + Ability.allowed?(current_user, :create_subgroup, parent) + else + Ability.allowed?(current_user, :create_group) + end + end + end + end + end +end diff --git a/lib/bulk_imports/groups/pipelines/group_pipeline.rb b/lib/bulk_imports/groups/pipelines/group_pipeline.rb new file mode 100644 index 00000000000..2b7d0ef7658 --- /dev/null +++ b/lib/bulk_imports/groups/pipelines/group_pipeline.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Pipelines + class GroupPipeline + include Pipeline + + extractor Common::Extractors::GraphqlExtractor, query: Graphql::GetGroupQuery + + transformer Common::Transformers::GraphqlCleanerTransformer + transformer Common::Transformers::UnderscorifyKeysTransformer + transformer Groups::Transformers::GroupAttributesTransformer + + loader Groups::Loaders::GroupLoader + end + end + end +end diff --git a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb new file mode 100644 index 00000000000..c3937cfe652 --- /dev/null +++ b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Transformers + class GroupAttributesTransformer + def initialize(options = {}) + @options = options + end + + def transform(context, data) + import_entity = find_by_full_path(data['full_path'], context.entities) + + data + .then { |data| transform_name(import_entity, data) } + .then { |data| transform_path(import_entity, data) } + .then { |data| transform_full_path(data) } + .then { |data| transform_parent(context, import_entity, data) } + .then { |data| transform_visibility_level(data) } + .then { |data| transform_project_creation_level(data) } + .then { |data| transform_subgroup_creation_level(data) } + end + + private + + def transform_name(import_entity, data) + data['name'] = import_entity.destination_name + data + end + + def transform_path(import_entity, data) + data['path'] = import_entity.destination_name.parameterize + data + end + + def transform_full_path(data) + data.delete('full_path') + data + end + + def transform_parent(context, import_entity, data) + current_user = context.current_user + namespace = Namespace.find_by_full_path(import_entity.destination_namespace) + + return data if namespace == current_user.namespace + + data['parent_id'] = namespace.id + data + end + + def transform_visibility_level(data) + visibility = data['visibility'] + + return data unless visibility.present? + + data['visibility_level'] = Gitlab::VisibilityLevel.string_options[visibility] + data.delete('visibility') + data + end + + def transform_project_creation_level(data) + project_creation_level = data['project_creation_level'] + + return data unless project_creation_level.present? + + data['project_creation_level'] = Gitlab::Access.project_creation_string_options[project_creation_level] + data + end + + def transform_subgroup_creation_level(data) + subgroup_creation_level = data['subgroup_creation_level'] + + return data unless subgroup_creation_level.present? + + data['subgroup_creation_level'] = Gitlab::Access.subgroup_creation_string_options[subgroup_creation_level] + data + end + + def find_by_full_path(full_path, entities) + entities.find { |entity| entity.source_full_path == full_path } + end + end + end + end +end diff --git a/lib/bulk_imports/importers/group_importer.rb b/lib/bulk_imports/importers/group_importer.rb new file mode 100644 index 00000000000..f053177d9fb --- /dev/null +++ b/lib/bulk_imports/importers/group_importer.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +# Imports a top level group into a destination +# Optionally imports into parent group +# Entity must be of type: 'group' & have parent_id: nil +# Subgroups not handled yet +module BulkImports + module Importers + class GroupImporter + def initialize(entity_id) + @entity_id = entity_id + end + + def execute + return if entity.parent + + bulk_import = entity.bulk_import + configuration = bulk_import.configuration + + context = BulkImports::Pipeline::Context.new( + current_user: bulk_import.user, + entities: [entity], + configuration: configuration + ) + + BulkImports::Groups::Pipelines::GroupPipeline.new.run(context) + end + + def entity + @entity ||= BulkImports::Entity.find(@entity_id) + end + end + end +end diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb new file mode 100644 index 00000000000..70e6030ea2c --- /dev/null +++ b/lib/bulk_imports/pipeline.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module BulkImports + module Pipeline + extend ActiveSupport::Concern + + included do + include Attributes + include Runner + end + end +end diff --git a/lib/bulk_imports/pipeline/attributes.rb b/lib/bulk_imports/pipeline/attributes.rb new file mode 100644 index 00000000000..ebfbaf6f6ba --- /dev/null +++ b/lib/bulk_imports/pipeline/attributes.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module BulkImports + module Pipeline + module Attributes + extend ActiveSupport::Concern + include Gitlab::ClassAttributes + + class_methods do + def extractor(klass, options = nil) + add_attribute(:extractors, klass, options) + end + + def transformer(klass, options = nil) + add_attribute(:transformers, klass, options) + end + + def loader(klass, options = nil) + add_attribute(:loaders, klass, options) + end + + def add_attribute(sym, klass, options) + class_attributes[sym] ||= [] + class_attributes[sym] << { klass: klass, options: options } + end + + def extractors + class_attributes[:extractors] + end + + def transformers + class_attributes[:transformers] + end + + def loaders + class_attributes[:loaders] + end + end + end + end +end diff --git a/lib/bulk_imports/pipeline/context.rb b/lib/bulk_imports/pipeline/context.rb new file mode 100644 index 00000000000..903f474ebbb --- /dev/null +++ b/lib/bulk_imports/pipeline/context.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module BulkImports + module Pipeline + class Context + include Gitlab::Utils::LazyAttributes + + Attribute = Struct.new(:name, :type) + + PIPELINE_ATTRIBUTES = [ + Attribute.new(:current_user, User), + Attribute.new(:entities, Array), + Attribute.new(:configuration, ::BulkImports::Configuration) + ].freeze + + def initialize(args) + assign_attributes(args) + end + + private + + PIPELINE_ATTRIBUTES.each do |attr| + lazy_attr_reader attr.name, type: attr.type + end + + def assign_attributes(values) + values.slice(*PIPELINE_ATTRIBUTES.map(&:name)).each do |name, value| + instance_variable_set("@#{name}", value) + end + end + end + end +end diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb new file mode 100644 index 00000000000..cf94b500612 --- /dev/null +++ b/lib/bulk_imports/pipeline/runner.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module BulkImports + module Pipeline + module Runner + extend ActiveSupport::Concern + + included do + attr_reader :extractors, :transformers, :loaders + + def initialize + @extractors = self.class.extractors.map(&method(:instantiate)) + @transformers = self.class.transformers.map(&method(:instantiate)) + @loaders = self.class.loaders.map(&method(:instantiate)) + + super + end + + def run(context) + extractors.each do |extractor| + extractor.extract(context).each do |entry| + transformers.each do |transformer| + entry = transformer.transform(context, entry) + end + + loaders.each do |loader| + loader.load(context, entry) + end + end + end + end + + def instantiate(class_config) + class_config[:klass].new(class_config[:options]) + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb index ecc05d9654a..82ea1ce26fb 100644 --- a/lib/gitlab/database/partitioning/monthly_strategy.rb +++ b/lib/gitlab/database/partitioning/monthly_strategy.rb @@ -17,23 +17,8 @@ module Gitlab end def current_partitions - result = connection.select_all(<<~SQL) - select - pg_class.relname, - parent_class.relname as base_table, - pg_get_expr(pg_class.relpartbound, inhrelid) as condition - from pg_class - inner join pg_inherits i on pg_class.oid = inhrelid - inner join pg_class parent_class on parent_class.oid = inhparent - inner join pg_namespace ON pg_namespace.oid = pg_class.relnamespace - where pg_namespace.nspname = #{connection.quote(Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA)} - and parent_class.relname = #{connection.quote(table_name)} - and pg_class.relispartition - order by pg_class.relname - SQL - - result.map do |record| - TimePartition.from_sql(table_name, record['relname'], record['condition']) + Gitlab::Database::PostgresPartition.for_parent_table(table_name).map do |partition| + TimePartition.from_sql(table_name, partition.name, partition.condition) end end diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb new file mode 100644 index 00000000000..0986372586b --- /dev/null +++ b/lib/gitlab/database/postgres_partition.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class PostgresPartition < ActiveRecord::Base + self.primary_key = :identifier + + belongs_to :postgres_partitioned_table, foreign_key: 'parent_identifier', primary_key: 'identifier' + + scope :by_identifier, ->(identifier) do + raise ArgumentError, "Partition name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + + find(identifier) + end + + scope :for_parent_table, ->(name) { where("parent_identifier = concat(current_schema(), '.', ?)", name).order(:name) } + + def to_s + name + end + end + end +end diff --git a/lib/gitlab/database/postgres_partitioned_table.rb b/lib/gitlab/database/postgres_partitioned_table.rb index 5856f63213a..58385821bde 100644 --- a/lib/gitlab/database/postgres_partitioned_table.rb +++ b/lib/gitlab/database/postgres_partitioned_table.rb @@ -7,6 +7,8 @@ module Gitlab self.primary_key = :identifier + has_many :postgres_partitions, foreign_key: 'parent_identifier', primary_key: 'identifier' + scope :by_identifier, ->(identifier) do raise ArgumentError, "Table name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ |