diff options
Diffstat (limited to 'lib/gitlab/pagination')
6 files changed, 113 insertions, 46 deletions
diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb index 39d6e016ac7..53faf8469f2 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb @@ -6,10 +6,7 @@ module Gitlab module InOperatorOptimization # rubocop: disable CodeReuse/ActiveRecord class QueryBuilder - UnsupportedScopeOrder = Class.new(StandardError) - RECURSIVE_CTE_NAME = 'recursive_keyset_cte' - RECORDS_COLUMN = 'records' # This class optimizes slow database queries (PostgreSQL specific) where the # IN SQL operator is used with sorting. @@ -42,26 +39,19 @@ module Gitlab # > array_mapping_scope: array_mapping_scope, # > finder_query: finder_query # > ).execute.limit(20) - def initialize(scope:, array_scope:, array_mapping_scope:, finder_query:, values: {}) + def initialize(scope:, array_scope:, array_mapping_scope:, finder_query: nil, values: {}) @scope, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(scope) - unless success - error_message = <<~MSG - The order on the scope does not support keyset pagination. You might need to define a custom Order object.\n - See https://docs.gitlab.com/ee/development/database/keyset_pagination.html#complex-order-configuration\n - Or the Gitlab::Pagination::Keyset::Order class for examples - MSG - raise(UnsupportedScopeOrder, error_message) - end + raise(UnsupportedScopeOrder) unless success @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) @array_scope = array_scope @array_mapping_scope = array_mapping_scope - @finder_query = finder_query @values = values @model = @scope.model @table_name = @model.table_name @arel_table = @model.arel_table + @finder_strategy = finder_query.present? ? Strategies::RecordLoaderStrategy.new(finder_query, model, order_by_columns) : Strategies::OrderValuesLoaderStrategy.new(model, order_by_columns) end def execute @@ -74,7 +64,7 @@ module Gitlab q = cte .apply_to(model.where({}) .with(selector_cte.to_arel)) - .select(result_collector_final_projections) + .select(finder_strategy.final_projections) .where("count <> 0") # filter out the initializer row model.from(q.arel.as(table_name)) @@ -82,13 +72,13 @@ module Gitlab private - attr_reader :array_scope, :scope, :order, :array_mapping_scope, :finder_query, :values, :model, :table_name, :arel_table + attr_reader :array_scope, :scope, :order, :array_mapping_scope, :finder_strategy, :values, :model, :table_name, :arel_table def initializer_query array_column_names = array_scope_columns.array_aggregated_column_names + order_by_columns.array_aggregated_column_names projections = [ - *result_collector_initializer_columns, + *finder_strategy.initializer_columns, *array_column_names, '0::bigint AS count' ] @@ -156,7 +146,7 @@ module Gitlab order_column_value_arrays = order_by_columns.replace_value_in_array_by_position_expressions select = [ - *result_collector_columns, + *finder_strategy.columns, *array_column_list, *order_column_value_arrays, "#{RECURSIVE_CTE_NAME}.count + 1" @@ -254,23 +244,6 @@ module Gitlab end.join(", ") end - def result_collector_initializer_columns - ["NULL::#{table_name} AS #{RECORDS_COLUMN}"] - end - - def result_collector_columns - query = finder_query - .call(*order_by_columns.array_lookup_expressions_by_position(RECURSIVE_CTE_NAME)) - .select("#{table_name}") - .limit(1) - - ["(#{query.to_sql})"] - end - - def result_collector_final_projections - ["(#{RECORDS_COLUMN}).*"] - end - def array_scope_columns @array_scope_columns ||= ArrayScopeColumns.new(array_scope.select_values) end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb new file mode 100644 index 00000000000..fc2b56048f6 --- /dev/null +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + module InOperatorOptimization + module Strategies + class OrderValuesLoaderStrategy + def initialize(model, order_by_columns) + @model = model + @order_by_columns = order_by_columns + end + + def initializer_columns + order_by_columns.map do |column| + column_name = column.original_column_name.to_s + type = model.columns_hash[column_name].sql_type + "NULL::#{type} AS #{column_name}" + end + end + + def columns + order_by_columns.array_lookup_expressions_by_position(QueryBuilder::RECURSIVE_CTE_NAME) + end + + def final_projections + order_by_columns.map(&:original_column_name) + end + + private + + attr_reader :model, :order_by_columns + end + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb new file mode 100644 index 00000000000..b12c33d6e51 --- /dev/null +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + module InOperatorOptimization + module Strategies + class RecordLoaderStrategy + RECORDS_COLUMN = 'records' + + def initialize(finder_query, model, order_by_columns) + @finder_query = finder_query + @order_by_columns = order_by_columns + @table_name = model.table_name + end + + def initializer_columns + ["NULL::#{table_name} AS #{RECORDS_COLUMN}"] + end + + def columns + query = finder_query + .call(*order_by_columns.array_lookup_expressions_by_position(QueryBuilder::RECURSIVE_CTE_NAME)) + .select("#{table_name}") + .limit(1) + + ["(#{query.to_sql})"] + end + + def final_projections + ["(#{RECORDS_COLUMN}).*"] + end + + private + + attr_reader :finder_query, :order_by_columns, :table_name + end + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/iterator.rb b/lib/gitlab/pagination/keyset/iterator.rb index 14807fa37c4..bcd17fd0d34 100644 --- a/lib/gitlab/pagination/keyset/iterator.rb +++ b/lib/gitlab/pagination/keyset/iterator.rb @@ -4,12 +4,11 @@ module Gitlab module Pagination module Keyset class Iterator - UnsupportedScopeOrder = Class.new(StandardError) - - def initialize(scope:, use_union_optimization: true, in_operator_optimization_options: nil) + def initialize(scope:, cursor: {}, use_union_optimization: true, in_operator_optimization_options: nil) @scope, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(scope) - raise(UnsupportedScopeOrder, 'The order on the scope does not support keyset pagination') unless success + raise(UnsupportedScopeOrder) unless success + @cursor = cursor @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) @use_union_optimization = in_operator_optimization_options ? false : use_union_optimization @in_operator_optimization_options = in_operator_optimization_options @@ -17,11 +16,9 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def each_batch(of: 1000) - cursor_attributes = {} - loop do current_scope = scope.dup - relation = order.apply_cursor_conditions(current_scope, cursor_attributes, keyset_options) + relation = order.apply_cursor_conditions(current_scope, cursor, keyset_options) relation = relation.reorder(order) unless @in_operator_optimization_options relation = relation.limit(of) @@ -30,14 +27,14 @@ module Gitlab last_record = relation.last break unless last_record - cursor_attributes = order.cursor_attributes_for_node(last_record) + @cursor = order.cursor_attributes_for_node(last_record) end end # rubocop: enable CodeReuse/ActiveRecord private - attr_reader :scope, :order + attr_reader :scope, :cursor, :order def keyset_options { diff --git a/lib/gitlab/pagination/keyset/paginator.rb b/lib/gitlab/pagination/keyset/paginator.rb index 1c71549d86a..1ff4589d8e1 100644 --- a/lib/gitlab/pagination/keyset/paginator.rb +++ b/lib/gitlab/pagination/keyset/paginator.rb @@ -19,8 +19,6 @@ module Gitlab FORWARD_DIRECTION = 'n' BACKWARD_DIRECTION = 'p' - UnsupportedScopeOrder = Class.new(StandardError) - # scope - ActiveRecord::Relation object with order by clause # cursor - Encoded cursor attributes as String. Empty value will requests the first page. # per_page - Number of items per page. @@ -167,7 +165,7 @@ module Gitlab def build_scope(scope) keyset_aware_scope, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(scope) - raise(UnsupportedScopeOrder, 'The order on the scope does not support keyset pagination') unless success + raise(UnsupportedScopeOrder) unless success keyset_aware_scope end diff --git a/lib/gitlab/pagination/keyset/unsupported_scope_order.rb b/lib/gitlab/pagination/keyset/unsupported_scope_order.rb new file mode 100644 index 00000000000..1571c00e130 --- /dev/null +++ b/lib/gitlab/pagination/keyset/unsupported_scope_order.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + class UnsupportedScopeOrder < StandardError + DEFAULT_ERROR_MESSAGE = <<~MSG + The order on the scope does not support keyset pagination. You might need to define a custom Order object.\n + See https://docs.gitlab.com/ee/development/database/keyset_pagination.html#complex-order-configuration\n + Or the Gitlab::Pagination::Keyset::Order class for examples + MSG + + def message + DEFAULT_ERROR_MESSAGE + end + end + end + end +end |