diff options
Diffstat (limited to 'spec/lib/gitlab/pagination/keyset')
8 files changed, 497 insertions, 0 deletions
diff --git a/spec/lib/gitlab/pagination/keyset/column_order_definition_spec.rb b/spec/lib/gitlab/pagination/keyset/column_order_definition_spec.rb index 6e9e987f90c..69384e0c501 100644 --- a/spec/lib/gitlab/pagination/keyset/column_order_definition_spec.rb +++ b/spec/lib/gitlab/pagination/keyset/column_order_definition_spec.rb @@ -185,4 +185,25 @@ RSpec.describe Gitlab::Pagination::Keyset::ColumnOrderDefinition do end end end + + describe "#order_direction_as_sql_string" do + let(:nulls_last_order) do + described_class.new( + attribute_name: :name, + column_expression: Project.arel_table[:name], + order_expression: Gitlab::Database.nulls_last_order('merge_request_metrics.merged_at', :desc), + reversed_order_expression: Gitlab::Database.nulls_first_order('merge_request_metrics.merged_at', :asc), + order_direction: :desc, + nullable: :nulls_last, # null values are always last + distinct: false + ) + end + + it { expect(project_name_column.order_direction_as_sql_string).to eq('ASC') } + it { expect(project_name_column.reverse.order_direction_as_sql_string).to eq('DESC') } + it { expect(project_name_lower_column.order_direction_as_sql_string).to eq('DESC') } + it { expect(project_name_lower_column.reverse.order_direction_as_sql_string).to eq('ASC') } + it { expect(nulls_last_order.order_direction_as_sql_string).to eq('DESC NULLS LAST') } + it { expect(nulls_last_order.reverse.order_direction_as_sql_string).to eq('ASC NULLS FIRST') } + end end diff --git a/spec/lib/gitlab/pagination/keyset/cursor_based_request_context_spec.rb b/spec/lib/gitlab/pagination/keyset/cursor_based_request_context_spec.rb new file mode 100644 index 00000000000..79de6f230ec --- /dev/null +++ b/spec/lib/gitlab/pagination/keyset/cursor_based_request_context_spec.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Pagination::Keyset::CursorBasedRequestContext do + let(:params) { { per_page: 2, cursor: 'eyJuYW1lIjoiR2l0TGFiIEluc3RhbmNlIiwiaWQiOiI1MiIsIl9rZCI6Im4ifQ==', order_by: :name, sort: :asc } } + let(:request) { double('request', url: 'http://localhost') } + let(:request_context) { double('request_context', header: nil, params: params, request: request) } + + describe '#per_page' do + subject(:per_page) { described_class.new(request_context).per_page } + + it { is_expected.to eq 2 } + end + + describe '#cursor' do + subject(:cursor) { described_class.new(request_context).cursor } + + it { is_expected.to eq 'eyJuYW1lIjoiR2l0TGFiIEluc3RhbmNlIiwiaWQiOiI1MiIsIl9rZCI6Im4ifQ==' } + end + + describe '#order_by' do + subject(:order_by) { described_class.new(request_context).order_by } + + it { is_expected.to eq({ name: :asc }) } + end + + describe '#apply_headers' do + let(:request) { double('request', url: "http://#{Gitlab.config.gitlab.host}/api/v4/projects?per_page=3") } + let(:params) { { per_page: 3 } } + let(:request_context) { double('request_context', header: nil, params: params, request: request) } + let(:cursor_for_next_page) { 'eyJuYW1lIjoiSDVicCIsImlkIjoiMjgiLCJfa2QiOiJuIn0=' } + + subject(:apply_headers) { described_class.new(request_context).apply_headers(cursor_for_next_page) } + + it 'sets Link header with same host/path as the original request' do + orig_uri = URI.parse(request_context.request.url) + + expect(request_context).to receive(:header).once do |name, header| + first_link, _ = /<([^>]+)>; rel="next"/.match(header).captures + + uri = URI.parse(first_link) + + expect(name).to eq('Link') + expect(uri.host).to eq(orig_uri.host) + expect(uri.path).to eq(orig_uri.path) + end + + apply_headers + end + + it 'sets Link header with a cursor to the next page' do + orig_uri = URI.parse(request_context.request.url) + + expect(request_context).to receive(:header).once do |name, header| + first_link, _ = /<([^>]+)>; rel="next"/.match(header).captures + + query = CGI.parse(URI.parse(first_link).query) + + expect(name).to eq('Link') + expect(query.except('cursor')).to eq(CGI.parse(orig_uri.query).except('cursor')) + expect(query['cursor']).to eq([cursor_for_next_page]) + end + + apply_headers + end + end +end diff --git a/spec/lib/gitlab/pagination/keyset/cursor_pager_spec.rb b/spec/lib/gitlab/pagination/keyset/cursor_pager_spec.rb new file mode 100644 index 00000000000..783e728b34c --- /dev/null +++ b/spec/lib/gitlab/pagination/keyset/cursor_pager_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Pagination::Keyset::CursorPager do + let(:relation) { Group.all.order(:name, :id) } + let(:per_page) { 3 } + let(:params) { { cursor: nil, per_page: per_page } } + let(:request_context) { double('request_context', params: params) } + let(:cursor_based_request_context) { Gitlab::Pagination::Keyset::CursorBasedRequestContext.new(request_context) } + + before_all do + create_list(:group, 7) + end + + describe '#paginate' do + subject(:paginated_result) { described_class.new(cursor_based_request_context).paginate(relation) } + + it 'returns the limited relation' do + expect(paginated_result).to eq(relation.limit(per_page)) + end + end + + describe '#finalize' do + subject(:finalize) do + service = described_class.new(cursor_based_request_context) + # we need to do this because `finalize` can only be called + # after `paginate` is called. Otherwise the `paginator` object won't be set. + service.paginate(relation) + service.finalize + end + + it 'passes information about next page to request' do + cursor_for_next_page = relation.keyset_paginate(**params).cursor_for_next_page + + expect_next_instance_of(Gitlab::Pagination::Keyset::HeaderBuilder, request_context) do |builder| + expect(builder).to receive(:add_next_page_header).with({ cursor: cursor_for_next_page }) + end + + finalize + end + + context 'when retrieving the last page' do + let(:relation) { Group.where('id > ?', Group.maximum(:id) - per_page).order(:name, :id) } + + it 'does not build information about the next page' do + expect(Gitlab::Pagination::Keyset::HeaderBuilder).not_to receive(:new) + + finalize + end + end + + context 'when retrieving an empty page' do + let(:relation) { Group.where('id > ?', Group.maximum(:id) + 1).order(:name, :id) } + + it 'does not build information about the next page' do + expect(Gitlab::Pagination::Keyset::HeaderBuilder).not_to receive(:new) + + finalize + end + end + end +end diff --git a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns_spec.rb b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns_spec.rb new file mode 100644 index 00000000000..2cebf0d9473 --- /dev/null +++ b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::ArrayScopeColumns do + let(:columns) { [:relative_position, :id] } + + subject(:array_scope_columns) { described_class.new(columns) } + + it 'builds array column names' do + expect(array_scope_columns.array_aggregated_column_names).to eq(%w[array_cte_relative_position_array array_cte_id_array]) + end + + context 'when no columns are given' do + let(:columns) { [] } + + it { expect { array_scope_columns }.to raise_error /No array columns were given/ } + end +end diff --git a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/column_data_spec.rb b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/column_data_spec.rb new file mode 100644 index 00000000000..4f200c9096f --- /dev/null +++ b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/column_data_spec.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::ColumnData do + subject(:column_data) { described_class.new('id', 'issue_id', Issue.arel_table) } + + describe '#array_aggregated_column_name' do + it { expect(column_data.array_aggregated_column_name).to eq('issues_id_array') } + end + + describe '#projection' do + it 'returns the Arel projection for the column with a new alias' do + expect(column_data.projection.to_sql).to eq('"issues"."id" AS issue_id') + end + end + + it 'accepts symbols for original_column_name and as' do + column_data = described_class.new(:id, :issue_id, Issue.arel_table) + + expect(column_data.projection.to_sql).to eq('"issues"."id" AS issue_id') + end +end diff --git a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_columns_spec.rb b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_columns_spec.rb new file mode 100644 index 00000000000..f4fa14e2261 --- /dev/null +++ b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_columns_spec.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::OrderByColumns do + let(:columns) do + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: :relative_position, + order_expression: Issue.arel_table[:relative_position].desc + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: :id, + order_expression: Issue.arel_table[:id].desc + ) + ] + end + + subject(:order_by_columns) { described_class.new(columns, Issue.arel_table) } + + describe '#array_aggregated_column_names' do + it { expect(order_by_columns.array_aggregated_column_names).to eq(%w[issues_relative_position_array issues_id_array]) } + end + + describe '#original_column_names' do + it { expect(order_by_columns.original_column_names).to eq(%w[relative_position id]) } + end + + describe '#cursor_values' do + it 'returns the keyset pagination cursor values from the column arrays as SQL expression' do + expect(order_by_columns.cursor_values('tbl')).to eq({ + "id" => "tbl.issues_id_array[position]", + "relative_position" => "tbl.issues_relative_position_array[position]" + }) + end + end +end diff --git a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder_spec.rb b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder_spec.rb new file mode 100644 index 00000000000..4ce51e37685 --- /dev/null +++ b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder_spec.rb @@ -0,0 +1,225 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::QueryBuilder do + let_it_be(:two_weeks_ago) { 2.weeks.ago } + let_it_be(:three_weeks_ago) { 3.weeks.ago } + let_it_be(:four_weeks_ago) { 4.weeks.ago } + let_it_be(:five_weeks_ago) { 5.weeks.ago } + + let_it_be(:top_level_group) { create(:group) } + let_it_be(:sub_group_1) { create(:group, parent: top_level_group) } + let_it_be(:sub_group_2) { create(:group, parent: top_level_group) } + let_it_be(:sub_sub_group_1) { create(:group, parent: sub_group_2) } + + let_it_be(:project_1) { create(:project, group: top_level_group) } + let_it_be(:project_2) { create(:project, group: top_level_group) } + + let_it_be(:project_3) { create(:project, group: sub_group_1) } + let_it_be(:project_4) { create(:project, group: sub_group_2) } + + let_it_be(:project_5) { create(:project, group: sub_sub_group_1) } + + let_it_be(:issues) do + [ + create(:issue, project: project_1, created_at: three_weeks_ago, relative_position: 5), + create(:issue, project: project_1, created_at: two_weeks_ago), + create(:issue, project: project_2, created_at: two_weeks_ago, relative_position: 15), + create(:issue, project: project_2, created_at: two_weeks_ago), + create(:issue, project: project_3, created_at: four_weeks_ago), + create(:issue, project: project_4, created_at: five_weeks_ago, relative_position: 10), + create(:issue, project: project_5, created_at: four_weeks_ago) + ] + end + + shared_examples 'correct ordering examples' do + let(:iterator) do + Gitlab::Pagination::Keyset::Iterator.new( + scope: scope.limit(batch_size), + in_operator_optimization_options: in_operator_optimization_options + ) + end + + it 'returns records in correct order' do + all_records = [] + iterator.each_batch(of: batch_size) do |records| + all_records.concat(records) + end + + expect(all_records).to eq(expected_order) + end + end + + context 'when ordering by issues.id DESC' do + let(:scope) { Issue.order(id: :desc) } + let(:expected_order) { issues.sort_by(&:id).reverse } + + let(:in_operator_optimization_options) do + { + array_scope: Project.where(namespace_id: top_level_group.self_and_descendants.select(:id)).select(:id), + array_mapping_scope: -> (id_expression) { Issue.where(Issue.arel_table[:project_id].eq(id_expression)) }, + finder_query: -> (id_expression) { Issue.where(Issue.arel_table[:id].eq(id_expression)) } + } + end + + context 'when iterating records one by one' do + let(:batch_size) { 1 } + + it_behaves_like 'correct ordering examples' + end + + context 'when iterating records with LIMIT 3' do + let(:batch_size) { 3 } + + it_behaves_like 'correct ordering examples' + end + + context 'when loading records at once' do + let(:batch_size) { issues.size + 1 } + + it_behaves_like 'correct ordering examples' + end + end + + context 'when ordering by issues.relative_position DESC NULLS LAST, id DESC' do + let(:scope) { Issue.order(order) } + let(:expected_order) { scope.to_a } + + let(:order) do + # NULLS LAST ordering requires custom Order object for keyset pagination: + # https://docs.gitlab.com/ee/development/database/keyset_pagination.html#complex-order-configuration + Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: :relative_position, + column_expression: Issue.arel_table[:relative_position], + order_expression: Gitlab::Database.nulls_last_order('relative_position', :desc), + reversed_order_expression: Gitlab::Database.nulls_first_order('relative_position', :asc), + order_direction: :desc, + nullable: :nulls_last, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: :id, + order_expression: Issue.arel_table[:id].desc, + nullable: :not_nullable, + distinct: true + ) + ]) + end + + let(:in_operator_optimization_options) do + { + array_scope: Project.where(namespace_id: top_level_group.self_and_descendants.select(:id)).select(:id), + array_mapping_scope: -> (id_expression) { Issue.where(Issue.arel_table[:project_id].eq(id_expression)) }, + finder_query: -> (_relative_position_expression, id_expression) { Issue.where(Issue.arel_table[:id].eq(id_expression)) } + } + end + + context 'when iterating records one by one' do + let(:batch_size) { 1 } + + it_behaves_like 'correct ordering examples' + end + + context 'when iterating records with LIMIT 3' do + let(:batch_size) { 3 } + + it_behaves_like 'correct ordering examples' + end + end + + context 'when ordering by issues.created_at DESC, issues.id ASC' do + let(:scope) { Issue.order(created_at: :desc, id: :asc) } + let(:expected_order) { issues.sort_by { |issue| [issue.created_at.to_f * -1, issue.id] } } + + let(:in_operator_optimization_options) do + { + array_scope: Project.where(namespace_id: top_level_group.self_and_descendants.select(:id)).select(:id), + array_mapping_scope: -> (id_expression) { Issue.where(Issue.arel_table[:project_id].eq(id_expression)) }, + finder_query: -> (_created_at_expression, id_expression) { Issue.where(Issue.arel_table[:id].eq(id_expression)) } + } + end + + context 'when iterating records one by one' do + let(:batch_size) { 1 } + + it_behaves_like 'correct ordering examples' + end + + context 'when iterating records with LIMIT 3' do + let(:batch_size) { 3 } + + it_behaves_like 'correct ordering examples' + end + + context 'when loading records at once' do + let(:batch_size) { issues.size + 1 } + + it_behaves_like 'correct ordering examples' + end + end + + context 'pagination support' do + let(:scope) { Issue.order(id: :desc) } + let(:expected_order) { issues.sort_by(&:id).reverse } + + let(:options) do + { + scope: scope, + array_scope: Project.where(namespace_id: top_level_group.self_and_descendants.select(:id)).select(:id), + array_mapping_scope: -> (id_expression) { Issue.where(Issue.arel_table[:project_id].eq(id_expression)) }, + finder_query: -> (id_expression) { Issue.where(Issue.arel_table[:id].eq(id_expression)) } + } + end + + context 'offset pagination' do + subject(:optimized_scope) { described_class.new(**options).execute } + + it 'paginates the scopes' do + first_page = optimized_scope.page(1).per(2) + expect(first_page).to eq(expected_order[0...2]) + + second_page = optimized_scope.page(2).per(2) + expect(second_page).to eq(expected_order[2...4]) + + third_page = optimized_scope.page(3).per(2) + expect(third_page).to eq(expected_order[4...6]) + end + end + + context 'keyset pagination' do + def paginator(cursor = nil) + scope.keyset_paginate(cursor: cursor, per_page: 2, keyset_order_options: options) + end + + it 'paginates correctly' do + first_page = paginator.records + expect(first_page).to eq(expected_order[0...2]) + + cursor_for_page_2 = paginator.cursor_for_next_page + + second_page = paginator(cursor_for_page_2).records + expect(second_page).to eq(expected_order[2...4]) + + cursor_for_page_3 = paginator(cursor_for_page_2).cursor_for_next_page + + third_page = paginator(cursor_for_page_3).records + expect(third_page).to eq(expected_order[4...6]) + end + end + end + + it 'raises error when unsupported scope is passed' do + scope = Issue.order(Issue.arel_table[:id].lower.desc) + + options = { + scope: scope, + array_scope: Project.where(namespace_id: top_level_group.self_and_descendants.select(:id)).select(:id), + array_mapping_scope: -> (id_expression) { Issue.where(Issue.arel_table[:project_id].eq(id_expression)) }, + finder_query: -> (id_expression) { Issue.where(Issue.arel_table[:id].eq(id_expression)) } + } + + expect { described_class.new(**options).execute }.to raise_error(/The order on the scope does not support keyset pagination/) + end +end diff --git a/spec/lib/gitlab/pagination/keyset/order_spec.rb b/spec/lib/gitlab/pagination/keyset/order_spec.rb index b867dd533e0..3c14d91fdfd 100644 --- a/spec/lib/gitlab/pagination/keyset/order_spec.rb +++ b/spec/lib/gitlab/pagination/keyset/order_spec.rb @@ -538,6 +538,47 @@ RSpec.describe Gitlab::Pagination::Keyset::Order do end it_behaves_like 'cursor attribute examples' + + context 'with projections' do + context 'when additional_projections is empty' do + let(:scope) { Project.select(:id, :namespace_id) } + + subject(:sql) { order.apply_cursor_conditions(scope, { id: '100' }).to_sql } + + it 'has correct projections' do + is_expected.to include('SELECT "projects"."id", "projects"."namespace_id" FROM "projects"') + end + end + + context 'when there are additional_projections' do + let(:order) do + order = Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'created_at_field', + column_expression: Project.arel_table[:created_at], + order_expression: Project.arel_table[:created_at].desc, + order_direction: :desc, + distinct: false, + add_to_projections: true + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id', + order_expression: Project.arel_table[:id].desc + ) + ]) + + order + end + + let(:scope) { Project.select(:id, :namespace_id).reorder(order) } + + subject(:sql) { order.apply_cursor_conditions(scope).to_sql } + + it 'has correct projections' do + is_expected.to include('SELECT "projects"."id", "projects"."namespace_id", "projects"."created_at" AS created_at_field FROM "projects"') + end + end + end end end end |