diff options
77 files changed, 1347 insertions, 148 deletions
diff --git a/app/assets/javascripts/error_tracking_settings/components/error_tracking_form.vue b/app/assets/javascripts/error_tracking_settings/components/error_tracking_form.vue index d86116aa315..9f77fe8cd59 100644 --- a/app/assets/javascripts/error_tracking_settings/components/error_tracking_form.vue +++ b/app/assets/javascripts/error_tracking_settings/components/error_tracking_form.vue @@ -32,12 +32,16 @@ export default { placeholder="https://mysentryserver.com" @input="updateApiHost" /> + <p class="form-text text-muted"> + {{ + s__( + "ErrorTracking|If you self-host Sentry, enter the full URL of your Sentry instance. If you're using Sentry's hosted solution, enter https://sentry.io", + ) + }} + </p> <!-- eslint-enable @gitlab/vue-i18n/no-bare-attribute-strings --> </div> </div> - <p class="form-text text-muted"> - {{ s__('ErrorTracking|Find your hostname in your Sentry account settings page') }} - </p> </div> <div class="form-group" :class="{ 'gl-show-field-errors': connectError }"> <label class="label-bold" for="error-tracking-token"> diff --git a/app/controllers/projects/raw_controller.rb b/app/controllers/projects/raw_controller.rb index 985587268c5..f39d98be516 100644 --- a/app/controllers/projects/raw_controller.rb +++ b/app/controllers/projects/raw_controller.rb @@ -4,11 +4,15 @@ class Projects::RawController < Projects::ApplicationController include ExtractsPath include SendsBlob + include StaticObjectExternalStorage + + prepend_before_action(only: [:show]) { authenticate_sessionless_user!(:blob) } before_action :require_non_empty_project before_action :assign_ref_vars before_action :authorize_download_code! - before_action :show_rate_limit, only: [:show] + before_action :show_rate_limit, only: [:show], unless: :external_storage_request? + before_action :redirect_to_external_storage, only: :show, if: :static_objects_external_storage_enabled? def show @blob = @repository.blob_at(@commit.id, @path) diff --git a/app/finders/pipelines_finder.rb b/app/finders/pipelines_finder.rb index 092a805f275..5a0d53d9683 100644 --- a/app/finders/pipelines_finder.rb +++ b/app/finders/pipelines_finder.rb @@ -25,6 +25,7 @@ class PipelinesFinder items = by_name(items) items = by_username(items) items = by_yaml_errors(items) + items = by_updated_at(items) sort_items(items) end @@ -128,6 +129,13 @@ class PipelinesFinder end # rubocop: enable CodeReuse/ActiveRecord + def by_updated_at(items) + items = items.updated_before(params[:updated_before]) if params[:updated_before].present? + items = items.updated_after(params[:updated_after]) if params[:updated_after].present? + + items + end + # rubocop: disable CodeReuse/ActiveRecord def sort_items(items) order_by = if ALLOWED_INDEXED_COLUMNS.include?(params[:order_by]) diff --git a/app/helpers/blob_helper.rb b/app/helpers/blob_helper.rb index 659f9778892..656e6039dbd 100644 --- a/app/helpers/blob_helper.rb +++ b/app/helpers/blob_helper.rb @@ -215,14 +215,29 @@ module BlobHelper return if blob.binary? || blob.stored_externally? title = _('Open raw') - link_to icon('file-code-o'), blob_raw_path, class: 'btn btn-sm has-tooltip', target: '_blank', rel: 'noopener noreferrer', title: title, data: { container: 'body' } + link_to sprite_icon('doc-code'), + external_storage_url_or_path(blob_raw_path), + class: 'btn btn-sm has-tooltip', + target: '_blank', + rel: 'noopener noreferrer', + aria: { label: title }, + title: title, + data: { container: 'body' } end def download_blob_button(blob) return if blob.empty? title = _('Download') - link_to sprite_icon('download'), blob_raw_path(inline: false), download: @path, class: 'btn btn-sm has-tooltip', target: '_blank', rel: 'noopener noreferrer', title: title, data: { container: 'body' } + link_to sprite_icon('download'), + external_storage_url_or_path(blob_raw_path(inline: false)), + download: @path, + class: 'btn btn-sm has-tooltip', + target: '_blank', + rel: 'noopener noreferrer', + aria: { label: title }, + title: title, + data: { container: 'body' } end def blob_render_error_reason(viewer) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index c3292d7524e..0b1d17a9e12 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -14,6 +14,7 @@ module Ci include HasRef include ShaAttribute include FromUnion + include UpdatedAtFilterable sha_attribute :source_sha sha_attribute :target_sha @@ -811,6 +812,10 @@ module Ci @persistent_ref ||= PersistentRef.new(pipeline: self) end + def find_successful_build_ids_by_names(names) + statuses.latest.success.where(name: names).pluck(:id) + end + private def pipeline_data diff --git a/app/models/concerns/ci/processable.rb b/app/models/concerns/ci/processable.rb index ed0087f34d4..c229358ad17 100644 --- a/app/models/concerns/ci/processable.rb +++ b/app/models/concerns/ci/processable.rb @@ -14,6 +14,8 @@ module Ci has_many :needs, class_name: 'Ci::BuildNeed', foreign_key: :build_id, inverse_of: :build accepts_nested_attributes_for :needs + + scope :preload_needs, -> { preload(:needs) } end def schedulable? diff --git a/app/services/ci/retry_pipeline_service.rb b/app/services/ci/retry_pipeline_service.rb index 1f747aac98f..7d01de9ee68 100644 --- a/app/services/ci/retry_pipeline_service.rb +++ b/app/services/ci/retry_pipeline_service.rb @@ -9,13 +9,23 @@ module Ci raise Gitlab::Access::AccessDeniedError end - pipeline.retryable_builds.find_each do |build| + needs = Set.new + + pipeline.retryable_builds.preload_needs.find_each do |build| next unless can?(current_user, :update_build, build) Ci::RetryBuildService.new(project, current_user) .reprocess!(build) + + needs += build.needs.map(&:name) end + # In a DAG, the dependencies may have already completed. Figure out + # which builds have succeeded and use them to update the pipeline. If we don't + # do this, then builds will be stuck in the created state since their dependencies + # will never run. + completed_build_ids = pipeline.find_successful_build_ids_by_names(needs) if needs.any? + pipeline.builds.latest.skipped.find_each do |skipped| retry_optimistic_lock(skipped) { |build| build.process } end @@ -26,7 +36,7 @@ module Ci Ci::ProcessPipelineService .new(pipeline) - .execute + .execute(completed_build_ids) end end end diff --git a/changelogs/unreleased/33099-updated_at-filters-for-pipelines-api.yml b/changelogs/unreleased/33099-updated_at-filters-for-pipelines-api.yml new file mode 100644 index 00000000000..799a0597afc --- /dev/null +++ b/changelogs/unreleased/33099-updated_at-filters-for-pipelines-api.yml @@ -0,0 +1,5 @@ +--- +title: Add updated_before and updated_after filters to the Pipelines API endpoint +merge_request: 21133 +author: +type: added diff --git a/changelogs/unreleased/9983-document-make-using-gitlab-auth-with-vault-easy.yml b/changelogs/unreleased/9983-document-make-using-gitlab-auth-with-vault-easy.yml new file mode 100644 index 00000000000..115c974d18c --- /dev/null +++ b/changelogs/unreleased/9983-document-make-using-gitlab-auth-with-vault-easy.yml @@ -0,0 +1,5 @@ +--- +title: Resolve Document - Make using GitLab auth with Vault easy +merge_request: 19980 +author: +type: other diff --git a/changelogs/unreleased/ab-keyset-pagination.yml b/changelogs/unreleased/ab-keyset-pagination.yml new file mode 100644 index 00000000000..bd80c27975d --- /dev/null +++ b/changelogs/unreleased/ab-keyset-pagination.yml @@ -0,0 +1,5 @@ +--- +title: Keyset pagination for REST API (Project endpoint) +merge_request: 21194 +author: +type: added diff --git a/changelogs/unreleased/allow-raw-blobs-to-be-served-from-external-storage.yml b/changelogs/unreleased/allow-raw-blobs-to-be-served-from-external-storage.yml new file mode 100644 index 00000000000..23995070c38 --- /dev/null +++ b/changelogs/unreleased/allow-raw-blobs-to-be-served-from-external-storage.yml @@ -0,0 +1,5 @@ +--- +title: Allow raw blobs to be served from an external storage +merge_request: 20936 +author: +type: added diff --git a/changelogs/unreleased/sentry-url-helper-text.yml b/changelogs/unreleased/sentry-url-helper-text.yml new file mode 100644 index 00000000000..9a71f219d6d --- /dev/null +++ b/changelogs/unreleased/sentry-url-helper-text.yml @@ -0,0 +1,5 @@ +--- +title: Update helper text for sentry error tracking settings +merge_request: 20663 +author: Rajendra Kadam +type: added diff --git a/changelogs/unreleased/sh-fix-pipeline-retry-dag.yml b/changelogs/unreleased/sh-fix-pipeline-retry-dag.yml new file mode 100644 index 00000000000..548d1bfec30 --- /dev/null +++ b/changelogs/unreleased/sh-fix-pipeline-retry-dag.yml @@ -0,0 +1,5 @@ +--- +title: Fix pipeline retry in a CI DAG +merge_request: 21296 +author: +type: fixed diff --git a/doc/administration/high_availability/README.md b/doc/administration/high_availability/README.md index dad83b633a2..d411fb7f20f 100644 --- a/doc/administration/high_availability/README.md +++ b/doc/administration/high_availability/README.md @@ -4,18 +4,19 @@ type: reference, concepts # Scaling and High Availability -GitLab supports several different types of clustering and high-availability. +GitLab supports a number of options for scaling your self-managed instance and configuring high availability (HA). The solution you choose will be based on the level of scalability and availability you require. The easiest solutions are scalable, but not necessarily highly available. -GitLab provides a service that is usually essential to most organizations: it +GitLab provides a service that is essential to most organizations: it enables people to collaborate on code in a timely fashion. Any downtime should -therefore be short and planned. Luckily, GitLab provides a solid setup even on -a single server without special measures. Due to the distributed nature -of Git, developers can still commit code locally even when GitLab is not +therefore be short and planned. Due to the distributed nature +of Git, developers can continue to commit code locally even when GitLab is not available. However, some GitLab features such as the issue tracker and -Continuous Integration are not available when GitLab is down. +continuous integration are not available when GitLab is down. +If you require all GitLab functionality to be highly available, +consider the options outlined below. **Keep in mind that all highly-available solutions come with a trade-off between cost/complexity and uptime**. The more uptime you want, the more complex the @@ -25,8 +26,8 @@ solution should balance the costs against the benefits. There are many options when choosing a highly-available GitLab architecture. We recommend engaging with GitLab Support to choose the best architecture for your -use case. This page contains some various options and guidelines based on -experience with GitLab.com and Enterprise Edition on-premises customers. +use case. This page contains recommendations based on +experience with GitLab.com and internal scale testing. For detailed insight into how GitLab scales and configures GitLab.com, you can watch [this 1 hour Q&A](https://www.youtube.com/watch?v=uCU8jdYzpac) diff --git a/doc/administration/restart_gitlab.md b/doc/administration/restart_gitlab.md index 9f95080654f..6f3c6028f71 100644 --- a/doc/administration/restart_gitlab.md +++ b/doc/administration/restart_gitlab.md @@ -8,6 +8,7 @@ If you want the TL;DR versions, jump to: - [Omnibus GitLab restart](#omnibus-gitlab-restart) - [Omnibus GitLab reconfigure](#omnibus-gitlab-reconfigure) - [Source installation restart](#installations-from-source) +- [Helm chart installation restart](#helm-chart-installations) ## Omnibus installations @@ -143,3 +144,16 @@ If you are using other init systems, like systemd, you can check the [chef]: https://www.chef.io/products/chef-infra/ "Chef official website" [src-service]: https://gitlab.com/gitlab-org/gitlab/blob/master/lib/support/init.d/gitlab "GitLab init service file" [gl-recipes]: https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/init "GitLab Recipes repository" + +## Helm chart installations + +There is no single command to restart the entire GitLab application installed via +the [cloud native Helm Chart](https://docs.gitlab.com/charts/). Usually, it should be +enough to restart a specific component separately (`gitaly`, `unicorn`, +`workhorse`, `gitlab-shell`, etc.) by deleting all the pods related to it: + +```bash +kubectl delete pods -l release=<helm release name>,app=<component name> +``` + +The release name can be obtained from the output of the `helm list` command. diff --git a/doc/api/pipelines.md b/doc/api/pipelines.md index 97dc316cc96..e1b2c12dd00 100644 --- a/doc/api/pipelines.md +++ b/doc/api/pipelines.md @@ -18,6 +18,8 @@ GET /projects/:id/pipelines | `yaml_errors`| boolean | no | Returns pipelines with invalid configurations | | `name`| string | no | The name of the user who triggered pipelines | | `username`| string | no | The username of the user who triggered pipelines | +| `updated_after` | datetime | no | Return pipelines updated after the specified date. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ | +| `updated_before` | datetime | no | Return pipelines updated before the specified date. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ | | `order_by`| string | no | Order pipelines by `id`, `status`, `ref`, `updated_at` or `user_id` (default: `id`) | | `sort` | string | no | Sort pipelines in `asc` or `desc` order (default: `desc`) | diff --git a/doc/integration/README.md b/doc/integration/README.md index 3f33aa94cb9..5cda537ac39 100644 --- a/doc/integration/README.md +++ b/doc/integration/README.md @@ -30,6 +30,7 @@ GitLab can be configured to authenticate access requests with the following auth - Use [OmniAuth](omniauth.md) to enable sign in via Twitter, GitHub, GitLab.com, Google, Bitbucket, Facebook, Shibboleth, SAML, Crowd, Azure or Authentiq ID. - Use GitLab as an [OpenID Connect](openid_connect_provider.md) identity provider. +- Authenticate to [Vault](vault.md) through GitLab OpenID Connect. - Configure GitLab as a [SAML](saml.md) 2.0 Service Provider. ## Security enhancements diff --git a/doc/integration/img/authorize_vault_with_gitlab_v12_6.png b/doc/integration/img/authorize_vault_with_gitlab_v12_6.png Binary files differnew file mode 100644 index 00000000000..dc5bc954cd7 --- /dev/null +++ b/doc/integration/img/authorize_vault_with_gitlab_v12_6.png diff --git a/doc/integration/img/gitlab_oauth_vault_v12_6.png b/doc/integration/img/gitlab_oauth_vault_v12_6.png Binary files differnew file mode 100644 index 00000000000..f952abc2c6d --- /dev/null +++ b/doc/integration/img/gitlab_oauth_vault_v12_6.png diff --git a/doc/integration/img/sign_into_vault_with_gitlab_v12_6.png b/doc/integration/img/sign_into_vault_with_gitlab_v12_6.png Binary files differnew file mode 100644 index 00000000000..8afa2c6aabd --- /dev/null +++ b/doc/integration/img/sign_into_vault_with_gitlab_v12_6.png diff --git a/doc/integration/img/signed_into_vault_via_oidc_v12_6.png b/doc/integration/img/signed_into_vault_via_oidc_v12_6.png Binary files differnew file mode 100644 index 00000000000..0ad81ef40e6 --- /dev/null +++ b/doc/integration/img/signed_into_vault_via_oidc_v12_6.png diff --git a/doc/integration/vault.md b/doc/integration/vault.md new file mode 100644 index 00000000000..4aca62b5fd1 --- /dev/null +++ b/doc/integration/vault.md @@ -0,0 +1,120 @@ +--- +type: reference, howto +--- + +# Vault Authentication with GitLab OpenID Connect + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/22323) in GitLab 9.0 + +[Vault](https://www.vaultproject.io/) is a secrets management application offered by HashiCorp. +It allows you to store and manage sensitive information such secret environment variables, encryption keys, and authentication tokens. +Vault offers Identity-based Access, which means Vault users can authenticate through several of their preferred cloud providers. + +In this document, we'll explain how Vault users can authenticate themselves through GitLab by utilizing our OpenID authentication feature. +The following assumes you already have Vault installed and running. + +1. **Get the OpenID Connect client ID and secret from GitLab:** + + First you'll need to create a GitLab application to obtain an application ID and secret for authenticating into Vault. To do this, sign in to GitLab and follow these steps: + + 1. On GitLab, click your avatar on the top-right corner, and select your user **Settings > Applications**. + 1. Fill out the application **Name** and [**Redirect URI**](https://www.vaultproject.io/docs/auth/jwt.html#redirect-uris), + making sure to select the **openid** scope. + 1. Save application. + 1. Copy client ID and secret, or keep the page open for reference. +  + +1. **Enable OIDC auth on Vault:** + + OpenID Connect is not enabled in Vault by default. This needs to be enabled in the terminal. + + Open a terminal session and run the following command to enable the OpenID Connect authentication provider in Vault: + + ```bash + vault auth enable oidc + ``` + + You should see the following output in the terminal: + + ```bash + Success! Enabled oidc auth method at: oidc/ + ``` + +1. **Write the OIDC config:** + + Next, Vault needs to be given the application ID and secret generated by Gitlab. + + In the terminal session, run the following command to give Vault access to the GitLab application you've just created with an OpenID scope. This allows Vault to authenticate through GitLab. + + Replace `your_application_id` and `your_secret` in the example below with the application ID and secret generated for your app: + + ```bash + $ vault write auth/oidc/config \ + oidc_discovery_url="https://gitlab.com" \ + oidc_client_id="your_application_id" \ + oidc_client_secret="your_secret" \ + default_role="demo" \ + bound_issuer="localhost" + ``` + + You should see the following output in the terminal: + + ```bash + Success! Data written to: auth/oidc/config + ``` + +1. **Write the OIDC Role Config:** + + Now that Vault has a GitLab application ID and secret, it needs to know the [**Redirect URIs**](https://www.vaultproject.io/docs/auth/jwt.html#redirect-uris) and scopes given to GitLab during the application creation process. The redirect URIs need to match where your Vault instance is running. The `oidc_scopes` field needs to include the `openid`. Similarly to the previous step, replace `your_application_id` with the generated application ID from GitLab: + + This configuration is saved under the name of the role you are creating. In this case, we are creating a `demo` role. Later, we'll show how you can access this role through the Vault CLI. + + ```bash + vault write auth/oidc/role/demo \ + user_claim="sub" \ + allowed_redirect_uris="http://localhost:8250/oidc/callback,http://127.0.0.1:8200/ui/vault/auth/oidc/oidc/callback" \ + bound_audiences="your_application_id" \ + role_type="oidc" \ + oidc_scopes="openid" \ + policies=demo \ + ttl=1h + ``` + +1. **Sign in to Vault:** + + 1. Go to your Vault UI (example: [http://127.0.0.1:8200/ui/vault/auth?with=oidc](http://127.0.0.1:8200/ui/vault/auth?with=oidc)). + 1. If the `OIDC` method is not currently selected, open the dropdown and select it. + 1. Click the **Sign in With GitLab** button, which will open a modal window: +  + + 1. Click **Authorize** on the modal to allow Vault to sign in through GitLab. This will redirect you back to your Vault UI as a signed-in user. +  + +1. **Sign in using the Vault CLI** (optional): + + Vault also allows you to sign in via their CLI. + + After writing the same configurations from above, you can run the command below in your terminal to sign in with the role configuration created in step 4 above: + + ```bash + vault login -method=oidc port=8250 role=demo + ``` + + Here is a short explaination of what this command does: + + 1. In the **Write the OIDC Role Config** (step 4), we created a role called `demo`. We set `role=demo` so Vault knows which configuration we'd like to login in with. + 1. To set Vault to use the `OIDC` sign-in method, we set `-method=oidc`. + 1. To set the port that GitLab should redirect to, we set `port=8250` or another port number that matches the port given to GitLab when listing [Redirect URIs](https://www.vaultproject.io/docs/auth/jwt.html#redirect-uris). + + Once you run the command above, it will present a link in the terminal. + Click the link in the terminal and a tab will open in the browser confirming you're signed into Vault via OIDC: + +  + + The terminal will output: + + ``` + Success! You are now authenticated. The token information displayed below + is already stored in the token helper. You do NOT need to run "vault login" + again. Future Vault requests will automatically use this token. + ``` diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 1e41e28c449..ead2ea34227 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -176,7 +176,7 @@ module API end class BasicProjectDetails < ProjectIdentity - include ::API::ProjectsRelationBuilder + include ::API::ProjectsBatchCounting expose :default_branch, if: -> (project, options) { Ability.allowed?(options[:current_user], :download_code, project) } # Avoids an N+1 query: https://github.com/mbleigh/acts-as-taggable-on/issues/91#issuecomment-168273770 @@ -418,7 +418,7 @@ module API options: { only_owned: true } ).execute - Entities::Project.prepare_relation(projects) + Entities::Project.preload_and_batch_count!(projects) end expose :shared_projects, using: Entities::Project do |group, options| @@ -428,7 +428,7 @@ module API options: { only_shared: true } ).execute - Entities::Project.prepare_relation(projects) + Entities::Project.preload_and_batch_count!(projects) end end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 6c88b61eee8..b9cfc16fb23 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -231,7 +231,7 @@ module API projects, options = with_custom_attributes(projects, options) - present options[:with].prepare_relation(projects), options + present options[:with].preload_and_batch_count!(projects), options end desc 'Get a list of subgroups in this group.' do diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 9c5b355e823..1b63e450a12 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -3,8 +3,33 @@ module API module Helpers module Pagination + # This returns an ActiveRecord relation def paginate(relation) - ::Gitlab::Pagination::OffsetPagination.new(self).paginate(relation) + Gitlab::Pagination::OffsetPagination.new(self).paginate(relation) + end + + # This applies pagination and executes the query + # It always returns an array instead of an ActiveRecord relation + def paginate_and_retrieve!(relation) + offset_or_keyset_pagination(relation).to_a + end + + private + + def offset_or_keyset_pagination(relation) + return paginate(relation) unless keyset_pagination_enabled? + + request_context = Gitlab::Pagination::Keyset::RequestContext.new(self) + + unless Gitlab::Pagination::Keyset.available?(request_context, relation) + return error!('Keyset pagination is not yet available for this type of request', 405) + end + + Gitlab::Pagination::Keyset.paginate(request_context, relation) + end + + def keyset_pagination_enabled? + params[:pagination] == 'keyset' && Feature.enabled?(:api_keyset_pagination, default_enabled: true) end end end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 7c87a9878bf..66a183173af 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -25,6 +25,8 @@ module API optional :yaml_errors, type: Boolean, desc: 'Returns pipelines with invalid configurations' optional :name, type: String, desc: 'The name of the user who triggered pipelines' optional :username, type: String, desc: 'The username of the user who triggered pipelines' + optional :updated_before, type: DateTime, desc: 'Return pipelines updated before the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ' + optional :updated_after, type: DateTime, desc: 'Return pipelines updated after the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ' optional :order_by, type: String, values: PipelinesFinder::ALLOWED_INDEXED_COLUMNS, default: 'id', desc: 'Order pipelines' optional :sort, type: String, values: %w[asc desc], default: 'desc', diff --git a/lib/api/projects.rb b/lib/api/projects.rb index a1fce9e8b20..ea7087d2f46 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -75,15 +75,17 @@ module API mutually_exclusive :import_url, :template_name, :template_project_id end - def load_projects + def find_projects ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute end - def present_projects(projects, options = {}) + # Prepare the full projects query + # None of this is supposed to actually execute any database query + def prepare_query(projects) projects = reorder_projects(projects) projects = apply_filters(projects) - projects = paginate(projects) - projects, options = with_custom_attributes(projects, options) + + projects, options = with_custom_attributes(projects) options = options.reverse_merge( with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, @@ -91,9 +93,23 @@ module API current_user: current_user, license: false ) + options[:with] = Entities::BasicProjectDetails if params[:simple] - present options[:with].prepare_relation(projects, options), options + projects = options[:with].preload_relation(projects, options) + + [projects, options] + end + + def prepare_and_present(project_relation) + projects, options = prepare_query(project_relation) + + projects = paginate_and_retrieve!(projects) + + # Refresh count caches + options[:with].execute_batch_counting(projects) + + present projects, options end def translate_params_for_compatibility(params) @@ -118,7 +134,7 @@ module API params[:user] = user - present_projects load_projects + prepare_and_present find_projects end desc 'Get projects starred by a user' do @@ -134,7 +150,7 @@ module API not_found!('User') unless user starred_projects = StarredProjectsFinder.new(user, params: project_finder_params, current_user: current_user).execute - present_projects starred_projects + prepare_and_present starred_projects end end @@ -150,7 +166,7 @@ module API use :with_custom_attributes end get do - present_projects load_projects + prepare_and_present find_projects end desc 'Create new project' do @@ -287,7 +303,7 @@ module API get ':id/forks' do forks = ForkProjectsFinder.new(user_project, params: project_finder_params, current_user: current_user).execute - present_projects forks + prepare_and_present forks end desc 'Check pages access of this project' diff --git a/lib/api/projects_batch_counting.rb b/lib/api/projects_batch_counting.rb new file mode 100644 index 00000000000..4e5124c8791 --- /dev/null +++ b/lib/api/projects_batch_counting.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module API + module ProjectsBatchCounting + extend ActiveSupport::Concern + + class_methods do + # This adds preloading to the query and executes batch counting + # Side-effect: The query will be executed during batch counting + def preload_and_batch_count!(projects_relation) + preload_relation(projects_relation).tap do |projects| + execute_batch_counting(projects) + end + end + + def execute_batch_counting(projects) + ::Projects::BatchForksCountService.new(forks_counting_projects(projects)).refresh_cache + + ::Projects::BatchOpenIssuesCountService.new(projects).refresh_cache + end + + def forks_counting_projects(projects) + projects + end + end + end +end diff --git a/lib/api/projects_relation_builder.rb b/lib/api/projects_relation_builder.rb deleted file mode 100644 index 263468c9aa6..00000000000 --- a/lib/api/projects_relation_builder.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -module API - module ProjectsRelationBuilder - extend ActiveSupport::Concern - - class_methods do - def prepare_relation(projects_relation, options = {}) - projects_relation = preload_relation(projects_relation, options) - execute_batch_counting(projects_relation) - projects_relation - end - - def preload_relation(projects_relation, options = {}) - projects_relation - end - - def forks_counting_projects(projects_relation) - projects_relation - end - - def batch_forks_counting(projects_relation) - ::Projects::BatchForksCountService.new(forks_counting_projects(projects_relation)).refresh_cache - end - - def batch_open_issues_counting(projects_relation) - ::Projects::BatchOpenIssuesCountService.new(projects_relation).refresh_cache - end - - def execute_batch_counting(projects_relation) - batch_forks_counting(projects_relation) - batch_open_issues_counting(projects_relation) - end - end - end -end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index e2f562c0843..a8869f907e6 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -169,6 +169,8 @@ module Gitlab case request_format when :archive archive_request? + when :blob + blob_request? else false end @@ -189,6 +191,10 @@ module Gitlab def archive_request? current_request.path.include?('/-/archive/') end + + def blob_request? + current_request.path.include?('/raw/') + end end end end diff --git a/lib/gitlab/pagination/keyset.rb b/lib/gitlab/pagination/keyset.rb new file mode 100644 index 00000000000..5bd45fa9b56 --- /dev/null +++ b/lib/gitlab/pagination/keyset.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + def self.paginate(request_context, relation) + Gitlab::Pagination::Keyset::Pager.new(request_context).paginate(relation) + end + + def self.available?(request_context, relation) + order_by = request_context.page.order_by + + # This is only available for Project and order-by id (asc/desc) + return false unless relation.klass == Project + return false unless order_by.size == 1 && order_by[:id] + + true + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/page.rb b/lib/gitlab/pagination/keyset/page.rb new file mode 100644 index 00000000000..735f54faf0f --- /dev/null +++ b/lib/gitlab/pagination/keyset/page.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + # A Page models the pagination information for a particular page of the collection + class Page + # Default number of records for a page + DEFAULT_PAGE_SIZE = 20 + + # Maximum number of records for a page + MAXIMUM_PAGE_SIZE = 100 + + attr_accessor :lower_bounds, :end_reached + attr_reader :order_by + + def initialize(order_by: {}, lower_bounds: nil, per_page: DEFAULT_PAGE_SIZE, end_reached: false) + @order_by = order_by.symbolize_keys + @lower_bounds = lower_bounds&.symbolize_keys + @per_page = per_page + @end_reached = end_reached + end + + # Number of records to return per page + def per_page + return DEFAULT_PAGE_SIZE if @per_page <= 0 + + [@per_page, MAXIMUM_PAGE_SIZE].min + end + + # Determine whether this page indicates the end of the collection + def end_reached? + @end_reached + end + + # Construct a Page for the next page + # Uses identical order_by/per_page information for the next page + def next(lower_bounds, end_reached) + dup.tap do |next_page| + next_page.lower_bounds = lower_bounds&.symbolize_keys + next_page.end_reached = end_reached + end + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/pager.rb b/lib/gitlab/pagination/keyset/pager.rb new file mode 100644 index 00000000000..99b125cc2a0 --- /dev/null +++ b/lib/gitlab/pagination/keyset/pager.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + class Pager + attr_reader :request + + def initialize(request) + @request = request + end + + def paginate(relation) + # Validate assumption: The last two columns must match the page order_by + validate_order!(relation) + + # This performs the database query and retrieves records + # We retrieve one record more to check if we have data beyond this page + all_records = relation.limit(page.per_page + 1).to_a # rubocop: disable CodeReuse/ActiveRecord + + records_for_page = all_records.first(page.per_page) + + # If we retrieved more records than belong on this page, + # we know there's a next page + there_is_more = all_records.size > records_for_page.size + apply_headers(records_for_page.last, there_is_more) + + records_for_page + end + + private + + def apply_headers(last_record_in_page, there_is_more) + end_reached = last_record_in_page.nil? || !there_is_more + lower_bounds = last_record_in_page&.slice(page.order_by.keys) + + next_page = page.next(lower_bounds, end_reached) + + request.apply_headers(next_page) + end + + def page + @page ||= request.page + end + + def validate_order!(rel) + present_order = rel.order_values.map { |val| [val.expr.name.to_sym, val.direction] }.last(2).to_h + + unless page.order_by == present_order + raise ArgumentError, "Page's order_by does not match the relation's order: #{present_order} vs #{page.order_by}" + end + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/request_context.rb b/lib/gitlab/pagination/keyset/request_context.rb new file mode 100644 index 00000000000..aeaed7587b3 --- /dev/null +++ b/lib/gitlab/pagination/keyset/request_context.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + class RequestContext + attr_reader :request + + DEFAULT_SORT_DIRECTION = :desc + PRIMARY_KEY = :id + + # A tie breaker is added as an additional order-by column + # to establish a well-defined order. We use the primary key + # column here. + TIE_BREAKER = { PRIMARY_KEY => DEFAULT_SORT_DIRECTION }.freeze + + def initialize(request) + @request = request + end + + # extracts Paging information from request parameters + def page + @page ||= Page.new(order_by: order_by, per_page: params[:per_page]) + end + + def apply_headers(next_page) + request.header('Links', pagination_links(next_page)) + end + + private + + def order_by + return TIE_BREAKER.dup unless params[:order_by] + + order_by = { params[:order_by].to_sym => params[:sort]&.to_sym || DEFAULT_SORT_DIRECTION } + + # Order by an additional unique key, we use the primary key here + order_by = order_by.merge(TIE_BREAKER) unless order_by[PRIMARY_KEY] + + order_by + end + + def params + @params ||= request.params + end + + def lower_bounds_params(page) + page.lower_bounds.each_with_object({}) do |(column, value), params| + filter = filter_with_comparator(page, column) + params[filter] = value + end + end + + def filter_with_comparator(page, column) + direction = page.order_by[column] + + if direction&.to_sym == :desc + "#{column}_before" + else + "#{column}_after" + end + end + + def page_href(page) + base_request_uri.tap do |uri| + uri.query = query_params_for(page).to_query + end.to_s + end + + def pagination_links(next_page) + return if next_page.end_reached? + + %(<#{page_href(next_page)}>; rel="next") + end + + def base_request_uri + @base_request_uri ||= URI.parse(request.request.url).tap do |uri| + uri.host = Gitlab.config.gitlab.host + uri.port = Gitlab.config.gitlab.port + end + end + + def query_params_for(page) + request.params.merge(lower_bounds_params(page)) + end + end + end + end +end diff --git a/lib/sentry/client.rb b/lib/sentry/client.rb index 708ace53f5b..450695aa545 100644 --- a/lib/sentry/client.rb +++ b/lib/sentry/client.rb @@ -206,7 +206,7 @@ module Sentry uri = URI(url) uri.path.squeeze!('/') - # Remove trailing spaces + # Remove trailing slash uri = uri.to_s.gsub(/\/\z/, '') uri diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 783766f3406..fdffac6dbc1 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -6993,7 +6993,7 @@ msgstr "" msgid "ErrorTracking|Connection has failed. Re-check Auth Token and try again." msgstr "" -msgid "ErrorTracking|Find your hostname in your Sentry account settings page" +msgid "ErrorTracking|If you self-host Sentry, enter the full URL of your Sentry instance. If you're using Sentry's hosted solution, enter https://sentry.io" msgstr "" msgid "ErrorTracking|No projects available" diff --git a/spec/controllers/projects/raw_controller_spec.rb b/spec/controllers/projects/raw_controller_spec.rb index ae9932174e8..ebc22043891 100644 --- a/spec/controllers/projects/raw_controller_spec.rb +++ b/spec/controllers/projects/raw_controller_spec.rb @@ -77,6 +77,24 @@ describe Projects::RawController do execute_raw_requests(requests: 6, project: project, file_path: file_path) end + context 'when receiving an external storage request' do + let(:token) { 'letmein' } + + before do + stub_application_setting( + static_objects_external_storage_url: 'https://cdn.gitlab.com', + static_objects_external_storage_auth_token: token + ) + end + + it 'does not prevent from accessing the raw file' do + request.headers['X-Gitlab-External-Storage-Token'] = token + execute_raw_requests(requests: 6, project: project, file_path: file_path) + + expect(response).to have_gitlab_http_status(200) + end + end + context 'when the request uses a different version of a commit' do it 'prevents from accessing the raw file' do # 3 times with the normal sha @@ -131,15 +149,74 @@ describe Projects::RawController do end end end + + context 'as a sessionless user' do + let_it_be(:project) { create(:project, :private, :repository) } + let_it_be(:user) { create(:user, static_object_token: 'very-secure-token') } + let_it_be(:file_path) { 'master/README.md' } + + before do + project.add_developer(user) + end + + context 'when no token is provided' do + it 'redirects to sign in page' do + execute_raw_requests(requests: 1, project: project, file_path: file_path) + + expect(response).to have_gitlab_http_status(302) + expect(response.location).to end_with('/users/sign_in') + end + end + + context 'when a token param is present' do + context 'when token is correct' do + it 'calls the action normally' do + execute_raw_requests(requests: 1, project: project, file_path: file_path, token: user.static_object_token) + + expect(response).to have_gitlab_http_status(200) + end + end + + context 'when token is incorrect' do + it 'redirects to sign in page' do + execute_raw_requests(requests: 1, project: project, file_path: file_path, token: 'foobar') + + expect(response).to have_gitlab_http_status(302) + expect(response.location).to end_with('/users/sign_in') + end + end + end + + context 'when a token header is present' do + context 'when token is correct' do + it 'calls the action normally' do + request.headers['X-Gitlab-Static-Object-Token'] = user.static_object_token + execute_raw_requests(requests: 1, project: project, file_path: file_path) + + expect(response).to have_gitlab_http_status(200) + end + end + + context 'when token is incorrect' do + it 'redirects to sign in page' do + request.headers['X-Gitlab-Static-Object-Token'] = 'foobar' + execute_raw_requests(requests: 1, project: project, file_path: file_path) + + expect(response).to have_gitlab_http_status(302) + expect(response.location).to end_with('/users/sign_in') + end + end + end + end end - def execute_raw_requests(requests:, project:, file_path:) + def execute_raw_requests(requests:, project:, file_path:, **params) requests.times do get :show, params: { namespace_id: project.namespace, project_id: project, id: file_path - } + }.merge(params) end end end diff --git a/spec/features/projects/blobs/blob_show_spec.rb b/spec/features/projects/blobs/blob_show_spec.rb index af6bb8c271f..5d86e4125df 100644 --- a/spec/features/projects/blobs/blob_show_spec.rb +++ b/spec/features/projects/blobs/blob_show_spec.rb @@ -611,4 +611,50 @@ describe 'File blob', :js do expect(page).to have_selector '.gpg-status-box.invalid' end end + + context 'when static objects external storage is enabled' do + before do + stub_application_setting(static_objects_external_storage_url: 'https://cdn.gitlab.com') + end + + context 'private project' do + let_it_be(:project) { create(:project, :repository, :private) } + let_it_be(:user) { create(:user) } + + before do + project.add_developer(user) + + sign_in(user) + visit_blob('README.md') + end + + it 'shows open raw and download buttons with external storage URL prepended and user token appended to their href' do + path = project_raw_path(project, 'master/README.md') + raw_uri = "https://cdn.gitlab.com#{path}?token=#{user.static_object_token}" + download_uri = "https://cdn.gitlab.com#{path}?inline=false&token=#{user.static_object_token}" + + aggregate_failures do + expect(page).to have_link 'Open raw', href: raw_uri + expect(page).to have_link 'Download', href: download_uri + end + end + end + + context 'public project' do + before do + visit_blob('README.md') + end + + it 'shows open raw and download buttons with external storage URL prepended to their href' do + path = project_raw_path(project, 'master/README.md') + raw_uri = "https://cdn.gitlab.com#{path}" + download_uri = "https://cdn.gitlab.com#{path}?inline=false" + + aggregate_failures do + expect(page).to have_link 'Open raw', href: raw_uri + expect(page).to have_link 'Download', href: download_uri + end + end + end + end end diff --git a/spec/finders/pipelines_finder_spec.rb b/spec/finders/pipelines_finder_spec.rb index 05d13a76e0e..116088f5141 100644 --- a/spec/finders/pipelines_finder_spec.rb +++ b/spec/finders/pipelines_finder_spec.rb @@ -170,41 +170,14 @@ describe PipelinesFinder do end end - context 'when order_by and sort are specified' do - context 'when order_by user_id' do - let(:params) { { order_by: 'user_id', sort: 'asc' } } - let(:users) { Array.new(2) { create(:user, developer_projects: [project]) } } - let!(:pipelines) { users.map { |user| create(:ci_pipeline, project: project, user: user) } } - - it 'sorts as user_id: :asc' do - is_expected.to match_array(pipelines) - end - - context 'when sort is invalid' do - let(:params) { { order_by: 'user_id', sort: 'invalid_sort' } } - - it 'sorts as user_id: :desc' do - is_expected.to eq(pipelines.sort_by { |p| -p.user.id }) - end - end - end - - context 'when order_by is invalid' do - let(:params) { { order_by: 'invalid_column', sort: 'asc' } } - let!(:pipelines) { create_list(:ci_pipeline, 2, project: project) } - - it 'sorts as id: :asc' do - is_expected.to eq(pipelines.sort_by { |p| p.id }) - end - end - - context 'when both are nil' do - let(:params) { { order_by: nil, sort: nil } } - let!(:pipelines) { create_list(:ci_pipeline, 2, project: project) } - - it 'sorts as id: :desc' do - is_expected.to eq(pipelines.sort_by { |p| -p.id }) - end + context 'when updated_at filters are specified' do + let(:params) { { updated_before: 1.day.ago, updated_after: 3.days.ago } } + let!(:pipeline1) { create(:ci_pipeline, project: project, updated_at: 2.days.ago) } + let!(:pipeline2) { create(:ci_pipeline, project: project, updated_at: 4.days.ago) } + let!(:pipeline3) { create(:ci_pipeline, project: project, updated_at: 1.hour.ago) } + + it 'returns deployments with matched updated_at' do + is_expected.to match_array([pipeline1]) end end @@ -249,5 +222,36 @@ describe PipelinesFinder do end end end + + describe 'ordering' do + using RSpec::Parameterized::TableSyntax + + let(:params) { { order_by: order_by, sort: sort } } + + let!(:pipeline_1) { create(:ci_pipeline, :scheduled, project: project, iid: 11, ref: 'master', created_at: Time.now, updated_at: Time.now, user: create(:user)) } + let!(:pipeline_2) { create(:ci_pipeline, :created, project: project, iid: 12, ref: 'feature', created_at: 1.day.ago, updated_at: 2.hours.ago, user: create(:user)) } + let!(:pipeline_3) { create(:ci_pipeline, :success, project: project, iid: 8, ref: 'patch', created_at: 2.days.ago, updated_at: 1.hour.ago, user: create(:user)) } + + where(:order_by, :sort, :ordered_pipelines) do + 'id' | 'asc' | [:pipeline_1, :pipeline_2, :pipeline_3] + 'id' | 'desc' | [:pipeline_3, :pipeline_2, :pipeline_1] + 'ref' | 'asc' | [:pipeline_2, :pipeline_1, :pipeline_3] + 'ref' | 'desc' | [:pipeline_3, :pipeline_1, :pipeline_2] + 'status' | 'asc' | [:pipeline_2, :pipeline_1, :pipeline_3] + 'status' | 'desc' | [:pipeline_3, :pipeline_1, :pipeline_2] + 'updated_at' | 'asc' | [:pipeline_2, :pipeline_3, :pipeline_1] + 'updated_at' | 'desc' | [:pipeline_1, :pipeline_3, :pipeline_2] + 'user_id' | 'asc' | [:pipeline_1, :pipeline_2, :pipeline_3] + 'user_id' | 'desc' | [:pipeline_3, :pipeline_2, :pipeline_1] + 'invalid' | 'asc' | [:pipeline_1, :pipeline_2, :pipeline_3] + 'id' | 'err' | [:pipeline_3, :pipeline_2, :pipeline_1] + end + + with_them do + it 'returns the pipelines ordered' do + expect(subject).to eq(ordered_pipelines.map { |name| public_send(name) }) + end + end + end end end diff --git a/spec/frontend/error_tracking_settings/components/error_tracking_form_spec.js b/spec/frontend/error_tracking_settings/components/error_tracking_form_spec.js index bff8ad0877a..21edcb7235a 100644 --- a/spec/frontend/error_tracking_settings/components/error_tracking_form_spec.js +++ b/spec/frontend/error_tracking_settings/components/error_tracking_form_spec.js @@ -49,7 +49,9 @@ describe('error tracking settings form', () => { it('is rendered with labels and placeholders', () => { const pageText = wrapper.text(); - expect(pageText).toContain('Find your hostname in your Sentry account settings page'); + expect(pageText).toContain( + "If you self-host Sentry, enter the full URL of your Sentry instance. If you're using Sentry's hosted solution, enter https://sentry.io", + ); expect(pageText).toContain( "After adding your Auth Token, use the 'Connect' button to load projects", ); diff --git a/spec/javascripts/ide/components/commit_sidebar/actions_spec.js b/spec/javascripts/ide/components/commit_sidebar/actions_spec.js index a3db3ee1b18..d02d8fa0253 100644 --- a/spec/javascripts/ide/components/commit_sidebar/actions_spec.js +++ b/spec/javascripts/ide/components/commit_sidebar/actions_spec.js @@ -1,9 +1,9 @@ import Vue from 'vue'; +import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; +import { projectData, branches } from 'spec/ide/mock_data'; import { createStore } from '~/ide/stores'; import commitActions from '~/ide/components/commit_sidebar/actions.vue'; import consts from '~/ide/stores/modules/commit/constants'; -import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; -import { projectData, branches } from 'spec/ide/mock_data'; const ACTION_UPDATE_COMMIT_ACTION = 'commit/updateCommitAction'; diff --git a/spec/javascripts/ide/components/commit_sidebar/form_spec.js b/spec/javascripts/ide/components/commit_sidebar/form_spec.js index b7a7afe4db4..fdbabf84e25 100644 --- a/spec/javascripts/ide/components/commit_sidebar/form_spec.js +++ b/spec/javascripts/ide/components/commit_sidebar/form_spec.js @@ -1,10 +1,10 @@ import Vue from 'vue'; -import store from '~/ide/stores'; -import CommitForm from '~/ide/components/commit_sidebar/form.vue'; -import { activityBarViews } from '~/ide/constants'; import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import getSetTimeoutPromise from 'spec/helpers/set_timeout_promise_helper'; import { projectData } from 'spec/ide/mock_data'; +import store from '~/ide/stores'; +import CommitForm from '~/ide/components/commit_sidebar/form.vue'; +import { activityBarViews } from '~/ide/constants'; import { resetStore } from '../../helpers'; describe('IDE commit form', () => { diff --git a/spec/javascripts/ide/components/commit_sidebar/list_collapsed_spec.js b/spec/javascripts/ide/components/commit_sidebar/list_collapsed_spec.js index 3c7d6192e2c..6eb912127d5 100644 --- a/spec/javascripts/ide/components/commit_sidebar/list_collapsed_spec.js +++ b/spec/javascripts/ide/components/commit_sidebar/list_collapsed_spec.js @@ -1,7 +1,7 @@ import Vue from 'vue'; +import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import store from '~/ide/stores'; import listCollapsed from '~/ide/components/commit_sidebar/list_collapsed.vue'; -import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import { file } from '../../helpers'; import { removeWhitespace } from '../../../helpers/text_helper'; diff --git a/spec/javascripts/ide/components/commit_sidebar/list_item_spec.js b/spec/javascripts/ide/components/commit_sidebar/list_item_spec.js index c1dcd4928a0..caf06b5e1d8 100644 --- a/spec/javascripts/ide/components/commit_sidebar/list_item_spec.js +++ b/spec/javascripts/ide/components/commit_sidebar/list_item_spec.js @@ -1,9 +1,9 @@ import Vue from 'vue'; +import { trimText } from 'spec/helpers/text_helper'; +import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import store from '~/ide/stores'; import listItem from '~/ide/components/commit_sidebar/list_item.vue'; import router from '~/ide/ide_router'; -import { trimText } from 'spec/helpers/text_helper'; -import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import { file, resetStore } from '../../helpers'; describe('Multi-file editor commit sidebar list item', () => { diff --git a/spec/javascripts/ide/components/commit_sidebar/list_spec.js b/spec/javascripts/ide/components/commit_sidebar/list_spec.js index b786be55019..81120f6d277 100644 --- a/spec/javascripts/ide/components/commit_sidebar/list_spec.js +++ b/spec/javascripts/ide/components/commit_sidebar/list_spec.js @@ -1,7 +1,7 @@ import Vue from 'vue'; +import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import store from '~/ide/stores'; import commitSidebarList from '~/ide/components/commit_sidebar/list.vue'; -import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import { file, resetStore } from '../../helpers'; describe('Multi-file editor commit sidebar list', () => { diff --git a/spec/javascripts/ide/components/commit_sidebar/message_field_spec.js b/spec/javascripts/ide/components/commit_sidebar/message_field_spec.js index af67991eadd..53508f52b2f 100644 --- a/spec/javascripts/ide/components/commit_sidebar/message_field_spec.js +++ b/spec/javascripts/ide/components/commit_sidebar/message_field_spec.js @@ -1,6 +1,6 @@ import Vue from 'vue'; -import CommitMessageField from '~/ide/components/commit_sidebar/message_field.vue'; import createComponent from 'spec/helpers/vue_mount_component_helper'; +import CommitMessageField from '~/ide/components/commit_sidebar/message_field.vue'; describe('IDE commit message field', () => { const Component = Vue.extend(CommitMessageField); diff --git a/spec/javascripts/ide/components/commit_sidebar/new_merge_request_option_spec.js b/spec/javascripts/ide/components/commit_sidebar/new_merge_request_option_spec.js index 5f2db695241..02caf689c50 100644 --- a/spec/javascripts/ide/components/commit_sidebar/new_merge_request_option_spec.js +++ b/spec/javascripts/ide/components/commit_sidebar/new_merge_request_option_spec.js @@ -1,9 +1,9 @@ import Vue from 'vue'; -import store from '~/ide/stores'; -import NewMergeRequestOption from '~/ide/components/commit_sidebar/new_merge_request_option.vue'; import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import { projectData, branches } from 'spec/ide/mock_data'; import { resetStore } from 'spec/ide/helpers'; +import NewMergeRequestOption from '~/ide/components/commit_sidebar/new_merge_request_option.vue'; +import store from '~/ide/stores'; import consts from '../../../../../app/assets/javascripts/ide/stores/modules/commit/constants'; describe('create new MR checkbox', () => { diff --git a/spec/javascripts/ide/components/commit_sidebar/radio_group_spec.js b/spec/javascripts/ide/components/commit_sidebar/radio_group_spec.js index db1988be3e1..b30f0e6822b 100644 --- a/spec/javascripts/ide/components/commit_sidebar/radio_group_spec.js +++ b/spec/javascripts/ide/components/commit_sidebar/radio_group_spec.js @@ -1,8 +1,8 @@ import Vue from 'vue'; -import store from '~/ide/stores'; -import radioGroup from '~/ide/components/commit_sidebar/radio_group.vue'; import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import { resetStore } from 'spec/ide/helpers'; +import store from '~/ide/stores'; +import radioGroup from '~/ide/components/commit_sidebar/radio_group.vue'; describe('IDE commit sidebar radio group', () => { let vm; diff --git a/spec/javascripts/ide/components/file_row_extra_spec.js b/spec/javascripts/ide/components/file_row_extra_spec.js index 86146fcef69..4c2f29f55dd 100644 --- a/spec/javascripts/ide/components/file_row_extra_spec.js +++ b/spec/javascripts/ide/components/file_row_extra_spec.js @@ -1,6 +1,6 @@ import Vue from 'vue'; -import { createStore } from '~/ide/stores'; import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; +import { createStore } from '~/ide/stores'; import FileRowExtra from '~/ide/components/file_row_extra.vue'; import { file, resetStore } from '../helpers'; diff --git a/spec/javascripts/ide/components/file_templates/bar_spec.js b/spec/javascripts/ide/components/file_templates/bar_spec.js index a688f7f69a6..5399ada94ae 100644 --- a/spec/javascripts/ide/components/file_templates/bar_spec.js +++ b/spec/javascripts/ide/components/file_templates/bar_spec.js @@ -1,7 +1,7 @@ import Vue from 'vue'; +import { mountComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import { createStore } from '~/ide/stores'; import Bar from '~/ide/components/file_templates/bar.vue'; -import { mountComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import { resetStore, file } from '../../helpers'; describe('IDE file templates bar component', () => { diff --git a/spec/javascripts/ide/components/ide_side_bar_spec.js b/spec/javascripts/ide/components/ide_side_bar_spec.js index 20ee20bc1d7..a2d15462ac5 100644 --- a/spec/javascripts/ide/components/ide_side_bar_spec.js +++ b/spec/javascripts/ide/components/ide_side_bar_spec.js @@ -1,8 +1,8 @@ import Vue from 'vue'; +import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import store from '~/ide/stores'; import ideSidebar from '~/ide/components/ide_side_bar.vue'; import { activityBarViews } from '~/ide/constants'; -import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import { resetStore } from '../helpers'; import { projectData } from '../mock_data'; diff --git a/spec/javascripts/ide/components/ide_spec.js b/spec/javascripts/ide/components/ide_spec.js index de4becec1cd..048db4a7533 100644 --- a/spec/javascripts/ide/components/ide_spec.js +++ b/spec/javascripts/ide/components/ide_spec.js @@ -1,7 +1,7 @@ import Vue from 'vue'; +import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import store from '~/ide/stores'; import ide from '~/ide/components/ide.vue'; -import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import { file, resetStore } from '../helpers'; import { projectData } from '../mock_data'; diff --git a/spec/javascripts/ide/components/ide_status_bar_spec.js b/spec/javascripts/ide/components/ide_status_bar_spec.js index bb8fb74c068..69f163574fb 100644 --- a/spec/javascripts/ide/components/ide_status_bar_spec.js +++ b/spec/javascripts/ide/components/ide_status_bar_spec.js @@ -1,8 +1,8 @@ import Vue from 'vue'; +import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import store from '~/ide/stores'; import ideStatusBar from '~/ide/components/ide_status_bar.vue'; import { rightSidebarViews } from '~/ide/constants'; -import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import { resetStore } from '../helpers'; import { projectData } from '../mock_data'; diff --git a/spec/javascripts/ide/components/nav_dropdown_button_spec.js b/spec/javascripts/ide/components/nav_dropdown_button_spec.js index 19b0071567a..0d63869fba2 100644 --- a/spec/javascripts/ide/components/nav_dropdown_button_spec.js +++ b/spec/javascripts/ide/components/nav_dropdown_button_spec.js @@ -1,8 +1,8 @@ import Vue from 'vue'; -import NavDropdownButton from '~/ide/components/nav_dropdown_button.vue'; -import store from '~/ide/stores'; import { trimText } from 'spec/helpers/text_helper'; import { mountComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; +import NavDropdownButton from '~/ide/components/nav_dropdown_button.vue'; +import store from '~/ide/stores'; import { resetStore } from '../helpers'; describe('NavDropdown', () => { diff --git a/spec/javascripts/ide/components/nav_dropdown_spec.js b/spec/javascripts/ide/components/nav_dropdown_spec.js index af6665bcd62..fe1d0ca371d 100644 --- a/spec/javascripts/ide/components/nav_dropdown_spec.js +++ b/spec/javascripts/ide/components/nav_dropdown_spec.js @@ -1,8 +1,8 @@ import $ from 'jquery'; import Vue from 'vue'; +import { mountComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import store from '~/ide/stores'; import NavDropdown from '~/ide/components/nav_dropdown.vue'; -import { mountComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; describe('IDE NavDropdown', () => { const Component = Vue.extend(NavDropdown); diff --git a/spec/javascripts/ide/components/new_dropdown/index_spec.js b/spec/javascripts/ide/components/new_dropdown/index_spec.js index aaebe88f314..03afe997fed 100644 --- a/spec/javascripts/ide/components/new_dropdown/index_spec.js +++ b/spec/javascripts/ide/components/new_dropdown/index_spec.js @@ -1,7 +1,7 @@ import Vue from 'vue'; +import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import store from '~/ide/stores'; import newDropdown from '~/ide/components/new_dropdown/index.vue'; -import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import { resetStore } from '../../helpers'; describe('new dropdown component', () => { diff --git a/spec/javascripts/ide/components/new_dropdown/modal_spec.js b/spec/javascripts/ide/components/new_dropdown/modal_spec.js index 0556feae46a..a1c00e99927 100644 --- a/spec/javascripts/ide/components/new_dropdown/modal_spec.js +++ b/spec/javascripts/ide/components/new_dropdown/modal_spec.js @@ -1,7 +1,7 @@ import Vue from 'vue'; +import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import { createStore } from '~/ide/stores'; import modal from '~/ide/components/new_dropdown/modal.vue'; -import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; describe('new file modal component', () => { const Component = Vue.extend(modal); diff --git a/spec/javascripts/ide/components/new_dropdown/upload_spec.js b/spec/javascripts/ide/components/new_dropdown/upload_spec.js index d19af6af2d7..4ebd0977832 100644 --- a/spec/javascripts/ide/components/new_dropdown/upload_spec.js +++ b/spec/javascripts/ide/components/new_dropdown/upload_spec.js @@ -1,6 +1,6 @@ import Vue from 'vue'; -import upload from '~/ide/components/new_dropdown/upload.vue'; import createComponent from 'spec/helpers/vue_mount_component_helper'; +import upload from '~/ide/components/new_dropdown/upload.vue'; describe('new dropdown upload', () => { let vm; diff --git a/spec/javascripts/ide/components/preview/navigator_spec.js b/spec/javascripts/ide/components/preview/navigator_spec.js index 576d2fae003..a5341997a3a 100644 --- a/spec/javascripts/ide/components/preview/navigator_spec.js +++ b/spec/javascripts/ide/components/preview/navigator_spec.js @@ -1,6 +1,6 @@ import Vue from 'vue'; -import ClientsideNavigator from '~/ide/components/preview/navigator.vue'; import mountComponent from 'spec/helpers/vue_mount_component_helper'; +import ClientsideNavigator from '~/ide/components/preview/navigator.vue'; describe('IDE clientside preview navigator', () => { let vm; diff --git a/spec/javascripts/ide/components/repo_commit_section_spec.js b/spec/javascripts/ide/components/repo_commit_section_spec.js index 6c726c1e154..917eb1438bd 100644 --- a/spec/javascripts/ide/components/repo_commit_section_spec.js +++ b/spec/javascripts/ide/components/repo_commit_section_spec.js @@ -1,8 +1,8 @@ import Vue from 'vue'; +import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import store from '~/ide/stores'; import router from '~/ide/ide_router'; import repoCommitSection from '~/ide/components/repo_commit_section.vue'; -import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; import { file, resetStore } from '../helpers'; describe('RepoCommitSection', () => { diff --git a/spec/javascripts/ide/components/shared/tokened_input_spec.js b/spec/javascripts/ide/components/shared/tokened_input_spec.js index b09bf760543..885fd976655 100644 --- a/spec/javascripts/ide/components/shared/tokened_input_spec.js +++ b/spec/javascripts/ide/components/shared/tokened_input_spec.js @@ -1,6 +1,6 @@ import Vue from 'vue'; -import TokenedInput from '~/ide/components/shared/tokened_input.vue'; import mountComponent from 'spec/helpers/vue_mount_component_helper'; +import TokenedInput from '~/ide/components/shared/tokened_input.vue'; const TEST_PLACEHOLDER = 'Searching in test'; const TEST_TOKENS = [ diff --git a/spec/javascripts/ide/stores/actions_spec.js b/spec/javascripts/ide/stores/actions_spec.js index 7e77b859fdd..246b3995395 100644 --- a/spec/javascripts/ide/stores/actions_spec.js +++ b/spec/javascripts/ide/stores/actions_spec.js @@ -1,3 +1,4 @@ +import MockAdapter from 'axios-mock-adapter'; import actions, { stageAllChanges, unstageAllChanges, @@ -18,7 +19,6 @@ import * as types from '~/ide/stores/mutation_types'; import router from '~/ide/ide_router'; import { resetStore, file } from '../helpers'; import testAction from '../../helpers/vuex_action_helper'; -import MockAdapter from 'axios-mock-adapter'; import eventHub from '~/ide/eventhub'; const store = createStore(); diff --git a/spec/javascripts/ide/stores/modules/branches/actions_spec.js b/spec/javascripts/ide/stores/modules/branches/actions_spec.js index 9c61ba3d1a6..4896eac46f4 100644 --- a/spec/javascripts/ide/stores/modules/branches/actions_spec.js +++ b/spec/javascripts/ide/stores/modules/branches/actions_spec.js @@ -1,8 +1,8 @@ import MockAdapter from 'axios-mock-adapter'; +import testAction from 'spec/helpers/vuex_action_helper'; import axios from '~/lib/utils/axios_utils'; import state from '~/ide/stores/modules/branches/state'; import * as types from '~/ide/stores/modules/branches/mutation_types'; -import testAction from 'spec/helpers/vuex_action_helper'; import { requestBranches, receiveBranchesError, diff --git a/spec/javascripts/ide/stores/modules/commit/actions_spec.js b/spec/javascripts/ide/stores/modules/commit/actions_spec.js index d464f30b947..cbc2401262f 100644 --- a/spec/javascripts/ide/stores/modules/commit/actions_spec.js +++ b/spec/javascripts/ide/stores/modules/commit/actions_spec.js @@ -1,3 +1,4 @@ +import { resetStore, file } from 'spec/ide/helpers'; import rootActions from '~/ide/stores/actions'; import { createStore } from '~/ide/stores'; import service from '~/ide/services'; @@ -7,7 +8,6 @@ import consts from '~/ide/stores/modules/commit/constants'; import * as mutationTypes from '~/ide/stores/modules/commit/mutation_types'; import * as actions from '~/ide/stores/modules/commit/actions'; import { commitActionTypes } from '~/ide/constants'; -import { resetStore, file } from 'spec/ide/helpers'; import testAction from '../../../../helpers/vuex_action_helper'; const TEST_COMMIT_SHA = '123456789'; diff --git a/spec/javascripts/ide/stores/modules/file_templates/actions_spec.js b/spec/javascripts/ide/stores/modules/file_templates/actions_spec.js index 548962c7a92..f049e6e01e4 100644 --- a/spec/javascripts/ide/stores/modules/file_templates/actions_spec.js +++ b/spec/javascripts/ide/stores/modules/file_templates/actions_spec.js @@ -1,9 +1,9 @@ import MockAdapter from 'axios-mock-adapter'; +import testAction from 'spec/helpers/vuex_action_helper'; import axios from '~/lib/utils/axios_utils'; import createState from '~/ide/stores/modules/file_templates/state'; import * as actions from '~/ide/stores/modules/file_templates/actions'; import * as types from '~/ide/stores/modules/file_templates/mutation_types'; -import testAction from 'spec/helpers/vuex_action_helper'; describe('IDE file templates actions', () => { let state; diff --git a/spec/javascripts/ide/stores/modules/pane/actions_spec.js b/spec/javascripts/ide/stores/modules/pane/actions_spec.js index 799bc89a0c3..9345a58746a 100644 --- a/spec/javascripts/ide/stores/modules/pane/actions_spec.js +++ b/spec/javascripts/ide/stores/modules/pane/actions_spec.js @@ -1,6 +1,6 @@ +import testAction from 'spec/helpers/vuex_action_helper'; import * as actions from '~/ide/stores/modules/pane/actions'; import * as types from '~/ide/stores/modules/pane/mutation_types'; -import testAction from 'spec/helpers/vuex_action_helper'; describe('IDE pane module actions', () => { const TEST_VIEW = { name: 'test' }; diff --git a/spec/lib/api/helpers/pagination_spec.rb b/spec/lib/api/helpers/pagination_spec.rb index 040ff1a8ebe..2d5bec2e752 100644 --- a/spec/lib/api/helpers/pagination_spec.rb +++ b/spec/lib/api/helpers/pagination_spec.rb @@ -5,10 +5,16 @@ require 'spec_helper' describe API::Helpers::Pagination do subject { Class.new.include(described_class).new } + let(:expected_result) { double("result", to_a: double) } + let(:relation) { double("relation") } + let(:params) { {} } + + before do + allow(subject).to receive(:params).and_return(params) + end + describe '#paginate' do - let(:relation) { double("relation") } let(:offset_pagination) { double("offset pagination") } - let(:expected_result) { double("result") } it 'delegates to OffsetPagination' do expect(::Gitlab::Pagination::OffsetPagination).to receive(:new).with(subject).and_return(offset_pagination) @@ -19,4 +25,50 @@ describe API::Helpers::Pagination do expect(result).to eq(expected_result) end end + + describe '#paginate_and_retrieve!' do + context 'for offset pagination' do + before do + allow(Gitlab::Pagination::Keyset).to receive(:available?).and_return(false) + end + + it 'delegates to paginate' do + expect(subject).to receive(:paginate).with(relation).and_return(expected_result) + + result = subject.paginate_and_retrieve!(relation) + + expect(result).to eq(expected_result.to_a) + end + end + + context 'for keyset pagination' do + let(:params) { { pagination: 'keyset' } } + let(:request_context) { double('request context') } + + before do + allow(Gitlab::Pagination::Keyset::RequestContext).to receive(:new).with(subject).and_return(request_context) + end + + context 'when keyset pagination is available' do + it 'delegates to KeysetPagination' do + expect(Gitlab::Pagination::Keyset).to receive(:available?).and_return(true) + expect(Gitlab::Pagination::Keyset).to receive(:paginate).with(request_context, relation).and_return(expected_result) + + result = subject.paginate_and_retrieve!(relation) + + expect(result).to eq(expected_result.to_a) + end + end + + context 'when keyset pagination is not available' do + it 'renders a 501 error if keyset pagination isnt available yet' do + expect(Gitlab::Pagination::Keyset).to receive(:available?).with(request_context, relation).and_return(false) + expect(Gitlab::Pagination::Keyset).not_to receive(:paginate) + expect(subject).to receive(:error!).with(/not yet available/, 405) + + subject.paginate_and_retrieve!(relation) + end + end + end + end end diff --git a/spec/lib/api/projects_batch_counting_spec.rb b/spec/lib/api/projects_batch_counting_spec.rb new file mode 100644 index 00000000000..6094952bb52 --- /dev/null +++ b/spec/lib/api/projects_batch_counting_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe API::ProjectsBatchCounting do + subject do + Class.new do + include ::API::ProjectsBatchCounting + end + end + + describe '.preload_and_batch_count!' do + let(:projects) { double } + let(:preloaded_projects) { double } + + it 'preloads the relation' do + allow(subject).to receive(:execute_batch_counting).with(preloaded_projects) + + expect(subject).to receive(:preload_relation).with(projects).and_return(preloaded_projects) + + expect(subject.preload_and_batch_count!(projects)).to eq(preloaded_projects) + end + + it 'executes batch counting' do + allow(subject).to receive(:preload_relation).with(projects).and_return(preloaded_projects) + + expect(subject).to receive(:execute_batch_counting).with(preloaded_projects) + + subject.preload_and_batch_count!(projects) + end + end + + describe '.execute_batch_counting' do + let(:projects) { create_list(:project, 2) } + let(:count_service) { double } + + it 'counts forks' do + allow(::Projects::BatchForksCountService).to receive(:new).with(projects).and_return(count_service) + + expect(count_service).to receive(:refresh_cache) + + subject.execute_batch_counting(projects) + end + + it 'counts open issues' do + allow(::Projects::BatchOpenIssuesCountService).to receive(:new).with(projects).and_return(count_service) + + expect(count_service).to receive(:refresh_cache) + + subject.execute_batch_counting(projects) + end + + context 'custom fork counting' do + subject do + Class.new do + include ::API::ProjectsBatchCounting + def self.forks_counting_projects(projects) + [projects.first] + end + end + end + + it 'counts forks for other projects' do + allow(::Projects::BatchForksCountService).to receive(:new).with([projects.first]).and_return(count_service) + + expect(count_service).to receive(:refresh_cache) + + subject.execute_batch_counting(projects) + end + end + end +end diff --git a/spec/lib/gitlab/auth/user_auth_finders_spec.rb b/spec/lib/gitlab/auth/user_auth_finders_spec.rb index dd8070c1240..125039edcf8 100644 --- a/spec/lib/gitlab/auth/user_auth_finders_spec.rb +++ b/spec/lib/gitlab/auth/user_auth_finders_spec.rb @@ -116,9 +116,9 @@ describe Gitlab::Auth::UserAuthFinders do end describe '#find_user_from_static_object_token' do - context 'when request format is archive' do + shared_examples 'static object request' do before do - env['SCRIPT_NAME'] = 'project/-/archive/master.zip' + env['SCRIPT_NAME'] = path end context 'when token header param is present' do @@ -126,7 +126,7 @@ describe Gitlab::Auth::UserAuthFinders do it 'returns the user' do request.headers['X-Gitlab-Static-Object-Token'] = user.static_object_token - expect(find_user_from_static_object_token(:archive)).to eq(user) + expect(find_user_from_static_object_token(format)).to eq(user) end end @@ -134,7 +134,7 @@ describe Gitlab::Auth::UserAuthFinders do it 'returns the user' do request.headers['X-Gitlab-Static-Object-Token'] = 'foobar' - expect { find_user_from_static_object_token(:archive) }.to raise_error(Gitlab::Auth::UnauthorizedError) + expect { find_user_from_static_object_token(format) }.to raise_error(Gitlab::Auth::UnauthorizedError) end end end @@ -144,7 +144,7 @@ describe Gitlab::Auth::UserAuthFinders do it 'returns the user' do set_param(:token, user.static_object_token) - expect(find_user_from_static_object_token(:archive)).to eq(user) + expect(find_user_from_static_object_token(format)).to eq(user) end end @@ -152,13 +152,27 @@ describe Gitlab::Auth::UserAuthFinders do it 'returns the user' do set_param(:token, 'foobar') - expect { find_user_from_static_object_token(:archive) }.to raise_error(Gitlab::Auth::UnauthorizedError) + expect { find_user_from_static_object_token(format) }.to raise_error(Gitlab::Auth::UnauthorizedError) end end end end - context 'when request format is not archive' do + context 'when request format is archive' do + it_behaves_like 'static object request' do + let_it_be(:path) { 'project/-/archive/master.zip' } + let_it_be(:format) { :archive } + end + end + + context 'when request format is blob' do + it_behaves_like 'static object request' do + let_it_be(:path) { 'project/raw/master/README.md' } + let_it_be(:format) { :blob } + end + end + + context 'when request format is not archive nor blob' do before do env['script_name'] = 'url' end diff --git a/spec/lib/gitlab/pagination/keyset/page_spec.rb b/spec/lib/gitlab/pagination/keyset/page_spec.rb new file mode 100644 index 00000000000..5c03224c05a --- /dev/null +++ b/spec/lib/gitlab/pagination/keyset/page_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Pagination::Keyset::Page do + describe '#per_page' do + it 'limits to a maximum of 100 records per page' do + per_page = described_class.new(per_page: 101).per_page + + expect(per_page).to eq(described_class::MAXIMUM_PAGE_SIZE) + end + + it 'uses default value when given 0' do + per_page = described_class.new(per_page: 0).per_page + + expect(per_page).to eq(described_class::DEFAULT_PAGE_SIZE) + end + + it 'uses default value when given negative values' do + per_page = described_class.new(per_page: -1).per_page + + expect(per_page).to eq(described_class::DEFAULT_PAGE_SIZE) + end + + it 'uses the given value if it is within range' do + per_page = described_class.new(per_page: 10).per_page + + expect(per_page).to eq(10) + end + end + + describe '#next' do + let(:page) { described_class.new(order_by: order_by, lower_bounds: lower_bounds, per_page: per_page, end_reached: end_reached) } + subject { page.next(new_lower_bounds, new_end_reached) } + + let(:order_by) { { id: :desc } } + let(:lower_bounds) { { id: 42 } } + let(:per_page) { 10 } + let(:end_reached) { false } + + let(:new_lower_bounds) { { id: 21 } } + let(:new_end_reached) { true } + + it 'copies over order_by' do + expect(subject.order_by).to eq(page.order_by) + end + + it 'copies over per_page' do + expect(subject.per_page).to eq(page.per_page) + end + + it 'dups the instance' do + expect(subject).not_to eq(page) + end + + it 'sets lower_bounds only on new instance' do + expect(subject.lower_bounds).to eq(new_lower_bounds) + expect(page.lower_bounds).to eq(lower_bounds) + end + + it 'sets end_reached only on new instance' do + expect(subject.end_reached?).to eq(new_end_reached) + expect(page.end_reached?).to eq(end_reached) + end + end +end diff --git a/spec/lib/gitlab/pagination/keyset/pager_spec.rb b/spec/lib/gitlab/pagination/keyset/pager_spec.rb new file mode 100644 index 00000000000..6d23fe2adcc --- /dev/null +++ b/spec/lib/gitlab/pagination/keyset/pager_spec.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Pagination::Keyset::Pager do + let(:relation) { Project.all.order(id: :asc) } + let(:request) { double('request', page: page, apply_headers: nil) } + let(:page) { Gitlab::Pagination::Keyset::Page.new(order_by: { id: :asc }, per_page: 3) } + let(:next_page) { double('next page') } + + before_all do + create_list(:project, 7) + end + + describe '#paginate' do + subject { described_class.new(request).paginate(relation) } + + it 'loads the result relation only once' do + expect do + subject + end.not_to exceed_query_limit(1) + end + + it 'passes information about next page to request' do + lower_bounds = relation.limit(page.per_page).last.slice(:id) + expect(page).to receive(:next).with(lower_bounds, false).and_return(next_page) + expect(request).to receive(:apply_headers).with(next_page) + + subject + end + + context 'when retrieving the last page' do + let(:relation) { Project.where('id > ?', Project.maximum(:id) - page.per_page).order(id: :asc) } + + it 'indicates this is the last page' do + expect(request).to receive(:apply_headers) do |next_page| + expect(next_page.end_reached?).to be_truthy + end + + subject + end + end + + context 'when retrieving an empty page' do + let(:relation) { Project.where('id > ?', Project.maximum(:id) + 1).order(id: :asc) } + + it 'indicates this is the last page' do + expect(request).to receive(:apply_headers) do |next_page| + expect(next_page.end_reached?).to be_truthy + end + + subject + end + end + + it 'returns an array with the loaded records' do + expect(subject).to eq(relation.limit(page.per_page).to_a) + end + + context 'validating the order clause' do + let(:page) { Gitlab::Pagination::Keyset::Page.new(order_by: { created_at: :asc }, per_page: 3) } + + it 'raises an error if has a different order clause than the page' do + expect { subject }.to raise_error(ArgumentError, /order_by does not match/) + end + end + end +end diff --git a/spec/lib/gitlab/pagination/keyset/request_context_spec.rb b/spec/lib/gitlab/pagination/keyset/request_context_spec.rb new file mode 100644 index 00000000000..344ef90efa3 --- /dev/null +++ b/spec/lib/gitlab/pagination/keyset/request_context_spec.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Pagination::Keyset::RequestContext do + let(:request) { double('request', params: params) } + + describe '#page' do + subject { described_class.new(request).page } + + context 'with only order_by given' do + let(:params) { { order_by: :id } } + + it 'extracts order_by/sorting information' do + page = subject + + expect(page.order_by).to eq(id: :desc) + end + end + + context 'with order_by and sort given' do + let(:params) { { order_by: :created_at, sort: :desc } } + + it 'extracts order_by/sorting information and adds tie breaker' do + page = subject + + expect(page.order_by).to eq(created_at: :desc, id: :desc) + end + end + + context 'with no order_by information given' do + let(:params) { {} } + + it 'defaults to tie breaker' do + page = subject + + expect(page.order_by).to eq({ id: :desc }) + end + end + + context 'with per_page params given' do + let(:params) { { per_page: 10 } } + + it 'extracts per_page information' do + page = subject + + expect(page.per_page).to eq(params[:per_page]) + end + end + end + + describe '#apply_headers' do + let(:request) { double('request', url: "http://#{Gitlab.config.gitlab.host}/api/v4/projects?foo=bar") } + let(:params) { { foo: 'bar' } } + let(:request_context) { double('request context', params: params, request: request) } + let(:next_page) { double('next page', order_by: { id: :asc }, lower_bounds: { id: 42 }, end_reached?: false) } + + subject { described_class.new(request_context).apply_headers(next_page) } + + it 'sets Links header with same host/path as the original request' do + orig_uri = URI.parse(request_context.request.url) + + expect(request_context).to receive(:header) do |name, header| + expect(name).to eq('Links') + + first_link, _ = /<([^>]+)>; rel="next"/.match(header).captures + + uri = URI.parse(first_link) + + expect(uri.host).to eq(orig_uri.host) + expect(uri.path).to eq(orig_uri.path) + end + + subject + end + + it 'sets Links header with a link to the next page' do + orig_uri = URI.parse(request_context.request.url) + + expect(request_context).to receive(:header) do |name, header| + expect(name).to eq('Links') + + first_link, _ = /<([^>]+)>; rel="next"/.match(header).captures + + query = CGI.parse(URI.parse(first_link).query) + + expect(query.except('id_after')).to eq(CGI.parse(orig_uri.query).except('id_after')) + expect(query['id_after']).to eq(['42']) + end + + subject + end + + context 'with descending order' do + let(:next_page) { double('next page', order_by: { id: :desc }, lower_bounds: { id: 42 }, end_reached?: false) } + + it 'sets Links header with a link to the next page' do + orig_uri = URI.parse(request_context.request.url) + + expect(request_context).to receive(:header) do |name, header| + expect(name).to eq('Links') + + first_link, _ = /<([^>]+)>; rel="next"/.match(header).captures + + query = CGI.parse(URI.parse(first_link).query) + + expect(query.except('id_before')).to eq(CGI.parse(orig_uri.query).except('id_before')) + expect(query['id_before']).to eq(['42']) + end + + subject + end + end + end +end diff --git a/spec/lib/gitlab/pagination/keyset_spec.rb b/spec/lib/gitlab/pagination/keyset_spec.rb new file mode 100644 index 00000000000..755c422c46a --- /dev/null +++ b/spec/lib/gitlab/pagination/keyset_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Pagination::Keyset do + describe '.paginate' do + subject { described_class.paginate(request_context, relation) } + + let(:request_context) { double } + let(:relation) { double } + let(:pager) { double } + let(:result) { double } + + it 'uses Pager to paginate the relation' do + expect(Gitlab::Pagination::Keyset::Pager).to receive(:new).with(request_context).and_return(pager) + expect(pager).to receive(:paginate).with(relation).and_return(result) + + expect(subject).to eq(result) + end + end + + describe '.available?' do + subject { described_class } + + let(:request_context) { double("request context", page: page)} + let(:page) { double("page", order_by: order_by) } + + shared_examples_for 'keyset pagination is available' do + it 'returns true for Project' do + expect(subject.available?(request_context, Project.all)).to be_truthy + end + + it 'return false for other types of relations' do + expect(subject.available?(request_context, User.all)).to be_falsey + end + end + + context 'with order-by id asc' do + let(:order_by) { { id: :asc } } + + it_behaves_like 'keyset pagination is available' + end + + context 'with order-by id desc' do + let(:order_by) { { id: :desc } } + + it_behaves_like 'keyset pagination is available' + end + + context 'with other order-by columns' do + let(:order_by) { { created_at: :desc, id: :desc } } + it 'returns false for Project' do + expect(subject.available?(request_context, Project.all)).to be_falsey + end + + it 'return false for other types of relations' do + expect(subject.available?(request_context, User.all)).to be_falsey + end + end + end +end diff --git a/spec/requests/api/deployments_spec.rb b/spec/requests/api/deployments_spec.rb index 986a44cc640..6e78b747a8c 100644 --- a/spec/requests/api/deployments_spec.rb +++ b/spec/requests/api/deployments_spec.rb @@ -30,6 +30,16 @@ describe API::Deployments do expect(json_response.last['iid']).to eq(deployment_3.iid) end + context 'with updated_at filters specified' do + it 'returns projects deployments with last update in specified datetime range' do + get api("/projects/#{project.id}/deployments", user), params: { updated_before: 30.minutes.ago, updated_after: 90.minutes.ago } + + expect(response).to have_gitlab_http_status(:ok) + expect(response).to include_pagination_headers + expect(json_response.first['id']).to eq(deployment_3.id) + end + end + describe 'ordering' do let(:order_by) { 'iid' } let(:sort) { 'desc' } diff --git a/spec/requests/api/pipelines_spec.rb b/spec/requests/api/pipelines_spec.rb index 2b34b64812d..a9d570b5696 100644 --- a/spec/requests/api/pipelines_spec.rb +++ b/spec/requests/api/pipelines_spec.rb @@ -237,6 +237,20 @@ describe API::Pipelines do end end + context 'when updated_at filters are specified' do + let!(:pipeline1) { create(:ci_pipeline, project: project, updated_at: 2.days.ago) } + let!(:pipeline2) { create(:ci_pipeline, project: project, updated_at: 4.days.ago) } + let!(:pipeline3) { create(:ci_pipeline, project: project, updated_at: 1.hour.ago) } + + it 'returns pipelines with last update date in specified datetime range' do + get api("/projects/#{project.id}/pipelines", user), params: { updated_before: 1.day.ago, updated_after: 3.days.ago } + + expect(response).to have_gitlab_http_status(:ok) + expect(response).to include_pagination_headers + expect(json_response.first['id']).to eq(pipeline1.id) + end + end + context 'when order_by and sort are specified' do context 'when order_by user_id' do before do diff --git a/spec/requests/api/projects_spec.rb b/spec/requests/api/projects_spec.rb index cda2dd7d2f4..5161de6de5e 100644 --- a/spec/requests/api/projects_spec.rb +++ b/spec/requests/api/projects_spec.rb @@ -155,6 +155,35 @@ describe API::Projects do project4 end + # This is a regression spec for https://gitlab.com/gitlab-org/gitlab/issues/37919 + context 'batch counting forks and open issues and refreshing count caches' do + # We expect to count these projects (only the ones on the first page, not all matching ones) + let(:projects) { Project.public_to_user(nil).order(id: :desc).first(per_page) } + let(:per_page) { 2 } + let(:count_service) { double } + + before do + # Create more projects, so we have more than one page + create_list(:project, 5, :public) + end + + it 'batch counts project forks' do + expect(::Projects::BatchForksCountService).to receive(:new).with(projects).and_return(count_service) + expect(count_service).to receive(:refresh_cache) + + get api("/projects?per_page=#{per_page}") + expect(response.status).to eq 200 + end + + it 'batch counts open issues' do + expect(::Projects::BatchOpenIssuesCountService).to receive(:new).with(projects).and_return(count_service) + expect(count_service).to receive(:refresh_cache) + + get api("/projects?per_page=#{per_page}") + expect(response.status).to eq 200 + end + end + context 'when unauthenticated' do it_behaves_like 'projects response' do let(:filter) { { search: project.name } } @@ -570,6 +599,87 @@ describe API::Projects do let(:projects) { Project.all } end end + + context 'with keyset pagination' do + let(:current_user) { user } + let(:projects) { [public_project, project, project2, project3] } + + context 'headers and records' do + let(:params) { { pagination: 'keyset', order_by: :id, sort: :asc, per_page: 1 } } + + it 'includes a pagination header with link to the next page' do + get api('/projects', current_user), params: params + + expect(response.header).to include('Links') + expect(response.header['Links']).to include('pagination=keyset') + expect(response.header['Links']).to include("id_after=#{public_project.id}") + end + + it 'contains only the first project with per_page = 1' do + get api('/projects', current_user), params: params + + expect(response).to have_gitlab_http_status(200) + expect(json_response).to be_an Array + expect(json_response.map { |p| p['id'] }).to contain_exactly(public_project.id) + end + + it 'does not include a link if the end has reached and there is no more data' do + get api('/projects', current_user), params: params.merge(id_after: project2.id) + + expect(response.header).not_to include('Links') + end + + it 'responds with 501 if order_by is different from id' do + get api('/projects', current_user), params: params.merge(order_by: :created_at) + + expect(response).to have_gitlab_http_status(405) + end + end + + context 'with descending sorting' do + let(:params) { { pagination: 'keyset', order_by: :id, sort: :desc, per_page: 1 } } + + it 'includes a pagination header with link to the next page' do + get api('/projects', current_user), params: params + + expect(response.header).to include('Links') + expect(response.header['Links']).to include('pagination=keyset') + expect(response.header['Links']).to include("id_before=#{project3.id}") + end + + it 'contains only the last project with per_page = 1' do + get api('/projects', current_user), params: params + + expect(response).to have_gitlab_http_status(200) + expect(json_response).to be_an Array + expect(json_response.map { |p| p['id'] }).to contain_exactly(project3.id) + end + end + + context 'retrieving the full relation' do + let(:params) { { pagination: 'keyset', order_by: :id, sort: :desc, per_page: 2 } } + + it 'returns all projects' do + url = '/projects' + requests = 0 + ids = [] + + while url && requests <= 5 # circuit breaker + requests += 1 + get api(url, current_user), params: params + + links = response.header['Links'] + url = links&.match(/<[^>]+(\/projects\?[^>]+)>; rel="next"/) do |match| + match[1] + end + + ids += JSON.parse(response.body).map { |p| p['id'] } + end + + expect(ids).to contain_exactly(*projects.map(&:id)) + end + end + end end describe 'POST /projects' do diff --git a/spec/services/ci/retry_pipeline_service_spec.rb b/spec/services/ci/retry_pipeline_service_spec.rb index 3cab3158cce..4b949761b8f 100644 --- a/spec/services/ci/retry_pipeline_service_spec.rb +++ b/spec/services/ci/retry_pipeline_service_spec.rb @@ -91,6 +91,25 @@ describe Ci::RetryPipelineService, '#execute' do end end + context 'when there is a failed test in a DAG' do + before do + create_build('build', :success, 0) + create_build('build2', :success, 0) + test_build = create_build('test', :failed, 1) + create(:ci_build_need, build: test_build, name: 'build') + create(:ci_build_need, build: test_build, name: 'build2') + end + + it 'retries the test' do + service.execute(pipeline) + + expect(build('build')).to be_success + expect(build('build2')).to be_success + expect(build('test')).to be_pending + expect(build('test').needs.map(&:name)).to match_array(%w(build build2)) + end + end + context 'when the last stage was skipepd' do before do create_build('build 1', :success, 0) |
