diff options
64 files changed, 1191 insertions, 749 deletions
diff --git a/.gitlab/ci/notify.gitlab-ci.yml b/.gitlab/ci/notify.gitlab-ci.yml index d927d9d585f..2c510773729 100644 --- a/.gitlab/ci/notify.gitlab-ci.yml +++ b/.gitlab/ci/notify.gitlab-ci.yml @@ -7,6 +7,8 @@ MERGE_REQUEST_URL: ${CI_MERGE_REQUEST_PROJECT_URL}/-/merge_requests/${CI_MERGE_REQUEST_IID} before_script: - apk update && apk add git curl bash + - echo "NOTIFY_CHANNEL is ${NOTIFY_CHANNEL}" + - echo "CI_PIPELINE_URL is ${CI_PIPELINE_URL}" notify-update-gitaly: extends: @@ -19,8 +21,6 @@ notify-update-gitaly: NOTIFY_CHANNEL: g_gitaly GITALY_UPDATE_BRANCH: release-tools/update-gitaly script: - - echo "NOTIFY_CHANNEL is ${NOTIFY_CHANNEL}" - - echo "CI_PIPELINE_URL is ${CI_PIPELINE_URL}" - scripts/slack ${NOTIFY_CHANNEL} "☠️ \`${GITALY_UPDATE_BRANCH}\` failed! ☠️ See ${CI_PIPELINE_URL} (triggered from ${MERGE_REQUEST_URL})" ci_failing "GitLab QA Bot" notify-security-pipeline: @@ -30,7 +30,17 @@ notify-security-pipeline: variables: NOTIFY_CHANNEL: f_upcoming_release script: - - echo "NOTIFY_CHANNEL is ${NOTIFY_CHANNEL}" - - echo "CI_PIPELINE_URL is ${CI_PIPELINE_URL}" # <!subteam^S0127FU8PDE> mentions the `@release-managers` group - scripts/slack ${NOTIFY_CHANNEL} "<!subteam^S0127FU8PDE> ☠️ Pipeline for merged result failed! ☠️ See ${CI_PIPELINE_URL} (triggered from ${MERGE_REQUEST_URL})" ci_failing "GitLab Release Tools Bot" + +notify-pipeline-failure: + extends: + - .notify-slack + rules: + - if: '$NOTIFY_PIPELINE_FAILURE_CHANNEL' + when: on_failure + allow_failure: true + variables: + NOTIFY_CHANNEL: "${NOTIFY_PIPELINE_FAILURE_CHANNEL}" + script: + - scripts/slack ${NOTIFY_CHANNEL} "❌ `${CI_COMMIT_REF_NAME}` pipeline failed! See ${CI_PIPELINE_URL}" ci_failing "notify-pipeline-failure" diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 8d0349fa117..da63e7e0a66 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -1520,6 +1520,12 @@ changes: ["vendor/gems/omniauth-gitlab/**/*"] - <<: *if-merge-request-labels-run-all-rspec +.vendor:rules:omniauth-salesforce: + rules: + - <<: *if-merge-request + changes: ["vendor/gems/omniauth-salesforce/**/*"] + - <<: *if-merge-request-labels-run-all-rspec + .vendor:rules:devise-pbkdf2-encryptable: rules: - <<: *if-merge-request diff --git a/.gitlab/ci/vendored-gems.gitlab-ci.yml b/.gitlab/ci/vendored-gems.gitlab-ci.yml index 4408a6e4624..03a6d1bcc0e 100644 --- a/.gitlab/ci/vendored-gems.gitlab-ci.yml +++ b/.gitlab/ci/vendored-gems.gitlab-ci.yml @@ -46,6 +46,14 @@ vendor omniauth-gitlab: include: vendor/gems/omniauth-gitlab/.gitlab-ci.yml strategy: depend +vendor omniauth-salesforce: + extends: + - .vendor:rules:omniauth-salesforce + needs: [] + trigger: + include: vendor/gems/omniauth-salesforce/.gitlab-ci.yml + strategy: depend + vendor devise-pbkdf2-encryptable: extends: - .vendor:rules:devise-pbkdf2-encryptable @@ -54,7 +54,7 @@ gem 'omniauth-twitter', '~> 1.4' gem 'omniauth_crowd', '~> 2.4.0', path: 'vendor/gems/omniauth_crowd' # See vendor/gems/omniauth_crowd/README.md gem 'omniauth-authentiq', '~> 0.3.3' gem 'gitlab-omniauth-openid-connect', '~> 0.9.0', require: 'omniauth_openid_connect' -gem 'omniauth-salesforce', '~> 1.0.5' +gem 'omniauth-salesforce', '~> 1.0.5', path: 'vendor/gems/omniauth-salesforce' # See gem README.md gem 'omniauth-atlassian-oauth2', '~> 0.2.0' gem 'rack-oauth2', '~> 1.21.2' gem 'jwt', '~> 2.1.0' diff --git a/Gemfile.lock b/Gemfile.lock index 2fa7558578b..941d9cb1538 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -40,6 +40,13 @@ PATH omniauth-oauth2 (~> 1.7.1) PATH + remote: vendor/gems/omniauth-salesforce + specs: + omniauth-salesforce (1.0.5) + omniauth (~> 1.0, < 3) + omniauth-oauth2 (~> 1.0) + +PATH remote: vendor/gems/omniauth_crowd specs: omniauth_crowd (2.4.0) @@ -926,9 +933,6 @@ GEM omniauth (>= 1.9, < 3) omniauth-oauth2-generic (0.2.2) omniauth-oauth2 (~> 1.0) - omniauth-salesforce (1.0.5) - omniauth (~> 1.0) - omniauth-oauth2 (~> 1.0) omniauth-saml (1.10.0) omniauth (~> 1.3, >= 1.3.2) ruby-saml (~> 1.7) @@ -1667,7 +1671,7 @@ DEPENDENCIES omniauth-gitlab (~> 4.0.0)! omniauth-google-oauth2 (~> 0.6.0) omniauth-oauth2-generic (~> 0.2.2) - omniauth-salesforce (~> 1.0.5) + omniauth-salesforce (~> 1.0.5)! omniauth-saml (~> 1.10) omniauth-shibboleth (~> 1.3.0) omniauth-twitter (~> 1.4) diff --git a/app/assets/javascripts/pages/groups/settings/repository/create_deploy_token/index.js b/app/assets/javascripts/pages/groups/settings/repository/create_deploy_token/index.js new file mode 100644 index 00000000000..6a7c6028c95 --- /dev/null +++ b/app/assets/javascripts/pages/groups/settings/repository/create_deploy_token/index.js @@ -0,0 +1 @@ +import '../show/index'; diff --git a/app/assets/javascripts/runner/group_runners/group_runners_app.vue b/app/assets/javascripts/runner/group_runners/group_runners_app.vue index a82411a2120..a2b666f4c6a 100644 --- a/app/assets/javascripts/runner/group_runners/group_runners_app.vue +++ b/app/assets/javascripts/runner/group_runners/group_runners_app.vue @@ -50,7 +50,8 @@ export default { props: { registrationToken: { type: String, - required: true, + required: false, + default: null, }, groupFullPath: { type: String, @@ -191,6 +192,7 @@ export default { /> <registration-dropdown + v-if="registrationToken" class="gl-ml-auto" :registration-token="registrationToken" :type="$options.GROUP_TYPE" diff --git a/app/controllers/groups/runners_controller.rb b/app/controllers/groups/runners_controller.rb index aeb54527c69..0ee6ddc2662 100644 --- a/app/controllers/groups/runners_controller.rb +++ b/app/controllers/groups/runners_controller.rb @@ -11,6 +11,7 @@ class Groups::RunnersController < Groups::ApplicationController def index finder = Ci::RunnersFinder.new(current_user: current_user, params: { group: @group }) @group_runners_limited_count = finder.execute.except(:limit, :offset).page.total_count_with_limit(:all, limit: 1000) + @group_runner_registration_token = @group.runners_token if can?(current_user, :register_group_runners, group) Gitlab::Tracking.event(self.class.name, 'index', user: current_user, namespace: @group) end diff --git a/app/controllers/groups/settings/repository_controller.rb b/app/controllers/groups/settings/repository_controller.rb index e09d8bf81cc..cb62ea2a543 100644 --- a/app/controllers/groups/settings/repository_controller.rb +++ b/app/controllers/groups/settings/repository_controller.rb @@ -17,13 +17,13 @@ module Groups def create_deploy_token result = Groups::DeployTokens::CreateService.new(@group, current_user, deploy_token_params).execute - @new_deploy_token = result[:deploy_token] if result[:status] == :success + @created_deploy_token = result[:deploy_token] respond_to do |format| format.json do # IMPORTANT: It's a security risk to expose the token value more than just once here! - json = API::Entities::DeployTokenWithToken.represent(@new_deploy_token).as_json + json = API::Entities::DeployTokenWithToken.represent(@created_deploy_token).as_json render json: json, status: result[:http_status] end format.html do @@ -32,6 +32,7 @@ module Groups end end else + @new_deploy_token = result[:deploy_token] respond_to do |format| format.json { render json: { message: result[:message] }, status: result[:http_status] } format.html do diff --git a/app/controllers/projects/settings/repository_controller.rb b/app/controllers/projects/settings/repository_controller.rb index a178b8f7aa3..43c6451577a 100644 --- a/app/controllers/projects/settings/repository_controller.rb +++ b/app/controllers/projects/settings/repository_controller.rb @@ -34,13 +34,13 @@ module Projects def create_deploy_token result = Projects::DeployTokens::CreateService.new(@project, current_user, deploy_token_params).execute - @new_deploy_token = result[:deploy_token] if result[:status] == :success + @created_deploy_token = result[:deploy_token] respond_to do |format| format.json do # IMPORTANT: It's a security risk to expose the token value more than just once here! - json = API::Entities::DeployTokenWithToken.represent(@new_deploy_token).as_json + json = API::Entities::DeployTokenWithToken.represent(@created_deploy_token).as_json render json: json, status: result[:http_status] end format.html do @@ -49,6 +49,7 @@ module Projects end end else + @new_deploy_token = result[:deploy_token] respond_to do |format| format.json { render json: { message: result[:message] }, status: result[:http_status] } format.html do diff --git a/app/helpers/ci/runners_helper.rb b/app/helpers/ci/runners_helper.rb index 852eaeca5e3..0de84c0d61f 100644 --- a/app/helpers/ci/runners_helper.rb +++ b/app/helpers/ci/runners_helper.rb @@ -84,7 +84,6 @@ module Ci def group_runners_data_attributes(group) { - registration_token: group.runners_token, group_id: group.id, group_full_path: group.full_path, runner_install_help_page: 'https://docs.gitlab.com/runner/install/', diff --git a/app/helpers/deploy_tokens_helper.rb b/app/helpers/deploy_tokens_helper.rb index d4da928edf1..597823cdac7 100644 --- a/app/helpers/deploy_tokens_helper.rb +++ b/app/helpers/deploy_tokens_helper.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true module DeployTokensHelper - def expand_deploy_tokens_section?(deploy_token) - deploy_token.persisted? || - deploy_token.errors.present? || + def expand_deploy_tokens_section?(new_deploy_token, created_deploy_token) + created_deploy_token || + new_deploy_token.errors.present? || Rails.env.test? end diff --git a/app/serializers/group_access_token_entity.rb b/app/serializers/group_access_token_entity.rb index e832eef1188..76dc3999a61 100644 --- a/app/serializers/group_access_token_entity.rb +++ b/app/serializers/group_access_token_entity.rb @@ -14,13 +14,13 @@ class GroupAccessTokenEntity < API::Entities::PersonalAccessToken group_id: group.path) end - expose :access_level do |token, options| + expose :role do |token, options| group = options.fetch(:group) next unless group next unless token.user - group.member(token.user)&.access_level + group.member(token.user)&.human_access end end # rubocop: enable Gitlab/NamespacedClass diff --git a/app/serializers/project_access_token_entity.rb b/app/serializers/project_access_token_entity.rb index b317057c952..695a5122348 100644 --- a/app/serializers/project_access_token_entity.rb +++ b/app/serializers/project_access_token_entity.rb @@ -15,13 +15,13 @@ class ProjectAccessTokenEntity < API::Entities::PersonalAccessToken project_id: project.path) end - expose :access_level do |token, options| + expose :role do |token, options| project = options.fetch(:project) next unless project next unless token.user - project.member(token.user)&.access_level + project.member(token.user)&.human_access end end # rubocop: enable Gitlab/NamespacedClass diff --git a/app/views/groups/runners/index.html.haml b/app/views/groups/runners/index.html.haml index a67a4f28c93..1146063969b 100644 --- a/app/views/groups/runners/index.html.haml +++ b/app/views/groups/runners/index.html.haml @@ -1,3 +1,3 @@ - page_title s_('Runners|Runners') -#js-group-runners{ data: group_runners_data_attributes(@group).merge( { group_runners_limited_count: @group_runners_limited_count } ) } +#js-group-runners{ data: group_runners_data_attributes(@group).merge( { group_runners_limited_count: @group_runners_limited_count, registration_token: @group_runner_registration_token } ) } diff --git a/app/views/shared/deploy_tokens/_index.html.haml b/app/views/shared/deploy_tokens/_index.html.haml index aa4a3deaac4..79bf35e2726 100644 --- a/app/views/shared/deploy_tokens/_index.html.haml +++ b/app/views/shared/deploy_tokens/_index.html.haml @@ -1,4 +1,4 @@ -- expanded = expand_deploy_tokens_section?(@new_deploy_token) +- expanded = expand_deploy_tokens_section?(@new_deploy_token, @created_deploy_token) %section.settings.no-animate#js-deploy-tokens{ class: ('expanded' if expanded), data: { qa_selector: 'deploy_tokens_settings_content' } } .settings-header @@ -8,11 +8,10 @@ %p = description .settings-content - - if @new_deploy_token.persisted? - = render 'shared/deploy_tokens/new_deploy_token', deploy_token: @new_deploy_token + - if @created_deploy_token + = render 'shared/deploy_tokens/new_deploy_token', deploy_token: @created_deploy_token %h5.gl-mt-0 = s_('DeployTokens|New deploy token') = render 'shared/deploy_tokens/form', group_or_project: group_or_project, token: @new_deploy_token, presenter: @deploy_tokens %hr = render 'shared/deploy_tokens/table', group_or_project: group_or_project, active_tokens: @deploy_tokens - diff --git a/config/feature_flags/development/ci_docker_image_pull_policy.yml b/config/feature_flags/development/ci_docker_image_pull_policy.yml deleted file mode 100644 index 5bdcdf03d27..00000000000 --- a/config/feature_flags/development/ci_docker_image_pull_policy.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: ci_docker_image_pull_policy -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85588 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/363186 -milestone: '15.1' -type: development -group: group::pipeline authoring -default_enabled: true diff --git a/config/feature_flags/development/tag_list_keyset_pagination.yml b/config/feature_flags/development/tag_list_keyset_pagination.yml deleted file mode 100644 index 52c21e22d9f..00000000000 --- a/config/feature_flags/development/tag_list_keyset_pagination.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: tag_list_keyset_pagination -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74239 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/345595 -milestone: '14.5' -type: development -group: group::source code -default_enabled: true diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index 46e36ce58a8..8cd2b26ee42 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -2308,11 +2308,11 @@ use Google Cloud's Kubernetes Engine (GKE) or AWS Elastic Kubernetes Service (EK and CPU requirements should translate to most other providers. We hope to update this in the future with further specific cloud provider details. -| Service | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | -|-----------------------------------------------|-------|-------------------------|-----------------|--------------|---------------------------------| -| Webservice | 4 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | 127.5 vCPU, 118 GB memory | -| Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 15.5 vCPU, 50 GB memory | -| Supporting services such as NGINX, Prometheus | 2 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 7.75 vCPU, 25 GB memory | +| Service | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | +|---------------------|-------|-------------------------|-----------------|--------------|---------------------------------| +| Webservice | 4 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | 127.5 vCPU, 118 GB memory | +| Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 15.5 vCPU, 50 GB memory | +| Supporting services | 2 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 7.75 vCPU, 25 GB memory | - For this setup, we **recommend** and regularly [test](index.md#validation-and-test-results) [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) and [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/). Other Kubernetes services may also work, but your mileage may vary. @@ -2359,7 +2359,7 @@ card "Kubernetes via Helm Charts" as kubernetes { collections "**Sidekiq** x4" as sidekiq #ff8dd1 } - card "**Supporting Services**" as support + card "**Supporting Services** x2" as support } card "**Internal Load Balancer**" as ilb #9370DB @@ -2425,29 +2425,42 @@ documents how to apply the calculated configuration to the Helm Chart. #### Webservice -Webservice pods typically need about 1 vCPU and 1.25 GB of memory _per worker_. -Each Webservice pod consumes roughly 4 vCPUs and 5 GB of memory using +Webservice pods typically need about 1 CPU and 1.25 GB of memory _per worker_. +Each Webservice pod consumes roughly 4 CPUs and 5 GB of memory using the [recommended topology](#cluster-topology) because four worker processes are created by default and each pod has other small processes running. For 10,000 users we recommend a total Puma worker count of around 80. With the [provided recommendations](#cluster-topology) this allows the deployment of up to 20 Webservice pods with 4 workers per pod and 5 pods per node. Expand available resources using -the ratio of 1 vCPU to 1.25 GB of memory _per each worker process_ for each additional +the ratio of 1 CPU to 1.25 GB of memory _per each worker process_ for each additional Webservice pod. For further information on resource usage, see the [Webservice resources](https://docs.gitlab.com/charts/charts/gitlab/webservice/#resources). #### Sidekiq -Sidekiq pods should generally have 1 vCPU and 2 GB of memory. +Sidekiq pods should generally have 0.9 CPU and 2 GB of memory. [The provided starting point](#cluster-topology) allows the deployment of up to -14 Sidekiq pods. Expand available resources using the 1 vCPU to 2GB memory +14 Sidekiq pods. Expand available resources using the 0.9 CPU to 2 GB memory ratio for each additional pod. For further information on resource usage, see the [Sidekiq resources](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/#resources). +### Supporting + +The Supporting Node Pool is designed to house all supporting deployments that don't need to be +on the Webservice and Sidekiq pools. + +This includes various deployments related to the Cloud Provider's implementation and supporting +GitLab deployments such as NGINX or [GitLab Shell](https://docs.gitlab.com/charts/charts/gitlab/gitlab-shell/). + +If you wish to make any additional deployments, such as for Monitoring, it's recommended +to deploy these in this pool where possible and not in the Webservice or Sidekiq pools, as the Supporting pool has been designed +specifically to accommodate several additional deployments. However, if your deployments don't fit into the +pool as given, you can increase the node pool accordingly. + <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 55e44abda13..07816dc7f8f 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -2306,11 +2306,11 @@ use Google Cloud's Kubernetes Engine (GKE) or AWS Elastic Kubernetes Service (EK and CPU requirements should translate to most other providers. We hope to update this in the future with further specific cloud provider details. -| Service | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | -|-----------------------------------------------|-------|-------------------------|-----------------|-----------------|---------------------------------| -| Webservice | 7 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | 223 vCPU, 206.5 GB memory | -| Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 15.5 vCPU, 50 GB memory | -| Supporting services such as NGINX, Prometheus | 2 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 7.75 vCPU, 25 GB memory | +| Service | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | +|---------------------|-------|-------------------------|-----------------|--------------|---------------------------------| +| Webservice | 7 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | 223 vCPU, 206.5 GB memory | +| Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 15.5 vCPU, 50 GB memory | +| Supporting services | 2 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 7.75 vCPU, 25 GB memory | - For this setup, we **recommend** and regularly [test](index.md#validation-and-test-results) [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) and [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/). Other Kubernetes services may also work, but your mileage may vary. @@ -2353,11 +2353,11 @@ card "Kubernetes via Helm Charts" as kubernetes { card "**External Load Balancer**" as elb #6a9be7 together { - collections "**Webservice** x4" as gitlab #32CD32 + collections "**Webservice** x7" as gitlab #32CD32 collections "**Sidekiq** x4" as sidekiq #ff8dd1 } - card "**Supporting Services**" as support + card "**Supporting Services** x2" as support } card "**Internal Load Balancer**" as ilb #9370DB @@ -2423,29 +2423,42 @@ documents how to apply the calculated configuration to the Helm Chart. #### Webservice -Webservice pods typically need about 1 vCPU and 1.25 GB of memory _per worker_. -Each Webservice pod consumes roughly 4 vCPUs and 5 GB of memory using +Webservice pods typically need about 1 CPU and 1.25 GB of memory _per worker_. +Each Webservice pod consumes roughly 4 CPUs and 5 GB of memory using the [recommended topology](#cluster-topology) because four worker processes are created by default and each pod has other small processes running. For 25,000 users we recommend a total Puma worker count of around 140. With the [provided recommendations](#cluster-topology) this allows the deployment of up to 35 Webservice pods with 4 workers per pod and 5 pods per node. Expand available resources using -the ratio of 1 vCPU to 1.25 GB of memory _per each worker process_ for each additional +the ratio of 1 CPU to 1.25 GB of memory _per each worker process_ for each additional Webservice pod. For further information on resource usage, see the [Webservice resources](https://docs.gitlab.com/charts/charts/gitlab/webservice/#resources). #### Sidekiq -Sidekiq pods should generally have 1 vCPU and 2 GB of memory. +Sidekiq pods should generally have 0.9 CPU and 2 GB of memory. [The provided starting point](#cluster-topology) allows the deployment of up to -14 Sidekiq pods. Expand available resources using the 1 vCPU to 2GB memory +14 Sidekiq pods. Expand available resources using the 0.9 CPU to 2 GB memory ratio for each additional pod. For further information on resource usage, see the [Sidekiq resources](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/#resources). +### Supporting + +The Supporting Node Pool is designed to house all supporting deployments that don't need to be +on the Webservice and Sidekiq pools. + +This includes various deployments related to the Cloud Provider's implementation and supporting +GitLab deployments such as NGINX or [GitLab Shell](https://docs.gitlab.com/charts/charts/gitlab/gitlab-shell/). + +If you wish to make any additional deployments, such as for Monitoring, it's recommended +to deploy these in this pool where possible and not in the Webservice or Sidekiq pools, as the Supporting pool has been designed +specifically to accommodate several additional deployments. However, if your deployments don't fit into the +pool as given, you can increase the node pool accordingly. + <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index b9b678ee623..0ef64bbdd55 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -1014,11 +1014,11 @@ use Google Cloud's Kubernetes Engine (GKE) or AWS Elastic Kubernetes Service (EK and CPU requirements should translate to most other providers. We hope to update this in the future with further specific cloud provider details. -| Service | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | -|-----------------------------------------------|-------|------------------------|-----------------|--------------|---------------------------------| -| Webservice | 3 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | 23.7 vCPU, 16.9 GB memory | -| Sidekiq | 1 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 3.9 vCPU, 11.8 GB memory | -| Supporting services such as NGINX, Prometheus | 2 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | 1.9 vCPU, 5.5 GB memory | +| Service | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | +|---------------------|-------|------------------------|-----------------|--------------|---------------------------------| +| Webservice | 3 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | 23.7 vCPU, 16.9 GB memory | +| Sidekiq | 2 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 7.8 vCPU, 25.9 GB memory | +| Supporting services | 2 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | 1.9 vCPU, 5.5 GB memory | - For this setup, we **recommend** and regularly [test](index.md#validation-and-test-results) [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) and [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/). Other Kubernetes services may also work, but your mileage may vary. @@ -1054,9 +1054,10 @@ card "Kubernetes via Helm Charts" as kubernetes { together { collections "**Webservice** x3" as gitlab #32CD32 - card "**Sidekiq**" as sidekiq #ff8dd1 - collections "**Supporting Services** x2" as support + collections "**Sidekiq** x2" as sidekiq #ff8dd1 } + + collections "**Supporting Services** x2" as support } card "**Gitaly**" as gitaly #FF8C00 @@ -1087,29 +1088,42 @@ documents how to apply the calculated configuration to the Helm Chart. #### Webservice -Webservice pods typically need about 1 vCPU and 1.25 GB of memory _per worker_. -Each Webservice pod consumes roughly 4 vCPUs and 5 GB of memory using +Webservice pods typically need about 1 CPU and 1.25 GB of memory _per worker_. +Each Webservice pod consumes roughly 4 CPUs and 5 GB of memory using the [recommended topology](#cluster-topology) because two worker processes are created by default and each pod has other small processes running. For 2,000 users we recommend a total Puma worker count of around 12. With the [provided recommendations](#cluster-topology) this allows the deployment of up to 3 Webservice pods with 4 workers per pod and 1 pod per node. Expand available resources using -the ratio of 1 vCPU to 1.25 GB of memory _per each worker process_ for each additional +the ratio of 1 CPU to 1.25 GB of memory _per each worker process_ for each additional Webservice pod. For further information on resource usage, see the [Webservice resources](https://docs.gitlab.com/charts/charts/gitlab/webservice/#resources). #### Sidekiq -Sidekiq pods should generally have 1 vCPU and 2 GB of memory. +Sidekiq pods should generally have 0.9 CPU and 2 GB of memory. [The provided starting point](#cluster-topology) allows the deployment of up to -4 Sidekiq pods. Expand available resources using the 1 vCPU to 2 GB memory +4 Sidekiq pods. Expand available resources using the 0.9 CPU to 2 GB memory ratio for each additional pod. For further information on resource usage, see the [Sidekiq resources](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/#resources). +### Supporting + +The Supporting Node Pool is designed to house all supporting deployments that don't need to be +on the Webservice and Sidekiq pools. + +This includes various deployments related to the Cloud Provider's implementation and supporting +GitLab deployments such as NGINX or [GitLab Shell](https://docs.gitlab.com/charts/charts/gitlab/gitlab-shell/). + +If you wish to make any additional deployments, such as for Monitoring, it's recommended +to deploy these in this pool where possible and not in the Webservice or Sidekiq pools, as the Supporting pool has been designed +specifically to accommodate several additional deployments. However, if your deployments don't fit into the +pool as given, you can increase the node pool accordingly. + <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 3ddd90dde31..50dde3084c4 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -2268,11 +2268,11 @@ use Google Cloud's Kubernetes Engine (GKE) or AWS Elastic Kubernetes Service (EK and CPU requirements should translate to most other providers. We hope to update this in the future with further specific cloud provider details. -| Service | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | -|-----------------------------------------------|-------|-------------------------|-----------------|--------------|---------------------------------| -| Webservice | 2 | 16 vCPU, 14.4 GB memory | `n1-highcpu-16` | `c5.4xlarge` | 31.8 vCPU, 24.8 GB memory | -| Sidekiq | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 11.8 vCPU, 38.9 GB memory | -| Supporting services such as NGINX, Prometheus | 2 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | 3.9 vCPU, 11.8 GB memory | +| Service | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | +|---------------------|-------|-------------------------|-----------------|--------------|---------------------------------| +| Webservice | 2 | 16 vCPU, 14.4 GB memory | `n1-highcpu-16` | `c5.4xlarge` | 31.8 vCPU, 24.8 GB memory | +| Sidekiq | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 11.8 vCPU, 38.9 GB memory | +| Supporting services | 2 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | 3.9 vCPU, 11.8 GB memory | - For this setup, we **recommend** and regularly [test](index.md#validation-and-test-results) [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) and [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/). Other Kubernetes services may also work, but your mileage may vary. @@ -2314,11 +2314,11 @@ card "Kubernetes via Helm Charts" as kubernetes { card "**External Load Balancer**" as elb #6a9be7 together { - collections "**Webservice** x4" as gitlab #32CD32 - collections "**Sidekiq** x4" as sidekiq #ff8dd1 + collections "**Webservice** x2" as gitlab #32CD32 + collections "**Sidekiq** x3" as sidekiq #ff8dd1 } - card "**Supporting Services**" as support + card "**Supporting Services** x2" as support } card "**Internal Load Balancer**" as ilb #9370DB @@ -2381,29 +2381,42 @@ documents how to apply the calculated configuration to the Helm Chart. #### Webservice -Webservice pods typically need about 1 vCPU and 1.25 GB of memory _per worker_. -Each Webservice pod consumes roughly 4 vCPUs and 5 GB of memory using +Webservice pods typically need about 1 CPU and 1.25 GB of memory _per worker_. +Each Webservice pod consumes roughly 4 CPUs and 5 GB of memory using the [recommended topology](#cluster-topology) because four worker processes are created by default and each pod has other small processes running. For 3,000 users we recommend a total Puma worker count of around 16. With the [provided recommendations](#cluster-topology) this allows the deployment of up to 4 Webservice pods with 4 workers per pod and 2 pods per node. Expand available resources using -the ratio of 1 vCPU to 1.25 GB of memory _per each worker process_ for each additional +the ratio of 1 CPU to 1.25 GB of memory _per each worker process_ for each additional Webservice pod. For further information on resource usage, see the [Webservice resources](https://docs.gitlab.com/charts/charts/gitlab/webservice/#resources). #### Sidekiq -Sidekiq pods should generally have 1 vCPU and 2 GB of memory. +Sidekiq pods should generally have 0.9 CPU and 2 GB of memory. [The provided starting point](#cluster-topology) allows the deployment of up to -8 Sidekiq pods. Expand available resources using the 1 vCPU to 2GB memory +8 Sidekiq pods. Expand available resources using the 0.9 CPU to 2 GB memory ratio for each additional pod. For further information on resource usage, see the [Sidekiq resources](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/#resources). +### Supporting + +The Supporting Node Pool is designed to house all supporting deployments that don't need to be +on the Webservice and Sidekiq pools. + +This includes various deployments related to the Cloud Provider's implementation and supporting +GitLab deployments such as NGINX or [GitLab Shell](https://docs.gitlab.com/charts/charts/gitlab/gitlab-shell/). + +If you wish to make any additional deployments, such as for Monitoring, it's recommended +to deploy these in this pool where possible and not in the Webservice or Sidekiq pools, as the Supporting pool has been designed +specifically to accommodate several additional deployments. However, if your deployments don't fit into the +pool as given, you can increase the node pool accordingly. + <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index cb96b521005..5a5ad469e1e 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -2322,11 +2322,11 @@ use Google Cloud's Kubernetes Engine (GKE) or AWS Elastic Kubernetes Service (EK and CPU requirements should translate to most other providers. We hope to update this in the future with further specific cloud provider details. -| Service | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | -|-----------------------------------------------|-------|-------------------------|-----------------|--------------|---------------------------------| -| Webservice | 16 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `m5.8xlarge` | 510 vCPU, 472 GB memory | -| Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 15.5 vCPU, 50 GB memory | -| Supporting services such as NGINX, Prometheus | 2 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 7.75 vCPU, 25 GB memory | +| Service | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | +|---------------------|-------|-------------------------|-----------------|--------------|---------------------------------| +| Webservice | 16 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `m5.8xlarge` | 510 vCPU, 472 GB memory | +| Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 15.5 vCPU, 50 GB memory | +| Supporting services | 2 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 7.75 vCPU, 25 GB memory | - For this setup, we **recommend** and regularly [test](index.md#validation-and-test-results) [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) and [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/). Other Kubernetes services may also work, but your mileage may vary. @@ -2369,11 +2369,11 @@ card "Kubernetes via Helm Charts" as kubernetes { card "**External Load Balancer**" as elb #6a9be7 together { - collections "**Webservice** x4" as gitlab #32CD32 + collections "**Webservice** x16" as gitlab #32CD32 collections "**Sidekiq** x4" as sidekiq #ff8dd1 } - card "**Supporting Services**" as support + card "**Supporting Services** x2" as support } card "**Internal Load Balancer**" as ilb #9370DB @@ -2439,29 +2439,42 @@ documents how to apply the calculated configuration to the Helm Chart. #### Webservice -Webservice pods typically need about 1 vCPU and 1.25 GB of memory _per worker_. -Each Webservice pod consumes roughly 4 vCPUs and 5 GB of memory using +Webservice pods typically need about 1 CPU and 1.25 GB of memory _per worker_. +Each Webservice pod consumes roughly 4 CPUs and 5 GB of memory using the [recommended topology](#cluster-topology) because four worker processes are created by default and each pod has other small processes running. For 50,000 users we recommend a total Puma worker count of around 320. With the [provided recommendations](#cluster-topology) this allows the deployment of up to 80 Webservice pods with 4 workers per pod and 5 pods per node. Expand available resources using -the ratio of 1 vCPU to 1.25 GB of memory _per each worker process_ for each additional +the ratio of 1 CPU to 1.25 GB of memory _per each worker process_ for each additional Webservice pod. For further information on resource usage, see the [Webservice resources](https://docs.gitlab.com/charts/charts/gitlab/webservice/#resources). #### Sidekiq -Sidekiq pods should generally have 1 vCPU and 2 GB of memory. +Sidekiq pods should generally have 0.9 CPU and 2 GB of memory. [The provided starting point](#cluster-topology) allows the deployment of up to -14 Sidekiq pods. Expand available resources using the 1 vCPU to 2GB memory +14 Sidekiq pods. Expand available resources using the 0.9 CPU to 2 GB memory ratio for each additional pod. For further information on resource usage, see the [Sidekiq resources](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/#resources). +### Supporting + +The Supporting Node Pool is designed to house all supporting deployments that don't need to be +on the Webservice and Sidekiq pools. + +This includes various deployments related to the Cloud Provider's implementation and supporting +GitLab deployments such as NGINX or [GitLab Shell](https://docs.gitlab.com/charts/charts/gitlab/gitlab-shell/). + +If you wish to make any additional deployments, such as for Monitoring, it's recommended +to deploy these in this pool where possible and not in the Webservice or Sidekiq pools, as the Supporting pool has been designed +specifically to accommodate several additional deployments. However, if your deployments don't fit into the +pool as given, you can increase the node pool accordingly. + <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 13f6ab80778..45352409aad 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -2289,11 +2289,11 @@ card "Kubernetes via Helm Charts" as kubernetes { card "**External Load Balancer**" as elb #6a9be7 together { - collections "**Webservice** x4" as gitlab #32CD32 - collections "**Sidekiq** x4" as sidekiq #ff8dd1 + collections "**Webservice** x5" as gitlab #32CD32 + collections "**Sidekiq** x3" as sidekiq #ff8dd1 } - card "**Supporting Services**" as support + card "**Supporting Services** x2" as support } card "**Internal Load Balancer**" as ilb #9370DB @@ -2356,29 +2356,42 @@ documents how to apply the calculated configuration to the Helm Chart. #### Webservice -Webservice pods typically need about 1 vCPU and 1.25 GB of memory _per worker_. -Each Webservice pod consumes roughly 4 vCPUs and 5 GB of memory using +Webservice pods typically need about 1 CPU and 1.25 GB of memory _per worker_. +Each Webservice pod consumes roughly 4 CPUs and 5 GB of memory using the [recommended topology](#cluster-topology) because four worker processes are created by default and each pod has other small processes running. For 5,000 users we recommend a total Puma worker count of around 40. With the [provided recommendations](#cluster-topology) this allows the deployment of up to 10 Webservice pods with 4 workers per pod and 2 pods per node. Expand available resources using -the ratio of 1 vCPU to 1.25 GB of memory _per each worker process_ for each additional +the ratio of 1 CPU to 1.25 GB of memory _per each worker process_ for each additional Webservice pod. For further information on resource usage, see the [Webservice resources](https://docs.gitlab.com/charts/charts/gitlab/webservice/#resources). #### Sidekiq -Sidekiq pods should generally have 1 vCPU and 2 GB of memory. +Sidekiq pods should generally have 0.9 CPU and 2 GB of memory. [The provided starting point](#cluster-topology) allows the deployment of up to -8 Sidekiq pods. Expand available resources using the 1 vCPU to 2GB memory +8 Sidekiq pods. Expand available resources using the 0.9 CPU to 2 GB memory ratio for each additional pod. For further information on resource usage, see the [Sidekiq resources](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/#resources). +### Supporting + +The Supporting Node Pool is designed to house all supporting deployments that don't need to be +on the Webservice and Sidekiq pools. + +This includes various deployments related to the Cloud Provider's implementation and supporting +GitLab deployments such as NGINX or [GitLab Shell](https://docs.gitlab.com/charts/charts/gitlab/gitlab-shell/). + +If you wish to make any additional deployments, such as for Monitoring, it's recommended +to deploy these in this pool where possible and not in the Webservice or Sidekiq pools, as the Supporting pool has been designed +specifically to accommodate several additional deployments. However, if your deployments don't fit into the +pool as given, you can increase the node pool accordingly. + <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md index f96d0434035..6f6ac2421ec 100644 --- a/doc/ci/yaml/index.md +++ b/doc/ci/yaml/index.md @@ -1896,13 +1896,10 @@ image: #### `image:pull_policy` > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21619) in GitLab 15.1 [with a flag](../../administration/feature_flags.md) named `ci_docker_image_pull_policy`. Disabled by default. +> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/363186) in GitLab 15.2. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/363186) in GitLab 15.4. [Feature flag `ci_docker_image_pull_policy`](https://gitlab.com/gitlab-org/gitlab/-/issues/363186) removed. > - Requires GitLab Runner 15.1 or later. -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, -ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `ci_docker_image_pull_policy`. -The feature is not ready for production use. - The pull policy that the runner uses to fetch the Docker image. **Keyword type**: Job keyword. You can use it only as part of a job or in the [`default` section](#default). @@ -3642,12 +3639,9 @@ in that container. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21619) in GitLab 15.1 [with a flag](../../administration/feature_flags.md) named `ci_docker_image_pull_policy`. Disabled by default. > - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/363186) in GitLab 15.2. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/363186) in GitLab 15.4. [Feature flag `ci_docker_image_pull_policy`](https://gitlab.com/gitlab-org/gitlab/-/issues/363186) removed. > - Requires GitLab Runner 15.1 or later. -FLAG: -On self-managed GitLab, by default this feature is available. To hide the feature, -ask an administrator to [disable the feature flag](../../administration/feature_flags.md) named `ci_docker_image_pull_policy`. - The pull policy that the runner uses to fetch the Docker image. **Keyword type**: Job keyword. You can use it only as part of a job or in the [`default` section](#default). diff --git a/lib/api/entities/ci/job_request/image.rb b/lib/api/entities/ci/job_request/image.rb index 83f64da6050..92d68269265 100644 --- a/lib/api/entities/ci/job_request/image.rb +++ b/lib/api/entities/ci/job_request/image.rb @@ -8,7 +8,7 @@ module API expose :name, :entrypoint expose :ports, using: Entities::Ci::JobRequest::Port - expose :pull_policy, if: ->(_) { ::Feature.enabled?(:ci_docker_image_pull_policy) } + expose :pull_policy end end end diff --git a/lib/api/entities/ci/job_request/service.rb b/lib/api/entities/ci/job_request/service.rb index 7d494c7e516..128591058fe 100644 --- a/lib/api/entities/ci/job_request/service.rb +++ b/lib/api/entities/ci/job_request/service.rb @@ -8,7 +8,7 @@ module API expose :name, :entrypoint expose :ports, using: Entities::Ci::JobRequest::Port - expose :pull_policy, if: ->(_) { ::Feature.enabled?(:ci_docker_image_pull_policy) } + expose :pull_policy expose :alias, :command expose :variables end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 613f7ff3370..84e31ca1fc6 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -11,10 +11,7 @@ module Gitlab include ::Gitlab::Ci::Config::Entry::Imageable validations do - validates :config, allowed_keys: IMAGEABLE_ALLOWED_KEYS, - if: :ci_docker_image_pull_policy_enabled? - validates :config, allowed_keys: IMAGEABLE_LEGACY_ALLOWED_KEYS, - unless: :ci_docker_image_pull_policy_enabled? + validates :config, allowed_keys: IMAGEABLE_ALLOWED_KEYS end def value @@ -25,7 +22,7 @@ module Gitlab name: @config[:name], entrypoint: @config[:entrypoint], ports: (ports_value if ports_defined?), - pull_policy: (ci_docker_image_pull_policy_enabled? ? pull_policy_value : nil) + pull_policy: pull_policy_value }.compact else {} diff --git a/lib/gitlab/ci/config/entry/imageable.rb b/lib/gitlab/ci/config/entry/imageable.rb index f045ee3d549..1aecfee9ab9 100644 --- a/lib/gitlab/ci/config/entry/imageable.rb +++ b/lib/gitlab/ci/config/entry/imageable.rb @@ -13,7 +13,6 @@ module Gitlab include ::Gitlab::Config::Entry::Configurable IMAGEABLE_ALLOWED_KEYS = %i[name entrypoint ports pull_policy].freeze - IMAGEABLE_LEGACY_ALLOWED_KEYS = %i[name entrypoint ports].freeze included do include ::Gitlab::Config::Entry::Validatable @@ -47,10 +46,6 @@ module Gitlab opt(:with_image_ports) end - def ci_docker_image_pull_policy_enabled? - ::Feature.enabled?(:ci_docker_image_pull_policy) - end - def skip_config_hash_validation? true end diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index 0e19447dff8..4b3a9990df4 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -11,14 +11,9 @@ module Gitlab include ::Gitlab::Ci::Config::Entry::Imageable ALLOWED_KEYS = %i[command alias variables].freeze - LEGACY_ALLOWED_KEYS = %i[command alias variables].freeze validations do - validates :config, allowed_keys: ALLOWED_KEYS + IMAGEABLE_ALLOWED_KEYS, - if: :ci_docker_image_pull_policy_enabled? - validates :config, allowed_keys: LEGACY_ALLOWED_KEYS + IMAGEABLE_LEGACY_ALLOWED_KEYS, - unless: :ci_docker_image_pull_policy_enabled? - + validates :config, allowed_keys: ALLOWED_KEYS + IMAGEABLE_ALLOWED_KEYS validates :command, array_of_strings: true, allow_nil: true validates :alias, type: String, allow_nil: true validates :alias, type: String, presence: true, unless: ->(record) { record.ports.blank? } @@ -43,7 +38,7 @@ module Gitlab { name: @config } elsif hash? @config.merge( - pull_policy: (pull_policy_value if ci_docker_image_pull_policy_enabled?) + pull_policy: pull_policy_value ).compact else {} diff --git a/lib/gitlab/pagination/gitaly_keyset_pager.rb b/lib/gitlab/pagination/gitaly_keyset_pager.rb index 1f1061fe4f1..d4de2791195 100644 --- a/lib/gitlab/pagination/gitaly_keyset_pager.rb +++ b/lib/gitlab/pagination/gitaly_keyset_pager.rb @@ -38,7 +38,7 @@ module Gitlab if finder.is_a?(BranchesFinder) Feature.enabled?(:branch_list_keyset_pagination, project) elsif finder.is_a?(TagsFinder) - Feature.enabled?(:tag_list_keyset_pagination, project) + true elsif finder.is_a?(::Repositories::TreeFinder) Feature.enabled?(:repository_tree_gitaly_pagination, project) else @@ -52,7 +52,7 @@ module Gitlab if finder.is_a?(BranchesFinder) Feature.enabled?(:branch_list_keyset_pagination, project) elsif finder.is_a?(TagsFinder) - Feature.enabled?(:tag_list_keyset_pagination, project) + true elsif finder.is_a?(::Repositories::TreeFinder) Feature.enabled?(:repository_tree_gitaly_pagination, project) else diff --git a/package.json b/package.json index 55cd582b9f9..20eb53f34b4 100644 --- a/package.json +++ b/package.json @@ -176,13 +176,13 @@ "url-loader": "^4.1.1", "uuid": "8.1.0", "visibilityjs": "^1.2.4", - "vue": "^2.7.10", + "vue": "2.6.14", "vue-apollo": "^3.0.7", - "vue-loader": "^15.10", + "vue-loader": "15.9.6", "vue-observe-visibility": "^1.0.0", "vue-resize": "^1.0.1", "vue-router": "3.4.9", - "vue-template-compiler": "^2.7.10", + "vue-template-compiler": "2.6.14", "vue-virtual-scroll-list": "^1.4.7", "vuedraggable": "^2.23.0", "vuex": "^3.6.2", diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/pipeline_with_image_pull_policy_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/pipeline_with_image_pull_policy_spec.rb index 47d16a6a3b1..f852a028dc5 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/pipeline_with_image_pull_policy_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/pipeline_with_image_pull_policy_spec.rb @@ -1,12 +1,7 @@ # frozen_string_literal: true module QA - # TODO: remove feature flag upon rollout completion - # FF rollout issue: https://gitlab.com/gitlab-org/gitlab/-/issues/363186 - RSpec.describe 'Verify', :runner, feature_flag: { - name: 'ci_docker_image_pull_policy', - scope: :global - } do + RSpec.describe 'Verify', :runner do describe 'Pipeline with image:pull_policy' do let(:runner_name) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(number: 8)}" } let(:job_name) { "test-job-#{pull_policies.join('-')}" } @@ -27,10 +22,6 @@ module QA end before do - Runtime::Feature.enable(:ci_docker_image_pull_policy) - # Give the feature some time to switch - sleep(30) - update_runner_policy(allowed_policies) add_ci_file Flow::Login.sign_in @@ -39,8 +30,6 @@ module QA end after do - Runtime::Feature.disable(:ci_docker_image_pull_policy) - runner.remove_via_api! end diff --git a/spec/frontend/runner/group_runners/group_runners_app_spec.js b/spec/frontend/runner/group_runners/group_runners_app_spec.js index 6ed375df338..e9376aca849 100644 --- a/spec/frontend/runner/group_runners/group_runners_app_spec.js +++ b/spec/frontend/runner/group_runners/group_runners_app_spec.js @@ -391,4 +391,36 @@ describe('GroupRunnersApp', () => { }); }); }); + + describe('when user has permission to register group runner', () => { + beforeEach(() => { + createComponent({ + propsData: { + registrationToken: mockRegistrationToken, + groupFullPath: mockGroupFullPath, + groupRunnersLimitedCount: mockGroupRunnersCount, + }, + }); + }); + + it('shows the register group runner button', () => { + expect(findRegistrationDropdown().exists()).toBe(true); + }); + }); + + describe('when user has no permission to register group runner', () => { + beforeEach(() => { + createComponent({ + propsData: { + registrationToken: null, + groupFullPath: mockGroupFullPath, + groupRunnersLimitedCount: mockGroupRunnersCount, + }, + }); + }); + + it('does not show the register group runner button', () => { + expect(findRegistrationDropdown().exists()).toBe(false); + }); + }); }); diff --git a/spec/helpers/ci/runners_helper_spec.rb b/spec/helpers/ci/runners_helper_spec.rb index 3b18572ad64..1b1edde8faf 100644 --- a/spec/helpers/ci/runners_helper_spec.rb +++ b/spec/helpers/ci/runners_helper_spec.rb @@ -131,17 +131,32 @@ RSpec.describe Ci::RunnersHelper do describe '#group_runners_data_attributes' do let(:group) { create(:group) } - it 'returns group data to render a runner list' do - expect(helper.group_runners_data_attributes(group)).to include( - registration_token: group.runners_token, - group_id: group.id, - group_full_path: group.full_path, - runner_install_help_page: 'https://docs.gitlab.com/runner/install/', - online_contact_timeout_secs: 7200, - stale_timeout_secs: 7889238, - empty_state_svg_path: start_with('/assets/illustrations/pipelines_empty'), - empty_state_filtered_svg_path: start_with('/assets/illustrations/magnifying-glass') - ) + context 'when user can register group runners' do + before do + allow(helper).to receive(:can?).with(user, :register_group_runners, group).and_return(true) + end + + it 'returns group data to render a runner list' do + expect(helper.group_runners_data_attributes(group)).to include( + group_id: group.id, + group_full_path: group.full_path, + runner_install_help_page: 'https://docs.gitlab.com/runner/install/', + online_contact_timeout_secs: 7200, + stale_timeout_secs: 7889238, + empty_state_svg_path: start_with('/assets/illustrations/pipelines_empty'), + empty_state_filtered_svg_path: start_with('/assets/illustrations/magnifying-glass') + ) + end + end + + context 'when user cannot register group runners' do + before do + allow(helper).to receive(:can?).with(user, :register_group_runners, group).and_return(false) + end + + it 'returns empty registration token' do + expect(helper.group_runners_data_attributes(group)).not_to include(registration_token: group.runners_token) + end end end diff --git a/spec/initializers/load_balancing_spec.rb b/spec/initializers/load_balancing_spec.rb new file mode 100644 index 00000000000..3a3e2bb560c --- /dev/null +++ b/spec/initializers/load_balancing_spec.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'load_balancing', :delete, :reestablished_active_record_base do + subject(:initialize_load_balancer) do + load Rails.root.join('config/initializers/load_balancing.rb') + end + + context 'for a clustered puma worker' do + let!(:group) { create(:group, name: 'my group') } + + before do + # Setup host-based load balancing + # Patch in our load balancer config, simply pointing at the test database twice + allow(Gitlab::Database::LoadBalancing::Configuration).to receive(:for_model) do |base_model| + db_host = base_model.connection_pool.db_config.host + + Gitlab::Database::LoadBalancing::Configuration.new(base_model, [db_host, db_host]) + end + + # Pretend we are in clustered environment + allow(Gitlab::Cluster::LifecycleEvents).to receive(:in_clustered_puma?).and_return(true) + + # Stub out middleware call, as not idempotent + allow(Gitlab::Application.instance.middleware).to receive(:use) + end + + after do + # reset load balancing to original state + allow(Gitlab::Database::LoadBalancing::Configuration).to receive(:for_model).and_call_original + allow(Gitlab::Cluster::LifecycleEvents).to receive(:in_clustered_puma?).and_call_original + + load Rails.root.join('config/initializers/load_balancing.rb') + end + + def simulate_puma_worker + pid = Process.fork do + # We call this in config/puma.rb + Gitlab::Cluster::LifecycleEvents.do_worker_start + + yield + end + + Process.waitpid(pid) + expect($?).to be_success + end + + it 'makes a query to a replica successfully' do + # Clear any previous sticky writes + ::Gitlab::Database::LoadBalancing::Session.clear_session + + initialize_load_balancer + + process_read, process_write = IO.pipe + + simulate_puma_worker do + process_read.close + + group = Group.find_by_name('my group') + process_write.write group.name + end + + process_write.close + expect(process_read.read).to eq(group.name) + end + + it 'makes a query to the primary successfully' do + initialize_load_balancer + + expect do + simulate_puma_worker do + Group.touch_all + end + + group.reload + end.to change(group, :updated_at) + end + end +end diff --git a/spec/lib/api/entities/ci/job_request/image_spec.rb b/spec/lib/api/entities/ci/job_request/image_spec.rb index fca3b5d3fa9..14d4a074fce 100644 --- a/spec/lib/api/entities/ci/job_request/image_spec.rb +++ b/spec/lib/api/entities/ci/job_request/image_spec.rb @@ -32,14 +32,4 @@ RSpec.describe API::Entities::Ci::JobRequest::Image do it 'returns the pull policy' do expect(subject[:pull_policy]).to eq(['if-not-present']) end - - context 'when the FF ci_docker_image_pull_policy is disabled' do - before do - stub_feature_flags(ci_docker_image_pull_policy: false) - end - - it 'does not return the pull policy' do - expect(subject).not_to have_key(:pull_policy) - end - end end diff --git a/spec/lib/api/entities/ci/job_request/service_spec.rb b/spec/lib/api/entities/ci/job_request/service_spec.rb index 86f2120c321..11350f7c41b 100644 --- a/spec/lib/api/entities/ci/job_request/service_spec.rb +++ b/spec/lib/api/entities/ci/job_request/service_spec.rb @@ -40,12 +40,4 @@ RSpec.describe API::Entities::Ci::JobRequest::Service do expect(subject[:ports]).to be_nil end end - - context 'when the FF ci_docker_image_pull_policy is disabled' do - before do - stub_feature_flags(ci_docker_image_pull_policy: false) - end - - it { is_expected.not_to have_key(:pull_policy) } - end end diff --git a/spec/lib/gitlab/ci/config/entry/image_spec.rb b/spec/lib/gitlab/ci/config/entry/image_spec.rb index 6121c28070f..b37498ba10a 100644 --- a/spec/lib/gitlab/ci/config/entry/image_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/image_spec.rb @@ -4,8 +4,6 @@ require 'spec_helper' RSpec.describe Gitlab::Ci::Config::Entry::Image do before do - stub_feature_flags(ci_docker_image_pull_policy: true) - entry.compose! end @@ -129,18 +127,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Image do it 'is valid' do expect(entry).to be_valid end - - context 'when the feature flag ci_docker_image_pull_policy is disabled' do - before do - stub_feature_flags(ci_docker_image_pull_policy: false) - entry.compose! - end - - it 'is not valid' do - expect(entry).not_to be_valid - expect(entry.errors).to include('image config contains unknown keys: pull_policy') - end - end end describe '#value' do @@ -150,19 +136,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Image do pull_policy: ['if-not-present'] ) end - - context 'when the feature flag ci_docker_image_pull_policy is disabled' do - before do - stub_feature_flags(ci_docker_image_pull_policy: false) - entry.compose! - end - - it 'is not valid' do - expect(entry.value).to eq( - name: 'image:1.0' - ) - end - end end end end diff --git a/spec/lib/gitlab/ci/config/entry/service_spec.rb b/spec/lib/gitlab/ci/config/entry/service_spec.rb index 821ab442d61..e36484bb0ae 100644 --- a/spec/lib/gitlab/ci/config/entry/service_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/service_spec.rb @@ -4,7 +4,6 @@ require 'spec_helper' RSpec.describe Gitlab::Ci::Config::Entry::Service do before do - stub_feature_flags(ci_docker_image_pull_policy: true) entry.compose! end @@ -149,18 +148,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Service do it 'is valid' do expect(entry).to be_valid end - - context 'when the feature flag ci_docker_image_pull_policy is disabled' do - before do - stub_feature_flags(ci_docker_image_pull_policy: false) - entry.compose! - end - - it 'is not valid' do - expect(entry).not_to be_valid - expect(entry.errors).to include('service config contains unknown keys: pull_policy') - end - end end describe '#value' do @@ -170,18 +157,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Service do pull_policy: ['if-not-present'] ) end - - context 'when the feature flag ci_docker_image_pull_policy is disabled' do - before do - stub_feature_flags(ci_docker_image_pull_policy: false) - end - - it 'is not valid' do - expect(entry.value).to eq( - name: 'postgresql:9.5' - ) - end - end end end end diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index 35af9ae6201..eafa8f8fb25 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -997,18 +997,6 @@ module Gitlab scheduling_type: :stage }) end - - context 'when the feature flag ci_docker_image_pull_policy is disabled' do - before do - stub_feature_flags(ci_docker_image_pull_policy: false) - end - - it { is_expected.not_to be_valid } - - it "returns no job" do - expect(processor.jobs).to eq({}) - end - end end context 'when a service has pull_policy' do @@ -1042,18 +1030,6 @@ module Gitlab scheduling_type: :stage }) end - - context 'when the feature flag ci_docker_image_pull_policy is disabled' do - before do - stub_feature_flags(ci_docker_image_pull_policy: false) - end - - it { is_expected.not_to be_valid } - - it "returns no job" do - expect(processor.jobs).to eq({}) - end - end end end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index b35d1ea019b..b4d6537b774 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -2,7 +2,7 @@ require "spec_helper" -RSpec.describe Gitlab::Git::Repository, :seed_helper do +RSpec.describe Gitlab::Git::Repository do include Gitlab::EncodingHelper include RepoHelpers using RSpec::Parameterized::TableSyntax @@ -21,13 +21,10 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end end - let(:mutable_repository) { Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '', 'group/project') } - let(:mutable_repository_path) { File.join(TestEnv.repos_path, mutable_repository.relative_path) } - let(:mutable_repository_rugged) { Rugged::Repository.new(mutable_repository_path) } - let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '', 'group/project') } - let(:repository_path) { File.join(TestEnv.repos_path, repository.relative_path) } - let(:repository_rugged) { Rugged::Repository.new(repository_path) } - let(:storage_path) { TestEnv.repos_path } + let_it_be(:repository) { create(:project, :repository).repository.raw } + + let(:mutable_project) { create(:project, :repository) } + let(:mutable_repository) { mutable_project.repository.raw } let(:user) { build(:user) } describe "Respond to" do @@ -61,8 +58,8 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do describe '#branch_names' do subject { repository.branch_names } - it 'has SeedRepo::Repo::BRANCHES.size elements' do - expect(subject.size).to eq(SeedRepo::Repo::BRANCHES.size) + it 'has TestRepo::BRANCH_SHA.size elements' do + expect(subject.size).to eq(TestEnv::BRANCH_SHA.size) end it 'returns UTF-8' do @@ -85,8 +82,8 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do it { is_expected.to be_kind_of Array } - it 'has SeedRepo::Repo::TAGS.size elements' do - expect(subject.size).to eq(SeedRepo::Repo::TAGS.size) + it 'has some elements' do + expect(subject.size).to be >= 1 end it 'returns UTF-8' do @@ -96,63 +93,24 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do describe '#last' do subject { super().last } - it { is_expected.to eq("v1.2.1") } + it { is_expected.to eq("v1.1.1") } end + it { is_expected.to include("v1.0.0") } it { is_expected.not_to include("v5.0.0") } - it 'gets the tag names from GitalyClient' do - expect_any_instance_of(Gitlab::GitalyClient::RefService).to receive(:tag_names) - subject - end - it_behaves_like 'wrapping gRPC errors', Gitlab::GitalyClient::RefService, :tag_names end describe '#tags' do subject { repository.tags } - it 'gets tags from GitalyClient' do - expect_next_instance_of(Gitlab::GitalyClient::RefService) do |service| - expect(service).to receive(:tags) - end - - subject - end - - context 'with sorting option' do - subject { repository.tags(sort_by: 'name_asc') } - - it 'gets tags from GitalyClient' do - expect_next_instance_of(Gitlab::GitalyClient::RefService) do |service| - expect(service).to receive(:tags).with(sort_by: 'name_asc', pagination_params: nil) - end - - subject - end - end - - context 'with pagination option' do - subject { repository.tags(pagination_params: { limit: 5, page_token: 'refs/tags/v1.0.0' }) } - - it 'gets tags from GitalyClient' do - expect_next_instance_of(Gitlab::GitalyClient::RefService) do |service| - expect(service).to receive(:tags).with( - sort_by: nil, - pagination_params: { limit: 5, page_token: 'refs/tags/v1.0.0' } - ) - end - - subject - end - end - it_behaves_like 'wrapping gRPC errors', Gitlab::GitalyClient::RefService, :tags end describe '#archive_metadata' do let(:storage_path) { '/tmp' } - let(:cache_key) { File.join(repository.gl_repository, SeedRepo::LastCommit::ID) } + let(:cache_key) { File.join(repository.gl_repository, TestEnv::BRANCH_SHA['master']) } let(:append_sha) { true } let(:ref) { 'master' } @@ -162,12 +120,12 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do let(:expected_extension) { 'tar.gz' } let(:expected_filename) { "#{expected_prefix}.#{expected_extension}" } let(:expected_path) { File.join(storage_path, cache_key, "@v2", expected_filename) } - let(:expected_prefix) { "gitlab-git-test-#{ref}-#{SeedRepo::LastCommit::ID}" } + let(:expected_prefix) { "gitlab-git-test-#{ref}-#{TestEnv::BRANCH_SHA['master']}" } subject(:metadata) { repository.archive_metadata(ref, storage_path, 'gitlab-git-test', format, append_sha: append_sha, path: path) } it 'sets CommitId to the commit SHA' do - expect(metadata['CommitId']).to eq(SeedRepo::LastCommit::ID) + expect(metadata['CommitId']).to start_with(TestEnv::BRANCH_SHA['master']) end it 'sets ArchivePrefix to the expected prefix' do @@ -175,7 +133,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end it 'sets ArchivePath to the expected globally-unique path' do - expect(expected_path).to include(File.join(repository.gl_repository, SeedRepo::LastCommit::ID)) + expect(expected_path).to include(File.join(repository.gl_repository, TestEnv::BRANCH_SHA['master'])) expect(metadata['ArchivePath']).to eq(expected_path) end @@ -190,7 +148,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do context 'append_sha varies archive path and filename' do where(:append_sha, :ref, :expected_prefix) do - sha = SeedRepo::LastCommit::ID + sha = TestEnv::BRANCH_SHA['master'] true | 'master' | "gitlab-git-test-master-#{sha}" true | sha | "gitlab-git-test-#{sha}-#{sha}" @@ -224,13 +182,13 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do describe '#size' do subject { repository.size } - it { is_expected.to be < 2 } + it { is_expected.to be > 0 } end describe '#to_s' do subject { repository.to_s } - it { is_expected.to eq("<Gitlab::Git::Repository: group/project>") } + it { is_expected.to eq("<Gitlab::Git::Repository: namespace1/project1>") } end describe '#object_directory_size' do @@ -259,26 +217,25 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do describe '#first' do subject { super().first } - it { is_expected.to eq('feature') } + it { is_expected.to eq(TestEnv::BRANCH_SHA.keys.min) } end describe '#last' do subject { super().last } - it { is_expected.to eq('v1.2.1') } + it { is_expected.to eq('v1.1.1') } end end describe '#submodule_url_for' do - let(:ref) { 'master' } + let(:ref) { 'submodule_inside_folder' } def submodule_url(path) repository.submodule_url_for(ref, path) end it { expect(submodule_url('six')).to eq('git://github.com/randx/six.git') } - it { expect(submodule_url('nested/six')).to eq('git://github.com/randx/six.git') } - it { expect(submodule_url('deeper/nested/six')).to eq('git://github.com/randx/six.git') } + it { expect(submodule_url('test_inside_folder/another_folder/six')).to eq('git://github.com/randx/six.git') } it { expect(submodule_url('invalid/path')).to eq(nil) } context 'uncommitted submodule dir' do @@ -288,7 +245,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end context 'tags' do - let(:ref) { 'v1.2.1' } + let(:ref) { 'v1.1.1' } it { expect(submodule_url('six')).to eq('git://github.com/randx/six.git') } end @@ -313,17 +270,15 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do urls = repository.submodule_urls_for(ref) expect(urls).to eq({ - "deeper/nested/six" => "git://github.com/randx/six.git", - "gitlab-grack" => "https://gitlab.com/gitlab-org/gitlab-grack.git", - "gitlab-shell" => "https://github.com/gitlabhq/gitlab-shell.git", - "nested/six" => "git://github.com/randx/six.git", + "gitlab-grack" => "https://gitlab.com/gitlab-org/gitlab-grack.git", + "gitlab-shell" => "https://github.com/gitlabhq/gitlab-shell.git", "six" => "git://github.com/randx/six.git" }) end end describe '#commit_count' do - it { expect(repository.commit_count("master")).to eq(25) } + it { expect(repository.commit_count("master")).to eq(37) } it { expect(repository.commit_count("feature")).to eq(9) } it { expect(repository.commit_count("does-not-exist")).to eq(0) } @@ -461,47 +416,12 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do describe '#has_local_branches?' do context 'check for local branches' do it { expect(repository.has_local_branches?).to eq(true) } - - context 'mutable' do - let(:repository) { mutable_repository } - - after do - ensure_seeds - end - - it 'returns false when there are no branches' do - # Sanity check - expect(repository.has_local_branches?).to eq(true) - - FileUtils.rm_rf(File.join(repository_path, 'packed-refs')) - heads_dir = File.join(repository_path, 'refs/heads') - FileUtils.rm_rf(heads_dir) - FileUtils.mkdir_p(heads_dir) - - repository.expire_has_local_branches_cache - expect(repository.has_local_branches?).to eq(false) - end - end - - context 'memoizes the value' do - it 'returns true' do - expect(repository).to receive(:uncached_has_local_branches?).once.and_call_original - - 2.times do - expect(repository.has_local_branches?).to eq(true) - end - end - end end end describe '#delete_refs' do let(:repository) { mutable_repository } - after do - ensure_seeds - end - it 'deletes the ref' do repository.delete_refs('refs/heads/feature') @@ -548,9 +468,8 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do subject { repository.refs_hash } it "has as many entries as branches and tags" do - expected_refs = SeedRepo::Repo::BRANCHES + SeedRepo::Repo::TAGS # We flatten in case a commit is pointed at by more than one branch and/or tag - expect(subject.values.flatten.size).to eq(expected_refs.size) + expect(subject.values.flatten.size).to be > 0 end it 'has valid commit ids as keys' do @@ -620,10 +539,6 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do ) end - after do - ensure_seeds - end - subject do repository.search_files_by_content(content, ref) end @@ -647,8 +562,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do let(:filter) { 'files\/.*\/.*\.rb' } it 'returns matched files' do - expect(result).to contain_exactly('files/links/regex.rb', - 'files/ruby/popen.rb', + expect(result).to contain_exactly('files/ruby/popen.rb', 'files/ruby/regex.rb', 'files/ruby/version_info.rb') end @@ -810,7 +724,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do after do # Erase our commits so other tests get the original repo - repository.write_ref(repository.root_ref, SeedRepo::LastCommit::ID) + repository.write_ref(repository.root_ref, TestEnv::BRANCH_SHA['master']) end context "where 'follow' == true" do @@ -963,16 +877,6 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do expect(log_commits).not_to include(commit_with_new_name) end end - - context "and 'path' includes a directory that used to be a file" do - let(:log_commits) do - repository.log(options.merge(ref: "refs/heads/fix-blob-path", path: "files/testdir/file.txt")) - end - - it "returns a list of commits" do - expect(log_commits.size).to eq(1) - end - end end context "where provides 'after' timestamp" do @@ -1036,7 +940,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do it 'returns a list of commits' do commits = repository.log({ all: true, limit: 50 }) - expect(commits.size).to eq(37) + expect(commits.size).to eq(50) end end end @@ -1047,7 +951,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end describe '#blobs' do - let_it_be(:commit_oid) { '4b4918a572fa86f9771e5ba40fbd48e1eb03e2c6' } + let_it_be(:commit_oid) { TestEnv::BRANCH_SHA['master'] } shared_examples 'a blob enumeration' do it 'enumerates blobs' do @@ -1063,7 +967,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do context 'single revision' do let(:revisions) { [commit_oid] } - let(:expected_blobs) { 53 } + let(:expected_blobs) { 52 } it_behaves_like 'a blob enumeration' end @@ -1093,48 +997,31 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do it_behaves_like 'a blob enumeration' end - - context 'partially blank revisions' do - let(:revisions) { [::Gitlab::Git::BLANK_SHA, commit_oid] } - let(:expected_blobs) { 53 } - - before do - expect_next_instance_of(Gitlab::GitalyClient::BlobService) do |service| - expect(service) - .to receive(:list_blobs) - .with([commit_oid], kind_of(Hash)) - .and_call_original - end - end - - it_behaves_like 'a blob enumeration' - end end describe '#new_blobs' do let(:repository) { mutable_repository } - let(:repository_rugged) { mutable_repository_rugged } - let(:blob) { create_blob('This is a new blob') } - let(:commit) { create_commit('nested/new-blob.txt' => blob) } - - def create_blob(content) - repository_rugged.write(content, :blob) - end + let(:commit) { create_commit('nested/new-blob.txt' => 'This is a new blob') } def create_commit(blobs) - author = { name: 'Test User', email: 'mail@example.com', time: Time.now } + commit_result = repository.multi_action( + user, + branch_name: 'a-new-branch', + message: 'Add a file', + actions: blobs.map do |path, content| + { + action: :create, + file_path: path, + content: content + } + end + ) - index = repository_rugged.index - blobs.each do |path, oid| - index.add(path: path, oid: oid, mode: 0100644) - end + # new_blobs only returns unreferenced blobs because it is used for hooks. + # Gitaly does not allow us to create loose objects via the RPC. + repository.delete_branch('a-new-branch') - Rugged::Commit.create(repository_rugged, - author: author, - committer: author, - message: "Message", - parents: [], - tree: index.write_tree(repository_rugged)) + commit_result.newrev end subject { repository.new_blobs(newrevs).to_a } @@ -1167,7 +1054,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do let(:newrevs) { commit } let(:expected_newrevs) { ['--not', '--all', '--not', newrevs] } let(:expected_blobs) do - [have_attributes(class: Gitlab::Git::Blob, id: blob, path: 'nested/new-blob.txt', size: 18)] + [have_attributes(class: Gitlab::Git::Blob, id: an_instance_of(String), path: 'nested/new-blob.txt', size: 18)] end it_behaves_like '#new_blobs with revisions' @@ -1177,20 +1064,19 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do let(:newrevs) { [commit] } let(:expected_newrevs) { ['--not', '--all', '--not'] + newrevs } let(:expected_blobs) do - [have_attributes(class: Gitlab::Git::Blob, id: blob, path: 'nested/new-blob.txt', size: 18)] + [have_attributes(class: Gitlab::Git::Blob, id: an_instance_of(String), path: 'nested/new-blob.txt', size: 18)] end it_behaves_like '#new_blobs with revisions' end context 'with multiple revisions' do - let(:another_blob) { create_blob('Another blob') } - let(:newrevs) { [commit, create_commit('another_path.txt' => another_blob)] } + let(:newrevs) { [commit, create_commit('another_path.txt' => 'Another blob')] } let(:expected_newrevs) { ['--not', '--all', '--not'] + newrevs.sort } let(:expected_blobs) do [ - have_attributes(class: Gitlab::Git::Blob, id: blob, path: 'nested/new-blob.txt', size: 18), - have_attributes(class: Gitlab::Git::Blob, id: another_blob, path: 'another_path.txt', size: 12) + have_attributes(class: Gitlab::Git::Blob, id: an_instance_of(String), path: 'nested/new-blob.txt', size: 18), + have_attributes(class: Gitlab::Git::Blob, id: an_instance_of(String), path: 'another_path.txt', size: 12) ] end @@ -1202,7 +1088,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do let(:expected_newrevs) { ['--not', '--all', '--not', commit] } let(:expected_blobs) do [ - have_attributes(class: Gitlab::Git::Blob, id: blob, path: 'nested/new-blob.txt', size: 18) + have_attributes(class: Gitlab::Git::Blob, id: an_instance_of(String), path: 'nested/new-blob.txt', size: 18) ] end @@ -1214,7 +1100,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do let(:expected_newrevs) { ['--not', '--all', '--not', commit] } let(:expected_blobs) do [ - have_attributes(class: Gitlab::Git::Blob, id: blob, path: 'nested/new-blob.txt', size: 18) + have_attributes(class: Gitlab::Git::Blob, id: an_instance_of(String), path: 'nested/new-blob.txt', size: 18) ] end @@ -1267,14 +1153,22 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do describe '#new_commits' do let(:repository) { mutable_repository } let(:new_commit) do - author = { name: 'Test User', email: 'mail@example.com', time: Time.now } + commit_result = repository.multi_action( + user, + branch_name: 'a-new-branch', + message: 'Message', + actions: [{ + action: :create, + file_path: 'some_file.txt', + content: 'This is a file' + }] + ) + + # new_commits only returns unreferenced commits because it is used for + # hooks. Gitaly does not allow us to create loose objects via the RPC. + repository.delete_branch('a-new-branch') - Rugged::Commit.create(repository_rugged, - author: author, - committer: author, - message: "Message", - parents: [], - tree: "4b825dc642cb6eb9a060e54bf8d69288fbee4904") + commit_result.newrev end let(:expected_commits) { 1 } @@ -1303,7 +1197,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do describe '#count_commits_between' do subject { repository.count_commits_between('feature', 'master') } - it { is_expected.to eq(17) } + it { is_expected.to eq(29) } end describe '#raw_changes_between' do @@ -1330,26 +1224,26 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end end - context 'with valid revs' do - let(:old_rev) { 'fa1b1e6c004a68b7d8763b86455da9e6b23e36d6' } - let(:new_rev) { '4b4918a572fa86f9771e5ba40fbd48e1eb03e2c6' } + context 'with valid revs', :aggregate_failures do + let(:old_rev) { TestEnv::BRANCH_SHA['feature'] } + let(:new_rev) { TestEnv::BRANCH_SHA['master'] } it 'returns the changes' do - expect(changes.size).to eq(9) - expect(changes.first.operation).to eq(:modified) - expect(changes.first.new_path).to eq('.gitmodules') + expect(changes.size).to eq(21) + expect(changes.first.operation).to eq(:deleted) + expect(changes.first.old_path).to eq('.DS_Store') expect(changes.last.operation).to eq(:added) - expect(changes.last.new_path).to eq('files/lfs/picture-invalid.png') + expect(changes.last.new_path).to eq('with space/README.md') end end end describe '#merge_base' do where(:from, :to, :result) do - '570e7b2abdd848b95f2f578043fc23bd6f6fd24d' | '40f4a7a617393735a95a0bb67b08385bc1e7c66d' | '570e7b2abdd848b95f2f578043fc23bd6f6fd24d' - '40f4a7a617393735a95a0bb67b08385bc1e7c66d' | '570e7b2abdd848b95f2f578043fc23bd6f6fd24d' | '570e7b2abdd848b95f2f578043fc23bd6f6fd24d' - '40f4a7a617393735a95a0bb67b08385bc1e7c66d' | 'foobar' | nil - 'foobar' | '40f4a7a617393735a95a0bb67b08385bc1e7c66d' | nil + 'master' | 'feature' | 'ae73cb07c9eeaf35924a10f713b364d32b2dd34f' + 'feature' | 'master' | 'ae73cb07c9eeaf35924a10f713b364d32b2dd34f' + 'master' | 'foobar' | nil + 'foobar' | 'master' | nil end with_them do @@ -1363,7 +1257,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do it 'returns the number of commits after timestamp' do options = { ref: 'master', after: Time.iso8601('2013-03-03T20:15:01+00:00') } - expect(repository.count_commits(options)).to eq(25) + expect(repository.count_commits(options)).to eq(37) end end @@ -1392,28 +1286,28 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end context 'with option :from and option :to' do - it 'returns the number of commits ahead for fix-mode..fix-blob-path' do - options = { from: 'fix-mode', to: 'fix-blob-path' } + it 'returns the number of commits ahead for master..feature' do + options = { from: 'master', to: 'feature' } - expect(repository.count_commits(options)).to eq(2) + expect(repository.count_commits(options)).to eq(1) end - it 'returns the number of commits ahead for fix-blob-path..fix-mode' do - options = { from: 'fix-blob-path', to: 'fix-mode' } + it 'returns the number of commits ahead for feature..master' do + options = { from: 'feature', to: 'master' } - expect(repository.count_commits(options)).to eq(1) + expect(repository.count_commits(options)).to eq(29) end context 'with option :left_right' do - it 'returns the number of commits for fix-mode...fix-blob-path' do - options = { from: 'fix-mode', to: 'fix-blob-path', left_right: true } + it 'returns the number of commits for master..feature' do + options = { from: 'master', to: 'feature', left_right: true } - expect(repository.count_commits(options)).to eq([1, 2]) + expect(repository.count_commits(options)).to eq([29, 1]) end context 'with max_count' do - it 'returns the number of commits with path' do - options = { from: 'fix-mode', to: 'fix-blob-path', left_right: true, max_count: 1 } + it 'returns the number of commits' do + options = { from: 'feature', to: 'master', left_right: true, max_count: 1 } expect(repository.count_commits(options)).to eq([1, 1]) end @@ -1433,7 +1327,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do it "returns the number of commits in the whole repository" do options = { all: true } - expect(repository.count_commits(options)).to eq(34) + expect(repository.count_commits(options)).to eq(314) end end @@ -1471,10 +1365,6 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do repository.create_branch('local_branch') end - after do - ensure_seeds - end - it 'returns the local and remote branches' do expect(subject.any? { |b| b.name == 'joe/remote_branch' }).to eq(true) expect(subject.any? { |b| b.name == 'local_branch' }).to eq(true) @@ -1486,7 +1376,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do describe '#branch_count' do it 'returns the number of branches' do - expect(repository.branch_count).to eq(11) + expect(repository.branch_count).to eq(TestEnv::BRANCH_SHA.size) end context 'with local and remote branches' do @@ -1497,10 +1387,6 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do repository.create_branch('local_branch') end - after do - ensure_seeds - end - it 'returns the count of local branches' do expect(repository.branch_count).to eq(repository.local_branches.count) end @@ -1543,21 +1429,16 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end context 'when no branch names are specified' do + let(:repository) { mutable_repository } + before do repository.create_branch('identical') end - after do - ensure_seeds - end - it 'returns all merged branch names except for identical one' do names = repository.merged_branch_names - expect(names).to include('merge-test') - expect(names).to include('fix-mode') - expect(names).not_to include('feature') - expect(names).not_to include('identical') + expect(names).to match_array(["'test'", "branch-merged", "flatten-dir", "improve/awesome", "merge-test"]) end end end @@ -1611,24 +1492,15 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end describe '#find_changed_paths' do - let(:commit_1) { 'fa1b1e6c004a68b7d8763b86455da9e6b23e36d6' } - let(:commit_2) { '4b4918a572fa86f9771e5ba40fbd48e1eb03e2c6' } + let(:commit_1) { TestEnv::BRANCH_SHA['with-executables'] } + let(:commit_2) { TestEnv::BRANCH_SHA['master'] } let(:commit_3) { '6f6d7e7ed97bb5f0054f2b1df789b39ca89b6ff9' } let(:commit_1_files) do - [ - Gitlab::Git::ChangedPath.new(status: :ADDED, path: "files/executables/ls"), - Gitlab::Git::ChangedPath.new(status: :ADDED, path: "files/executables/touch"), - Gitlab::Git::ChangedPath.new(status: :ADDED, path: "files/links/regex.rb"), - Gitlab::Git::ChangedPath.new(status: :ADDED, path: "files/links/ruby-style-guide.md"), - Gitlab::Git::ChangedPath.new(status: :ADDED, path: "files/links/touch"), - Gitlab::Git::ChangedPath.new(status: :MODIFIED, path: ".gitmodules"), - Gitlab::Git::ChangedPath.new(status: :ADDED, path: "deeper/nested/six"), - Gitlab::Git::ChangedPath.new(status: :ADDED, path: "nested/six") - ] + [Gitlab::Git::ChangedPath.new(status: :ADDED, path: "files/executables/ls")] end let(:commit_2_files) do - [Gitlab::Git::ChangedPath.new(status: :ADDED, path: "bin/executable")] + [Gitlab::Git::ChangedPath.new(status: :ADDED, path: "bar/branch-test.txt")] end let(:commit_3_files) do @@ -1676,7 +1548,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do let(:not_existed_branch) { repository.ls_files("not_existed_branch") } it "read every file paths of master branch" do - expect(master_file_paths.length).to equal(40) + expect(master_file_paths.length).to equal(38) end it "reads full file paths of master branch" do @@ -1701,11 +1573,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end describe "#copy_gitattributes" do - let(:attributes_path) { File.join(SEED_STORAGE_PATH, TEST_REPO_PATH, 'info/attributes') } - - after do - FileUtils.rm_rf(attributes_path) if Dir.exist?(attributes_path) - end + let(:repository) { mutable_repository } it "raises an error with invalid ref" do expect { repository.copy_gitattributes("invalid") }.to raise_error(Gitlab::Git::Repository::InvalidRef) @@ -1728,63 +1596,10 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do repository end end - - context "with no .gitattrbutes" do - before do - repository.copy_gitattributes("master") - end - - it "does not have an info/attributes" do - expect(File.exist?(attributes_path)).to be_falsey - end - end - - context "with .gitattrbutes" do - before do - repository.copy_gitattributes("gitattributes") - end - - it "has an info/attributes" do - expect(File.exist?(attributes_path)).to be_truthy - end - - it "has the same content in info/attributes as .gitattributes" do - contents = File.open(attributes_path, "rb") { |f| f.read } - expect(contents).to eq("*.md binary\n") - end - end - - context "with updated .gitattrbutes" do - before do - repository.copy_gitattributes("gitattributes") - repository.copy_gitattributes("gitattributes-updated") - end - - it "has an info/attributes" do - expect(File.exist?(attributes_path)).to be_truthy - end - - it "has the updated content in info/attributes" do - contents = File.read(attributes_path) - expect(contents).to eq("*.txt binary\n") - end - end - - context "with no .gitattrbutes in HEAD but with previous info/attributes" do - before do - repository.copy_gitattributes("gitattributes") - repository.copy_gitattributes("master") - end - - it "does not have an info/attributes" do - expect(File.exist?(attributes_path)).to be_falsey - end - end end describe '#gitattribute' do - let(:project) { create(:project, :repository) } - let(:repository) { project.repository } + let(:repository) { mutable_repository } context 'with gitattributes' do before do @@ -1863,10 +1678,6 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do repository.create_branch('local_branch') end - after do - ensure_seeds - end - it 'returns the local branches' do expect(repository.local_branches.any? { |branch| branch.name == 'remote_branch' }).to eq(false) expect(repository.local_branches.any? { |branch| branch.name == 'local_branch' }).to eq(true) @@ -1935,7 +1746,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do describe '#languages' do it 'returns exactly the expected results' do - languages = repository.languages('4b4918a572fa86f9771e5ba40fbd48e1eb03e2c6') + languages = repository.languages(TestEnv::BRANCH_SHA['master']) expect(languages).to match_array([ { value: a_value_within(0.1).of(66.7), label: "Ruby", color: "#701516", highlight: "#701516" }, @@ -1973,12 +1784,9 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do describe '#fetch_source_branch!' do let(:local_ref) { 'refs/merge-requests/1/head' } + let(:repository) { create(:project, :repository).repository.raw } let(:source_repository) { mutable_repository } - after do - ensure_seeds - end - context 'when the branch exists' do context 'when the commit does not exist locally' do let(:source_branch) { 'new-branch-for-fetch-source-branch' } @@ -2004,14 +1812,14 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do context 'when the commit exists locally' do let(:source_branch) { 'master' } - let(:expected_oid) { SeedRepo::LastCommit::ID } + let(:expected_oid) { TestEnv::BRANCH_SHA['master'] } it 'writes the ref' do # Sanity check: the commit should already exist expect(repository.commit(expected_oid)).not_to be_nil expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(true) - expect(repository.commit(local_ref).sha).to eq(expected_oid) + expect(repository.commit(local_ref).sha).to start_with(expected_oid) end end end @@ -2067,9 +1875,9 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end it 'writes other refs' do - repository.write_ref('refs/heads/feature', SeedRepo::Commit::ID) + repository.write_ref('refs/heads/feature', TestEnv::BRANCH_SHA['master']) - expect(repository.commit('feature').sha).to eq(SeedRepo::Commit::ID) + expect(repository.commit('feature').sha).to start_with(TestEnv::BRANCH_SHA['master']) end end @@ -2107,28 +1915,28 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do it 'returns nil for an empty repo' do project = create(:project) - expect(project.repository.refs_by_oid(oid: SeedRepo::Commit::ID, limit: 0)).to be_nil + expect(project.repository.refs_by_oid(oid: TestEnv::BRANCH_SHA['master'], limit: 0)).to be_nil end end describe '#set_full_path' do + let(:full_path) { 'some/path' } + before do - repository.set_full_path(full_path: repository_path) + repository.set_full_path(full_path: full_path) end - context 'is given a path' do - it 'writes it to disk' do - repository.set_full_path(full_path: "not-the/real-path.git") + it 'writes full_path to gitaly' do + repository.set_full_path(full_path: "not-the/real-path.git") - expect(repository.full_path).to eq('not-the/real-path.git') - end + expect(repository.full_path).to eq('not-the/real-path.git') end context 'it is given an empty path' do it 'does not write it to disk' do repository.set_full_path(full_path: "") - expect(repository.full_path).to eq(repository_path) + expect(repository.full_path).to eq(full_path) end end @@ -2200,10 +2008,6 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do repository.create_branch(target_branch, '6d394385cf567f80a8fd85055db1ab4c5295806f') end - after do - ensure_seeds - end - it 'can perform a merge' do merge_commit_id = nil result = repository.merge(user, source_sha, target_branch, 'Test merge') do |commit_id| @@ -2240,10 +2044,6 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do repository.create_branch(target_branch, branch_head) end - after do - ensure_seeds - end - subject { repository.ff_merge(user, source_sha, target_branch) } shared_examples '#ff_merge' do @@ -2297,14 +2097,10 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do let(:repository) { mutable_repository } before do - repository.write_ref("refs/delete/a", "0b4bc9a49b562e85de7cc9e834518ea6828729b9") - repository.write_ref("refs/also-delete/b", "12d65c8dd2b2676fa3ac47d955accc085a37a9c1") - repository.write_ref("refs/keep/c", "6473c90867124755509e100d0d35ebdc85a0b6ae") - repository.write_ref("refs/also-keep/d", "0b4bc9a49b562e85de7cc9e834518ea6828729b9") - end - - after do - ensure_seeds + repository.write_ref("refs/delete/a", TestEnv::BRANCH_SHA['master']) + repository.write_ref("refs/also-delete/b", TestEnv::BRANCH_SHA['master']) + repository.write_ref("refs/keep/c", TestEnv::BRANCH_SHA['master']) + repository.write_ref("refs/also-keep/d", TestEnv::BRANCH_SHA['master']) end it 'deletes all refs except those with the specified prefixes' do @@ -2327,11 +2123,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do it 'saves a bundle to disk' do repository.bundle_to_disk(save_path) - success = system( - *%W(#{Gitlab.config.git.bin_path} -C #{repository_path} bundle verify #{save_path}), - [:out, :err] => '/dev/null' - ) - expect(success).to be true + expect(File).to exist(save_path) end end @@ -2381,41 +2173,22 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do describe '#checksum' do it 'calculates the checksum for non-empty repo' do - expect(repository.checksum).to eq '51d0a9662681f93e1fee547a6b7ba2bcaf716059' - end - - it 'returns 0000000000000000000000000000000000000000 for an empty repo' do - FileUtils.rm_rf(File.join(storage_path, 'empty-repo.git')) - - system(git_env, *%W(#{Gitlab.config.git.bin_path} init --bare empty-repo.git), - chdir: storage_path, - out: '/dev/null', - err: '/dev/null') - - empty_repo = described_class.new('default', 'empty-repo.git', '', 'group/empty-repo') - - expect(empty_repo.checksum).to eq '0000000000000000000000000000000000000000' + expect(repository.checksum.length).to be(40) + expect(Gitlab::Git.blank_ref?(repository.checksum)).to be false end - it 'raises Gitlab::Git::Repository::InvalidRepository error for non-valid git repo' do - FileUtils.rm_rf(File.join(storage_path, 'non-valid.git')) - - system(git_env, *%W(#{Gitlab.config.git.bin_path} clone --bare #{TEST_REPO_PATH} non-valid.git), - chdir: SEED_STORAGE_PATH, - out: '/dev/null', - err: '/dev/null') - - File.truncate(File.join(storage_path, 'non-valid.git/HEAD'), 0) + it 'returns a blank sha for an empty repo' do + repository = create(:project, :empty_repo).repository - non_valid = described_class.new('default', 'non-valid.git', '', 'a/non-valid') - - expect { non_valid.checksum }.to raise_error(Gitlab::Git::Repository::InvalidRepository) + expect(Gitlab::Git.blank_ref?(repository.checksum)).to be true end - it 'raises Gitlab::Git::Repository::NoRepository error when there is no repo' do - broken_repo = described_class.new('default', 'a/path.git', '', 'a/path') + it 'raises NoRepository for a non-existent repo' do + repository = create(:project).repository - expect { broken_repo.checksum }.to raise_error(Gitlab::Git::Repository::NoRepository) + expect do + repository.checksum + end.to raise_error(described_class::NoRepository) end end @@ -2430,7 +2203,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do describe '#squash' do let(:branch_name) { 'fix' } - let(:start_sha) { '4b4918a572fa86f9771e5ba40fbd48e1eb03e2c6' } + let(:start_sha) { TestEnv::BRANCH_SHA['master'] } let(:end_sha) { '12d65c8dd2b2676fa3ac47d955accc085a37a9c1' } subject do @@ -2482,7 +2255,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do after do # Erase our commits so other tests get the original repo - repository.write_ref(repository.root_ref, SeedRepo::LastCommit::ID) + repository.write_ref(repository.root_ref, TestEnv::BRANCH_SHA['master']) end it 'does not include the renamed file in the sparse checkout' do @@ -2535,9 +2308,9 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end describe '#disconnect_alternates' do - let(:project) { create(:project, :repository) } + let(:project) { mutable_project } + let(:repository) { mutable_repository } let(:pool_repository) { create(:pool_repository) } - let(:repository) { project.repository } let(:object_pool) { pool_repository.object_pool } before do @@ -2570,8 +2343,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end describe '#rename' do - let(:project) { create(:project, :repository) } - let(:repository) { project.repository } + let(:repository) { mutable_repository } it 'moves the repository' do checksum = repository.checksum @@ -2586,15 +2358,14 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end describe '#remove' do - let(:project) { create(:project, :repository) } - let(:repository) { project.repository } + let(:repository) { mutable_repository } it 'removes the repository' do expect(repository.exists?).to be true repository.remove - expect(repository.raw_repository.exists?).to be false + expect(repository.exists?).to be false end context 'when the repository does not exist' do @@ -2605,15 +2376,14 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do repository.remove - expect(repository.raw_repository.exists?).to be false + expect(repository.exists?).to be false end end end describe '#import_repository' do - let_it_be(:project) { create(:project) } + let_it_be(:repository) { create(:project).repository } - let(:repository) { project.repository } let(:url) { 'http://invalid.invalid' } it 'raises an error if a relative path is provided' do @@ -2639,11 +2409,9 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do describe '#replicate' do let(:new_repository) do - Gitlab::Git::Repository.new('test_second_storage', TEST_REPO_PATH, '', 'group/project') + Gitlab::Git::Repository.new('test_second_storage', repository.relative_path, '', 'group/project') end - let(:new_repository_path) { File.join(TestEnv::SECOND_STORAGE_PATH, new_repository.relative_path) } - subject { new_repository.replicate(repository) } before do @@ -2677,7 +2445,8 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end context 'with keep-around refs' do - let(:sha) { SeedRepo::Commit::ID } + let(:repository) { mutable_repository } + let(:sha) { TestEnv::BRANCH_SHA['master'] } let(:keep_around_ref) { "refs/keep-around/#{sha}" } let(:tmp_ref) { "refs/tmp/#{SecureRandom.hex}" } diff --git a/spec/requests/api/ci/runner/jobs_request_post_spec.rb b/spec/requests/api/ci/runner/jobs_request_post_spec.rb index 930d7690550..e5555eea487 100644 --- a/spec/requests/api/ci/runner/jobs_request_post_spec.rb +++ b/spec/requests/api/ci/runner/jobs_request_post_spec.rb @@ -833,22 +833,6 @@ RSpec.describe API::Ci::Runner, :clean_gitlab_redis_shared_state do 'image' => { 'name' => 'ruby', 'pull_policy' => ['if-not-present'], 'entrypoint' => nil, 'ports' => [] } ) end - - context 'when the FF ci_docker_image_pull_policy is disabled' do - before do - stub_feature_flags(ci_docker_image_pull_policy: false) - end - - it 'returns the image without pull policy' do - request_job - - expect(response).to have_gitlab_http_status(:created) - expect(json_response).to include( - 'id' => job.id, - 'image' => { 'name' => 'ruby', 'entrypoint' => nil, 'ports' => [] } - ) - end - end end context 'when service has pull_policy' do @@ -873,23 +857,6 @@ RSpec.describe API::Ci::Runner, :clean_gitlab_redis_shared_state do 'ports' => [], 'pull_policy' => ['if-not-present'], 'variables' => [] }] ) end - - context 'when the FF ci_docker_image_pull_policy is disabled' do - before do - stub_feature_flags(ci_docker_image_pull_policy: false) - end - - it 'returns the service without pull policy' do - request_job - - expect(response).to have_gitlab_http_status(:created) - expect(json_response).to include( - 'id' => job.id, - 'services' => [{ 'alias' => nil, 'command' => nil, 'entrypoint' => nil, 'name' => 'postgres:11.9', - 'ports' => [], 'variables' => [] }] - ) - end - end end describe 'a job with excluded artifacts' do diff --git a/spec/requests/api/tags_spec.rb b/spec/requests/api/tags_spec.rb index e81e9e0bf2f..fbedb69a640 100644 --- a/spec/requests/api/tags_spec.rb +++ b/spec/requests/api/tags_spec.rb @@ -17,10 +17,6 @@ RSpec.describe API::Tags do end describe 'GET /projects/:id/repository/tags', :use_clean_rails_memory_store_caching do - before do - stub_feature_flags(tag_list_keyset_pagination: false) - end - let(:route) { "/projects/#{project_id}/repository/tags" } context 'sorting' do @@ -154,50 +150,44 @@ RSpec.describe API::Tags do end end - context 'with keyset pagination on', :aggregate_errors do - before do - stub_feature_flags(tag_list_keyset_pagination: true) - end - - context 'with keyset pagination option' do - let(:base_params) { { pagination: 'keyset' } } + context 'with keyset pagination option', :aggregate_errors do + let(:base_params) { { pagination: 'keyset' } } - context 'with gitaly pagination params' do - context 'with high limit' do - let(:params) { base_params.merge(per_page: 100) } + context 'with gitaly pagination params' do + context 'with high limit' do + let(:params) { base_params.merge(per_page: 100) } - it 'returns all repository tags' do - get api(route, user), params: params + it 'returns all repository tags' do + get api(route, user), params: params - expect(response).to have_gitlab_http_status(:ok) - expect(response).to match_response_schema('public_api/v4/tags') - expect(response.headers).not_to include('Link') - tag_names = json_response.map { |x| x['name'] } - expect(tag_names).to match_array(project.repository.tag_names) - end + expect(response).to have_gitlab_http_status(:ok) + expect(response).to match_response_schema('public_api/v4/tags') + expect(response.headers).not_to include('Link') + tag_names = json_response.map { |x| x['name'] } + expect(tag_names).to match_array(project.repository.tag_names) end + end - context 'with low limit' do - let(:params) { base_params.merge(per_page: 2) } + context 'with low limit' do + let(:params) { base_params.merge(per_page: 2) } - it 'returns limited repository tags' do - get api(route, user), params: params + it 'returns limited repository tags' do + get api(route, user), params: params - expect(response).to have_gitlab_http_status(:ok) - expect(response).to match_response_schema('public_api/v4/tags') - expect(response.headers).to include('Link') - tag_names = json_response.map { |x| x['name'] } - expect(tag_names).to match_array(%w(v1.1.0 v1.1.1)) - end + expect(response).to have_gitlab_http_status(:ok) + expect(response).to match_response_schema('public_api/v4/tags') + expect(response.headers).to include('Link') + tag_names = json_response.map { |x| x['name'] } + expect(tag_names).to match_array(%w(v1.1.0 v1.1.1)) end + end - context 'with missing page token' do - let(:params) { base_params.merge(page_token: 'unknown') } + context 'with missing page token' do + let(:params) { base_params.merge(page_token: 'unknown') } - it_behaves_like '422 response' do - let(:request) { get api(route, user), params: params } - let(:message) { 'Invalid page token: refs/tags/unknown' } - end + it_behaves_like '422 response' do + let(:request) { get api(route, user), params: params } + let(:message) { 'Invalid page token: refs/tags/unknown' } end end end diff --git a/spec/serializers/group_access_token_entity_spec.rb b/spec/serializers/group_access_token_entity_spec.rb index 39b587c7df7..05609dc3c7a 100644 --- a/spec/serializers/group_access_token_entity_spec.rb +++ b/spec/serializers/group_access_token_entity_spec.rb @@ -27,7 +27,7 @@ RSpec.describe GroupAccessTokenEntity do scopes: token.scopes, user_id: token.user_id, revoke_path: expected_revoke_path, - access_level: ::Gitlab::Access::DEVELOPER + role: 'Developer' )) expect(json).not_to include(:token) @@ -48,7 +48,7 @@ RSpec.describe GroupAccessTokenEntity do scopes: token.scopes, user_id: token.user_id, revoke_path: expected_revoke_path, - access_level: nil + role: nil )) expect(json).not_to include(:token) diff --git a/spec/serializers/project_access_token_entity_spec.rb b/spec/serializers/project_access_token_entity_spec.rb index 616aa45e9d5..4b5b4d4d77d 100644 --- a/spec/serializers/project_access_token_entity_spec.rb +++ b/spec/serializers/project_access_token_entity_spec.rb @@ -28,7 +28,7 @@ RSpec.describe ProjectAccessTokenEntity do scopes: token.scopes, user_id: token.user_id, revoke_path: expected_revoke_path, - access_level: ::Gitlab::Access::DEVELOPER + role: 'Developer' )) expect(json).not_to include(:token) @@ -52,7 +52,7 @@ RSpec.describe ProjectAccessTokenEntity do scopes: token.scopes, user_id: token.user_id, revoke_path: expected_revoke_path, - access_level: nil + role: nil )) expect(json).not_to include(:token) diff --git a/spec/support/helpers/gitaly_setup.rb b/spec/support/helpers/gitaly_setup.rb index 278dc79e1d0..81a01f1ca7f 100644 --- a/spec/support/helpers/gitaly_setup.rb +++ b/spec/support/helpers/gitaly_setup.rb @@ -344,6 +344,7 @@ module GitalySetup def spawn_gitaly(toml = nil) check_gitaly_config! + spawning_process = Process.pid pids = [] if toml @@ -364,6 +365,8 @@ module GitalySetup # running until `make test` cleans it up. next if ENV['GITALY_PID_FILE'] + next if Process.pid != spawning_process + pids.each { |pid| stop(pid) } end rescue StandardError diff --git a/spec/support/helpers/test_env.rb b/spec/support/helpers/test_env.rb index 6f9027ea79a..6ec7d491152 100644 --- a/spec/support/helpers/test_env.rb +++ b/spec/support/helpers/test_env.rb @@ -23,7 +23,7 @@ module TestEnv 'merged-target' => '21751bf', 'markdown' => '0ed8c6c', 'lfs' => '55bc176', - 'master' => 'b83d6e3', + 'master' => 'b83d6e391c22777fca1ed3012fce84f633d7fed0', 'merge-test' => '5937ac0', "'test'" => 'e56497b', 'orphaned-branch' => '45127a9', diff --git a/spec/support/shared_examples/features/deploy_token_shared_examples.rb b/spec/support/shared_examples/features/deploy_token_shared_examples.rb index 25dfe089f51..79ad5bd6c7f 100644 --- a/spec/support/shared_examples/features/deploy_token_shared_examples.rb +++ b/spec/support/shared_examples/features/deploy_token_shared_examples.rb @@ -30,6 +30,27 @@ RSpec.shared_examples 'a deploy token in settings' do expect(page).to have_selector("input[name='deploy-token-user'][value='deployer']") expect(page).to have_selector("input[name='deploy-token'][readonly='readonly']") end + + expect(find("input#deploy_token_name").value).to eq nil + expect(find("input#deploy_token_read_repository").checked?).to eq false + end + + context "with form errors" do + before do + visit page_path + fill_in "deploy_token_name", with: "new_deploy_key" + fill_in "deploy_token_username", with: "deployer" + click_button "Create deploy token" + end + + it "shows form errors" do + expect(page).to have_text("Scopes can't be blank") + end + + it "keeps form inputs" do + expect(find("input#deploy_token_name").value).to eq "new_deploy_key" + expect(find("input#deploy_token_username").value).to eq "deployer" + end end context 'when User#time_display_relative is false', :js do diff --git a/vendor/gems/omniauth-salesforce/.gitlab-ci.yml b/vendor/gems/omniauth-salesforce/.gitlab-ci.yml new file mode 100644 index 00000000000..4e18980d01c --- /dev/null +++ b/vendor/gems/omniauth-salesforce/.gitlab-ci.yml @@ -0,0 +1,28 @@ +workflow: + rules: + - if: $CI_MERGE_REQUEST_ID + +.rspec: + cache: + key: omniauth-salesforce + paths: + - vendor/gems/omniauth-salesforce/vendor/ruby + before_script: + - cd vendor/gems/omniauth-salesforce + - ruby -v # Print out ruby version for debugging + - gem install bundler --no-document # Bundler is not installed with the image + - bundle config set --local path 'vendor' # Install dependencies into ./vendor/ruby + - bundle config set with 'development' + - bundle config set --local frozen 'true' # Disallow Gemfile.lock changes on CI + - bundle config # Show bundler configuration + - bundle install -j $(nproc) + script: + - bundle exec rspec + +rspec-2.7: + image: "ruby:2.7" + extends: .rspec + +rspec-3.0: + image: "ruby:3.0" + extends: .rspec diff --git a/vendor/gems/omniauth-salesforce/Gemfile b/vendor/gems/omniauth-salesforce/Gemfile new file mode 100755 index 00000000000..0ac6c2141a7 --- /dev/null +++ b/vendor/gems/omniauth-salesforce/Gemfile @@ -0,0 +1,12 @@ +source 'https://rubygems.org' + +# Specify your gem's dependencies in omniauth-salesforce.gemspec +gemspec + +group :development, :test do + gem 'guard' + gem 'guard-rspec' + gem 'guard-bundler' + gem 'rb-fsevent' + gem 'growl' +end diff --git a/vendor/gems/omniauth-salesforce/Gemfile.lock b/vendor/gems/omniauth-salesforce/Gemfile.lock new file mode 100644 index 00000000000..0eb983bcd76 --- /dev/null +++ b/vendor/gems/omniauth-salesforce/Gemfile.lock @@ -0,0 +1,121 @@ +PATH + remote: . + specs: + omniauth-salesforce (1.0.5) + omniauth (~> 1.0, < 3) + omniauth-oauth2 (~> 1.0) + +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.1) + public_suffix (>= 2.0.2, < 6.0) + coderay (1.1.3) + crack (0.4.5) + rexml + diff-lcs (1.5.0) + docile (1.4.0) + faraday (2.5.2) + faraday-net_http (>= 2.0, < 3.1) + ruby2_keywords (>= 0.0.4) + faraday-net_http (3.0.0) + ffi (1.15.5) + formatador (1.1.0) + growl (1.0.3) + guard (2.18.0) + formatador (>= 0.2.4) + listen (>= 2.7, < 4.0) + lumberjack (>= 1.0.12, < 2.0) + nenv (~> 0.1) + notiffany (~> 0.0) + pry (>= 0.13.0) + shellany (~> 0.0) + thor (>= 0.18.1) + guard-bundler (3.0.0) + bundler (>= 2.1, < 3) + guard (~> 2.2) + guard-compat (~> 1.1) + guard-compat (1.2.1) + guard-rspec (4.7.3) + guard (~> 2.1) + guard-compat (~> 1.1) + rspec (>= 2.99.0, < 4.0) + hashdiff (1.0.1) + hashie (5.0.0) + jwt (2.4.1) + listen (3.7.1) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) + lumberjack (1.2.8) + method_source (1.0.0) + multi_xml (0.6.0) + nenv (0.3.0) + notiffany (0.1.3) + nenv (~> 0.1) + shellany (~> 0.0) + oauth2 (2.0.7) + faraday (>= 0.17.3, < 3.0) + jwt (>= 1.0, < 3.0) + multi_xml (~> 0.5) + rack (>= 1.2, < 3) + rash_alt (>= 0.4, < 1) + version_gem (~> 1.1) + omniauth (1.9.2) + hashie (>= 3.4.6) + rack (>= 1.6.2, < 3) + omniauth-oauth2 (1.7.3) + oauth2 (>= 1.4, < 3) + omniauth (>= 1.9, < 3) + pry (0.14.1) + coderay (~> 1.1) + method_source (~> 1.0) + public_suffix (5.0.0) + rack (2.2.4) + rack-test (2.0.2) + rack (>= 1.3) + rash_alt (0.4.12) + hashie (>= 3.4) + rb-fsevent (0.11.1) + rb-inotify (0.10.1) + ffi (~> 1.0) + rexml (3.2.5) + rspec (2.99.0) + rspec-core (~> 2.99.0) + rspec-expectations (~> 2.99.0) + rspec-mocks (~> 2.99.0) + rspec-core (2.99.2) + rspec-expectations (2.99.2) + diff-lcs (>= 1.1.3, < 2.0) + rspec-mocks (2.99.4) + ruby2_keywords (0.0.5) + shellany (0.0.1) + simplecov (0.21.2) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-html (0.12.3) + simplecov_json_formatter (0.1.4) + thor (1.2.1) + version_gem (1.1.0) + webmock (3.18.1) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) + +PLATFORMS + ruby + +DEPENDENCIES + growl + guard + guard-bundler + guard-rspec + omniauth-salesforce! + rack-test + rb-fsevent + rspec (~> 2.7) + simplecov + webmock + +BUNDLED WITH + 2.3.20 diff --git a/vendor/gems/omniauth-salesforce/Guardfile b/vendor/gems/omniauth-salesforce/Guardfile new file mode 100755 index 00000000000..2aaba26cc43 --- /dev/null +++ b/vendor/gems/omniauth-salesforce/Guardfile @@ -0,0 +1,10 @@ +guard 'rspec', :version => 2 do + watch(%r{^spec/.+_spec\.rb$}) + watch(%r{^lib/(.+)\.rb$}) { |m| "spec/#{m[1]}_spec.rb" } + watch('spec/spec_helper.rb') { "spec" } +end + +guard 'bundler' do + watch('Gemfile') + watch('omniauth-salesforce.gemspec') +end diff --git a/vendor/gems/omniauth-salesforce/LICENSE.md b/vendor/gems/omniauth-salesforce/LICENSE.md new file mode 100644 index 00000000000..e662c786216 --- /dev/null +++ b/vendor/gems/omniauth-salesforce/LICENSE.md @@ -0,0 +1,5 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file diff --git a/vendor/gems/omniauth-salesforce/README.md b/vendor/gems/omniauth-salesforce/README.md new file mode 100755 index 00000000000..df59e99bc55 --- /dev/null +++ b/vendor/gems/omniauth-salesforce/README.md @@ -0,0 +1,60 @@ +# omniauth-salesforce + +This is fork of [omniauth-salesforce](https://github.com/realdoug/omniauth-salesforce) to support: + +1. OmniAuth v1 and v2. OmniAuth v2 disables GET requests by default + and defaults to POST. GitLab already has patched v1 to use POST, + but other dependencies need to be updated: + https://gitlab.com/gitlab-org/gitlab/-/issues/30073. + +There is active discussion with the gem owner (via email) about adding some GitLab employees as gem +authors so tha they can push changes. If that happens, the updated/canonical gem +should be used in favor of this vendored fork. + +[OmniAuth](https://github.com/intridea/omniauth) Strategy for [salesforce.com](salesforce.com). + +Note: This is a fork of the [original](https://github.com/richardvanhook/omniauth-salesforce) project and is now the main repository for the omniauth-salesforce gem. + +## See it in action + +[http://omniauth-salesforce-example.herokuapp.com](http://omniauth-salesforce-example.herokuapp.com) + +[Source for above app](https://github.com/richardvanhook/omniauth-salesforce-example) + +## Basic Usage + +```ruby +require "sinatra" +require "omniauth" +require "omniauth-salesforce" + +class MyApplication < Sinatra::Base + use Rack::Session + use OmniAuth::Builder do + provider :salesforce, ENV['SALESFORCE_KEY'], ENV['SALESFORCE_SECRET'] + end +end +``` + +## Including other sites + +```ruby +use OmniAuth::Builder do + provider :salesforce, + ENV['SALESFORCE_KEY'], + ENV['SALESFORCE_SECRET'] + provider OmniAuth::Strategies::SalesforceSandbox, + ENV['SALESFORCE_SANDBOX_KEY'], + ENV['SALESFORCE_SANDBOX_SECRET'] + provider OmniAuth::Strategies::SalesforcePreRelease, + ENV['SALESFORCE_PRERELEASE_KEY'], + ENV['SALESFORCE_PRERELEASE_SECRET'] + provider OmniAuth::Strategies::DatabaseDotCom, + ENV['DATABASE_DOT_COM_KEY'], + ENV['DATABASE_DOT_COM_SECRET'] +end +``` + +## Resources + +* [Article: Digging Deeper into OAuth 2.0 on Force.com](http://wiki.developerforce.com/index.php/Digging_Deeper_into_OAuth_2.0_on_Force.com) diff --git a/vendor/gems/omniauth-salesforce/Rakefile b/vendor/gems/omniauth-salesforce/Rakefile new file mode 100755 index 00000000000..ca978a9cf0d --- /dev/null +++ b/vendor/gems/omniauth-salesforce/Rakefile @@ -0,0 +1,12 @@ +#!/usr/bin/env rake +require "bundler/gem_tasks" +require 'rspec/core/rake_task' + +desc 'Default: run specs.' +task :default => :spec + +desc "Run specs" +RSpec::Core::RakeTask.new + +desc 'Run specs' +task :default => :spec diff --git a/vendor/gems/omniauth-salesforce/lib/omniauth-salesforce.rb b/vendor/gems/omniauth-salesforce/lib/omniauth-salesforce.rb new file mode 100755 index 00000000000..20a5937d762 --- /dev/null +++ b/vendor/gems/omniauth-salesforce/lib/omniauth-salesforce.rb @@ -0,0 +1,2 @@ +require "omniauth-salesforce/version" +require 'omniauth/strategies/salesforce' diff --git a/vendor/gems/omniauth-salesforce/lib/omniauth-salesforce/version.rb b/vendor/gems/omniauth-salesforce/lib/omniauth-salesforce/version.rb new file mode 100755 index 00000000000..51f0084840e --- /dev/null +++ b/vendor/gems/omniauth-salesforce/lib/omniauth-salesforce/version.rb @@ -0,0 +1,5 @@ +module OmniAuth + module Salesforce + VERSION = "1.0.5" + end +end diff --git a/vendor/gems/omniauth-salesforce/lib/omniauth/strategies/salesforce.rb b/vendor/gems/omniauth-salesforce/lib/omniauth/strategies/salesforce.rb new file mode 100755 index 00000000000..f2d811aeb9e --- /dev/null +++ b/vendor/gems/omniauth-salesforce/lib/omniauth/strategies/salesforce.rb @@ -0,0 +1,97 @@ +require 'omniauth-oauth2' +require 'openssl' +require 'base64' + +module OmniAuth + module Strategies + class Salesforce < OmniAuth::Strategies::OAuth2 + + MOBILE_USER_AGENTS = 'webos|ipod|iphone|ipad|android|blackberry|mobile' + + option :client_options, { + :site => 'https://login.salesforce.com', + :authorize_url => '/services/oauth2/authorize', + :token_url => '/services/oauth2/token' + } + option :authorize_options, [ + :scope, + :display, + :immediate, + :state, + :prompt + ] + + def request_phase + req = Rack::Request.new(@env) + options.update(req.params) + ua = req.user_agent.to_s + if !options.has_key?(:display) + mobile_request = ua.downcase =~ Regexp.new(MOBILE_USER_AGENTS) + options[:display] = mobile_request ? 'touch' : 'page' + end + super + end + + def auth_hash + signed_value = access_token.params['id'] + access_token.params['issued_at'] + raw_expected_signature = OpenSSL::HMAC.digest('sha256', options.client_secret.to_s, signed_value) + expected_signature = Base64.strict_encode64 raw_expected_signature + signature = access_token.params['signature'] + fail! "Salesforce user id did not match signature!" unless signature == expected_signature + super + end + + uid { raw_info['id'] } + + info do + { + 'name' => raw_info['display_name'], + 'email' => raw_info['email'], + 'nickname' => raw_info['nick_name'], + 'first_name' => raw_info['first_name'], + 'last_name' => raw_info['last_name'], + 'location' => '', + 'description' => '', + 'image' => raw_info['photos']['thumbnail'] + "?oauth_token=#{access_token.token}", + 'phone' => '', + 'urls' => raw_info['urls'] + } + end + + credentials do + hash = {'token' => access_token.token} + hash.merge!('instance_url' => access_token.params["instance_url"]) + hash.merge!('refresh_token' => access_token.refresh_token) if access_token.refresh_token + hash + end + + def raw_info + access_token.options[:mode] = :header + @raw_info ||= access_token.post(access_token['id']).parsed + end + + extra do + raw_info.merge({ + 'instance_url' => access_token.params['instance_url'], + 'pod' => access_token.params['instance_url'], + 'signature' => access_token.params['signature'], + 'issued_at' => access_token.params['issued_at'] + }) + end + + end + + class SalesforceSandbox < OmniAuth::Strategies::Salesforce + default_options[:client_options][:site] = 'https://test.salesforce.com' + end + + class DatabaseDotCom < OmniAuth::Strategies::Salesforce + default_options[:client_options][:site] = 'https://login.database.com' + end + + class SalesforcePreRelease < OmniAuth::Strategies::Salesforce + default_options[:client_options][:site] = 'https://prerellogin.pre.salesforce.com/' + end + + end +end diff --git a/vendor/gems/omniauth-salesforce/omniauth-salesforce.gemspec b/vendor/gems/omniauth-salesforce/omniauth-salesforce.gemspec new file mode 100755 index 00000000000..0b9cfbd73ff --- /dev/null +++ b/vendor/gems/omniauth-salesforce/omniauth-salesforce.gemspec @@ -0,0 +1,24 @@ +# -*- encoding: utf-8 -*- +require File.expand_path('../lib/omniauth-salesforce/version', __FILE__) + +Gem::Specification.new do |gem| + gem.authors = ["Richard Vanhook"] + gem.email = ["rvanhook@salesforce.com"] + gem.description = %q{OmniAuth strategy for salesforce.com.} + gem.summary = %q{OmniAuth strategy for salesforce.com.} + gem.homepage = "https://github.com/realdoug/omniauth-salesforce" + + gem.files = Dir.glob("lib/**/*.*") + gem.test_files = Dir.glob("spec/**/**/*.*") + gem.name = "omniauth-salesforce" + gem.require_paths = ["lib"] + gem.version = OmniAuth::Salesforce::VERSION + gem.license = "MIT" + + gem.add_dependency 'omniauth', '~> 1.0', '< 3' + gem.add_dependency 'omniauth-oauth2', '~> 1.0' + gem.add_development_dependency 'rspec', '~> 2.7' + gem.add_development_dependency 'rack-test' + gem.add_development_dependency 'simplecov' + gem.add_development_dependency 'webmock' +end diff --git a/vendor/gems/omniauth-salesforce/spec/omniauth/strategies/salesforce_spec.rb b/vendor/gems/omniauth-salesforce/spec/omniauth/strategies/salesforce_spec.rb new file mode 100755 index 00000000000..9b5afbdbe25 --- /dev/null +++ b/vendor/gems/omniauth-salesforce/spec/omniauth/strategies/salesforce_spec.rb @@ -0,0 +1,219 @@ +require 'spec_helper' + +describe OmniAuth::Strategies::Salesforce do + strategy = nil + before do + OmniAuth.config.test_mode = true + rack_app = [] + rack_app.stub :call + strategy = OmniAuth::Strategies::Salesforce.new rack_app, 'Consumer Key', 'Consumer Secret' + end + describe "request_phase" do + env = nil + before do + env = { + 'rack.session' => {}, + 'HTTP_USER_AGENT' => 'unknown', + 'REQUEST_METHOD' => 'GET', + 'rack.input' => '', + 'rack.url_scheme' => 'http', + 'SERVER_NAME' => 'server.example', + 'QUERY_STRING' => 'code=xxxx', + 'SCRIPT_NAME' => '', + 'SERVER_PORT' => 80 + } + end + context "when using a mobile browser" do + user_agents = { + :Pre => "Mozilla/5.0 (webOS/1.4.0; U; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Version/1.0 Safari/532.2 Pre/1.1", + :iPod => "Mozilla/5.0 (iPod; U; CPU like Mac OS X; en) AppleWebKit/420.1 (KHTML, like Gecko) Version/3.0 Mobile/4A93 Safari/419.3", + :iPhone => "Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1A543 Safari/419.3", + :iPad => "Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10", + :Nexus => "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", + :myTouch => "Mozilla/5.0 (Linux; U; Android 1.6; en-us; WOWMobile myTouch 3G Build/unknown) AppleWebKit/528.5+ (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", + :Storm => "BlackBerry9530/4.7.0.148 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/105", + :Torch => "Mozilla/5.0 (BlackBerry; U; BlackBerry 9810; en-US) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.0.0 Mobile Safari/534.11+", + :generic_mobile => "some mobile device" + } + user_agents.each_pair do |name, agent| + context "with the user agent from a #{name.to_s}" do + before do + env['HTTP_USER_AGENT'] = agent + strategy.call!(env) + strategy.request_phase + end + subject {strategy.options} + it "sets the :display option to 'touch'" do + subject[:display].should == 'touch' + end + end + end + end + context "when using a desktop browser" do + user_agents = { + :Chrome => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1042.0 Safari/535.21", + :Safari => "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; de-at) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1", + :IE => "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)", + :anything_else => "unknown" + } + user_agents.each_pair do |name, agent| + context "with the user agent from #{name.to_s}" do + before do + env['HTTP_USER_AGENT'] = agent + strategy.call!(env) + strategy.request_phase + end + subject {strategy.options} + it "sets the :display option to 'page'" do + subject[:display].should == 'page' + end + end + end + end + end + describe "callback phase" do + raw_info = nil + before do + raw_info = { + 'id' => 'salesforce id', + 'display_name' => 'display name', + 'email' => 'email', + 'nick_name' => 'nick name', + 'first_name' => 'first name', + 'last_name' => 'last name', + 'photos' => {'thumbnail' => '/thumbnail/url'}, + 'urls'=> { + "enterprise" => "https://salesforce.example/services", + "metadata" => "https://salesforce.example/services" + } + } + client = OAuth2::Client.new 'id', 'secret', {:site => 'example.com'} + access_token = OAuth2::AccessToken.from_hash client, { + 'access_token' => 'token', + 'instance_url' => 'http://instance.salesforce.example', + 'signature' => 'invalid', + 'issued_at' => '1296458209517' + } + strategy.stub(:raw_info) { raw_info } + strategy.stub(:access_token) { access_token } + end + describe "uid" do + it "sets the id" do + strategy.uid.should == raw_info['id'] + end + end + describe "info" do + subject { strategy.info } + it "returns an info hash" do + subject.should_not be_nil + end + it "sets name" do + subject['name'].should == raw_info['display_name'] + end + it "sets email" do + subject['email'].should == raw_info['email'] + end + it "sets nickname" do + subject['nickname'].should == raw_info['nick_name'] + end + it "sets first_name" do + subject['first_name'].should == raw_info['first_name'] + end + it "sets last_name" do + subject['last_name'].should == raw_info['last_name'] + end + it "sets location" do + subject['location'].should == '' + end + it "sets description" do + subject['description'].should == '' + end + it "sets image" do + subject['image'].should == raw_info['photos']['thumbnail'] + "?oauth_token=#{strategy.access_token.token}" + end + it "sets phone" do + subject['phone'].should == '' + end + it "sets urls" do + subject['urls'].should == raw_info['urls'] + end + end + describe "credentials" do + subject { strategy.credentials } + it "sets token" do + subject['token'].should == strategy.access_token.token + end + it "sets instance_url" do + subject['instance_url'].should == strategy.access_token.params["instance_url"] + end + context "given a refresh token" do + it "sets refresh_token" do + subject['refresh_token'].should == strategy.access_token.refresh_token + end + end + context "when not given a refresh token" do + it "does not set a refresh token" do + subject['refresh_token'].should be_nil + end + end + end + describe "extra" do + subject { strategy.extra } + it "sets instance_url" do + subject['instance_url'].should == strategy.access_token.params['instance_url'] + end + it "sets pod" do + subject['pod'].should == strategy.access_token.params['instance_url'] + end + it "sets signature" do + subject['signature'].should == strategy.access_token.params['signature'] + end + it "sets issued_at" do + subject['issued_at'].should == strategy.access_token.params['issued_at'] + end + end + describe "user id validation" do + client_id = nil + issued_at = nil + signature = nil + instance_url = 'http://instance.salesforce.example' + before do + client_id = "https://login.salesforce.com/id/00Dd0000000d45TEBQ/005d0000000fyGPCCY" + issued_at = "1331142541514" + signature = Base64.strict_encode64(OpenSSL::HMAC.digest('sha256', strategy.options.client_secret.to_s, client_id + issued_at)) + end + context "when the signature does not match" do + before do + access_token = OAuth2::AccessToken.from_hash strategy.access_token.client, { + 'id' => 'forged client id', + 'refresh_token' => 'anything', + 'issued_at' => issued_at, + 'instance_url' => 'http://instance.salesforce.example', + 'signature' => signature + } + strategy.stub(:access_token) { access_token } + end + it "should call fail!" do + strategy.should_receive(:fail!) + strategy.auth_hash + end + end + context "when the signature does match" do + before do + access_token = OAuth2::AccessToken.from_hash strategy.access_token.client, { + 'id' => client_id, + 'refresh_token' => 'anything', + 'issued_at' => issued_at, + 'instance_url' => 'http://instance.salesforce.example', + 'signature' => signature + } + strategy.stub(:access_token) { access_token } + end + it "should not fail" do + strategy.should_not_receive(:fail!) + strategy.auth_hash + end + end + end + end +end diff --git a/vendor/gems/omniauth-salesforce/spec/spec_helper.rb b/vendor/gems/omniauth-salesforce/spec/spec_helper.rb new file mode 100755 index 00000000000..0b00358fa05 --- /dev/null +++ b/vendor/gems/omniauth-salesforce/spec/spec_helper.rb @@ -0,0 +1,16 @@ +$:.unshift File.expand_path('..', __FILE__) +$:.unshift File.expand_path('../../lib', __FILE__) +require 'simplecov' +SimpleCov.start +require 'rspec' +require 'rack/test' +require 'webmock/rspec' +require 'omniauth' +require 'omniauth-salesforce' + +RSpec.configure do |config| + config.include WebMock::API + config.include Rack::Test::Methods + config.extend OmniAuth::Test::StrategyMacros, :type => :strategy +end + diff --git a/workhorse/internal/upstream/routes.go b/workhorse/internal/upstream/routes.go index 0bd86cfe039..899e5d9835e 100644 --- a/workhorse/internal/upstream/routes.go +++ b/workhorse/internal/upstream/routes.go @@ -288,7 +288,7 @@ func configureRoutes(u *upstream) { u.route("PUT", apiProjectPattern+`/packages/debian/`, requestBodyUploader), // RPM Artifact Repository - u.route("POST", apiProjectPattern+`packages/rpm/`, requestBodyUploader), + u.route("POST", apiProjectPattern+`/packages/rpm/`, requestBodyUploader), // Gem Artifact Repository u.route("POST", apiProjectPattern+`/packages/rubygems/`, requestBodyUploader), diff --git a/yarn.lock b/yarn.lock index 49ca190fdeb..0ab35d67ec2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -302,7 +302,7 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.16.7", "@babel/parser@^7.16.8", "@babel/parser@^7.18.4", "@babel/parser@^7.18.5": +"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.16.7", "@babel/parser@^7.16.8", "@babel/parser@^7.18.5": version "7.18.11" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.18.11.tgz#68bb07ab3d380affa9a3f96728df07969645d2d9" integrity sha512-9JKn5vN+hDt0Hdqn1PiJ2guflwP+B6Ga8qbDuoF0PzzVhrzsKIJo8yGqVk6CmMHiMei9w1C1Bp9IMJSIK+HPIQ== @@ -2214,15 +2214,6 @@ semver "^6.3.0" tsutils "^3.17.1" -"@vue/compiler-sfc@2.7.10": - version "2.7.10" - resolved "https://registry.yarnpkg.com/@vue/compiler-sfc/-/compiler-sfc-2.7.10.tgz#3fe08e780053a3bbf41328c65ae5dfdee0385206" - integrity sha512-55Shns6WPxlYsz4WX7q9ZJBL77sKE1ZAYNYStLs6GbhIOMrNtjMvzcob6gu3cGlfpCR4bT7NXgyJ3tly2+Hx8Q== - dependencies: - "@babel/parser" "^7.18.4" - postcss "^8.4.14" - source-map "^0.6.1" - "@vue/component-compiler-utils@^3.1.0": version "3.3.0" resolved "https://registry.yarnpkg.com/@vue/component-compiler-utils/-/component-compiler-utils-3.3.0.tgz#f9f5fb53464b0c37b2c8d2f3fbfe44df60f61dc9" @@ -3985,11 +3976,6 @@ cssstyle@^2.3.0: dependencies: cssom "~0.3.6" -csstype@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.0.tgz#4ddcac3718d787cf9df0d1b7d15033925c8f29f2" - integrity sha512-uX1KG+x9h5hIJsaKR9xHUeUraxf8IODOwq9JLNPq6BwB04a/xgpq3rcx47l5BZu5zBPlgD342tdke3Hom/nJRA== - custom-jquery-matchers@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/custom-jquery-matchers/-/custom-jquery-matchers-2.1.0.tgz#e5988fa9715c416b0986b372563f872d9e91e024" @@ -6304,7 +6290,7 @@ hastscript@^7.0.0: property-information "^6.0.0" space-separated-tokens "^2.0.0" -he@^1.2.0: +he@^1.1.0: version "1.2.0" resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== @@ -11928,10 +11914,10 @@ vue-hot-reload-api@^2.3.0: resolved "https://registry.yarnpkg.com/vue-hot-reload-api/-/vue-hot-reload-api-2.3.4.tgz#532955cc1eb208a3d990b3a9f9a70574657e08f2" integrity sha512-BXq3jwIagosjgNVae6tkHzzIk6a8MHFtzAdwhnV5VlvPTFxDCvIttgSiHWjdGoTJvXtmRu5HacExfdarRcFhog== -vue-loader@^15.10: - version "15.10.0" - resolved "https://registry.yarnpkg.com/vue-loader/-/vue-loader-15.10.0.tgz#2a12695c421a2a2cc2138f05a949d04ed086e38b" - integrity sha512-VU6tuO8eKajrFeBzMssFUP9SvakEeeSi1BxdTH5o3+1yUyrldp8IERkSdXlMI2t4kxF2sqYUDsQY+WJBxzBmZg== +vue-loader@15.9.6: + version "15.9.6" + resolved "https://registry.yarnpkg.com/vue-loader/-/vue-loader-15.9.6.tgz#f4bb9ae20c3a8370af3ecf09b8126d38ffdb6b8b" + integrity sha512-j0cqiLzwbeImIC6nVIby2o/ABAWhlppyL/m5oJ67R5MloP0hj/DtFgb0Zmq3J9CG7AJ+AXIvHVnJAPBvrLyuDg== dependencies: "@vue/component-compiler-utils" "^3.1.0" hash-sum "^1.0.2" @@ -11969,13 +11955,13 @@ vue-style-loader@^4.1.0: hash-sum "^1.0.2" loader-utils "^1.0.2" -vue-template-compiler@^2.7.10: - version "2.7.10" - resolved "https://registry.yarnpkg.com/vue-template-compiler/-/vue-template-compiler-2.7.10.tgz#9e20f35b2fdccacacf732dd7dedb49bf65f4556b" - integrity sha512-QO+8R9YRq1Gudm8ZMdo/lImZLJVUIAM8c07Vp84ojdDAf8HmPJc7XB556PcXV218k2AkKznsRz6xB5uOjAC4EQ== +vue-template-compiler@2.6.14: + version "2.6.14" + resolved "https://registry.yarnpkg.com/vue-template-compiler/-/vue-template-compiler-2.6.14.tgz#a2f0e7d985670d42c9c9ee0d044fed7690f4f763" + integrity sha512-ODQS1SyMbjKoO1JBJZojSw6FE4qnh9rIpUZn2EUT86FKizx9uH5z6uXiIrm4/Nb/gwxTi/o17ZDEGWAXHvtC7g== dependencies: de-indent "^1.0.2" - he "^1.2.0" + he "^1.1.0" vue-template-es2015-compiler@^1.9.0: version "1.9.1" @@ -11987,13 +11973,10 @@ vue-virtual-scroll-list@^1.4.7: resolved "https://registry.yarnpkg.com/vue-virtual-scroll-list/-/vue-virtual-scroll-list-1.4.7.tgz#12ee26833885f5bb4d37dc058085ccf3ce5b5a74" integrity sha512-R8bk+k7WMGGoFQ9xF0krGCAlZhQjbJOkDUX+YZD2J+sHQWTzDtmTLS6kiIJToOHK1d/8QPGiD8fd9w0lDP4arg== -vue@^2.7.10: - version "2.7.10" - resolved "https://registry.yarnpkg.com/vue/-/vue-2.7.10.tgz#ae516cc6c88e1c424754468844218fdd5e280f40" - integrity sha512-HmFC70qarSHPXcKtW8U8fgIkF6JGvjEmDiVInTkKZP0gIlEPhlVlcJJLkdGIDiNkIeA2zJPQTWJUI4iWe+AVfg== - dependencies: - "@vue/compiler-sfc" "2.7.10" - csstype "^3.1.0" +vue@2.6.14: + version "2.6.14" + resolved "https://registry.yarnpkg.com/vue/-/vue-2.6.14.tgz#e51aa5250250d569a3fbad3a8a5a687d6036e235" + integrity sha512-x2284lgYvjOMj3Za7kqzRcUSxBboHqtgRE2zlos1qWaOye5yUmHn42LB1250NJBLRwEcdrB0JRwyPTEPhfQjiQ== vuedraggable@^2.23.0: version "2.23.0" |