diff options
34 files changed, 541 insertions, 242 deletions
diff --git a/.rubocop.yml b/.rubocop.yml index a26e9ab986b..981b71e582f 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -607,6 +607,12 @@ Migration/CreateTableWithForeignKeys: Exclude: - !ruby/regexp /\Adb\/(?:post_)?migrate\/(?:201[0-9]\d+|20200[0-8][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])_.+\.rb\z/ +Migration/PreventIndexCreation: + Exclude: + - !ruby/regexp /\Adb\/(post_)?migrate\/201.*\.rb\z/ + - !ruby/regexp /\Adb\/(post_)?migrate\/2020.*\.rb\z/ + - !ruby/regexp /\Adb\/(post_)?migrate\/20210[1-6].*\.rb\z/ + Gitlab/RailsLogger: Exclude: - 'spec/**/*.rb' diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 28c55f889ba..8be1d8ee85d 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -49893735ed64feebc208f4efe90bebbb8bbb02ad +5fdd1ba64d79df3a46c74f29d17faf7927650887 @@ -394,7 +394,7 @@ group :development, :test do end group :development, :test, :danger do - gem 'gitlab-dangerfiles', '~> 2.2.1', require: false + gem 'gitlab-dangerfiles', '~> 2.2.2', require: false end group :development, :test, :coverage do diff --git a/Gemfile.lock b/Gemfile.lock index 91564d4faae..07bd72755d0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -468,7 +468,7 @@ GEM terminal-table (~> 1.5, >= 1.5.1) gitlab-chronic (0.10.5) numerizer (~> 0.2) - gitlab-dangerfiles (2.2.1) + gitlab-dangerfiles (2.2.2) danger (>= 8.3.1) danger-gitlab (>= 8.0.0) gitlab-experiment (0.6.1) @@ -1487,7 +1487,7 @@ DEPENDENCIES gitaly (~> 14.1.0.pre.rc2) github-markup (~> 1.7.0) gitlab-chronic (~> 0.10.5) - gitlab-dangerfiles (~> 2.2.1) + gitlab-dangerfiles (~> 2.2.2) gitlab-experiment (~> 0.6.1) gitlab-fog-azure-rm (~> 1.1.1) gitlab-labkit (~> 0.20.0) diff --git a/app/helpers/personal_access_tokens_helper.rb b/app/helpers/personal_access_tokens_helper.rb new file mode 100644 index 00000000000..5cc8d21096f --- /dev/null +++ b/app/helpers/personal_access_tokens_helper.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module PersonalAccessTokensHelper + def personal_access_token_expiration_enforced? + false + end +end diff --git a/app/views/projects/settings/ci_cd/show.html.haml b/app/views/projects/settings/ci_cd/show.html.haml index 15752287fc3..a4a8ac3e1f9 100644 --- a/app/views/projects/settings/ci_cd/show.html.haml +++ b/app/views/projects/settings/ci_cd/show.html.haml @@ -105,6 +105,6 @@ = expanded ? _('Collapse') : _('Expand') %p = _("Control which projects can use the CI_JOB_TOKEN CI/CD variable for API access to this project. It is a security risk to disable this feature, because unauthorized projects may attempt to retrieve an active token and access the API.") - = link_to _('Learn more'), help_page_path('api/index', anchor: 'gitlab-cicd-job-token-scope'), target: '_blank', rel: 'noopener noreferrer' + = link_to _('Learn more'), help_page_path('api/index', anchor: 'limit-gitlab-cicd-job-token-access'), target: '_blank', rel: 'noopener noreferrer' .settings-content = render 'ci/token_access/index' diff --git a/app/views/shared/access_tokens/_table.html.haml b/app/views/shared/access_tokens/_table.html.haml index 462f15c6078..1f08bff9858 100644 --- a/app/views/shared/access_tokens/_table.html.haml +++ b/app/views/shared/access_tokens/_table.html.haml @@ -1,11 +1,15 @@ - no_active_tokens_message = local_assigns.fetch(:no_active_tokens_message, _('This user has no active %{type}.') % { type: type_plural }) - impersonation = local_assigns.fetch(:impersonation, false) - project = local_assigns.fetch(:project, false) +- personal = !impersonation && !project %hr %h5 = _('Active %{type} (%{token_length})') % { type: type_plural, token_length: active_tokens.length } +- if personal && !personal_access_token_expiration_enforced? + %p.profile-settings-content + = _("Personal access tokens are not revoked upon expiration.") - if impersonation %p.profile-settings-content = _("To see all the user's personal access tokens you must impersonate them first.") @@ -16,6 +20,7 @@ %thead %tr %th= _('Token name') + %th= _('Scopes') %th= s_('AccessTokens|Created') %th = _('Last Used') @@ -23,12 +28,12 @@ %th= _('Expires') - if project %th= _('Role') - %th= _('Scopes') %th %tbody - active_tokens.each do |token| %tr %td= token.name + %td= token.scopes.present? ? token.scopes.join(', ') : _('no scopes selected') %td= token.created_at.to_date.to_s(:medium) %td - if token.last_used_at? @@ -47,8 +52,7 @@ %span.token-never-expires-label= _('Never') - if project %td= project.project_member(token.user).human_access - %td= token.scopes.present? ? token.scopes.join(', ') : _('no scopes selected') - %td= link_to _('Revoke'), revoke_route_helper.call(token), method: :put, class: 'gl-button btn btn-danger btn-sm float-right qa-revoke-button', data: { confirm: _('Are you sure you want to revoke this %{type}? This action cannot be undone.') % { type: type } } + %td= link_to _('Revoke'), revoke_route_helper.call(token), method: :put, class: "gl-button btn btn-danger btn-sm float-right qa-revoke-button #{'btn-danger-secondary' unless token.expires?}", data: { confirm: _('Are you sure you want to revoke this %{type}? This action cannot be undone.') % { type: type } } - else .settings-message.text-center = no_active_tokens_message diff --git a/app/views/shared/deploy_tokens/_form.html.haml b/app/views/shared/deploy_tokens/_form.html.haml index 976776ccc62..5d351bd11fd 100644 --- a/app/views/shared/deploy_tokens/_form.html.haml +++ b/app/views/shared/deploy_tokens/_form.html.haml @@ -38,7 +38,7 @@ - if packages_registry_enabled?(group_or_project) %fieldset.form-group.form-check - = f.check_box :read_package_registry, class: 'form-check-input' + = f.check_box :read_package_registry, class: 'form-check-input', data: { qa_selector: 'deploy_token_read_package_registry_checkbox' } = f.label :read_package_registry, 'read_package_registry', class: 'label-bold form-check-label' .text-secondary= s_('DeployTokens|Allows read access to the package registry.') diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 12b1a37c820..26afe36893a 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -1103,7 +1103,7 @@ :urgency: :low :resource_boundary: :unknown :weight: 1 - :idempotent: true + :idempotent: :tags: [] - :name: jira_connect:jira_connect_sync_builds :worker_name: JiraConnect::SyncBuildsWorker @@ -1112,7 +1112,7 @@ :urgency: :low :resource_boundary: :unknown :weight: 1 - :idempotent: true + :idempotent: :tags: - :exclude_from_kubernetes - :name: jira_connect:jira_connect_sync_deployments @@ -1122,7 +1122,7 @@ :urgency: :low :resource_boundary: :unknown :weight: 1 - :idempotent: true + :idempotent: :tags: - :exclude_from_kubernetes - :name: jira_connect:jira_connect_sync_feature_flags @@ -1132,7 +1132,7 @@ :urgency: :low :resource_boundary: :unknown :weight: 1 - :idempotent: true + :idempotent: :tags: - :exclude_from_kubernetes - :name: jira_connect:jira_connect_sync_merge_request @@ -1142,7 +1142,7 @@ :urgency: :low :resource_boundary: :unknown :weight: 1 - :idempotent: true + :idempotent: :tags: [] - :name: jira_connect:jira_connect_sync_project :worker_name: JiraConnect::SyncProjectWorker @@ -1151,7 +1151,7 @@ :urgency: :low :resource_boundary: :unknown :weight: 1 - :idempotent: true + :idempotent: :tags: - :exclude_from_kubernetes - :name: jira_importer:jira_import_advance_stage diff --git a/app/workers/jira_connect/sync_branch_worker.rb b/app/workers/jira_connect/sync_branch_worker.rb index 4e8566d86c9..bd6023ccdd1 100644 --- a/app/workers/jira_connect/sync_branch_worker.rb +++ b/app/workers/jira_connect/sync_branch_worker.rb @@ -1,16 +1,17 @@ # frozen_string_literal: true module JiraConnect - class SyncBranchWorker + class SyncBranchWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker sidekiq_options retry: 3 queue_namespace :jira_connect feature_category :integrations + data_consistency :delayed, feature_flag: :load_balancing_for_jira_connect_workers loggable_arguments 1, 2 + worker_has_external_dependencies! - idempotent! def perform(project_id, branch_name, commit_shas, update_sequence_id) project = Project.find_by_id(project_id) diff --git a/app/workers/jira_connect/sync_builds_worker.rb b/app/workers/jira_connect/sync_builds_worker.rb index 11a3b598035..4193b8ccdd7 100644 --- a/app/workers/jira_connect/sync_builds_worker.rb +++ b/app/workers/jira_connect/sync_builds_worker.rb @@ -1,18 +1,18 @@ # frozen_string_literal: true module JiraConnect - class SyncBuildsWorker + class SyncBuildsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker sidekiq_options retry: 3 - idempotent! - worker_has_external_dependencies! - queue_namespace :jira_connect feature_category :integrations + data_consistency :delayed, feature_flag: :load_balancing_for_jira_connect_workers tags :exclude_from_kubernetes + worker_has_external_dependencies! + def perform(pipeline_id, sequence_id) pipeline = Ci::Pipeline.find_by_id(pipeline_id) diff --git a/app/workers/jira_connect/sync_deployments_worker.rb b/app/workers/jira_connect/sync_deployments_worker.rb index 9f75b1161f0..d8d556d7102 100644 --- a/app/workers/jira_connect/sync_deployments_worker.rb +++ b/app/workers/jira_connect/sync_deployments_worker.rb @@ -1,18 +1,18 @@ # frozen_string_literal: true module JiraConnect - class SyncDeploymentsWorker + class SyncDeploymentsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker sidekiq_options retry: 3 - idempotent! - worker_has_external_dependencies! - queue_namespace :jira_connect feature_category :integrations + data_consistency :delayed, feature_flag: :load_balancing_for_jira_connect_workers tags :exclude_from_kubernetes + worker_has_external_dependencies! + def perform(deployment_id, sequence_id) deployment = Deployment.find_by_id(deployment_id) diff --git a/app/workers/jira_connect/sync_feature_flags_worker.rb b/app/workers/jira_connect/sync_feature_flags_worker.rb index 0d8d3d3142e..919c76ab4c7 100644 --- a/app/workers/jira_connect/sync_feature_flags_worker.rb +++ b/app/workers/jira_connect/sync_feature_flags_worker.rb @@ -1,18 +1,18 @@ # frozen_string_literal: true module JiraConnect - class SyncFeatureFlagsWorker + class SyncFeatureFlagsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker sidekiq_options retry: 3 - idempotent! - worker_has_external_dependencies! - queue_namespace :jira_connect feature_category :integrations + data_consistency :delayed, feature_flag: :load_balancing_for_jira_connect_workers tags :exclude_from_kubernetes + worker_has_external_dependencies! + def perform(feature_flag_id, sequence_id) feature_flag = ::Operations::FeatureFlag.find_by_id(feature_flag_id) diff --git a/app/workers/jira_connect/sync_merge_request_worker.rb b/app/workers/jira_connect/sync_merge_request_worker.rb index bf31df2271f..f7115212792 100644 --- a/app/workers/jira_connect/sync_merge_request_worker.rb +++ b/app/workers/jira_connect/sync_merge_request_worker.rb @@ -1,14 +1,14 @@ # frozen_string_literal: true module JiraConnect - class SyncMergeRequestWorker + class SyncMergeRequestWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker sidekiq_options retry: 3 queue_namespace :jira_connect feature_category :integrations - idempotent! + data_consistency :delayed, feature_flag: :load_balancing_for_jira_connect_workers worker_has_external_dependencies! diff --git a/app/workers/jira_connect/sync_project_worker.rb b/app/workers/jira_connect/sync_project_worker.rb index dfff0c4b3b6..bca05b9311f 100644 --- a/app/workers/jira_connect/sync_project_worker.rb +++ b/app/workers/jira_connect/sync_project_worker.rb @@ -1,15 +1,16 @@ # frozen_string_literal: true module JiraConnect - class SyncProjectWorker + class SyncProjectWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker sidekiq_options retry: 3 queue_namespace :jira_connect feature_category :integrations + data_consistency :delayed, feature_flag: :load_balancing_for_jira_connect_workers tags :exclude_from_kubernetes - idempotent! + worker_has_external_dependencies! MERGE_REQUEST_LIMIT = 400 diff --git a/config/feature_flags/development/load_balancing_for_jira_connect_workers.yml b/config/feature_flags/development/load_balancing_for_jira_connect_workers.yml new file mode 100644 index 00000000000..03c6de7309f --- /dev/null +++ b/config/feature_flags/development/load_balancing_for_jira_connect_workers.yml @@ -0,0 +1,8 @@ +--- +name: load_balancing_for_jira_connect_workers +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64715 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/335420 +milestone: '14.1' +type: development +group: group::ecosystem +default_enabled: false diff --git a/doc/api/index.md b/doc/api/index.md index 76d31c35b96..3080816e363 100644 --- a/doc/api/index.md +++ b/doc/api/index.md @@ -245,53 +245,55 @@ your [runners](../ci/runners/README.md) to be secure. Avoid: If you have an insecure GitLab Runner configuration, you increase the risk that someone tries to steal tokens from other jobs. -#### GitLab CI/CD job token scope +#### Limit GitLab CI/CD job token access -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/328553) in GitLab 14.1. - -- [Deployed behind a feature flag](../user/feature_flags.md), disabled by default. -- Disabled on GitLab.com. -- Not recommended for production use. -- To use in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-ci-job-token-scope). **(FREE SELF)** +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/328553) in GitLab 14.1. +> - [Deployed behind a feature flag](../user/feature_flags.md), disabled by default. +> - Disabled on GitLab.com. +> - Not recommended for production use. +> - To use in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-ci-job-token-scope-limit). **(FREE SELF)** This in-development feature might not be available for your use. There can be [risks when enabling features still in development](../user/feature_flags.md#risks-when-enabling-features-still-in-development). Refer to this feature's version history for more details. -CI job token can access only projects that are defined in its scope. -You can configure the scope via project settings. - -The CI job token scope consists in a allowlist of projects that are authorized by maintainers to be -accessible via a CI job token. By default a scope only contains the same project where the token -comes from. Other projects can be added and removed by maintainers. +You can limit the access scope of a project's CI/CD job token to increase the +job token's security. A job token might give extra permissions that aren't necessary +to access specific resources. Limiting the job token access scope reduces the risk of a leaked +token being used to access private data that the user associated to the job can access. -You can configure the scope via project settings. +Control the job token access scope with an allowlist of other projects authorized +to be accessed by authenticating with the current project's job token. By default +the token scope only allows access to the same project where the token comes from. +Other projects can be added and removed by maintainers with access to both projects. -Since GitLab 14.1 this setting is enabled by default for new projects. Existing projects are -recommended to enable this feature and configure which projects are authorized to be accessed -by a job token. +This setting is enabled by default for all new projects, and disabled by default in projects +created before GitLab 14.1. It is strongly recommended that project maintainers enable this +setting at all times, and configure the allowlist for cross-project access if needed. -The CI job token scope limits the risks that a leaked token is used to access private data that -the user associated to the job can access to. +For example, when the setting is enabled, jobs in a pipeline in project `A` have +a `CI_JOB_TOKEN` scope limited to project `A`. If the job needs to use the token +to make an API request to project `B`, then `B` must be added to the allowlist for `A`. -When the job token scope feature is enabled in the project settings, only the projects in scope -will be allowed to be accessed by a job token. If the job token scope feature is disabled, any -projects can be accessed, as long as the user associated to the job has permissions. +To enable and configure the job token scope limit: -For example. If a project `A` has a running job with a `CI_JOB_TOKEN`, its scope is defined by -project `A`. If the job wants to use the `CI_JOB_TOKEN` to access data from project `B` or -trigger some actions in that project, then project `B` must be in the job token scope for `A`. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **Token Access**. +1. Toggle **Limit CI_JOB_TOKEN access** to enabled. +1. (Optional) Add existing projects to the token's access scope. The user adding a + project must have the [maintainer role](../user/permissions.md) in both projects. -A job token might give extra permissions that aren't necessary to access specific resources. -There is [a proposal](https://gitlab.com/groups/gitlab-org/-/epics/3559) to redesign the feature -for more strategic control of the access permissions. +If the job token scope limit is disabled, the token can potentially be used to authenticate +API requests to all projects accessible to the user that triggered the job. -<!-- Add this at the end of the file --> +There is [a proposal](https://gitlab.com/groups/gitlab-org/-/epics/3559) to improve +the feature with more strategic control of the access permissions. -#### Enable or disable CI Job Token Scope **(FREE SELF)** +##### Enable or disable CI job token scope limit **(FREE SELF)** -This is under development and not ready for production use. It is -deployed behind a feature flag that is **disabled by default**. +The GitLab CI/CD job token access scope limit is under development and not ready for production +use. It is deployed behind a feature flag that is **disabled by default**. [GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) can enable it. diff --git a/doc/user/packages/index.md b/doc/user/packages/index.md index f0bf2fc3363..9158b5cc674 100644 --- a/doc/user/packages/index.md +++ b/doc/user/packages/index.md @@ -17,6 +17,7 @@ The Package Registry supports the following formats: | [Composer](composer_repository/index.md) | 13.2+ | | [Conan](conan_repository/index.md) | 12.6+ | | [Go](go_proxy/index.md) | 13.1+ | +| [Helm](helm_repository/index.md) | 14.1+ | | [Maven](maven_repository/index.md) | 11.3+ | | [npm](npm_registry/index.md) | 11.7+ | | [NuGet](nuget_repository/index.md) | 12.8+ | @@ -40,7 +41,6 @@ guides you through the process. | Conda | [#36891](https://gitlab.com/gitlab-org/gitlab/-/issues/36891) | | CRAN | [#36892](https://gitlab.com/gitlab-org/gitlab/-/issues/36892) | | Debian | [Draft: Merge Request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50438) | -| Helm | [#18997](https://gitlab.com/gitlab-org/gitlab/-/issues/18997) | | Opkg | [#36894](https://gitlab.com/gitlab-org/gitlab/-/issues/36894) | | P2 | [#36895](https://gitlab.com/gitlab-org/gitlab/-/issues/36895) | | Puppet | [#36897](https://gitlab.com/gitlab-org/gitlab/-/issues/36897) | diff --git a/doc/user/packages/maven_repository/index.md b/doc/user/packages/maven_repository/index.md index 7f84da61dc2..70b9c28da76 100644 --- a/doc/user/packages/maven_repository/index.md +++ b/doc/user/packages/maven_repository/index.md @@ -813,6 +813,19 @@ You can play around with the regex and try your version strings on [this regular ## Troubleshooting +To improve performance, Maven caches files related to a package. If you encounter issues, clear +the cache with these commands: + +```shell +rm -rf ~/.m2/repository +``` + +If you're using Gradle, run this command to clear the cache: + +```shell +rm -rf ~/.gradle/caches +``` + ### Review network trace logs If you are having issues with the Maven Repository, you may want to review network trace logs. diff --git a/doc/user/packages/npm_registry/index.md b/doc/user/packages/npm_registry/index.md index abac38af065..1e5c294689b 100644 --- a/doc/user/packages/npm_registry/index.md +++ b/doc/user/packages/npm_registry/index.md @@ -455,6 +455,14 @@ Due to a bug in npm 6.9.0, deleting distribution tags fails. Make sure your npm When troubleshooting npm issues, first run the same command with the `--verbose` flag to confirm what registry you are hitting. +To improve performance, npm caches files related to a package. Note that npm doesn't remove data by +itself. The cache grows as new packages are installed. If you encounter issues, clear the cache with +this command: + +```shell +npm cache clean --force +``` + ### Error running Yarn with the Package Registry for npm registry If you are using [Yarn](https://classic.yarnpkg.com/en/) with the npm registry, you may get diff --git a/doc/user/packages/nuget_repository/index.md b/doc/user/packages/nuget_repository/index.md index 6c76d5e4067..46cfd763668 100644 --- a/doc/user/packages/nuget_repository/index.md +++ b/doc/user/packages/nuget_repository/index.md @@ -422,3 +422,12 @@ CLI (`dotnet`): - `dotnet nuget push`: Upload a package to the registry. - `nuget install`: Install a package from the registry. - `dotnet add`: Install a package from the registry. + +## Troubleshooting + +To improve performance, NuGet caches files related to a package. If you encounter issues, clear the +cache with this command: + +```shell +nuget locals all -clear +``` diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 221e383a0d0..0e1b6604f2e 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -13092,6 +13092,9 @@ msgstr "" msgid "EscalationPolicies|The escalation policy could not be updated. Please try again" msgstr "" +msgid "EscalationPolicies|This policy has no escalation rules." +msgstr "" + msgid "EscalationPolicies|mins" msgstr "" @@ -22721,6 +22724,9 @@ msgstr "" msgid "On-call schedules" msgstr "" +msgid "OnCallScheduless|Any escalation rules that are using this schedule will also be deleted." +msgstr "" + msgid "OnCallSchedules|1 day" msgstr "" @@ -23872,6 +23878,9 @@ msgstr "" msgid "Personal Access Token prefix" msgstr "" +msgid "Personal access tokens are not revoked upon expiration." +msgstr "" + msgid "Personal project creation is not allowed. Please contact your administrator with questions" msgstr "" diff --git a/qa/qa/page/project/settings/deploy_tokens.rb b/qa/qa/page/project/settings/deploy_tokens.rb index b26cae86d8b..db1f6f68ec6 100644 --- a/qa/qa/page/project/settings/deploy_tokens.rb +++ b/qa/qa/page/project/settings/deploy_tokens.rb @@ -9,6 +9,7 @@ module QA element :deploy_token_name_field element :deploy_token_expires_at_field element :deploy_token_read_repository_checkbox + element :deploy_token_read_package_registry_checkbox element :deploy_token_read_registry_checkbox element :create_deploy_token_button end @@ -27,8 +28,9 @@ module QA fill_element(:deploy_token_expires_at_field, expires_at.to_s + "\n") end - def fill_scopes(read_repository:, read_registry:) + def fill_scopes(read_repository: false, read_registry: false, read_package_registry: false) check_element(:deploy_token_read_repository_checkbox) if read_repository + check_element(:deploy_token_read_package_registry_checkbox) if read_package_registry check_element(:deploy_token_read_registry_checkbox) if read_registry end diff --git a/qa/qa/resource/deploy_token.rb b/qa/qa/resource/deploy_token.rb index 0ba8dbbf287..cd638ad2f85 100644 --- a/qa/qa/resource/deploy_token.rb +++ b/qa/qa/resource/deploy_token.rb @@ -37,7 +37,7 @@ module QA setting.expand_deploy_tokens do |page| page.fill_token_name(name) page.fill_token_expires_at(expires_at) - page.fill_scopes(read_repository: true, read_registry: false) + page.fill_scopes(read_repository: true, read_package_registry: true) page.add_token end diff --git a/qa/qa/resource/package.rb b/qa/qa/resource/package.rb index 1009353a296..0e8c3ee95de 100644 --- a/qa/qa/resource/package.rb +++ b/qa/qa/resource/package.rb @@ -15,11 +15,10 @@ module QA end attribute :id do - packages = project.packages - - return unless (this_package = packages&.find { |package| package[:name] == "#{project.path_with_namespace}/#{name}" }) # rubocop:disable Cop/AvoidReturnFromBlocks + this_package = project.packages + &.find { |package| package[:name] == name } - this_package[:id] + this_package.try(:fetch, :id) end def fabricate! diff --git a/qa/qa/specs/features/browser_ui/5_package/maven_gradle_repository_spec.rb b/qa/qa/specs/features/browser_ui/5_package/maven_gradle_repository_spec.rb index 05ed19df763..22d2fb2d8de 100644 --- a/qa/qa/specs/features/browser_ui/5_package/maven_gradle_repository_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/maven_gradle_repository_spec.rb @@ -3,39 +3,45 @@ module QA RSpec.describe 'Package', :orchestrated, :packages do describe 'Maven Repository with Gradle' do + using RSpec::Parameterized::TableSyntax include Runtime::Fixtures let(:group_id) { 'com.gitlab.qa' } let(:artifact_id) { 'maven_gradle' } let(:package_name) { "#{group_id}/#{artifact_id}".tr('.', '/') } - let(:auth_token) do - unless Page::Main::Menu.perform(&:signed_in?) - Flow::Login.sign_in - end + let(:package_version) { '1.3.7' } - Resource::PersonalAccessToken.fabricate!.token - end + let(:personal_access_token) { Runtime::Env.personal_access_token } - let(:project) do + let(:package_project) do Resource::Project.fabricate_via_api! do |project| project.name = 'maven-with-gradle-project' project.initialize_with_readme = true + project.visibility = :private + end + end + + let(:client_project) do + Resource::Project.fabricate_via_api! do |client_project| + client_project.name = 'gradle_client' + client_project.initialize_with_readme = true + client_project.group = package_project.group end end let(:package) do Resource::Package.init do |package| package.name = package_name - package.project = project + package.project = package_project end end - let!(:runner) do + let(:runner) do Resource::Runner.fabricate! do |runner| runner.name = "qa-runner-#{Time.now.to_i}" - runner.tags = ["runner-for-#{project.name}"] + runner.tags = ["runner-for-#{package_project.group.name}"] runner.executor = :docker - runner.project = project + runner.token = package_project.group.runners_token end end @@ -44,87 +50,200 @@ module QA "#{uri.scheme}://#{uri.host}:#{uri.port}" end + let(:project_deploy_token) do + Resource::DeployToken.fabricate_via_browser_ui! do |deploy_token| + deploy_token.name = 'maven-with-gradle-deploy-token' + deploy_token.project = package_project + end + end + + let(:package_gitlab_ci_file) do + { + file_path: '.gitlab-ci.yml', + content: + <<~YAML + deploy: + image: gradle:6.5-jdk11 + script: + - 'gradle publish' + only: + - "#{package_project.default_branch}" + tags: + - "runner-for-#{package_project.group.name}" + YAML + } + end + + let(:package_build_gradle_file) do + { + file_path: 'build.gradle', + content: + <<~EOF + plugins { + id 'java' + id 'maven-publish' + } + + publishing { + publications { + library(MavenPublication) { + groupId '#{group_id}' + artifactId '#{artifact_id}' + version '#{package_version}' + from components.java + } + } + repositories { + maven { + url "#{gitlab_address_with_port}/api/v4/projects/#{package_project.id}/packages/maven" + credentials(HttpHeaderCredentials) { + name = "Private-Token" + value = "#{personal_access_token}" + } + authentication { + header(HttpHeaderAuthentication) + } + } + } + } + EOF + } + end + + let(:client_gitlab_ci_file) do + { + file_path: '.gitlab-ci.yml', + content: + <<~YAML + build: + image: gradle:6.5-jdk11 + script: + - 'gradle build' + only: + - "#{client_project.default_branch}" + tags: + - "runner-for-#{client_project.group.name}" + YAML + } + end + + before do + Flow::Login.sign_in_unless_signed_in + runner + end + after do runner.remove_via_api! package.remove_via_api! + package_project.remove_via_api! + client_project.remove_via_api! end - it 'publishes a maven package via gradle', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/1074' do - Resource::Repository::Commit.fabricate_via_api! do |commit| - commit.project = project - commit.commit_message = 'Add .gitlab-ci.yml' - commit.add_files([{ - file_path: '.gitlab-ci.yml', - content: - <<~YAML - deploy: - image: gradle:6.5-jdk11 - script: - - 'gradle publish' - only: - - "#{project.default_branch}" - tags: - - "runner-for-#{project.name}" - YAML - }, - { - file_path: 'build.gradle', - content: - <<~EOF - plugins { - id 'java' - id 'maven-publish' - } - - publishing { - publications { - library(MavenPublication) { - groupId '#{group_id}' - artifactId '#{artifact_id}' - from components.java - } - } - repositories { - maven { - url "#{gitlab_address_with_port}/api/v4/projects/#{project.id}/packages/maven" - credentials(HttpHeaderCredentials) { - name = "Private-Token" - value = "#{auth_token}" - } - authentication { - header(HttpHeaderAuthentication) - } - } - } - } - EOF - }]) - end - - project.visit! - Flow::Pipeline.visit_latest_pipeline + where(:authentication_token_type, :maven_header_name) do + :personal_access_token | 'Private-Token' + :ci_job_token | 'Job-Token' + :project_deploy_token | 'Deploy-Token' + end - Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.click_job('deploy') + with_them do + let(:token) do + case authentication_token_type + when :personal_access_token + "\"#{personal_access_token}\"" + when :ci_job_token + 'System.getenv("CI_JOB_TOKEN")' + when :project_deploy_token + "\"#{project_deploy_token.password}\"" + end end - Page::Project::Job::Show.perform do |job| - expect(job).to be_successful(timeout: 800) + let(:client_build_gradle_file) do + { + file_path: 'build.gradle', + content: + <<~EOF + plugins { + id 'java' + id 'application' + } + + repositories { + jcenter() + maven { + url "#{gitlab_address_with_port}/api/v4/projects/#{package_project.id}/packages/maven" + name "GitLab" + credentials(HttpHeaderCredentials) { + name = '#{maven_header_name}' + value = #{token} + } + authentication { + header(HttpHeaderAuthentication) + } + } + } + + dependencies { + implementation group: '#{group_id}', name: '#{artifact_id}', version: '#{package_version}' + testImplementation 'junit:junit:4.12' + } + + application { + mainClassName = 'gradle_maven_app.App' + } + EOF + } end - Page::Project::Menu.perform(&:click_packages_link) + it "pushes and pulls a maven package via gradle using #{params[:authentication_token_type]}", testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/1074' do + # pushing + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = package_project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files([package_gitlab_ci_file, package_build_gradle_file]) + end - Page::Project::Packages::Index.perform do |index| - expect(index).to have_package(package_name) + package_project.visit! - index.click_package(package_name) - end + Flow::Pipeline.visit_latest_pipeline + + Page::Project::Pipeline::Show.perform do |pipeline| + pipeline.click_job('deploy') + end + + Page::Project::Job::Show.perform do |job| + expect(job).to be_successful(timeout: 800) + end + + Page::Project::Menu.perform(&:click_packages_link) + + Page::Project::Packages::Index.perform do |index| + expect(index).to have_package(package_name) + + index.click_package(package_name) + end + + Page::Project::Packages::Show.perform do |show| + expect(show).to have_package_info(package_name, package_version) + end + + # pulling + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = client_project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files([client_gitlab_ci_file, client_build_gradle_file]) + end + + client_project.visit! + + Flow::Pipeline.visit_latest_pipeline - Page::Project::Packages::Show.perform(&:click_delete) + Page::Project::Pipeline::Show.perform do |pipeline| + pipeline.click_job('build') + end - Page::Project::Packages::Index.perform do |index| - expect(index).to have_content("Package deleted successfully") - expect(index).not_to have_package(package_name) + Page::Project::Job::Show.perform do |job| + expect(job).to be_successful(timeout: 800) + end end end end diff --git a/rubocop/cop/migration/prevent_index_creation.rb b/rubocop/cop/migration/prevent_index_creation.rb new file mode 100644 index 00000000000..c90f911d24e --- /dev/null +++ b/rubocop/cop/migration/prevent_index_creation.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true +require_relative '../../migration_helpers' + +module RuboCop + module Cop + module Migration + # Cop that checks if new indexes are introduced to forbidden tables. + class PreventIndexCreation < RuboCop::Cop::Cop + include MigrationHelpers + + FORBIDDEN_TABLES = %i[ci_builds].freeze + + MSG = "Adding new index to #{FORBIDDEN_TABLES.join(", ")} is forbidden, see https://gitlab.com/gitlab-org/gitlab/-/issues/332886" + + def_node_matcher :add_index?, <<~PATTERN + (send nil? :add_index (sym #forbidden_tables?) ...) + PATTERN + + def_node_matcher :add_concurrent_index?, <<~PATTERN + (send nil? :add_concurrent_index (sym #forbidden_tables?) ...) + PATTERN + + def forbidden_tables?(node) + FORBIDDEN_TABLES.include?(node) + end + + def on_def(node) + return unless in_migration?(node) + + node.each_descendant(:send) do |send_node| + add_offense(send_node, location: :selector) if offense?(send_node) + end + end + + def offense?(node) + add_index?(node) || add_concurrent_index?(node) + end + end + end + end +end diff --git a/spec/rubocop/cop/migration/prevent_index_creation_spec.rb b/spec/rubocop/cop/migration/prevent_index_creation_spec.rb new file mode 100644 index 00000000000..a3965f54bbd --- /dev/null +++ b/spec/rubocop/cop/migration/prevent_index_creation_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require_relative '../../../../rubocop/cop/migration/prevent_index_creation' + +RSpec.describe RuboCop::Cop::Migration::PreventIndexCreation do + subject(:cop) { described_class.new } + + context 'when in migration' do + before do + allow(cop).to receive(:in_migration?).and_return(true) + end + + context 'when adding an index to a forbidden table' do + it 'registers an offense when add_index is used' do + expect_offense(<<~RUBY) + def change + add_index :ci_builds, :protected + ^^^^^^^^^ Adding new index to ci_builds is forbidden, see https://gitlab.com/gitlab-org/gitlab/-/issues/332886 + end + RUBY + end + + it 'registers an offense when add_concurrent_index is used' do + expect_offense(<<~RUBY) + def change + add_concurrent_index :ci_builds, :protected + ^^^^^^^^^^^^^^^^^^^^ Adding new index to ci_builds is forbidden, see https://gitlab.com/gitlab-org/gitlab/-/issues/332886 + end + RUBY + end + end + + context 'when adding an index to a regular table' do + it 'does not register an offense' do + expect_no_offenses(<<~RUBY) + def change + add_index :ci_pipelines, :locked + end + RUBY + end + end + end + + context 'when outside of migration' do + it 'does not register an offense' do + expect_no_offenses('def change; add_index :table, :column; end') + end + end +end diff --git a/spec/workers/jira_connect/sync_branch_worker_spec.rb b/spec/workers/jira_connect/sync_branch_worker_spec.rb index 7c715f36fb4..f70a09c1ee2 100644 --- a/spec/workers/jira_connect/sync_branch_worker_spec.rb +++ b/spec/workers/jira_connect/sync_branch_worker_spec.rb @@ -5,6 +5,11 @@ require 'spec_helper' RSpec.describe JiraConnect::SyncBranchWorker do include AfterNextHelpers + it_behaves_like 'worker with data consistency', + described_class, + feature_flag: :load_balancing_for_jira_connect_workers, + data_consistency: :delayed + describe '#perform' do let_it_be(:group) { create(:group) } let_it_be(:project) { create(:project, :repository, group: group) } @@ -15,65 +20,59 @@ RSpec.describe JiraConnect::SyncBranchWorker do let(:commit_shas) { %w(b83d6e3 5a62481) } let(:update_sequence_id) { 1 } + def perform + described_class.new.perform(project_id, branch_name, commit_shas, update_sequence_id) + end + def expect_jira_sync_service_execute(args) - expect_next_instances_of(JiraConnect::SyncService, IdempotentWorkerHelper::WORKER_EXEC_TIMES) do |instance| - expect(instance).to receive(:execute).with(args) - end + expect_next(JiraConnect::SyncService).to receive(:execute).with(args) end - it_behaves_like 'an idempotent worker' do - let(:job_args) { [project_id, branch_name, commit_shas, update_sequence_id] } + it 'calls JiraConnect::SyncService#execute' do + expect_jira_sync_service_execute( + branches: [instance_of(Gitlab::Git::Branch)], + commits: project.commits_by(oids: commit_shas), + update_sequence_id: update_sequence_id + ) - before do - stub_request(:post, 'https://sample.atlassian.net/rest/devinfo/0.10/bulk').to_return(status: 200, body: '', headers: {}) - end + perform + end + + context 'without branch name' do + let(:branch_name) { nil } it 'calls JiraConnect::SyncService#execute' do expect_jira_sync_service_execute( - branches: [instance_of(Gitlab::Git::Branch)], + branches: nil, commits: project.commits_by(oids: commit_shas), update_sequence_id: update_sequence_id ) - subject - end - - context 'without branch name' do - let(:branch_name) { nil } - - it 'calls JiraConnect::SyncService#execute' do - expect_jira_sync_service_execute( - branches: nil, - commits: project.commits_by(oids: commit_shas), - update_sequence_id: update_sequence_id - ) - - subject - end + perform end + end - context 'without commits' do - let(:commit_shas) { nil } + context 'without commits' do + let(:commit_shas) { nil } - it 'calls JiraConnect::SyncService#execute' do - expect_jira_sync_service_execute( - branches: [instance_of(Gitlab::Git::Branch)], - commits: nil, - update_sequence_id: update_sequence_id - ) + it 'calls JiraConnect::SyncService#execute' do + expect_jira_sync_service_execute( + branches: [instance_of(Gitlab::Git::Branch)], + commits: nil, + update_sequence_id: update_sequence_id + ) - subject - end + perform end + end - context 'when project no longer exists' do - let(:project_id) { non_existing_record_id } + context 'when project no longer exists' do + let(:project_id) { non_existing_record_id } - it 'does not call JiraConnect::SyncService' do - expect(JiraConnect::SyncService).not_to receive(:new) + it 'does not call JiraConnect::SyncService' do + expect(JiraConnect::SyncService).not_to receive(:new) - subject - end + perform end end end diff --git a/spec/workers/jira_connect/sync_builds_worker_spec.rb b/spec/workers/jira_connect/sync_builds_worker_spec.rb index a3327716dc4..e60d1c58966 100644 --- a/spec/workers/jira_connect/sync_builds_worker_spec.rb +++ b/spec/workers/jira_connect/sync_builds_worker_spec.rb @@ -5,6 +5,11 @@ require 'spec_helper' RSpec.describe ::JiraConnect::SyncBuildsWorker do include AfterNextHelpers + it_behaves_like 'worker with data consistency', + described_class, + feature_flag: :load_balancing_for_jira_connect_workers, + data_consistency: :delayed + describe '#perform' do let_it_be(:pipeline) { create(:ci_pipeline) } diff --git a/spec/workers/jira_connect/sync_deployments_worker_spec.rb b/spec/workers/jira_connect/sync_deployments_worker_spec.rb index 4e2fd72c61a..62b879e0130 100644 --- a/spec/workers/jira_connect/sync_deployments_worker_spec.rb +++ b/spec/workers/jira_connect/sync_deployments_worker_spec.rb @@ -5,6 +5,11 @@ require 'spec_helper' RSpec.describe ::JiraConnect::SyncDeploymentsWorker do include AfterNextHelpers + it_behaves_like 'worker with data consistency', + described_class, + feature_flag: :load_balancing_for_jira_connect_workers, + data_consistency: :delayed + describe '#perform' do let_it_be(:deployment) { create(:deployment) } diff --git a/spec/workers/jira_connect/sync_feature_flags_worker_spec.rb b/spec/workers/jira_connect/sync_feature_flags_worker_spec.rb index e003febaa3b..1c2f9126b71 100644 --- a/spec/workers/jira_connect/sync_feature_flags_worker_spec.rb +++ b/spec/workers/jira_connect/sync_feature_flags_worker_spec.rb @@ -5,6 +5,11 @@ require 'spec_helper' RSpec.describe ::JiraConnect::SyncFeatureFlagsWorker do include AfterNextHelpers + it_behaves_like 'worker with data consistency', + described_class, + feature_flag: :load_balancing_for_jira_connect_workers, + data_consistency: :delayed + describe '#perform' do let_it_be(:feature_flag) { create(:operations_feature_flag) } diff --git a/spec/workers/jira_connect/sync_merge_request_worker_spec.rb b/spec/workers/jira_connect/sync_merge_request_worker_spec.rb index 6a0a0744f6f..7e403de54bb 100644 --- a/spec/workers/jira_connect/sync_merge_request_worker_spec.rb +++ b/spec/workers/jira_connect/sync_merge_request_worker_spec.rb @@ -5,6 +5,11 @@ require 'spec_helper' RSpec.describe JiraConnect::SyncMergeRequestWorker do include AfterNextHelpers + it_behaves_like 'worker with data consistency', + described_class, + feature_flag: :load_balancing_for_jira_connect_workers, + data_consistency: :delayed + describe '#perform' do let_it_be(:group) { create(:group) } let_it_be(:project) { create(:project, :repository, group: group) } @@ -14,29 +19,24 @@ RSpec.describe JiraConnect::SyncMergeRequestWorker do let(:merge_request_id) { merge_request.id } let(:update_sequence_id) { 1 } - it_behaves_like 'an idempotent worker' do - let(:job_args) { [merge_request_id, update_sequence_id] } - - before do - stub_request(:post, 'https://sample.atlassian.net/rest/devinfo/0.10/bulk').to_return(status: 200, body: '', headers: {}) - end + def perform + described_class.new.perform(merge_request_id, update_sequence_id) + end - it 'calls JiraConnect::SyncService#execute' do - expect_next_instances_of(JiraConnect::SyncService, IdempotentWorkerHelper::WORKER_EXEC_TIMES) do |service| - expect(service).to receive(:execute).with(merge_requests: [merge_request], update_sequence_id: update_sequence_id) - end + it 'calls JiraConnect::SyncService#execute' do + expect_next(JiraConnect::SyncService).to receive(:execute) + .with(merge_requests: [merge_request], update_sequence_id: update_sequence_id) - subject - end + perform + end - context 'when MR no longer exists' do - let(:merge_request_id) { non_existing_record_id } + context 'when MR no longer exists' do + let(:merge_request_id) { non_existing_record_id } - it 'does not call JiraConnect::SyncService' do - expect(JiraConnect::SyncService).not_to receive(:new) + it 'does not call JiraConnect::SyncService' do + expect(JiraConnect::SyncService).not_to receive(:new) - subject - end + perform end end end diff --git a/spec/workers/jira_connect/sync_project_worker_spec.rb b/spec/workers/jira_connect/sync_project_worker_spec.rb index 5c0e7e7609c..727d22d8b08 100644 --- a/spec/workers/jira_connect/sync_project_worker_spec.rb +++ b/spec/workers/jira_connect/sync_project_worker_spec.rb @@ -3,6 +3,13 @@ require 'spec_helper' RSpec.describe JiraConnect::SyncProjectWorker, factory_default: :keep do + include AfterNextHelpers + + it_behaves_like 'worker with data consistency', + described_class, + feature_flag: :load_balancing_for_jira_connect_workers, + data_consistency: :delayed + describe '#perform' do let_it_be(:project) { create_default(:project).freeze } @@ -14,6 +21,22 @@ RSpec.describe JiraConnect::SyncProjectWorker, factory_default: :keep do let(:jira_connect_sync_service) { JiraConnect::SyncService.new(project) } let(:job_args) { [project.id, update_sequence_id] } let(:update_sequence_id) { 1 } + let(:request_path) { '/rest/devinfo/0.10/bulk' } + let(:request_body) do + { + repositories: [ + Atlassian::JiraConnect::Serializers::RepositoryEntity.represent( + project, + merge_requests: [mr_with_jira_description, mr_with_jira_title], + update_sequence_id: update_sequence_id + ) + ] + } + end + + def perform(project_id, update_sequence_id) + described_class.new.perform(project_id, update_sequence_id) + end before do stub_request(:post, 'https://sample.atlassian.net/rest/devinfo/0.10/bulk').to_return(status: 200, body: '', headers: {}) @@ -24,54 +47,37 @@ RSpec.describe JiraConnect::SyncProjectWorker, factory_default: :keep do context 'when the project is not found' do it 'does not raise an error' do - expect { described_class.new.perform('non_existing_record_id', update_sequence_id) }.not_to raise_error + expect { perform('non_existing_record_id', update_sequence_id) }.not_to raise_error end end it 'avoids N+1 database queries' do - control_count = ActiveRecord::QueryRecorder.new { described_class.new.perform(project.id, update_sequence_id) }.count + control_count = ActiveRecord::QueryRecorder.new { perform(project.id, update_sequence_id) }.count create(:merge_request, :unique_branches, title: 'TEST-123') - expect { described_class.new.perform(project.id, update_sequence_id) }.not_to exceed_query_limit(control_count) + expect { perform(project.id, update_sequence_id) }.not_to exceed_query_limit(control_count) end - it_behaves_like 'an idempotent worker' do - let(:request_path) { '/rest/devinfo/0.10/bulk' } - let(:request_body) do - { - repositories: [ - Atlassian::JiraConnect::Serializers::RepositoryEntity.represent( - project, - merge_requests: [mr_with_jira_description, mr_with_jira_title], - update_sequence_id: update_sequence_id - ) - ] - } - end - - it 'sends the request with custom update_sequence_id' do - allow_next_instances_of(Atlassian::JiraConnect::Client, IdempotentWorkerHelper::WORKER_EXEC_TIMES) do |client| - expect(client).to receive(:post).with(request_path, request_body) - end + it 'sends the request with custom update_sequence_id' do + allow_next(Atlassian::JiraConnect::Client).to receive(:post) + .with(request_path, request_body) - subject - end + perform(project.id, update_sequence_id) + end - context 'when the number of merge requests to sync is higher than the limit' do - let!(:most_recent_merge_request) { create(:merge_request, :unique_branches, description: 'TEST-323', title: 'TEST-123') } + context 'when the number of merge requests to sync is higher than the limit' do + let!(:most_recent_merge_request) { create(:merge_request, :unique_branches, description: 'TEST-323', title: 'TEST-123') } - before do - stub_const("#{described_class}::MERGE_REQUEST_LIMIT", 1) - end + before do + stub_const("#{described_class}::MERGE_REQUEST_LIMIT", 1) + end - it 'syncs only the most recent merge requests within the limit' do - expect(jira_connect_sync_service).to receive(:execute) - .exactly(IdempotentWorkerHelper::WORKER_EXEC_TIMES).times - .with(merge_requests: [most_recent_merge_request], update_sequence_id: update_sequence_id) + it 'syncs only the most recent merge requests within the limit' do + expect(jira_connect_sync_service).to receive(:execute) + .with(merge_requests: [most_recent_merge_request], update_sequence_id: update_sequence_id) - subject - end + perform(project.id, update_sequence_id) end end end |