diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-08-15 09:09:36 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-08-15 09:09:36 +0300 |
commit | 024f77efd68833bb78540ff9b4c7b4ec4b9dfe39 (patch) | |
tree | 4b564d2de286b3032907c66cabee92fe6b6754fc | |
parent | b291dca0e5062e05572e0f106ea499b47d908d35 (diff) |
Add latest changes from gitlab-org/gitlab@master
38 files changed, 486 insertions, 192 deletions
diff --git a/.rubocop_todo/rspec/missing_feature_category.yml b/.rubocop_todo/rspec/missing_feature_category.yml index 7a1d6ab67a3..0d625f92c3f 100644 --- a/.rubocop_todo/rspec/missing_feature_category.yml +++ b/.rubocop_todo/rspec/missing_feature_category.yml @@ -2743,7 +2743,6 @@ RSpec/MissingFeatureCategory: - 'spec/lib/bulk_imports/common/extractors/json_extractor_spec.rb' - 'spec/lib/bulk_imports/common/extractors/ndjson_extractor_spec.rb' - 'spec/lib/bulk_imports/common/extractors/rest_extractor_spec.rb' - - 'spec/lib/bulk_imports/common/graphql/get_members_query_spec.rb' - 'spec/lib/bulk_imports/common/pipelines/badges_pipeline_spec.rb' - 'spec/lib/bulk_imports/common/pipelines/entity_finisher_spec.rb' - 'spec/lib/bulk_imports/common/pipelines/lfs_objects_pipeline_spec.rb' diff --git a/GITLAB_KAS_VERSION b/GITLAB_KAS_VERSION index 009c4912efc..a8ec175ad83 100644 --- a/GITLAB_KAS_VERSION +++ b/GITLAB_KAS_VERSION @@ -1 +1 @@ -v16.3.0-rc6 +v16.3.0 diff --git a/app/controllers/groups/dependency_proxy/application_controller.rb b/app/controllers/groups/dependency_proxy/application_controller.rb index 300a82eed78..12c3679cf2a 100644 --- a/app/controllers/groups/dependency_proxy/application_controller.rb +++ b/app/controllers/groups/dependency_proxy/application_controller.rb @@ -24,7 +24,7 @@ module Groups case user_or_deploy_token when User @authentication_result = Gitlab::Auth::Result.new(user_or_deploy_token, nil, :user, []) - sign_in(user_or_deploy_token) + sign_in(user_or_deploy_token) unless user_or_deploy_token.project_bot? when DeployToken @authentication_result = Gitlab::Auth::Result.new(user_or_deploy_token, nil, :deploy_token, []) end diff --git a/app/graphql/types/notes/position_type_enum.rb b/app/graphql/types/notes/position_type_enum.rb index 157b0b4b884..b585d531192 100644 --- a/app/graphql/types/notes/position_type_enum.rb +++ b/app/graphql/types/notes/position_type_enum.rb @@ -8,6 +8,7 @@ module Types value 'text', description: "Text file." value 'image', description: "An image." + value 'file', description: "Unknown file type." end end end diff --git a/app/views/admin/application_settings/_gitpod.html.haml b/app/views/admin/application_settings/_gitpod.html.haml index 09817a9172f..25d267bad11 100644 --- a/app/views/admin/application_settings/_gitpod.html.haml +++ b/app/views/admin/application_settings/_gitpod.html.haml @@ -23,7 +23,7 @@ = f.label :gitpod_url, s_('Gitpod|Gitpod URL'), class: 'label-bold' = f.text_field :gitpod_url, class: 'form-control gl-form-input', placeholder: s_('Gitpod|https://gitpod.example.com') .form-text.text-muted + - help_link = link_to('', help_page_path('integration/gitpod', anchor: 'enable-gitpod-in-your-user-settings', target: '_blank', rel: 'noopener noreferrer')) = s_('Gitpod|The URL to your Gitpod instance configured to read your GitLab projects, such as https://gitpod.example.com.') - - link_start = '<a href="%{url}">'.html_safe % { url: help_page_path('integration/gitpod', anchor: 'enable-gitpod-in-your-user-settings') } - = s_('Gitpod|To use the integration, each user must also enable Gitpod on their GitLab account. %{link_start}How do I enable it?%{link_end} ').html_safe % { link_start: link_start, link_end: '</a>'.html_safe } + = safe_format(s_('Gitpod|To use the integration, each user must also enable Gitpod on their GitLab account. %{help_link_start}How do I enable it?%{help_link_end}'), tag_pair(help_link, :help_link_start, :help_link_end)) = f.submit _('Save changes'), pajamas_button: true diff --git a/app/views/admin/application_settings/_snowplow.html.haml b/app/views/admin/application_settings/_snowplow.html.haml index 4e7d9b8ab21..46a30f631b8 100644 --- a/app/views/admin/application_settings/_snowplow.html.haml +++ b/app/views/admin/application_settings/_snowplow.html.haml @@ -6,8 +6,9 @@ = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded ? _('Collapse') : _('Expand') %p - - link_start = '<a href="%{url}">'.html_safe % { url: help_page_path('development/snowplow/index') } - = html_escape(_('Configure %{link} to track events. %{link_start}Learn more.%{link_end}')) % { link: link_to('Snowplow', 'https://snowplowanalytics.com/', target: '_blank', rel: 'noopener noreferrer').html_safe, link_start: link_start, link_end: '</a>'.html_safe } + - help_link = link_to('', help_page_path('development/snowplow/index'), target: '_blank', rel: 'noopener noreferrer') + - snowplow_link = link_to('', 'https://snowplow.io/', target: '_blank', rel: 'noopener noreferrer') + = safe_format(_('Configure %{snowplow_link_start}Snowplow%{snowplow_link_end} to track events. %{help_link_start}Learn more.%{help_link_end}'), tag_pair(snowplow_link, :snowplow_link_start, :snowplow_link_end), tag_pair(help_link, :help_link_start, :help_link_end)) .settings-content = gitlab_ui_form_for @application_setting, url: general_admin_application_settings_path(anchor: 'js-snowplow-settings'), html: { class: 'fieldset-form', id: 'snowplow-settings' } do |f| = form_errors(@application_setting) if expanded diff --git a/app/views/admin/application_settings/ci_cd.html.haml b/app/views/admin/application_settings/ci_cd.html.haml index 9b36a983f26..f8161465924 100644 --- a/app/views/admin/application_settings/ci_cd.html.haml +++ b/app/views/admin/application_settings/ci_cd.html.haml @@ -9,8 +9,8 @@ .settings-content - if ci_variable_protected_by_default? %p.settings-message.text-center.gl-mb-0 - - link_start = '<a href="%{url}">'.html_safe % { url: help_page_path('ci/variables/index', anchor: 'protect-a-cicd-variable') } - = s_('Environment variables on this GitLab instance are configured to be %{link_start}protected%{link_end} by default.').html_safe % { link_start: link_start, link_end: '</a>'.html_safe } + - help_link = link_to('', help_page_path('ci/variables/index', anchor: 'protect-a-cicd-variable', target: '_blank', rel: 'noopener noreferrer')) + = safe_format(s_('Environment variables on this GitLab instance are configured to be %{help_link_start}protected%{help_link_end} by default.'), tag_pair(help_link, :help_link_start, :help_link_end)) #js-instance-variables{ data: { endpoint: admin_ci_variables_path, maskable_regex: ci_variable_maskable_regex, protected_by_default: ci_variable_protected_by_default?.to_s} } %section.settings.as-ci-cd.no-animate#js-ci-cd-settings{ class: ('expanded' if expanded_by_default?) } diff --git a/app/views/admin/application_settings/service_usage_data.html.haml b/app/views/admin/application_settings/service_usage_data.html.haml index 634d006e736..9f73099465c 100644 --- a/app/views/admin/application_settings/service_usage_data.html.haml +++ b/app/views/admin/application_settings/service_usage_data.html.haml @@ -23,9 +23,7 @@ title: _('Service Ping payload not found in the application cache')) do |c| - c.with_body do - - enable_service_ping_link_url = help_page_path('administration/settings/usage_statistics', anchor: 'enable-or-disable-usage-statistics') - - enable_service_ping_link = '<a href="%{url}">'.html_safe % { url: enable_service_ping_link_url } - - generate_manually_link_url = help_page_path('development/internal_analytics/service_ping/troubleshooting', anchor: 'generate-service-ping') - - generate_manually_link = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: generate_manually_link_url } + - enable_service_ping_link = link_to('', help_page_path('administration/settings/usage_statistics', anchor: 'enable-or-disable-usage-statistics'), target: '_blank', rel: 'noopener noreferrer') + - generate_manually_link = link_to('', help_page_path('development/internal_analytics/service_ping/troubleshooting', anchor: 'generate-service-ping'), target: '_blank', rel: 'noopener noreferrer') - = html_escape(s_('%{enable_service_ping_link_start}Enable%{link_end} or %{generate_manually_link_start}generate%{link_end} Service Ping to preview and download service usage data payload.')) % { enable_service_ping_link_start: enable_service_ping_link, generate_manually_link_start: generate_manually_link, link_end: '</a>'.html_safe } + = safe_format(s_('%{enable_service_ping_link_start}Enable%{enable_service_ping_link_end} or %{generate_manually_link_start}generate%{generate_manually_link_end} Service Ping to preview and download service usage data payload.'), tag_pair(enable_service_ping_link, :enable_service_ping_link_start, :enable_service_ping_link_end), tag_pair(generate_manually_link, :generate_manually_link_start, :generate_manually_link_end)) diff --git a/app/views/ci/variables/_header.html.haml b/app/views/ci/variables/_header.html.haml index dfcf8f39533..886edbd0687 100644 --- a/app/views/ci/variables/_header.html.haml +++ b/app/views/ci/variables/_header.html.haml @@ -6,5 +6,5 @@ = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded ? _('Collapse') : _('Expand') -%p +%p.gl-text-secondary = render "ci/variables/content", entity: @entity, variable_limit: @variable_limit diff --git a/app/views/groups/edit.html.haml b/app/views/groups/edit.html.haml index 1c256bd57c5..c11154cbd75 100644 --- a/app/views/groups/edit.html.haml +++ b/app/views/groups/edit.html.haml @@ -11,7 +11,7 @@ = _('Naming, visibility') = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = _('Collapse') - %p + %p.gl-text-secondary = _('Update your group name, description, avatar, and visibility.') = link_to _('Learn more about groups.'), help_page_path('user/group/index') .settings-content @@ -23,7 +23,7 @@ = _('Permissions and group features') = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded ? _('Collapse') : _('Expand') - %p + %p.gl-text-secondary = _('Configure advanced permissions, Large File Storage, two-factor authentication, and customer relations settings.') .settings-content = render 'groups/settings/permissions' @@ -38,7 +38,7 @@ = s_('GroupSettings|Badges') = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded ? _('Collapse') : _('Expand') - %p.gl-text-secondary.gl-mb-0 + %p.gl-text-secondary = s_('GroupSettings|Customize this group\'s badges.') = link_to s_('GroupSettings|What are badges?'), help_page_path('user/project/badges') .settings-content @@ -55,7 +55,7 @@ = _('Advanced') = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded ? _('Collapse') : _('Expand') - %p + %p.gl-text-secondary = _('Perform advanced options such as changing path, transferring, exporting, or removing the group.') .settings-content = render 'groups/settings/advanced' diff --git a/app/views/groups/settings/ci_cd/show.html.haml b/app/views/groups/settings/ci_cd/show.html.haml index 7b6e50ffd36..f9ade00a300 100644 --- a/app/views/groups/settings/ci_cd/show.html.haml +++ b/app/views/groups/settings/ci_cd/show.html.haml @@ -14,7 +14,7 @@ = _("General pipelines") = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded ? _('Collapse') : _('Expand') - %p + %p.gl-text-secondary = _("Customize your pipeline configuration.") .settings-content = render 'groups/settings/ci_cd/form', group: @group @@ -31,7 +31,7 @@ = _('Runners') = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded ? _('Collapse') : _('Expand') - %p + %p.gl-text-secondary = _("Runners are processes that pick up and execute CI/CD jobs for GitLab.") = link_to s_('What is GitLab Runner?'), 'https://docs.gitlab.com/runner/', target: '_blank', rel: 'noopener noreferrer' .settings-content @@ -43,7 +43,7 @@ = _('Auto DevOps') = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded ? _('Collapse') : _('Expand') - %p + %p.gl-text-secondary - auto_devops_url = help_page_path('topics/autodevops/index') - quickstart_url = help_page_path('topics/autodevops/cloud_deployments/auto_devops_with_gke') - auto_devops_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: auto_devops_url } diff --git a/app/views/groups/settings/integrations/index.html.haml b/app/views/groups/settings/integrations/index.html.haml index 3c1a38d9997..d0d3b1bf137 100644 --- a/app/views/groups/settings/integrations/index.html.haml +++ b/app/views/groups/settings/integrations/index.html.haml @@ -6,5 +6,5 @@ %h3= s_('Integrations|Group-level integration management') - integrations_link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: integrations_help_page_path } - %p= s_("Integrations|GitLab administrators can set up integrations that all projects in a group inherit and use by default. These integrations apply to all projects that don't already use custom settings. You can override custom settings for a project if the settings are necessary at that level. Learn more about %{integrations_link_start}group-level integration management%{link_end}.").html_safe % { integrations_link_start: integrations_link_start, link_end: "</a>".html_safe } + %p.gl-text-secondary= s_("Integrations|GitLab administrators can set up integrations that all projects in a group inherit and use by default. These integrations apply to all projects that don't already use custom settings. You can override custom settings for a project if the settings are necessary at that level. Learn more about %{integrations_link_start}group-level integration management%{link_end}.").html_safe % { integrations_link_start: integrations_link_start, link_end: "</a>".html_safe } = render 'shared/integrations/index', integrations: @integrations diff --git a/app/views/groups/settings/repository/_default_branch.html.haml b/app/views/groups/settings/repository/_default_branch.html.haml index e8aa809a6ca..b04a3fe50ae 100644 --- a/app/views/groups/settings/repository/_default_branch.html.haml +++ b/app/views/groups/settings/repository/_default_branch.html.haml @@ -4,7 +4,7 @@ = _('Default branch') = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded_by_default? ? _('Collapse') : _('Expand') - %p + %p.gl-text-secondary = s_('GroupSettings|Set the initial name and protections for the default branch of new repositories created in the group.') .settings-content = gitlab_ui_form_for @group, url: group_path(@group, anchor: 'js-default-branch-name'), html: { class: 'fieldset-form' } do |f| diff --git a/app/views/shared/deploy_tokens/_index.html.haml b/app/views/shared/deploy_tokens/_index.html.haml index 382fa650e4c..ccffc3ec923 100644 --- a/app/views/shared/deploy_tokens/_index.html.haml +++ b/app/views/shared/deploy_tokens/_index.html.haml @@ -5,7 +5,7 @@ %h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only= s_('DeployTokens|Deploy tokens') = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded ? _('Collapse') : _('Expand') - %p + %p.gl-text-secondary = description .settings-content #new-deploy-token-alert diff --git a/config/feature_flags/development/enable_exclusive_lease_double_lock_rw.yml b/config/feature_flags/development/enable_exclusive_lease_double_lock_rw.yml new file mode 100644 index 00000000000..1c735e32317 --- /dev/null +++ b/config/feature_flags/development/enable_exclusive_lease_double_lock_rw.yml @@ -0,0 +1,8 @@ +--- +name: enable_exclusive_lease_double_lock_rw +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/128083 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/421156 +milestone: '16.3' +type: development +group: group::scalability +default_enabled: false diff --git a/config/feature_flags/development/scan_execution_bot_users.yml b/config/feature_flags/development/scan_execution_bot_users.yml index 70890ee4ba8..ca06e666e67 100644 --- a/config/feature_flags/development/scan_execution_bot_users.yml +++ b/config/feature_flags/development/scan_execution_bot_users.yml @@ -5,4 +5,4 @@ rollout_issue_url: milestone: '16.0' type: development group: group::security policies -default_enabled: false +default_enabled: true diff --git a/config/feature_flags/development/use_cluster_shared_state_for_exclusive_lease.yml b/config/feature_flags/development/use_cluster_shared_state_for_exclusive_lease.yml new file mode 100644 index 00000000000..6ef9e80551a --- /dev/null +++ b/config/feature_flags/development/use_cluster_shared_state_for_exclusive_lease.yml @@ -0,0 +1,8 @@ +--- +name: use_cluster_shared_state_for_exclusive_lease +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/128083 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/421156 +milestone: '16.3' +type: development +group: group::scalability +default_enabled: false diff --git a/config/metrics/counts_28d/20210216181150_projects_jira_active.yml b/config/metrics/counts_28d/20210216181150_projects_jira_active.yml index c0cbc111c6a..585e7812247 100644 --- a/config/metrics/counts_28d/20210216181150_projects_jira_active.yml +++ b/config/metrics/counts_28d/20210216181150_projects_jira_active.yml @@ -16,5 +16,6 @@ tier: - free - premium - ultimate -performance_indicator_type: [] +performance_indicator_type: +- customer_health_score milestone: "<13.9" diff --git a/config/metrics/counts_28d/20210216181152_projects_jira_dvcs_cloud_active.yml b/config/metrics/counts_28d/20210216181152_projects_jira_dvcs_cloud_active.yml index fdbcab9749d..3169c02624a 100644 --- a/config/metrics/counts_28d/20210216181152_projects_jira_dvcs_cloud_active.yml +++ b/config/metrics/counts_28d/20210216181152_projects_jira_dvcs_cloud_active.yml @@ -17,5 +17,6 @@ tier: - free - premium - ultimate -performance_indicator_type: [] +performance_indicator_type: +- customer_health_score milestone: "<13.9" diff --git a/config/metrics/counts_28d/20210216181154_projects_jira_dvcs_server_active.yml b/config/metrics/counts_28d/20210216181154_projects_jira_dvcs_server_active.yml index 0efd1ec9f52..d0d730381a6 100644 --- a/config/metrics/counts_28d/20210216181154_projects_jira_dvcs_server_active.yml +++ b/config/metrics/counts_28d/20210216181154_projects_jira_dvcs_server_active.yml @@ -17,5 +17,6 @@ tier: - free - premium - ultimate -performance_indicator_type: [] +performance_indicator_type: +- customer_health_score milestone: "<13.9" diff --git a/config/metrics/counts_28d/20210216184322_i_code_review_user_approve_mr_monthly.yml b/config/metrics/counts_28d/20210216184322_i_code_review_user_approve_mr_monthly.yml index 10075e0b60a..91635edbe68 100644 --- a/config/metrics/counts_28d/20210216184322_i_code_review_user_approve_mr_monthly.yml +++ b/config/metrics/counts_28d/20210216184322_i_code_review_user_approve_mr_monthly.yml @@ -21,3 +21,5 @@ tier: - premium - ultimate milestone: "<13.9" +performance_indicator_type: +- customer_health_score diff --git a/config/redis.yml.example b/config/redis.yml.example index 950a98c9fd6..fff53b7757c 100644 --- a/config/redis.yml.example +++ b/config/redis.yml.example @@ -6,6 +6,9 @@ development: cluster_cache: cluster: - redis://localhost:7001 + cluster_shared_state: + cluster: + - redis://localhost:7001 feature_flag: cluster: - redis://localhost:7001 @@ -20,6 +23,9 @@ test: cluster_cache: cluster: - redis://localhost:7001 + cluster_shared_state: + cluster: + - redis://localhost:7001 feature_flag: cluster: - redis://localhost:7001 diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index d5920b2969e..26638b11498 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -26438,6 +26438,7 @@ Type of file the position refers to. | Value | Description | | ----- | ----------- | +| <a id="diffpositiontypefile"></a>`file` | Unknown file type. | | <a id="diffpositiontypeimage"></a>`image` | An image. | | <a id="diffpositiontypetext"></a>`text` | Text file. | diff --git a/doc/user/packages/dependency_proxy/index.md b/doc/user/packages/dependency_proxy/index.md index ebe87332948..9cb252a664b 100644 --- a/doc/user/packages/dependency_proxy/index.md +++ b/doc/user/packages/dependency_proxy/index.md @@ -69,6 +69,7 @@ Prerequisites: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) in GitLab 13.7 [with a flag](../../../administration/feature_flags.md) named `dependency_proxy_for_private_groups`. Enabled by default. > - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/276777) the feature flag `dependency_proxy_for_private_groups` in GitLab 15.0. +> - Support for group access tokens [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362991) in GitLab 16.3. Because the Dependency Proxy is storing Docker images in a space associated with your group, you must authenticate against the Dependency Proxy. @@ -87,6 +88,7 @@ You can authenticate using: - Your GitLab username and password. - A [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `read_registry` and `write_registry`. - A [group deploy token](../../../user/project/deploy_tokens/index.md) with the scope set to `read_registry` and `write_registry`. +- A [group access token](../../../user/group/settings/group_access_tokens.md) for the group, with the scope set to `read_registry` and `write_registry`. Users accessing the Dependency Proxy with a personal access token or username and password must have at least the Guest role for the group they pull images from. diff --git a/lib/bulk_imports/common/graphql/get_members_query.rb b/lib/bulk_imports/common/graphql/get_members_query.rb index 00977f694d7..8fa8d7f4c0b 100644 --- a/lib/bulk_imports/common/graphql/get_members_query.rb +++ b/lib/bulk_imports/common/graphql/get_members_query.rb @@ -14,7 +14,7 @@ module BulkImports <<-GRAPHQL query($full_path: ID!, $cursor: String, $per_page: Int) { portable: #{context.entity.entity_type}(fullPath: $full_path) { - members: #{members_type}(relations: [DIRECT, INHERITED], first: $per_page, after: $cursor) { + members: #{members_type}(relations: #{relations}, first: $per_page, after: $cursor) { page_info: pageInfo { next_page: endCursor has_next_page: hasNextPage @@ -66,6 +66,14 @@ module BulkImports 'projectMembers' end end + + def relations + if context.entity.group? + "[DIRECT INHERITED SHARED_FROM_GROUPS]" + else + "[DIRECT INHERITED INVITED_GROUPS SHARED_INTO_ANCESTORS]" + end + end end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 0b18a337707..8679f17eb9b 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -12,6 +12,8 @@ module Gitlab # ExclusiveLease. # class ExclusiveLease + include Gitlab::Utils::StrongMemoize + PREFIX = 'gitlab:exclusive_lease' NoKey = Class.new(ArgumentError) @@ -31,7 +33,7 @@ module Gitlab EOS def self.get_uuid(key) - Gitlab::Redis::SharedState.with do |redis| + with_read_redis do |redis| redis.get(redis_shared_state_key(key)) || false end end @@ -61,7 +63,7 @@ module Gitlab def self.cancel(key, uuid) return unless key.present? - Gitlab::Redis::SharedState.with do |redis| + with_write_redis do |redis| redis.eval(LUA_CANCEL_SCRIPT, keys: [ensure_prefixed_key(key)], argv: [uuid]) end end @@ -84,6 +86,21 @@ module Gitlab redis.del(key) end end + + Gitlab::Redis::ClusterSharedState.with do |redis| + redis.scan_each(match: redis_shared_state_key(scope)).each do |key| + redis.del(key) + end + end + end + + def self.use_cluster_shared_state? + Gitlab::SafeRequestStore[:use_cluster_shared_state] ||= + Feature.enabled?(:use_cluster_shared_state_for_exclusive_lease) + end + + def self.use_double_lock? + Gitlab::SafeRequestStore[:use_double_lock] ||= Feature.enabled?(:enable_exclusive_lease_double_lock_rw) end def initialize(key, uuid: nil, timeout:) @@ -95,10 +112,23 @@ module Gitlab # Try to obtain the lease. Return lease UUID on success, # false if the lease is already taken. def try_obtain + return try_obtain_with_new_lock if self.class.use_cluster_shared_state? + # Performing a single SET is atomic - Gitlab::Redis::SharedState.with do |redis| - redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid - end + obtained = set_lease(Gitlab::Redis::SharedState) && @uuid + + # traffic to new store is minimal since only the first lock holder can run SETNX in ClusterSharedState + return false unless obtained + return obtained unless self.class.use_double_lock? + return obtained if same_store # 2nd setnx will surely fail if store are the same + + second_lock_obtained = set_lease(Gitlab::Redis::ClusterSharedState) && @uuid + + # cancel is safe since it deletes key only if value matches uuid + # i.e. it will not delete the held lock on ClusterSharedState + cancel unless second_lock_obtained + + second_lock_obtained end # This lease is waiting to obtain @@ -109,7 +139,7 @@ module Gitlab # Try to renew an existing lease. Return lease UUID on success, # false if the lease is taken by a different UUID or inexistent. def renew - Gitlab::Redis::SharedState.with do |redis| + self.class.with_write_redis do |redis| result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_shared_state_key], argv: [@uuid, @timeout]) result == @uuid end @@ -117,7 +147,7 @@ module Gitlab # Returns true if the key for this lease is set. def exists? - Gitlab::Redis::SharedState.with do |redis| + self.class.with_read_redis do |redis| redis.exists?(@redis_shared_state_key) # rubocop:disable CodeReuse/ActiveRecord end end @@ -126,17 +156,66 @@ module Gitlab # # This method will return `nil` if no TTL could be obtained. def ttl - Gitlab::Redis::SharedState.with do |redis| + self.class.with_read_redis do |redis| ttl = redis.ttl(@redis_shared_state_key) ttl if ttl > 0 end end + # rubocop:disable CodeReuse/ActiveRecord + def self.with_write_redis(&blk) + if use_cluster_shared_state? + result = Gitlab::Redis::ClusterSharedState.with(&blk) + Gitlab::Redis::SharedState.with(&blk) + + result + elsif use_double_lock? + result = Gitlab::Redis::SharedState.with(&blk) + Gitlab::Redis::ClusterSharedState.with(&blk) + + result + else + Gitlab::Redis::SharedState.with(&blk) + end + end + + def self.with_read_redis(&blk) + if use_cluster_shared_state? + Gitlab::Redis::ClusterSharedState.with(&blk) + elsif use_double_lock? + Gitlab::Redis::SharedState.with(&blk) || Gitlab::Redis::ClusterSharedState.with(&blk) + else + Gitlab::Redis::SharedState.with(&blk) + end + end + # rubocop:enable CodeReuse/ActiveRecord + # Gives up this lease, allowing it to be obtained by others. def cancel self.class.cancel(@redis_shared_state_key, @uuid) end + + private + + def set_lease(redis_class) + redis_class.with do |redis| + redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) + end + end + + def try_obtain_with_new_lock + # checks shared-state to avoid 2 versions of the application acquiring 1 lock + # wait for held lock to expire or yielded in case any process on old version is running + return false if Gitlab::Redis::SharedState.with { |c| c.exists?(@redis_shared_state_key) } # rubocop:disable CodeReuse/ActiveRecord + + set_lease(Gitlab::Redis::ClusterSharedState) && @uuid + end + + def same_store + Gitlab::Redis::ClusterSharedState.with(&:id) == Gitlab::Redis::SharedState.with(&:id) # rubocop:disable CodeReuse/ActiveRecord + end + strong_memoize_attr :same_store end end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 06bce7649bf..351e5399a05 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -10,6 +10,7 @@ module Gitlab ALL_CLASSES = [ Gitlab::Redis::Cache, Gitlab::Redis::ClusterCache, + Gitlab::Redis::ClusterSharedState, Gitlab::Redis::DbLoadBalancing, Gitlab::Redis::FeatureFlag, Gitlab::Redis::Queues, diff --git a/lib/gitlab/redis/cluster_shared_state.rb b/lib/gitlab/redis/cluster_shared_state.rb new file mode 100644 index 00000000000..678566a0c9c --- /dev/null +++ b/lib/gitlab/redis/cluster_shared_state.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class ClusterSharedState < ::Gitlab::Redis::Wrapper + class << self + def config_fallback + SharedState + end + end + end + end +end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index aa6762a68ef..1959fd07c4e 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -718,7 +718,7 @@ msgstr "" msgid "%{emailPrefix}@company.com" msgstr "" -msgid "%{enable_service_ping_link_start}Enable%{link_end} or %{generate_manually_link_start}generate%{link_end} Service Ping to preview and download service usage data payload." +msgid "%{enable_service_ping_link_start}Enable%{enable_service_ping_link_end} or %{generate_manually_link_start}generate%{generate_manually_link_end} Service Ping to preview and download service usage data payload." msgstr "" msgid "%{extra} more downstream pipelines" @@ -12282,10 +12282,10 @@ msgstr "" msgid "Configure %{italic_start}What's new%{italic_end} drawer and content." msgstr "" -msgid "Configure %{link} to track events. %{link_start}Learn more.%{link_end}" +msgid "Configure %{repository_checks_link_start}repository checks%{link_end} and %{housekeeping_link_start}housekeeping%{link_end} on repositories." msgstr "" -msgid "Configure %{repository_checks_link_start}repository checks%{link_end} and %{housekeeping_link_start}housekeeping%{link_end} on repositories." +msgid "Configure %{snowplow_link_start}Snowplow%{snowplow_link_end} to track events. %{help_link_start}Learn more.%{help_link_end}" msgstr "" msgid "Configure CAPTCHAs, IP address limits, and other anti-spam measures." @@ -18057,7 +18057,7 @@ msgstr "" msgid "Environment scope" msgstr "" -msgid "Environment variables on this GitLab instance are configured to be %{link_start}protected%{link_end} by default." +msgid "Environment variables on this GitLab instance are configured to be %{help_link_start}protected%{help_link_end} by default." msgstr "" msgid "Environment:" @@ -21653,7 +21653,7 @@ msgstr "" msgid "Gitpod|To use Gitpod you must first enable the feature in the integrations section of your %{linkStart}user preferences%{linkEnd}." msgstr "" -msgid "Gitpod|To use the integration, each user must also enable Gitpod on their GitLab account. %{link_start}How do I enable it?%{link_end} " +msgid "Gitpod|To use the integration, each user must also enable Gitpod on their GitLab account. %{help_link_start}How do I enable it?%{help_link_end}" msgstr "" msgid "Gitpod|https://gitpod.example.com" @@ -22616,7 +22616,7 @@ msgstr "" msgid "GroupSettings|Compliance frameworks" msgstr "" -msgid "GroupSettings|Configure analytics features for this group" +msgid "GroupSettings|Configure analytics features for this group." msgstr "" msgid "GroupSettings|Configure compliance frameworks to make them available to projects in this group. %{linkStart}What are compliance frameworks?%{linkEnd}" diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/composer_registry_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/composer_registry_spec.rb index 5d37bbd1f38..8d0efc31787 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/composer_registry_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/composer_registry_spec.rb @@ -68,7 +68,14 @@ module QA package.remove_via_api! end - it 'publishes a composer package and deletes it', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348016' do + it( + 'publishes a composer package and deletes it', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348016', + quarantine: { + type: :broken, + issue: "https://gitlab.com/gitlab-org/gitlab/-/issues/421885" + } + ) do Page::Project::Menu.perform(&:go_to_package_registry) Page::Project::Packages::Index.perform do |index| diff --git a/spec/controllers/groups/dependency_proxy_auth_controller_spec.rb b/spec/controllers/groups/dependency_proxy_auth_controller_spec.rb index ed79712f828..344e15557ce 100644 --- a/spec/controllers/groups/dependency_proxy_auth_controller_spec.rb +++ b/spec/controllers/groups/dependency_proxy_auth_controller_spec.rb @@ -17,93 +17,78 @@ RSpec.describe Groups::DependencyProxyAuthController do end end - context 'with valid JWT' do - context 'user' do - let_it_be(:user) { create(:user) } + context 'with JWT' do + let(:jwt) { build_jwt(user) } + let(:token_header) { "Bearer #{jwt.encoded}" } - let(:jwt) { build_jwt(user) } - let(:token_header) { "Bearer #{jwt.encoded}" } + before do + request.headers['HTTP_AUTHORIZATION'] = token_header + end + + context 'with valid JWT' do + context 'user' do + let_it_be(:user) { create(:user) } - before do - request.headers['HTTP_AUTHORIZATION'] = token_header + it { is_expected.to have_gitlab_http_status(:success) } end - it { is_expected.to have_gitlab_http_status(:success) } - end + context 'group bot user' do + let_it_be(:user) { create(:user, :project_bot) } - context 'deploy token' do - let_it_be(:user) { create(:deploy_token) } + it { is_expected.to have_gitlab_http_status(:success) } + end - let(:jwt) { build_jwt(user) } - let(:token_header) { "Bearer #{jwt.encoded}" } + context 'deploy token' do + let_it_be(:user) { create(:deploy_token) } - before do - request.headers['HTTP_AUTHORIZATION'] = token_header + it { is_expected.to have_gitlab_http_status(:success) } end - - it { is_expected.to have_gitlab_http_status(:success) } end - end - context 'with invalid JWT' do - context 'bad user' do - let(:jwt) { build_jwt(double('bad_user', id: 999)) } - let(:token_header) { "Bearer #{jwt.encoded}" } + context 'with invalid JWT' do + context 'bad user' do + let(:jwt) { build_jwt(double('bad_user', id: 999)) } - before do - request.headers['HTTP_AUTHORIZATION'] = token_header + it { is_expected.to have_gitlab_http_status(:unauthorized) } end - it { is_expected.to have_gitlab_http_status(:unauthorized) } - end + context 'token with no user id' do + let(:token_header) { "Bearer #{build_jwt.encoded}" } - context 'token with no user id' do - let(:token_header) { "Bearer #{build_jwt.encoded}" } + before do + request.headers['HTTP_AUTHORIZATION'] = token_header + end - before do - request.headers['HTTP_AUTHORIZATION'] = token_header + it { is_expected.to have_gitlab_http_status(:unauthorized) } end - it { is_expected.to have_gitlab_http_status(:unauthorized) } - end - - context 'expired token' do - let_it_be(:user) { create(:user) } + context 'expired token' do + let_it_be(:user) { create(:user) } - let(:jwt) { build_jwt(user, expire_time: Time.zone.now - 1.hour) } - let(:token_header) { "Bearer #{jwt.encoded}" } + let(:jwt) { build_jwt(user, expire_time: Time.zone.now - 1.hour) } - before do - request.headers['HTTP_AUTHORIZATION'] = token_header + it { is_expected.to have_gitlab_http_status(:unauthorized) } end - it { is_expected.to have_gitlab_http_status(:unauthorized) } - end - - context 'expired deploy token' do - let_it_be(:user) { create(:deploy_token, :expired) } + context 'group bot user from an expired token' do + let_it_be(:user) { create(:user, :project_bot) } - let(:jwt) { build_jwt(user) } - let(:token_header) { "Bearer #{jwt.encoded}" } + let(:jwt) { build_jwt(user, expire_time: Time.zone.now - 1.hour) } - before do - request.headers['HTTP_AUTHORIZATION'] = token_header + it { is_expected.to have_gitlab_http_status(:unauthorized) } end - it { is_expected.to have_gitlab_http_status(:unauthorized) } - end + context 'expired deploy token' do + let_it_be(:user) { create(:deploy_token, :expired) } - context 'revoked deploy token' do - let_it_be(:user) { create(:deploy_token, :revoked) } + it { is_expected.to have_gitlab_http_status(:unauthorized) } + end - let(:jwt) { build_jwt(user) } - let(:token_header) { "Bearer #{jwt.encoded}" } + context 'revoked deploy token' do + let_it_be(:user) { create(:deploy_token, :revoked) } - before do - request.headers['HTTP_AUTHORIZATION'] = token_header + it { is_expected.to have_gitlab_http_status(:unauthorized) } end - - it { is_expected.to have_gitlab_http_status(:unauthorized) } end end end diff --git a/spec/lib/bulk_imports/common/graphql/get_members_query_spec.rb b/spec/lib/bulk_imports/common/graphql/get_members_query_spec.rb index e3a7335a238..bcc2d6fd5ed 100644 --- a/spec/lib/bulk_imports/common/graphql/get_members_query_spec.rb +++ b/spec/lib/bulk_imports/common/graphql/get_members_query_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe BulkImports::Common::Graphql::GetMembersQuery do +RSpec.describe BulkImports::Common::Graphql::GetMembersQuery, feature_category: :importers do let(:entity) { create(:bulk_import_entity, :group_entity) } let(:tracker) { create(:bulk_import_tracker, entity: entity) } let(:context) { BulkImports::Pipeline::Context.new(tracker) } @@ -41,6 +41,7 @@ RSpec.describe BulkImports::Common::Graphql::GetMembersQuery do it 'queries group & group members' do expect(query.to_s).to include('group') expect(query.to_s).to include('groupMembers') + expect(query.to_s).to include('SHARED_FROM_GROUPS') end end @@ -50,6 +51,7 @@ RSpec.describe BulkImports::Common::Graphql::GetMembersQuery do it 'queries project & project members' do expect(query.to_s).to include('project') expect(query.to_s).to include('projectMembers') + expect(query.to_s).to include('INVITED_GROUPS SHARED_INTO_ANCESTORS') end end end diff --git a/spec/lib/gitlab/exclusive_lease_spec.rb b/spec/lib/gitlab/exclusive_lease_spec.rb index 968d26e1c38..c8325c5b359 100644 --- a/spec/lib/gitlab/exclusive_lease_spec.rb +++ b/spec/lib/gitlab/exclusive_lease_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' -RSpec.describe Gitlab::ExclusiveLease, :clean_gitlab_redis_shared_state do +RSpec.describe Gitlab::ExclusiveLease, :request_store, :clean_gitlab_redis_shared_state, + :clean_gitlab_redis_cluster_shared_state, feature_category: :shared do let(:unique_key) { SecureRandom.hex(10) } describe '#try_obtain' do @@ -19,6 +20,67 @@ RSpec.describe Gitlab::ExclusiveLease, :clean_gitlab_redis_shared_state do sleep(2 * timeout) # lease should have expired now expect(lease.try_obtain).to be_present end + + context 'when migrating across stores' do + let(:lease) { described_class.new(unique_key, timeout: 3600) } + + before do + stub_feature_flags(use_cluster_shared_state_for_exclusive_lease: false) + allow(lease).to receive(:same_store).and_return(false) + end + + it 'acquires 2 locks' do + # stub first SETNX + Gitlab::Redis::SharedState.with { |r| expect(r).to receive(:set).and_return(true) } + Gitlab::Redis::ClusterSharedState.with { |r| expect(r).to receive(:set).and_call_original } + + expect(lease.try_obtain).to be_truthy + end + + it 'rollback first lock if second lock is not acquired' do + Gitlab::Redis::ClusterSharedState.with do |r| + expect(r).to receive(:set).and_return(false) + expect(r).to receive(:eval).and_call_original + end + + Gitlab::Redis::SharedState.with do |r| + expect(r).to receive(:set).and_call_original + expect(r).to receive(:eval).and_call_original + end + + expect(lease.try_obtain).to be_falsey + end + end + + context 'when cutting over to ClusterSharedState' do + context 'when lock is not acquired' do + it 'waits for existing holder to yield the lock' do + Gitlab::Redis::ClusterSharedState.with { |r| expect(r).to receive(:set).and_call_original } + Gitlab::Redis::SharedState.with { |r| expect(r).not_to receive(:set) } + + lease = described_class.new(unique_key, timeout: 3600) + expect(lease.try_obtain).to be_truthy + end + end + + context 'when lock is still acquired' do + let(:lease) { described_class.new(unique_key, timeout: 3600) } + + before do + # simulates cutover where some application's feature-flag has not updated + stub_feature_flags(use_cluster_shared_state_for_exclusive_lease: false) + lease.try_obtain + stub_feature_flags(use_cluster_shared_state_for_exclusive_lease: true) + end + + it 'waits for existing holder to yield the lock' do + Gitlab::Redis::ClusterSharedState.with { |r| expect(r).not_to receive(:set) } + Gitlab::Redis::SharedState.with { |r| expect(r).not_to receive(:set) } + + expect(lease.try_obtain).to be_falsey + end + end + end end describe '.redis_shared_state_key' do @@ -42,131 +104,159 @@ RSpec.describe Gitlab::ExclusiveLease, :clean_gitlab_redis_shared_state do end end - describe '#renew' do - it 'returns true when we have the existing lease' do - lease = described_class.new(unique_key, timeout: 3600) - expect(lease.try_obtain).to be_present - expect(lease.renew).to be_truthy - end + shared_examples 'write operations' do + describe '#renew' do + it 'returns true when we have the existing lease' do + lease = described_class.new(unique_key, timeout: 3600) + expect(lease.try_obtain).to be_present + expect(lease.renew).to be_truthy + end - it 'returns false when we dont have a lease' do - lease = described_class.new(unique_key, timeout: 3600) - expect(lease.renew).to be_falsey + it 'returns false when we dont have a lease' do + lease = described_class.new(unique_key, timeout: 3600) + expect(lease.renew).to be_falsey + end end - end - describe '#exists?' do - it 'returns true for an existing lease' do - lease = described_class.new(unique_key, timeout: 3600) - lease.try_obtain + describe 'cancellation' do + def new_lease(key) + described_class.new(key, timeout: 3600) + end - expect(lease.exists?).to eq(true) - end + shared_examples 'cancelling a lease' do + let(:lease) { new_lease(unique_key) } - it 'returns false for a lease that does not exist' do - lease = described_class.new(unique_key, timeout: 3600) + it 'releases the held lease' do + uuid = lease.try_obtain + expect(uuid).to be_present + expect(new_lease(unique_key).try_obtain).to eq(false) - expect(lease.exists?).to eq(false) - end - end + cancel_lease(uuid) - describe '.get_uuid' do - it 'gets the uuid if lease with the key associated exists' do - uuid = described_class.new(unique_key, timeout: 3600).try_obtain + expect(new_lease(unique_key).try_obtain).to be_present + end + end - expect(described_class.get_uuid(unique_key)).to eq(uuid) - end + describe '.cancel' do + def cancel_lease(uuid) + described_class.cancel(release_key, uuid) + end - it 'returns false if the lease does not exist' do - expect(described_class.get_uuid(unique_key)).to be false - end - end + context 'when called with the unprefixed key' do + it_behaves_like 'cancelling a lease' do + let(:release_key) { unique_key } + end + end - describe 'cancellation' do - def new_lease(key) - described_class.new(key, timeout: 3600) - end + context 'when called with the prefixed key' do + it_behaves_like 'cancelling a lease' do + let(:release_key) { described_class.redis_shared_state_key(unique_key) } + end + end - shared_examples 'cancelling a lease' do - let(:lease) { new_lease(unique_key) } + it 'does not raise errors when given a nil key' do + expect { described_class.cancel(nil, nil) }.not_to raise_error + end + end - it 'releases the held lease' do - uuid = lease.try_obtain - expect(uuid).to be_present - expect(new_lease(unique_key).try_obtain).to eq(false) + describe '#cancel' do + def cancel_lease(_uuid) + lease.cancel + end - cancel_lease(uuid) + it_behaves_like 'cancelling a lease' - expect(new_lease(unique_key).try_obtain).to be_present - end - end + it 'is safe to call even if the lease was never obtained' do + lease = new_lease(unique_key) - describe '.cancel' do - def cancel_lease(uuid) - described_class.cancel(release_key, uuid) - end + lease.cancel - context 'when called with the unprefixed key' do - it_behaves_like 'cancelling a lease' do - let(:release_key) { unique_key } + expect(new_lease(unique_key).try_obtain).to be_present end end + end - context 'when called with the prefixed key' do - it_behaves_like 'cancelling a lease' do - let(:release_key) { described_class.redis_shared_state_key(unique_key) } - end - end + describe '.reset_all!' do + it 'removes all existing lease keys from redis' do + uuid = described_class.new(unique_key, timeout: 3600).try_obtain - it 'does not raise errors when given a nil key' do - expect { described_class.cancel(nil, nil) }.not_to raise_error + expect(described_class.get_uuid(unique_key)).to eq(uuid) + + described_class.reset_all! + + expect(described_class.get_uuid(unique_key)).to be_falsey end end + end - describe '#cancel' do - def cancel_lease(_uuid) - lease.cancel + shared_examples 'read operations' do + describe '#exists?' do + it 'returns true for an existing lease' do + lease = described_class.new(unique_key, timeout: 3600) + lease.try_obtain + + expect(lease.exists?).to eq(true) end - it_behaves_like 'cancelling a lease' + it 'returns false for a lease that does not exist' do + lease = described_class.new(unique_key, timeout: 3600) + + expect(lease.exists?).to eq(false) + end + end - it 'is safe to call even if the lease was never obtained' do - lease = new_lease(unique_key) + describe '.get_uuid' do + it 'gets the uuid if lease with the key associated exists' do + uuid = described_class.new(unique_key, timeout: 3600).try_obtain - lease.cancel + expect(described_class.get_uuid(unique_key)).to eq(uuid) + end - expect(new_lease(unique_key).try_obtain).to be_present + it 'returns false if the lease does not exist' do + expect(described_class.get_uuid(unique_key)).to be false end end - end - describe '#ttl' do - it 'returns the TTL of the Redis key' do - lease = described_class.new('kittens', timeout: 100) - lease.try_obtain + describe '#ttl' do + it 'returns the TTL of the Redis key' do + lease = described_class.new('kittens', timeout: 100) + lease.try_obtain - expect(lease.ttl <= 100).to eq(true) - end + expect(lease.ttl <= 100).to eq(true) + end - it 'returns nil when the lease does not exist' do - lease = described_class.new('kittens', timeout: 10) + it 'returns nil when the lease does not exist' do + lease = described_class.new('kittens', timeout: 10) - expect(lease.ttl).to be_nil + expect(lease.ttl).to be_nil + end end end - describe '.reset_all!' do - it 'removes all existing lease keys from redis' do - uuid = described_class.new(unique_key, timeout: 3600).try_obtain - - expect(described_class.get_uuid(unique_key)).to eq(uuid) + context 'when migrating across stores' do + before do + stub_feature_flags(use_cluster_shared_state_for_exclusive_lease: false) + end - described_class.reset_all! + it_behaves_like 'read operations' + it_behaves_like 'write operations' + end - expect(described_class.get_uuid(unique_key)).to be_falsey + context 'when feature flags are all disabled' do + before do + stub_feature_flags( + use_cluster_shared_state_for_exclusive_lease: false, + enable_exclusive_lease_double_lock_rw: false + ) end + + it_behaves_like 'read operations' + it_behaves_like 'write operations' end + it_behaves_like 'read operations' + it_behaves_like 'write operations' + describe '.throttle' do it 'prevents repeated execution of the block' do number = 0 @@ -244,4 +334,74 @@ RSpec.describe Gitlab::ExclusiveLease, :clean_gitlab_redis_shared_state do described_class.throttle(1, count: 48, period: 1.day) {} end end + + describe 'transitions between feature-flag toggles' do + shared_examples 'retains behaviours across transitions' do |flag| + it 'retains read behaviour' do + lease = described_class.new(unique_key, timeout: 3600) + uuid = lease.try_obtain + + expect(lease.ttl).not_to eq(nil) + expect(lease.exists?).to be_truthy + expect(described_class.get_uuid(unique_key)).to eq(uuid) + + # simulates transition + stub_feature_flags({ flag => true }) + Gitlab::SafeRequestStore.clear! + + expect(lease.ttl).not_to eq(nil) + expect(lease.exists?).to be_truthy + expect(described_class.get_uuid(unique_key)).to eq(uuid) + end + + it 'retains renew behaviour' do + lease = described_class.new(unique_key, timeout: 3600) + lease.try_obtain + + expect(lease.renew).to be_truthy + + # simulates transition + stub_feature_flags({ flag => true }) + Gitlab::SafeRequestStore.clear! + + expect(lease.renew).to be_truthy + end + + it 'retains renew behaviour' do + lease = described_class.new(unique_key, timeout: 3600) + uuid = lease.try_obtain + lease.cancel + + # proves successful cancellation + expect(lease.try_obtain).to eq(uuid) + + # simulates transition + stub_feature_flags({ flag => true }) + Gitlab::SafeRequestStore.clear! + + expect(lease.try_obtain).to be_falsey + lease.cancel + expect(lease.try_obtain).to eq(uuid) + end + end + + context 'when enabling enable_exclusive_lease_double_lock_rw' do + before do + stub_feature_flags( + enable_exclusive_lease_double_lock_rw: false, + use_cluster_shared_state_for_exclusive_lease: false + ) + end + + it_behaves_like 'retains behaviours across transitions', :enable_exclusive_lease_double_lock_rw + end + + context 'when enabling use_cluster_shared_state_for_exclusive_lease' do + before do + stub_feature_flags(use_cluster_shared_state_for_exclusive_lease: false) + end + + it_behaves_like 'retains behaviours across transitions', :use_cluster_shared_state_for_exclusive_lease + end + end end diff --git a/spec/lib/gitlab/redis/cluster_shared_state_spec.rb b/spec/lib/gitlab/redis/cluster_shared_state_spec.rb new file mode 100644 index 00000000000..11a574c79c4 --- /dev/null +++ b/spec/lib/gitlab/redis/cluster_shared_state_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Redis::ClusterSharedState, feature_category: :redis do + include_examples "redis_new_instance_shared_examples", 'cluster_shared_state', Gitlab::Redis::SharedState +end diff --git a/spec/workers/background_migration/ci_database_worker_spec.rb b/spec/workers/background_migration/ci_database_worker_spec.rb index 3f2977a0aaa..496e7830c94 100644 --- a/spec/workers/background_migration/ci_database_worker_spec.rb +++ b/spec/workers/background_migration/ci_database_worker_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' -RSpec.describe BackgroundMigration::CiDatabaseWorker, :clean_gitlab_redis_shared_state, feature_category: :database do +RSpec.describe BackgroundMigration::CiDatabaseWorker, :clean_gitlab_redis_shared_state, + :clean_gitlab_redis_cluster_shared_state, feature_category: :database do before do skip_if_shared_database(:ci) end diff --git a/spec/workers/background_migration_worker_spec.rb b/spec/workers/background_migration_worker_spec.rb index 32ee6708736..4cffbe5be97 100644 --- a/spec/workers/background_migration_worker_spec.rb +++ b/spec/workers/background_migration_worker_spec.rb @@ -2,6 +2,7 @@ require 'spec_helper' -RSpec.describe BackgroundMigrationWorker, :clean_gitlab_redis_shared_state, feature_category: :database do +RSpec.describe BackgroundMigrationWorker, :clean_gitlab_redis_shared_state, + :clean_gitlab_redis_cluster_shared_state, feature_category: :database do it_behaves_like 'it runs background migration jobs', 'main' end diff --git a/spec/workers/integrations/slack_event_worker_spec.rb b/spec/workers/integrations/slack_event_worker_spec.rb index 6754801a2bd..019d68b40e0 100644 --- a/spec/workers/integrations/slack_event_worker_spec.rb +++ b/spec/workers/integrations/slack_event_worker_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' -RSpec.describe Integrations::SlackEventWorker, :clean_gitlab_redis_shared_state, feature_category: :integrations do +RSpec.describe Integrations::SlackEventWorker, :clean_gitlab_redis_shared_state, + :clean_gitlab_redis_cluster_shared_state, feature_category: :integrations do describe '.event?' do subject { described_class.event?(event) } diff --git a/workhorse/.tool-versions b/workhorse/.tool-versions index bf7f6cc9184..6d1d0fb759f 100644 --- a/workhorse/.tool-versions +++ b/workhorse/.tool-versions @@ -1 +1 @@ -golang 1.20.6 +golang 1.20.7 |