Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2023-06-07 03:10:26 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2023-06-07 03:10:26 +0300
commit1e19d757e8a088e9d4aa67cc092fda87aba1cd35 (patch)
treeb183775b7355dae057ab0d2e5076df86cdc8a77d
parentb04f912deb494b6dc0d0dba36776a5e53f622b43 (diff)
Add latest changes from gitlab-org/gitlab@master
-rw-r--r--.gitleaksignore2
-rw-r--r--app/assets/javascripts/ci/ci_variable_list/components/ci_variable_modal.vue32
-rw-r--r--app/models/ci/secure_file.rb1
-rw-r--r--app/models/commit_status.rb10
-rw-r--r--app/models/namespace.rb6
-rw-r--r--app/models/namespace/root_storage_statistics.rb6
-rw-r--r--app/uploaders/ci/secure_file_uploader.rb4
-rw-r--r--app/views/ci/variables/_content.html.haml10
-rw-r--r--app/views/ci/variables/_index.html.haml11
-rw-r--r--app/views/users/_profile_basic_info.html.haml2
-rw-r--r--app/workers/concerns/worker_attributes.rb2
-rw-r--r--config/feature_flags/worker/defer_sidekiq_workers_on_database_health_signal.yml8
-rw-r--r--config/gitleaks.toml3
-rw-r--r--doc/architecture/blueprints/runner_tokens/index.md119
-rw-r--r--doc/ci/runners/new_creation_workflow.md152
-rw-r--r--doc/install/docker.md14
-rw-r--r--doc/update/index.md2
-rw-r--r--lib/gitlab/ci/secure_files/migration_helper.rb33
-rw-r--r--lib/gitlab/database/health_status/context.rb13
-rw-r--r--lib/gitlab/sidekiq_logging/structured_logger.rb2
-rw-r--r--lib/gitlab/sidekiq_middleware.rb2
-rw-r--r--lib/gitlab/sidekiq_middleware/defer_jobs.rb64
-rw-r--r--lib/tasks/gitlab/ci_secure_files/migrate.rake23
-rw-r--r--locale/gitlab.pot14
-rw-r--r--package.json2
-rw-r--r--qa/qa/specs/features/browser_ui/5_package/container_registry/self_managed/container_registry_spec.rb8
-rw-r--r--qa/qa/specs/features/browser_ui/5_package/dependency_proxy/dependency_proxy_spec.rb1
-rw-r--r--spec/frontend/ci/ci_variable_list/components/ci_variable_modal_spec.js14
-rw-r--r--spec/lib/gitlab/ci/secure_files/migration_helper_spec.rb44
-rw-r--r--spec/lib/gitlab/database/health_status_spec.rb101
-rw-r--r--spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb1
-rw-r--r--spec/lib/gitlab/sidekiq_middleware/defer_jobs_spec.rb76
-rw-r--r--spec/models/namespace/root_storage_statistics_spec.rb6
-rw-r--r--spec/models/namespace_spec.rb46
-rw-r--r--spec/support/shared_contexts/lib/gitlab/sidekiq_logging/structured_logger_shared_context.rb3
-rw-r--r--spec/support/shared_examples/features/variable_list_shared_examples.rb2
-rw-r--r--spec/tasks/gitlab/ci_secure_files/migrate_rake_spec.rb46
-rw-r--r--spec/workers/concerns/worker_attributes_spec.rb18
-rw-r--r--yarn.lock8
39 files changed, 646 insertions, 265 deletions
diff --git a/.gitleaksignore b/.gitleaksignore
index df50b4bdfae..09288cde3ef 100644
--- a/.gitleaksignore
+++ b/.gitleaksignore
@@ -1,3 +1 @@
-5f8440d74ba194204935669f6f98fe9c08a21200:doc/architecture/blueprints/runner_tokens/index.md:gitlab-rrt:504
-a349496b88d2add528669f5566ef458d90fc7fba:doc/architecture/blueprints/runner_tokens/index.md:generic-api-key:516
afedb913baf4203aa688421873fdb9f94649578e:doc/api/users.md:generic-api-key:2201
diff --git a/app/assets/javascripts/ci/ci_variable_list/components/ci_variable_modal.vue b/app/assets/javascripts/ci/ci_variable_list/components/ci_variable_modal.vue
index b3ecaceba69..b45e55fcb4b 100644
--- a/app/assets/javascripts/ci/ci_variable_list/components/ci_variable_modal.vue
+++ b/app/assets/javascripts/ci/ci_variable_list/components/ci_variable_modal.vue
@@ -154,7 +154,9 @@ export default {
return createJoinedEnvironments(this.variables, this.environments, this.newEnvironments);
},
maskedFeedback() {
- return this.displayMaskedError ? __('This variable can not be masked.') : '';
+ return this.displayMaskedError
+ ? __('This variable value does not meet the masking requirements.')
+ : '';
},
maskedState() {
if (this.displayMaskedError) {
@@ -190,6 +192,11 @@ export default {
variableValidationState() {
return this.variable.value === '' || (this.tokenValidationState && this.maskedState);
},
+ variableValueHelpText() {
+ return this.variable.masked
+ ? __('Value must meet regular expression requirements to be masked.')
+ : '';
+ },
},
watch: {
variable: {
@@ -324,6 +331,7 @@ export default {
:label="__('Value')"
label-for="ci-variable-value"
:state="variableValidationState"
+ :description="variableValueHelpText"
:invalid-feedback="variableValidationFeedback"
>
<gl-form-textarea
@@ -423,17 +431,19 @@ export default {
>
{{ __('Mask variable') }}
<p class="gl-mt-2 text-secondary">
- {{ __('Variable will be masked in job logs.') }}
- <span
- :class="{
- 'bold text-plain': displayMaskedError,
- }"
- >
- {{ __('Requires values to meet regular expression requirements.') }}</span
+ <gl-sprintf
+ :message="
+ __(
+ 'Mask this variable in job logs if it meets %{linkStart}regular expression requirements%{linkEnd}.',
+ )
+ "
>
- <gl-link target="_blank" :href="maskedEnvironmentVariablesLink">{{
- __('Learn more.')
- }}</gl-link>
+ <template #link="{ content }"
+ ><gl-link target="_blank" :href="maskedEnvironmentVariablesLink">{{
+ content
+ }}</gl-link>
+ </template>
+ </gl-sprintf>
</p>
</gl-form-checkbox>
<gl-form-checkbox
diff --git a/app/models/ci/secure_file.rb b/app/models/ci/secure_file.rb
index 5e273e0fd4b..37f4d620da6 100644
--- a/app/models/ci/secure_file.rb
+++ b/app/models/ci/secure_file.rb
@@ -29,6 +29,7 @@ module Ci
scope :order_by_created_at, -> { order(created_at: :desc) }
scope :project_id_in, ->(ids) { where(project_id: ids) }
+ scope :with_files_stored_locally, -> { where(file_store: Ci::SecureFileUploader::Store::LOCAL) }
def checksum_algorithm
CHECKSUM_ALGORITHM
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index 6dfea7ef9a7..f26831c1049 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -317,6 +317,16 @@ class CommitStatus < Ci::ApplicationRecord
ci_stage&.name
end
+ # For AiAction
+ def to_ability_name
+ 'build'
+ end
+
+ # For AiAction
+ def resource_parent
+ project
+ end
+
private
def unrecoverable_failure?
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index 15966709c94..fdd88cdc365 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -405,6 +405,12 @@ class Namespace < ApplicationRecord
Project.where(namespace: namespace)
end
+ # Includes projects from this namespace and projects from all subgroups
+ # that belongs to this namespace, except the ones that are soft deleted
+ def all_projects_except_soft_deleted
+ all_projects.not_aimed_for_deletion
+ end
+
def has_parent?
parent_id.present? || parent.present?
end
diff --git a/app/models/namespace/root_storage_statistics.rb b/app/models/namespace/root_storage_statistics.rb
index c7670351f4b..8af0cf2767c 100644
--- a/app/models/namespace/root_storage_statistics.rb
+++ b/app/models/namespace/root_storage_statistics.rb
@@ -21,7 +21,7 @@ class Namespace::RootStorageStatistics < ApplicationRecord
scope :for_namespace_ids, ->(namespace_ids) { where(namespace_id: namespace_ids) }
- delegate :all_projects, to: :namespace
+ delegate :all_projects_except_soft_deleted, to: :namespace
enum notification_level: {
storage_remaining: 100,
@@ -76,7 +76,7 @@ class Namespace::RootStorageStatistics < ApplicationRecord
end
def for_forks_statistics
- all_projects
+ all_projects_except_soft_deleted
.joins([:statistics, :fork_network])
.where('fork_networks.root_project_id != projects.id')
.group('projects.visibility_level')
@@ -92,7 +92,7 @@ class Namespace::RootStorageStatistics < ApplicationRecord
end
def from_project_statistics
- all_projects
+ all_projects_except_soft_deleted
.joins('INNER JOIN project_statistics ps ON ps.project_id = projects.id')
.select(
'COALESCE(SUM(ps.storage_size), 0) AS storage_size',
diff --git a/app/uploaders/ci/secure_file_uploader.rb b/app/uploaders/ci/secure_file_uploader.rb
index 09d9b3abafb..85a285ff581 100644
--- a/app/uploaders/ci/secure_file_uploader.rb
+++ b/app/uploaders/ci/secure_file_uploader.rb
@@ -6,6 +6,10 @@ module Ci
storage_location :ci_secure_files
+ # TODO: Remove this line
+ # See https://gitlab.com/gitlab-org/gitlab/-/issues/232917
+ alias_method :upload, :model
+
# Use Lockbox to encrypt/decrypt the stored file (registers CarrierWave callbacks)
encrypt(key: :key)
diff --git a/app/views/ci/variables/_content.html.haml b/app/views/ci/variables/_content.html.haml
index 65e57d68288..f7ab495111a 100644
--- a/app/views/ci/variables/_content.html.haml
+++ b/app/views/ci/variables/_content.html.haml
@@ -1,12 +1,2 @@
= format(s_('CiVariables|Variables store information, like passwords and secret keys, that you can use in job scripts. Each %{entity} can define a maximum of %{limit} variables.'), entity: entity, limit: variable_limit).html_safe
= link_to _('Learn more.'), help_page_path('ci/variables/index'), target: '_blank', rel: 'noopener noreferrer'
-%p
- = _('Variables can have several attributes.')
- = link_to _('Learn more.'), help_page_path('ci/variables/index', anchor: 'define-a-cicd-variable-in-the-ui'), target: '_blank', rel: 'noopener noreferrer'
-%ul
- %li
- = html_escape(_('%{code_open}Protected:%{code_close} Only exposed to protected branches or protected tags.')) % { code_open: '<code>'.html_safe, code_close: '</code>'.html_safe }
- %li
- = html_escape(_('%{code_open}Masked:%{code_close} Hidden in job logs. Must match masking requirements.')) % { code_open: '<code>'.html_safe, code_close: '</code>'.html_safe }
- %li
- = html_escape(_('%{code_open}Expanded:%{code_close} Variables with %{code_open}$%{code_close} will be treated as the start of a reference to another variable.')) % { code_open: '<code>'.html_safe, code_close: '</code>'.html_safe }
diff --git a/app/views/ci/variables/_index.html.haml b/app/views/ci/variables/_index.html.haml
index da2c8a71dcd..5eed4e92386 100644
--- a/app/views/ci/variables/_index.html.haml
+++ b/app/views/ci/variables/_index.html.haml
@@ -2,6 +2,17 @@
- is_group = !@group.nil?
- is_project = !@project.nil?
+%p
+ = _('Variables can have several attributes.')
+ = link_to _('Learn more.'), help_page_path('ci/variables/index', anchor: 'define-a-cicd-variable-in-the-ui'), target: '_blank', rel: 'noopener noreferrer'
+%ul
+ %li
+ = html_escape(_('%{code_open}Protected:%{code_close} Only exposed to protected branches or protected tags.')) % { code_open: '<code>'.html_safe, code_close: '</code>'.html_safe }
+ %li
+ = html_escape(_('%{code_open}Masked:%{code_close} Hidden in job logs. Must match masking requirements.')) % { code_open: '<code>'.html_safe, code_close: '</code>'.html_safe }
+ %li
+ = html_escape(_('%{code_open}Expanded:%{code_close} Variables with %{code_open}$%{code_close} will be treated as the start of a reference to another variable.')) % { code_open: '<code>'.html_safe, code_close: '</code>'.html_safe }
+
#js-ci-variables{ data: { endpoint: save_endpoint,
is_project: is_project.to_s,
project_id: @project&.id || '',
diff --git a/app/views/users/_profile_basic_info.html.haml b/app/views/users/_profile_basic_info.html.haml
index c916b6c3d45..51736b4970d 100644
--- a/app/views/users/_profile_basic_info.html.haml
+++ b/app/views/users/_profile_basic_info.html.haml
@@ -6,4 +6,4 @@
= s_('UserProfile|User ID: %{id}') % { id: @user.id }
= clipboard_button(title: s_('UserProfile|Copy user ID'), text: @user.id)
= render 'middle_dot_divider', stacking: true do
- = s_('Member since %{date}') % { date: @user.created_at.to_date.to_s(:long) }
+ = s_('Member since %{date}') % { date: l(@user.created_at, format: :long) }
diff --git a/app/workers/concerns/worker_attributes.rb b/app/workers/concerns/worker_attributes.rb
index 41cdf43bbbb..c260e06607c 100644
--- a/app/workers/concerns/worker_attributes.rb
+++ b/app/workers/concerns/worker_attributes.rb
@@ -201,8 +201,6 @@ module WorkerAttributes
end
def defer_on_database_health_signal?
- return false unless Feature.enabled?(:defer_sidekiq_workers_on_database_health_signal, type: :worker)
-
database_health_check_attrs.present?
end
diff --git a/config/feature_flags/worker/defer_sidekiq_workers_on_database_health_signal.yml b/config/feature_flags/worker/defer_sidekiq_workers_on_database_health_signal.yml
deleted file mode 100644
index 7e8575074c0..00000000000
--- a/config/feature_flags/worker/defer_sidekiq_workers_on_database_health_signal.yml
+++ /dev/null
@@ -1,8 +0,0 @@
----
-name: defer_sidekiq_workers_on_database_health_signal
-introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121261
-rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/412990
-milestone: '16.1'
-type: worker
-group: group::database
-default_enabled: false
diff --git a/config/gitleaks.toml b/config/gitleaks.toml
index be02a43aa9c..24761dfd783 100644
--- a/config/gitleaks.toml
+++ b/config/gitleaks.toml
@@ -18,4 +18,7 @@ path = "/gitleaks.toml"
"glpat--8GMtG8Mf4EnMJzmAWDU",
# doc/development/sec/token_revocation_api.md
"glpat--tG84EGK33nMLLDE70zU",
+ # doc/ci/runners/new_creation_workflow.md
+ "GR1348941C6YcZVddc8kjtdU-yWYD",
+ "glrt-2CR8_eVxiioB1QmzPZwa",
]
diff --git a/doc/architecture/blueprints/runner_tokens/index.md b/doc/architecture/blueprints/runner_tokens/index.md
index 39130e3384b..fd5f9fe15e1 100644
--- a/doc/architecture/blueprints/runner_tokens/index.md
+++ b/doc/architecture/blueprints/runner_tokens/index.md
@@ -440,124 +440,7 @@ scope.
## FAQ
-### Will my runner registration workflow break?
-
-If no action is taken before your GitLab instance is upgraded to 16.6, then your runner registration
-workflow will break.
-Until then, both the new and the old workflow will coexist side-by-side.
-
-To avoid a broken workflow, you need to first create a runner in the GitLab runners admin page.
-After that, you'll need to replace the registration token you're using in your runner registration
-workflow with the obtained runner authentication token.
-
-### Can I use the old runner registration process after 15.6?
-
-- If you're using GitLab.com, you'll be able to manually re-enable the previous runner registration process in the top-level group settings until GitLab 16.8.
-- If you're running GitLab self-managed, you'll be able re-enable the previous runner registration process in admin settings until GitLab 17.0.
-
-### What is the new runner registration process?
-
-When the new runner registration process is introduced, you will:
-
-1. Create a runner directly in the GitLab UI.
-1. Receive an authentication token in return.
-1. Use the authentication token instead of the registration token, whenever you need to register a runner with this
- configuration. Runner managers registered in multiple hosts will appear under the same runner in the GitLab UI,
- but with an identifying system ID.
-
-This has added benefits such as preserved ownership records for runners, and minimizes
-impact on users.
-The addition of a unique system ID ensures that you can reuse the same authentication token across
-multiple runners.
-For example, in an auto-scaling scenario where a runner manager spawns a runner process with a
-fixed authentication token.
-This ID generates once at the runner's startup, persists in a sidecar file, and is sent to the
-GitLab instance when requesting jobs.
-This allows the GitLab instance to display which system executed a given job.
-
-### What is the estimated timeframe for the planned changes?
-
-- In GitLab 15.10, we plan to implement runner creation directly in the runners administration page,
- and prepare the runner to follow the new workflow.
-- In GitLab 16.6, we plan to disable registration tokens.
-- In GitLab 17.0, we plan to completely remove support for runner registration tokens.
-
-### How will the `gitlab-runner register` command syntax change?
-
-The `gitlab-runner register` command will stop accepting registration tokens and instead accept new
-authentication tokens generated in the GitLab runners administration page.
-These authentication tokens are recognizable by their `glrt-` prefix.
-
-Example command for GitLab 15.9:
-
-```shell
-gitlab-runner register
- --non-interactive \
- --executor "shell" \
- --url "https://gitlab.com/" \
- --tag-list "shell,mac,gdk,test" \
- --run-untagged "false" \
- --locked "false" \
- --access-level "not_protected" \
- --registration-token "GR1348941C6YcZVddc8kjtdU-yWYD"
-```
-
-In GitLab 16.0, the runner will be created in the UI where some of its attributes can be
-pre-configured by the creator.
-Examples are the tag list, locked status, or access level. These are no longer accepted as arguments
-to `register`. The following example shows the new command:
-
-```shell
-gitlab-runner register
- --non-interactive \
- --executor "shell" \
- --url "https://gitlab.com/" \
- --token "glrt-2CR8_eVxiioB1QmzPZwa"
-```
-
-### How does this change impact auto-scaling scenarios?
-
-In auto-scaling scenarios such as GitLab Runner Operator or GitLab Runner Helm Chart, the
-registration token is replaced with the authentication token generated from the UI.
-This means that the same runner configuration is reused across jobs, instead of creating a runner
-for each job.
-The specific runner can be identified by the unique system ID that is generated when the runner
-process is started.
-
-### Will existing runners continue to work?
-
-Yes, existing runners will continue to work as usual. This change only affects registration of new runners.
-
-### Can runners still be created programmatically?
-
-A new [POST /user/runners REST API](../../../api/users.md#create-a-runner) was introduced in
-GitLab 15.11, which allows a runner to be created in the context of an authenticated user. This should only be used in
-scenarios where the runner configuration is dynamic, or not reusable. If the runner configuration is static, it is
-preferable to reuse the authentication token of an existing runner.
-
-The following snippet shows how a group runner could be created and registered with a
-[Group Access Token](../../../user/group/settings/group_access_tokens.md) using the new creation flow.
-The process is very similar when using [Project Access Tokens](../../../user/project/settings/project_access_tokens.md)
-or [Personal Access Tokens](../../../user/profile/personal_access_tokens.md):
-
-```shell
-# `GROUP_ID` contains the numerical ID of the group where the runner will be created
-# `GITLAB_TOKEN` can be a Personal Access Token for a group owner, or a Group Access Token on the respective group
-# created with `owner` access and `api` scope.
-#
-# The output will be parsed by `jq` to extract the token of the newly created runner
-RUNNER_TOKEN=$(curl --silent --method POST "https://gitlab.com/api/v4/user/runners" \
- --header "private-token: $GITLAB_TOKEN" \
- --header 'content-type: application/json' \
- --data "{\"runner_type\":\"group_type\",\"group_id\":\"$GROUP_ID\",\"description\":\"My runner\",\"tag-list\":\"java,linux\"}" \
- | jq -r '.token')
-
-gitlab-runner register
- --non-interactive \
- --executor "shell" \
- --url "https://gitlab.com/" \
- --token "$RUNNER_TOKEN"
-```
+Please follow [the user documentation](../../../ci/runners/new_creation_workflow.md).
## Status
diff --git a/doc/ci/runners/new_creation_workflow.md b/doc/ci/runners/new_creation_workflow.md
new file mode 100644
index 00000000000..3f1e8c13add
--- /dev/null
+++ b/doc/ci/runners/new_creation_workflow.md
@@ -0,0 +1,152 @@
+---
+stage: Verify
+group: Runner
+info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments
+---
+
+# Migrating to the new runner registration workflow **(FREE)**
+
+DISCLAIMER:
+This page contains information related to upcoming products, features, and functionality.
+It is important to note that the information presented is for informational purposes only.
+Please do not rely on this information for purchasing or planning purposes.
+As with all projects, the items mentioned on this page are subject to change or delay.
+The development, release, and timing of any products, features, or functionality remain at the
+sole discretion of GitLab Inc.
+
+In GitLab 16.0 we introduced a new runner creation workflow,
+the previous workflow that uses registration tokens is deprecated
+and will be removed in GitLab 17.0.
+
+For more information about the implementation for the new workflow, see the:
+
+- [Next GitLab Runner Token Architecture](../../architecture/blueprints/runner_tokens/index.md) for information about the technical design and reasons for the new token architecture.
+- [Development epic](https://gitlab.com/groups/gitlab-org/-/epics/7663) for the most accurate information about the current development status.
+
+## Feedback
+
+If you experience problems with the new runner registration workflow,
+and the following information is not sufficient,
+or if you have concerns about it,
+you can reach out to us in the [feedback issue](https://gitlab.com/gitlab-org/gitlab/-/issues/387993).
+
+## Will my runner registration workflow break?
+
+If no action is taken before your GitLab instance is upgraded to 16.6, then your runner registration
+workflow will break.
+Until then, both the new and the old workflow will coexist side-by-side.
+
+To avoid a broken workflow, you must:
+
+1. [Create a shared runner](register_runner.md#for-a-shared-runner) and obtain the authentication token.
+1. Replace the registration token in your runner registration workflow with the
+authentication token.
+
+## Can I use the old runner registration process after 16.6?
+
+- On GitLab.com, you'll be able to manually re-enable the previous runner registration process in the top-level group settings until GitLab 16.8.
+- On GitLab self-managed, you'll be able manually re-enable the previous runner registration process in the Admin Area settings until GitLab 17.0.
+
+## What is the new runner registration process?
+
+When the new runner registration process is introduced, you will:
+
+1. Create a runner directly in the GitLab UI.
+1. Receive an authentication token in return.
+1. Use the authentication token instead of the registration token, whenever you need to register a runner with this
+ configuration. Runner managers registered in multiple hosts will appear under the same runner in the GitLab UI,
+ but with an identifying system ID.
+
+This has added benefits such as preserved ownership records for runners, and minimizes
+impact on users.
+The addition of a unique system ID ensures that you can reuse the same authentication token across
+multiple runners.
+For example, in an auto-scaling scenario where a runner manager spawns a runner process with a
+fixed authentication token.
+This ID generates once at the runner's startup, persists in a sidecar file, and is sent to the
+GitLab instance when requesting jobs.
+This allows the GitLab instance to display which system executed a given job.
+
+## What is the estimated timeframe for the planned changes?
+
+- In GitLab 15.10, we plan to implement runner creation directly in the runners administration page,
+ and prepare the runner to follow the new workflow.
+- In GitLab 16.6, we plan to disable registration tokens.
+- In GitLab 17.0, we plan to completely remove support for runner registration tokens.
+
+## How will the `gitlab-runner register` command syntax change?
+
+The `gitlab-runner register` command will stop accepting registration tokens and instead accept new
+authentication tokens generated in the GitLab runners administration page.
+These authentication tokens are recognizable by their `glrt-` prefix.
+
+Example command for GitLab 15.9:
+
+```shell
+gitlab-runner register
+ --non-interactive \
+ --executor "shell" \
+ --url "https://gitlab.com/" \
+ --tag-list "shell,mac,gdk,test" \
+ --run-untagged "false" \
+ --locked "false" \
+ --access-level "not_protected" \
+ --registration-token "GR1348941C6YcZVddc8kjtdU-yWYD"
+```
+
+In GitLab 16.0, the runner will be created in the UI where some of its attributes can be
+pre-configured by the creator.
+Examples are the tag list, locked status, or access level. These are no longer accepted as arguments
+to `register`. The following example shows the new command:
+
+```shell
+gitlab-runner register
+ --non-interactive \
+ --executor "shell" \
+ --url "https://gitlab.com/" \
+ --token "glrt-2CR8_eVxiioB1QmzPZwa"
+```
+
+## How does this change impact auto-scaling scenarios?
+
+In auto-scaling scenarios such as GitLab Runner Operator or GitLab Runner Helm Chart, the
+registration token is replaced with the authentication token generated from the UI.
+This means that the same runner configuration is reused across jobs, instead of creating a runner
+for each job.
+The specific runner can be identified by the unique system ID that is generated when the runner
+process is started.
+
+## Will existing runners continue to work?
+
+Yes, existing runners will continue to work as usual. This change only affects registration of new runners.
+
+## Can runners still be created programmatically?
+
+A new [POST /user/runners REST API](../../api/users.md#create-a-runner) was introduced in
+GitLab 15.11, which allows a runner to be created in the context of an authenticated user. This should only be used in
+scenarios where the runner configuration is dynamic, or not reusable. If the runner configuration is static, it is
+preferable to reuse the authentication token of an existing runner.
+
+The following snippet shows how a group runner could be created and registered with a
+[Group Access Token](../../user/group/settings/group_access_tokens.md) using the new creation flow.
+The process is very similar when using [Project Access Tokens](../../user/project/settings/project_access_tokens.md)
+or [Personal Access Tokens](../../user/profile/personal_access_tokens.md):
+
+```shell
+# `GROUP_ID` contains the numerical ID of the group where the runner will be created
+# `GITLAB_TOKEN` can be a Personal Access Token for a group owner, or a Group Access Token on the respective group
+# created with `owner` access and `api` scope.
+#
+# The output will be parsed by `jq` to extract the token of the newly created runner
+RUNNER_TOKEN=$(curl --silent --method POST "https://gitlab.com/api/v4/user/runners" \
+ --header "private-token: $GITLAB_TOKEN" \
+ --header 'content-type: application/json' \
+ --data "{\"runner_type\":\"group_type\",\"group_id\":\"$GROUP_ID\",\"description\":\"My runner\",\"tag-list\":\"java,linux\"}" \
+ | jq -r '.token')
+
+gitlab-runner register
+ --non-interactive \
+ --executor "shell" \
+ --url "https://gitlab.com/" \
+ --token "$RUNNER_TOKEN"
+```
diff --git a/doc/install/docker.md b/doc/install/docker.md
index d387a4d0abb..0d5b0e5d7b0 100644
--- a/doc/install/docker.md
+++ b/doc/install/docker.md
@@ -774,3 +774,17 @@ xargs: tail: terminated by signal 6
```
Removing old log files helps fix the error, and ensures a clean startup of the instance.
+
+### ThreadError can't create Thread Operation not permitted
+
+```plaintext
+can't create Thread: Operation not permitted
+```
+
+This error occurs when running a container built with newer `glibc` versions on a
+[host that doesn't have support for the new clone3 function](https://github.com/moby/moby/issues/42680). In GitLab 16.0 and later, the container image includes
+the Ubuntu 22.04 GitLab Linux package which is built with this newer `glibc`.
+
+This problem is fixed with newer container runtime tools like [Docker 20.10.10](https://github.com/moby/moby/pull/42836).
+
+To resolve this issue, update Docker to version 20.10.10 or later.
diff --git a/doc/update/index.md b/doc/update/index.md
index 93d3edd2ef5..67c17bef7fb 100644
--- a/doc/update/index.md
+++ b/doc/update/index.md
@@ -287,6 +287,8 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap
- Sidekiq jobs are only routed to `default` and `mailers` queues by default, and as a result,
every Sidekiq process also listens to those queues to ensure all jobs are processed across
all queues. This behavior does not apply if you have configured the [routing rules](../administration/sidekiq/processing_specific_job_classes.md#routing-rules).
+- Docker 20.10.10 or later is required to run the GitLab Docker image. Older versions
+ [throw errors on startup](../install/docker.md#threaderror-cant-create-thread-operation-not-permitted).
### 15.11.1
diff --git a/lib/gitlab/ci/secure_files/migration_helper.rb b/lib/gitlab/ci/secure_files/migration_helper.rb
new file mode 100644
index 00000000000..13796f2476e
--- /dev/null
+++ b/lib/gitlab/ci/secure_files/migration_helper.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module SecureFiles
+ class MigrationHelper
+ class << self
+ def migrate_to_remote_storage(&block)
+ migrate_in_batches(
+ ::Ci::SecureFile.with_files_stored_locally,
+ ::Ci::SecureFileUploader::Store::REMOTE,
+ &block
+ )
+ end
+
+ private
+
+ def batch_size
+ ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i
+ end
+
+ def migrate_in_batches(files, store, &block)
+ files.find_each(batch_size: batch_size) do |file| # rubocop:disable CodeReuse/ActiveRecord
+ file.file.migrate!(store)
+
+ yield file if block
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/health_status/context.rb b/lib/gitlab/database/health_status/context.rb
index a1d9efead1f..717257a84ad 100644
--- a/lib/gitlab/database/health_status/context.rb
+++ b/lib/gitlab/database/health_status/context.rb
@@ -7,7 +7,7 @@ module Gitlab
attr_reader :status_checker, :connection, :tables, :gitlab_schema
# status_checker: the caller object which checks for database health status
- # eg: batched_migration
+ # eg: BackgroundMigration::BatchedMigration or DeferJobs::DatabaseHealthStatusChecker
def initialize(status_checker, connection, tables, gitlab_schema)
@status_checker = status_checker
@connection = connection
@@ -16,16 +16,11 @@ module Gitlab
end
def status_checker_info
- data = {
+ {
status_checker_id: status_checker.id,
- status_checker_type: status_checker.class.name
+ status_checker_type: status_checker.class.name,
+ job_class_name: status_checker.job_class_name
}
-
- if status_checker.is_a?(Gitlab::Database::BackgroundMigration::BatchedMigration)
- data[:job_class_name] = status_checker.job_class_name
- end
-
- data
end
end
end
diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb
index 03c8981440b..c4566a6dc2a 100644
--- a/lib/gitlab/sidekiq_logging/structured_logger.rb
+++ b/lib/gitlab/sidekiq_logging/structured_logger.rb
@@ -86,6 +86,8 @@ module Gitlab
payload['message'] = "#{message}: #{job_status}: #{payload['duration_s']} sec"
payload['job_status'] = job_status
+ payload['job_deferred_by'] = job['deferred_by'] if job['deferred']
+
Gitlab::ExceptionLogFormatter.format!(job_exception, payload) if job_exception
db_duration = ActiveRecord::LogSubscriber.runtime
diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb
index e389b7ae645..ec2a6472809 100644
--- a/lib/gitlab/sidekiq_middleware.rb
+++ b/lib/gitlab/sidekiq_middleware.rb
@@ -36,7 +36,7 @@ module Gitlab
chain.add ::Gitlab::SidekiqVersioning::Middleware
chain.add ::Gitlab::SidekiqStatus::ServerMiddleware
chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Server
- # DuplicateJobs::Server should be placed at the bottom, but before the SidekiqServerMiddleware,
+ # DuplicateJobs::Server should be placed at the bottom, but before the SidekiqServerMiddleware,
# so we can compare the latest WAL location against replica
chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Server
chain.add ::Gitlab::Database::LoadBalancing::SidekiqServerMiddleware
diff --git a/lib/gitlab/sidekiq_middleware/defer_jobs.rb b/lib/gitlab/sidekiq_middleware/defer_jobs.rb
index 76b44046a81..0a12667865c 100644
--- a/lib/gitlab/sidekiq_middleware/defer_jobs.rb
+++ b/lib/gitlab/sidekiq_middleware/defer_jobs.rb
@@ -3,30 +3,72 @@
module Gitlab
module SidekiqMiddleware
class DeferJobs
- include Sidekiq::ServerMiddleware
-
DELAY = ENV.fetch("SIDEKIQ_DEFER_JOBS_DELAY", 5.minutes)
FEATURE_FLAG_PREFIX = "defer_sidekiq_jobs"
- # This middleware will defer jobs indefinitely until the `defer_sidekiq_jobs_#{worker_name}` feature flag
- # is turned off (or when Feature.enabled? returns false by chance while using `percentage of time` value)
+ DatabaseHealthStatusChecker = Struct.new(:id, :job_class_name)
+
+ # There are 2 scenarios under which this middleware defers a job
+ # 1. defer_sidekiq_jobs_#{worker_name} FF, jobs are deferred indefinitely until this feature flag
+ # is turned off or when Feature.enabled? returns false by chance while using `percentage of time` value.
+ # 2. Gitlab::Database::HealthStatus, on evaluating the db health status if it returns any indicator
+ # with stop signal, the jobs will be delayed by 'x' seconds (set in worker).
def call(worker, job, _queue)
- if defer_job?(worker)
- job['deferred'] = true # for logging job_status
- worker.class.perform_in(DELAY, *job['args'])
+ # ActiveJobs have wrapped class stored in 'wrapped' key
+ resolved_class = job['wrapped']&.safe_constantize || worker.class
+ defer_job, delay, deferred_by = defer_job_info(resolved_class, job)
+
+ if !!defer_job
+ # Referred in job_logger's 'log_job_done' method to compute proper 'job_status'
+ job['deferred'] = true
+ job['deferred_by'] = deferred_by
+
+ worker.class.perform_in(delay, *job['args'])
counter.increment({ worker: worker.class.name })
+
+ # This breaks the middleware chain and return
return
end
yield
end
- def defer_job?(worker)
- Feature.enabled?(:"#{FEATURE_FLAG_PREFIX}_#{worker.class.name}", type: :worker,
- default_enabled_if_undefined: false)
+ private
+
+ def defer_job_info(worker_class, job)
+ if defer_job_by_ff?(worker_class)
+ [true, DELAY, :feature_flag]
+ elsif defer_job_by_database_health_signal?(job, worker_class)
+ [true, worker_class.database_health_check_attrs[:delay_by], :database_health_check]
+ end
+ end
+
+ def defer_job_by_ff?(worker_class)
+ Feature.enabled?(
+ :"#{FEATURE_FLAG_PREFIX}_#{worker_class.name}",
+ type: :worker,
+ default_enabled_if_undefined: false
+ )
end
- private
+ def defer_job_by_database_health_signal?(job, worker_class)
+ unless worker_class.respond_to?(:defer_on_database_health_signal?) &&
+ worker_class.defer_on_database_health_signal?
+ return false
+ end
+
+ health_check_attrs = worker_class.database_health_check_attrs
+ job_base_model = Gitlab::Database.schemas_to_base_models[health_check_attrs[:gitlab_schema]].first
+
+ health_context = Gitlab::Database::HealthStatus::Context.new(
+ DatabaseHealthStatusChecker.new(job['jid'], worker_class.name),
+ job_base_model.connection,
+ health_check_attrs[:gitlab_schema],
+ health_check_attrs[:tables]
+ )
+
+ Gitlab::Database::HealthStatus.evaluate(health_context).any?(&:stop?)
+ end
def counter
@counter ||= Gitlab::Metrics.counter(:sidekiq_jobs_deferred_total, 'The number of jobs deferred')
diff --git a/lib/tasks/gitlab/ci_secure_files/migrate.rake b/lib/tasks/gitlab/ci_secure_files/migrate.rake
new file mode 100644
index 00000000000..8de1b7da6be
--- /dev/null
+++ b/lib/tasks/gitlab/ci_secure_files/migrate.rake
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+desc "GitLab | CI Secure Files | Migrate Secure Files to remote storage"
+namespace :gitlab do
+ namespace :ci_secure_files do
+ task migrate: :environment do
+ require 'logger'
+
+ logger = Logger.new($stdout)
+ logger.info('Starting transfer of Secure Files to object storage')
+
+ begin
+ Gitlab::Ci::SecureFiles::MigrationHelper.migrate_to_remote_storage do |file|
+ message = "Transferred Secure File ID #{file.id} (#{file.name}) to object storage"
+
+ logger.info(message)
+ end
+ rescue StandardError => e
+ logger.error("Failed to migrate: #{e.message}")
+ end
+ end
+ end
+end
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index deaf1e2fa2d..ef2498be7b2 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -27677,6 +27677,9 @@ msgstr ""
msgid "Marks to do as done."
msgstr ""
+msgid "Mask this variable in job logs if it meets %{linkStart}regular expression requirements%{linkEnd}."
+msgstr ""
+
msgid "Mask variable"
msgstr ""
@@ -38739,9 +38742,6 @@ msgstr[1] ""
msgid "Requires a verified GitLab email address."
msgstr ""
-msgid "Requires values to meet regular expression requirements."
-msgstr ""
-
msgid "Requires you to deploy or set up cloud-hosted Sentry."
msgstr ""
@@ -46939,7 +46939,7 @@ msgstr ""
msgid "This user is the author of this %{noteable}."
msgstr ""
-msgid "This variable can not be masked."
+msgid "This variable value does not meet the masking requirements."
msgstr ""
msgid "This vulnerability was automatically resolved because its vulnerability type was disabled in this project or removed from GitLab's default ruleset."
@@ -49681,6 +49681,9 @@ msgstr ""
msgid "Value might contain a variable reference"
msgstr ""
+msgid "Value must meet regular expression requirements to be masked."
+msgstr ""
+
msgid "Value stream"
msgstr ""
@@ -49825,9 +49828,6 @@ msgstr ""
msgid "Variable value will be evaluated as raw string."
msgstr ""
-msgid "Variable will be masked in job logs."
-msgstr ""
-
msgid "Variables"
msgstr ""
diff --git a/package.json b/package.json
index 52e69615701..03585ff772f 100644
--- a/package.json
+++ b/package.json
@@ -56,7 +56,7 @@
"@gitlab/cluster-client": "^1.2.0",
"@gitlab/favicon-overlay": "2.0.0",
"@gitlab/fonts": "^1.2.0",
- "@gitlab/svgs": "3.50.0",
+ "@gitlab/svgs": "3.51.0",
"@gitlab/ui": "64.2.3",
"@gitlab/visual-review-tools": "1.7.3",
"@gitlab/web-ide": "0.0.1-dev-20230524134151",
diff --git a/qa/qa/specs/features/browser_ui/5_package/container_registry/self_managed/container_registry_spec.rb b/qa/qa/specs/features/browser_ui/5_package/container_registry/self_managed/container_registry_spec.rb
index 9cf448aa8b7..800b3a01dc6 100644
--- a/qa/qa/specs/features/browser_ui/5_package/container_registry/self_managed/container_registry_spec.rb
+++ b/qa/qa/specs/features/browser_ui/5_package/container_registry/self_managed/container_registry_spec.rb
@@ -27,13 +27,6 @@ module QA
end
end
- let(:registry_repository) do
- Resource::RegistryRepository.fabricate! do |repository|
- repository.name = project.path_with_namespace.to_s
- repository.project = project
- end
- end
-
let!(:runner) do
Resource::ProjectRunner.fabricate! do |runner|
runner.name = "qa-runner-#{Time.now.to_i}"
@@ -51,7 +44,6 @@ module QA
end
after do
- registry_repository&.remove_via_api!
runner.remove_via_api!
end
diff --git a/qa/qa/specs/features/browser_ui/5_package/dependency_proxy/dependency_proxy_spec.rb b/qa/qa/specs/features/browser_ui/5_package/dependency_proxy/dependency_proxy_spec.rb
index 026fd3c5d10..235c3604523 100644
--- a/qa/qa/specs/features/browser_ui/5_package/dependency_proxy/dependency_proxy_spec.rb
+++ b/qa/qa/specs/features/browser_ui/5_package/dependency_proxy/dependency_proxy_spec.rb
@@ -53,7 +53,6 @@ module QA
end
after do
- project.remove_via_api!
runner.remove_via_api!
end
diff --git a/spec/frontend/ci/ci_variable_list/components/ci_variable_modal_spec.js b/spec/frontend/ci/ci_variable_list/components/ci_variable_modal_spec.js
index b6ffde9b33f..40b04ca3e00 100644
--- a/spec/frontend/ci/ci_variable_list/components/ci_variable_modal_spec.js
+++ b/spec/frontend/ci/ci_variable_list/components/ci_variable_modal_spec.js
@@ -458,7 +458,8 @@ describe('Ci variable modal', () => {
});
describe('Validations', () => {
- const maskError = 'This variable can not be masked.';
+ const maskError = 'This variable value does not meet the masking requirements.';
+ const helpText = 'Value must meet regular expression requirements to be masked.';
describe('when the variable is raw', () => {
const [variable] = mockVariables;
@@ -488,6 +489,10 @@ describe('Ci variable modal', () => {
expect(findModal().text()).toContain(maskError);
});
+
+ it('does not show the masked variable help text', () => {
+ expect(findModal().text()).not.toContain(helpText);
+ });
});
describe('when the mask state is invalid', () => {
@@ -510,8 +515,9 @@ describe('Ci variable modal', () => {
expect(findAddorUpdateButton().attributes('disabled')).toBeDefined();
});
- it('shows the correct error text', () => {
+ it('shows the correct error text and help text', () => {
expect(findModal().text()).toContain(maskError);
+ expect(findModal().text()).toContain(helpText);
});
it('sends the correct tracking event', () => {
@@ -578,6 +584,10 @@ describe('Ci variable modal', () => {
});
});
+ it('shows the help text', () => {
+ expect(findModal().text()).toContain(helpText);
+ });
+
it('does not disable the submit button', () => {
expect(findAddorUpdateButton().attributes('disabled')).toBeUndefined();
});
diff --git a/spec/lib/gitlab/ci/secure_files/migration_helper_spec.rb b/spec/lib/gitlab/ci/secure_files/migration_helper_spec.rb
new file mode 100644
index 00000000000..8f1b300ae98
--- /dev/null
+++ b/spec/lib/gitlab/ci/secure_files/migration_helper_spec.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Ci::SecureFiles::MigrationHelper, feature_category: :mobile_devops do
+ before do
+ stub_ci_secure_file_object_storage
+ end
+
+ describe '.migrate_to_remote_storage' do
+ let!(:local_file) { create(:ci_secure_file) }
+
+ subject { described_class.migrate_to_remote_storage }
+
+ it 'migrates remote files to remote storage' do
+ subject
+
+ expect(local_file.reload.file_store).to eq(Ci::SecureFileUploader::Store::REMOTE)
+ end
+ end
+
+ describe '.migrate_in_batches' do
+ let!(:local_file) { create(:ci_secure_file) }
+ let!(:storage) { Ci::SecureFileUploader::Store::REMOTE }
+
+ subject { described_class.migrate_to_remote_storage }
+
+ it 'migrates the given file to the given storage backend' do
+ expect_next_found_instance_of(Ci::SecureFile) do |instance|
+ expect(instance).to receive_message_chain(:file, :migrate!).with(storage)
+ end
+
+ described_class.send(:migrate_in_batches, Ci::SecureFile.all, storage)
+ end
+
+ it 'calls the given block for each migrated file' do
+ expect_next_found_instance_of(Ci::SecureFile) do |instance|
+ expect(instance).to receive(:metadata)
+ end
+
+ described_class.send(:migrate_in_batches, Ci::SecureFile.all, storage, &:metadata)
+ end
+ end
+end
diff --git a/spec/lib/gitlab/database/health_status_spec.rb b/spec/lib/gitlab/database/health_status_spec.rb
index 8d7717beaff..bc923635b1d 100644
--- a/spec/lib/gitlab/database/health_status_spec.rb
+++ b/spec/lib/gitlab/database/health_status_spec.rb
@@ -12,9 +12,10 @@ RSpec.describe Gitlab::Database::HealthStatus, feature_category: :database do
end
describe '.evaluate' do
- subject(:evaluate) { described_class.evaluate(migration.health_context, [autovacuum_indicator_class]) }
+ subject(:evaluate) { described_class.evaluate(health_context, [autovacuum_indicator_class]) }
let(:migration) { build(:batched_background_migration, :active) }
+ let(:health_context) { migration.health_context }
let(:health_status) { described_class }
let(:autovacuum_indicator_class) { health_status::Indicators::AutovacuumActiveOnTable }
@@ -25,20 +26,20 @@ RSpec.describe Gitlab::Database::HealthStatus, feature_category: :database do
let(:patroni_apdex_indicator) { instance_double(patroni_apdex_indicator_class) }
before do
- allow(autovacuum_indicator_class).to receive(:new).with(migration.health_context).and_return(autovacuum_indicator)
+ allow(autovacuum_indicator_class).to receive(:new).with(health_context).and_return(autovacuum_indicator)
end
context 'with default indicators' do
- subject(:evaluate) { described_class.evaluate(migration.health_context) }
+ subject(:evaluate) { described_class.evaluate(health_context) }
it 'returns a collection of signals' do
normal_signal = instance_double("#{health_status}::Signals::Normal", log_info?: false)
not_available_signal = instance_double("#{health_status}::Signals::NotAvailable", log_info?: false)
expect(autovacuum_indicator).to receive(:evaluate).and_return(normal_signal)
- expect(wal_indicator_class).to receive(:new).with(migration.health_context).and_return(wal_indicator)
+ expect(wal_indicator_class).to receive(:new).with(health_context).and_return(wal_indicator)
expect(wal_indicator).to receive(:evaluate).and_return(not_available_signal)
- expect(patroni_apdex_indicator_class).to receive(:new).with(migration.health_context)
+ expect(patroni_apdex_indicator_class).to receive(:new).with(health_context)
.and_return(patroni_apdex_indicator)
expect(patroni_apdex_indicator).to receive(:evaluate).and_return(not_available_signal)
@@ -46,7 +47,7 @@ RSpec.describe Gitlab::Database::HealthStatus, feature_category: :database do
end
end
- it 'returns a collection of signals' do
+ it 'returns the signal of the given indicator' do
signal = instance_double("#{health_status}::Signals::Normal", log_info?: false)
expect(autovacuum_indicator).to receive(:evaluate).and_return(signal)
@@ -54,28 +55,78 @@ RSpec.describe Gitlab::Database::HealthStatus, feature_category: :database do
expect(evaluate).to contain_exactly(signal)
end
- it 'logs interesting signals' do
- signal = instance_double(
- "#{health_status}::Signals::Stop",
- log_info?: true,
- indicator_class: autovacuum_indicator_class,
- short_name: 'Stop',
- reason: 'Test Exception'
- )
+ context 'with stop signals' do
+ let(:stop_signal) do
+ instance_double(
+ "#{health_status}::Signals::Stop",
+ log_info?: true,
+ indicator_class: autovacuum_indicator_class,
+ short_name: 'Stop',
+ reason: 'Test Exception'
+ )
+ end
- expect(autovacuum_indicator).to receive(:evaluate).and_return(signal)
+ before do
+ allow(autovacuum_indicator).to receive(:evaluate).and_return(stop_signal)
+ end
- expect(Gitlab::Database::HealthStatus::Logger).to receive(:info).with(
- status_checker_id: migration.id,
- status_checker_type: 'Gitlab::Database::BackgroundMigration::BatchedMigration',
- job_class_name: migration.job_class_name,
- health_status_indicator: autovacuum_indicator_class.to_s,
- indicator_signal: 'Stop',
- signal_reason: 'Test Exception',
- message: "#{migration} signaled: #{signal}"
- )
+ context 'with batched migrations as the status checker' do
+ it 'captures BatchedMigration class name in the log' do
+ expect(Gitlab::Database::HealthStatus::Logger).to receive(:info).with(
+ status_checker_id: migration.id,
+ status_checker_type: 'Gitlab::Database::BackgroundMigration::BatchedMigration',
+ job_class_name: migration.job_class_name,
+ health_status_indicator: autovacuum_indicator_class.to_s,
+ indicator_signal: 'Stop',
+ signal_reason: 'Test Exception',
+ message: "#{migration} signaled: #{stop_signal}"
+ )
- evaluate
+ evaluate
+ end
+ end
+
+ context 'with sidekiq deferred job as the status checker' do
+ let(:deferred_worker) do
+ Class.new do
+ def self.name
+ 'TestDeferredWorker'
+ end
+
+ include ApplicationWorker
+ end
+ end
+
+ let(:deferred_worker_health_checker) do
+ Gitlab::SidekiqMiddleware::DeferJobs::DatabaseHealthStatusChecker.new(
+ 123,
+ deferred_worker.name
+ )
+ end
+
+ let(:health_context) do
+ Gitlab::Database::HealthStatus::Context.new(
+ deferred_worker_health_checker,
+ ActiveRecord::Base.connection,
+ :gitlab_main,
+ [:users]
+ )
+ end
+
+ it 'captures sidekiq job class in the log' do
+ expect(Gitlab::Database::HealthStatus::Logger).to receive(:info).with(
+ status_checker_id: deferred_worker_health_checker.id,
+ status_checker_type: 'Gitlab::SidekiqMiddleware::DeferJobs::DatabaseHealthStatusChecker',
+ job_class_name: deferred_worker_health_checker.job_class_name,
+ health_status_indicator: autovacuum_indicator_class.to_s,
+ indicator_signal: 'Stop',
+ signal_reason: 'Test Exception',
+ message: "#{deferred_worker_health_checker} signaled: #{stop_signal}"
+ )
+
+ evaluate
+ end
+ end
end
it 'does not log signals of no interest' do
diff --git a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb
index 96897cea9c0..1c23a619b38 100644
--- a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb
+++ b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb
@@ -435,6 +435,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do
call_subject(job, 'test_queue') do
job['deferred'] = true
+ job['deferred_by'] = :feature_flag
end
end
end
diff --git a/spec/lib/gitlab/sidekiq_middleware/defer_jobs_spec.rb b/spec/lib/gitlab/sidekiq_middleware/defer_jobs_spec.rb
index 6dcf9aaeb63..195a79c22ec 100644
--- a/spec/lib/gitlab/sidekiq_middleware/defer_jobs_spec.rb
+++ b/spec/lib/gitlab/sidekiq_middleware/defer_jobs_spec.rb
@@ -5,7 +5,7 @@ require 'spec_helper'
RSpec.describe Gitlab::SidekiqMiddleware::DeferJobs, feature_category: :scalability do
let(:job) { { 'jid' => 123, 'args' => [456] } }
let(:queue) { 'test_queue' }
- let(:worker) do
+ let(:deferred_worker) do
Class.new do
def self.name
'TestDeferredWorker'
@@ -14,7 +14,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::DeferJobs, feature_category: :scalabil
end
end
- let(:worker2) do
+ let(:undeferred_worker) do
Class.new do
def self.name
'UndeferredWorker'
@@ -26,21 +26,29 @@ RSpec.describe Gitlab::SidekiqMiddleware::DeferJobs, feature_category: :scalabil
subject { described_class.new }
before do
- stub_const('TestDeferredWorker', worker)
- stub_const('UndeferredWorker', worker2)
+ stub_const('TestDeferredWorker', deferred_worker)
+ stub_const('UndeferredWorker', undeferred_worker)
end
describe '#call' do
- context 'when sidekiq_defer_jobs feature flag is enabled for a worker' do
- before do
- stub_feature_flags("defer_sidekiq_jobs_#{TestDeferredWorker.name}": true)
- stub_feature_flags("defer_sidekiq_jobs_#{UndeferredWorker.name}": false)
- end
+ context 'with worker not opted for database health check' do
+ context 'when sidekiq_defer_jobs feature flag is enabled for a worker' do
+ before do
+ stub_feature_flags("defer_sidekiq_jobs_#{TestDeferredWorker.name}": true)
+ stub_feature_flags("defer_sidekiq_jobs_#{UndeferredWorker.name}": false)
+ end
+
+ context 'for the affected worker' do
+ it 'defers the job' do
+ expect(TestDeferredWorker).to receive(:perform_in).with(described_class::DELAY, *job['args'])
+ expect { |b| subject.call(TestDeferredWorker.new, job, queue, &b) }.not_to yield_control
+ end
+ end
- context 'for the affected worker' do
- it 'defers the job' do
- expect(TestDeferredWorker).to receive(:perform_in).with(described_class::DELAY, *job['args'])
- expect { |b| subject.call(TestDeferredWorker.new, job, queue, &b) }.not_to yield_control
+ context 'for other workers' do
+ it 'runs the job normally' do
+ expect { |b| subject.call(UndeferredWorker.new, job, queue, &b) }.to yield_control
+ end
end
it 'increments the counter' do
@@ -51,22 +59,52 @@ RSpec.describe Gitlab::SidekiqMiddleware::DeferJobs, feature_category: :scalabil
end
end
- context 'for other workers' do
+ context 'when sidekiq_defer_jobs feature flag is disabled' do
+ before do
+ stub_feature_flags("defer_sidekiq_jobs_#{TestDeferredWorker.name}": false)
+ stub_feature_flags("defer_sidekiq_jobs_#{UndeferredWorker.name}": false)
+ end
+
it 'runs the job normally' do
+ expect { |b| subject.call(TestDeferredWorker.new, job, queue, &b) }.to yield_control
expect { |b| subject.call(UndeferredWorker.new, job, queue, &b) }.to yield_control
end
end
end
- context 'when sidekiq_defer_jobs feature flag is disabled' do
+ context 'with worker opted for database health check' do
+ let(:health_signal_attrs) { { gitlab_schema: :gitlab_main, delay: 1.minute, tables: [:users] } }
+
+ around do |example|
+ with_sidekiq_server_middleware do |chain|
+ chain.add described_class
+ Sidekiq::Testing.inline! { example.run }
+ end
+ end
+
before do
stub_feature_flags("defer_sidekiq_jobs_#{TestDeferredWorker.name}": false)
- stub_feature_flags("defer_sidekiq_jobs_#{UndeferredWorker.name}": false)
+
+ TestDeferredWorker.defer_on_database_health_signal(*health_signal_attrs.values)
+ end
+
+ context 'without any stop signal from database health check' do
+ it 'runs the job normally' do
+ expect { |b| subject.call(TestDeferredWorker.new, job, queue, &b) }.to yield_control
+ end
end
- it 'runs the job normally' do
- expect { |b| subject.call(TestDeferredWorker.new, job, queue, &b) }.to yield_control
- expect { |b| subject.call(UndeferredWorker.new, job, queue, &b) }.to yield_control
+ context 'with stop signal from database health check' do
+ before do
+ stop_signal = instance_double("Gitlab::Database::HealthStatus::Signals::Stop", stop?: true)
+ allow(Gitlab::Database::HealthStatus).to receive(:evaluate).and_return([stop_signal])
+ end
+
+ it 'defers the job by set time' do
+ expect(TestDeferredWorker).to receive(:perform_in).with(health_signal_attrs[:delay], *job['args'])
+
+ TestDeferredWorker.perform_async(*job['args'])
+ end
end
end
end
diff --git a/spec/models/namespace/root_storage_statistics_spec.rb b/spec/models/namespace/root_storage_statistics_spec.rb
index 3b6062c0d8a..a7f21e3a07f 100644
--- a/spec/models/namespace/root_storage_statistics_spec.rb
+++ b/spec/models/namespace/root_storage_statistics_spec.rb
@@ -6,7 +6,7 @@ RSpec.describe Namespace::RootStorageStatistics, type: :model do
it { is_expected.to belong_to :namespace }
it { is_expected.to have_one(:route).through(:namespace) }
- it { is_expected.to delegate_method(:all_projects).to(:namespace) }
+ it { is_expected.to delegate_method(:all_projects_except_soft_deleted).to(:namespace) }
context 'scopes' do
describe '.for_namespace_ids' do
@@ -28,9 +28,10 @@ RSpec.describe Namespace::RootStorageStatistics, type: :model do
let(:project1) { create(:project, namespace: namespace) }
let(:project2) { create(:project, namespace: namespace) }
+ let(:project3) { create(:project, namespace: namespace, marked_for_deletion_at: 1.day.ago, pending_delete: true) }
shared_examples 'project data refresh' do
- it 'aggregates project statistics' do
+ it 'aggregates eligible project statistics' do
root_storage_statistics.recalculate!
root_storage_statistics.reload
@@ -97,6 +98,7 @@ RSpec.describe Namespace::RootStorageStatistics, type: :model do
context 'with project statistics' do
let!(:project_stat1) { create(:project_statistics, project: project1, with_data: true, size_multiplier: 100) }
let!(:project_stat2) { create(:project_statistics, project: project2, with_data: true, size_multiplier: 200) }
+ let!(:project_stat3) { create(:project_statistics, project: project3, with_data: true, size_multiplier: 300) }
it_behaves_like 'project data refresh'
it_behaves_like 'does not include personal snippets'
diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb
index c9995122bb2..aea7a3b52c9 100644
--- a/spec/models/namespace_spec.rb
+++ b/spec/models/namespace_spec.rb
@@ -1745,6 +1745,52 @@ RSpec.describe Namespace, feature_category: :groups_and_projects do
end
end
+ describe '#all_projects_except_soft_deleted' do
+ context 'when namespace is a group' do
+ let_it_be(:namespace) { create(:group) }
+ let_it_be(:child) { create(:group, parent: namespace) }
+ let_it_be(:project1) { create(:project_empty_repo, namespace: namespace) }
+ let_it_be(:project2) { create(:project_empty_repo, namespace: child) }
+ let_it_be(:other_project) { create(:project_empty_repo) }
+
+ before do
+ reload_models(namespace, child)
+ end
+
+ it { expect(namespace.all_projects_except_soft_deleted.to_a).to match_array([project2, project1]) }
+ it { expect(child.all_projects_except_soft_deleted.to_a).to match_array([project2]) }
+
+ context 'with soft deleted projects' do
+ let_it_be(:delayed_deletion_project) { create(:project, namespace: child, marked_for_deletion_at: Date.current) }
+
+ it 'skips delayed deletion project' do
+ expect(namespace.all_projects_except_soft_deleted.to_a).to match_array([project2, project1])
+ end
+ end
+ end
+
+ context 'when namespace is a user namespace' do
+ let_it_be(:user) { create(:user) }
+ let_it_be(:user_namespace) { create(:namespace, owner: user) }
+ let_it_be(:project) { create(:project, namespace: user_namespace) }
+ let_it_be(:other_project) { create(:project_empty_repo) }
+
+ before do
+ reload_models(user_namespace)
+ end
+
+ it { expect(user_namespace.all_projects_except_soft_deleted.to_a).to match_array([project]) }
+
+ context 'with soft deleted projects' do
+ let_it_be(:delayed_deletion_project) { create(:project, namespace: user_namespace, marked_for_deletion_at: Date.current) }
+
+ it 'skips delayed deletion project' do
+ expect(user_namespace.all_projects_except_soft_deleted.to_a).to match_array([project])
+ end
+ end
+ end
+ end
+
describe '#all_projects' do
context 'with use_traversal_ids feature flag enabled' do
before do
diff --git a/spec/support/shared_contexts/lib/gitlab/sidekiq_logging/structured_logger_shared_context.rb b/spec/support/shared_contexts/lib/gitlab/sidekiq_logging/structured_logger_shared_context.rb
index 88339df3475..1b50ef3fcff 100644
--- a/spec/support/shared_contexts/lib/gitlab/sidekiq_logging/structured_logger_shared_context.rb
+++ b/spec/support/shared_contexts/lib/gitlab/sidekiq_logging/structured_logger_shared_context.rb
@@ -68,7 +68,8 @@ RSpec.shared_context 'structured_logger' do
let(:deferred_payload) do
end_payload.merge(
'message' => 'TestWorker JID-da883554ee4fe414012f5f42: deferred: 0.0 sec',
- 'job_status' => 'deferred'
+ 'job_status' => 'deferred',
+ 'job_deferred_by' => :feature_flag
)
end
diff --git a/spec/support/shared_examples/features/variable_list_shared_examples.rb b/spec/support/shared_examples/features/variable_list_shared_examples.rb
index 1211c9d19e6..d206f669b3e 100644
--- a/spec/support/shared_examples/features/variable_list_shared_examples.rb
+++ b/spec/support/shared_examples/features/variable_list_shared_examples.rb
@@ -186,7 +186,7 @@ RSpec.shared_examples 'variable list' do
click_button('Add variable')
fill_variable('empty_mask_key', '???', protected: true, masked: true) do
- expect(page).to have_content('This variable can not be masked')
+ expect(page).to have_content('This variable value does not meet the masking requirements.')
expect(find_button('Add variable', disabled: true)).to be_present
end
end
diff --git a/spec/tasks/gitlab/ci_secure_files/migrate_rake_spec.rb b/spec/tasks/gitlab/ci_secure_files/migrate_rake_spec.rb
new file mode 100644
index 00000000000..ed6b5914f3e
--- /dev/null
+++ b/spec/tasks/gitlab/ci_secure_files/migrate_rake_spec.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'rake_helper'
+
+RSpec.describe 'gitlab:ci_secure_files', feature_category: :mobile_devops do
+ let_it_be(:local_file) { create(:ci_secure_file) }
+
+ let(:logger) { instance_double(Logger) }
+ let(:helper) { double }
+
+ before(:all) do
+ Rake.application.rake_require 'tasks/gitlab/ci_secure_files/migrate'
+ end
+
+ before do
+ allow(Logger).to receive(:new).with($stdout).and_return(logger)
+ end
+
+ describe 'gitlab:ci_secure_files:migrate' do
+ subject { run_rake_task('gitlab:ci_secure_files:migrate') }
+
+ it 'invokes the migration helper to move files to object storage' do
+ expect(Gitlab::Ci::SecureFiles::MigrationHelper).to receive(:migrate_to_remote_storage).and_yield(local_file)
+ expect(logger).to receive(:info).with('Starting transfer of Secure Files to object storage')
+ expect(logger).to receive(:info).with(/Transferred Secure File ID #{local_file.id}/)
+
+ subject
+ end
+
+ context 'when an error is raised while migrating' do
+ let(:error_message) { 'Something went wrong' }
+
+ before do
+ allow(Gitlab::Ci::SecureFiles::MigrationHelper).to receive(:migrate_to_remote_storage).and_raise(StandardError,
+ error_message)
+ end
+
+ it 'logs the error' do
+ expect(logger).to receive(:info).with('Starting transfer of Secure Files to object storage')
+ expect(logger).to receive(:error).with("Failed to migrate: #{error_message}")
+
+ subject
+ end
+ end
+ end
+end
diff --git a/spec/workers/concerns/worker_attributes_spec.rb b/spec/workers/concerns/worker_attributes_spec.rb
index cee8e0fb0ad..959cb62c6fb 100644
--- a/spec/workers/concerns/worker_attributes_spec.rb
+++ b/spec/workers/concerns/worker_attributes_spec.rb
@@ -157,23 +157,5 @@ RSpec.describe WorkerAttributes, feature_category: :shared do
context 'when defer_on_database_health_signal is not set' do
it { is_expected.to be(false) }
end
-
- context 'when FF `defer_sidekiq_workers_on_database_health_signal` is disabled' do
- before do
- stub_feature_flags(defer_sidekiq_workers_on_database_health_signal: false)
- end
-
- context 'when defer_on_database_health_signal is set' do
- before do
- worker.defer_on_database_health_signal(:gitlab_main, 1.minute, [:users])
- end
-
- it { is_expected.to be(false) }
- end
-
- context 'when defer_on_database_health_signal is not set' do
- it { is_expected.to be(false) }
- end
- end
end
end
diff --git a/yarn.lock b/yarn.lock
index dd74947b3b0..1198913d1ec 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1122,10 +1122,10 @@
stylelint-declaration-strict-value "1.8.0"
stylelint-scss "4.2.0"
-"@gitlab/svgs@3.50.0":
- version "3.50.0"
- resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-3.50.0.tgz#5ef7c0dabcd32ff32b5aaa3f576cf2bba7d5b466"
- integrity sha512-Ssw+TXeAJd/LRKovx/P5RHi1ofPTdroFidej9vdyIVhcGZh7lZYt+qCBq4FfCXGefK86jisyVmNuStdpZxGHng==
+"@gitlab/svgs@3.51.0":
+ version "3.51.0"
+ resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-3.51.0.tgz#c6d63c7e16b71b167662696662efc1b0587acaaf"
+ integrity sha512-IyMcZsLJ7AYoyHhWimHKsP/Swk0MlOLxHxgP9kp8Nc/xPC8p/JRVzlWU9vehSTcoYzu55alnKYABJe4fhUw2aQ==
"@gitlab/ui@64.2.3":
version "64.2.3"