Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2022-08-31 21:10:17 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2022-08-31 21:10:17 +0300
commitfebc637ca98cb72901745fc125154bcffc517e46 (patch)
tree02bd564235b5df4529377aacdc05ad73f09c387c
parent19044caf6695065ff64e26355e830dbdc6719e54 (diff)
Add latest changes from gitlab-org/gitlab@master
-rw-r--r--.gitignore2
-rw-r--r--.gitlab-ci.yml2
-rw-r--r--.gitlab/ci/rails.gitlab-ci.yml12
-rw-r--r--.gitlab/ci/setup.gitlab-ci.yml42
-rw-r--r--.gitlab/ci/static-analysis.gitlab-ci.yml2
-rw-r--r--app/services/projects/destroy_service.rb4
-rw-r--r--app/services/snippets/bulk_destroy_service.rb4
-rw-r--r--app/services/users/destroy_service.rb4
-rw-r--r--db/docs/spam_logs.yml6
-rw-r--r--doc/user/application_security/dast/index.md52
-rw-r--r--doc/user/application_security/security_dashboard/index.md7
-rw-r--r--doc/user/application_security/vulnerability_report/pipeline.md14
-rw-r--r--lib/api/helpers/groups_helpers.rb3
-rw-r--r--lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb10
-rw-r--r--lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb18
-rw-r--r--lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb9
-rw-r--r--package.json2
-rw-r--r--scripts/rspec_helpers.sh19
-rw-r--r--spec/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy_spec.rb21
-rw-r--r--spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb96
-rw-r--r--spec/requests/api/groups_spec.rb38
-rw-r--r--spec/services/projects/destroy_service_spec.rb12
-rw-r--r--spec/services/snippets/bulk_destroy_service_spec.rb4
-rw-r--r--spec/services/users/destroy_service_spec.rb2
-rw-r--r--workhorse/internal/upstream/routes.go4
-rw-r--r--workhorse/upload_test.go3
26 files changed, 146 insertions, 246 deletions
diff --git a/.gitignore b/.gitignore
index 0bd718f254b..234593b944e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -95,8 +95,6 @@ jsdoc/
webpack-dev-server.json
/.nvimrc
.solargraph.yml
-/tmp/matching_foss_tests.txt
-/tmp/matching_tests.txt
ee/changelogs/unreleased-ee
/sitespeed-result
tags.lock
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cb912870307..f832b88b821 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -90,6 +90,8 @@ variables:
RSPEC_PACKED_TESTS_MAPPING_PATH: crystalball/packed-mapping.json
RSPEC_PROFILING_FOLDER_PATH: rspec/profiling
FRONTEND_FIXTURES_MAPPING_PATH: crystalball/frontend_fixtures_mapping.json
+ RSPEC_CHANGED_FILES_PATH: rspec/changed_files.txt
+ RSPEC_MATCHING_TESTS_PATH: rspec/matching_tests.txt
RSPEC_LAST_RUN_RESULTS_FILE: rspec/rspec_last_run_results.txt
JUNIT_RESULT_FILE: rspec/junit_rspec.xml
JUNIT_RETRY_FILE: rspec/junit_rspec-retry.xml
diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml
index 9c06c390da0..9b8eb5571a7 100644
--- a/.gitlab/ci/rails.gitlab-ci.yml
+++ b/.gitlab/ci/rails.gitlab-ci.yml
@@ -8,11 +8,13 @@
.base-script:
script:
+ - source ./scripts/rspec_helpers.sh
# Only install knapsack after bundle install! Otherwise oddly some native
# gems could not be found under some circumstance. No idea why, hours wasted.
- run_timed_command "gem install knapsack --no-document"
- - run_timed_command "scripts/gitaly-test-spawn"
- - source ./scripts/rspec_helpers.sh
+ - echo -e "\e[0Ksection_start:`date +%s`:gitaly-test-spawn[collapsed=true]\r\e[0KStarting Gitaly"
+ - run_timed_command "scripts/gitaly-test-spawn" # Do not use 'bundle exec' here
+ - echo -e "\e[0Ksection_end:`date +%s`:gitaly-test-spawn\r\e[0K"
.minimal-rspec-tests:
variables:
@@ -966,7 +968,7 @@ rspec fail-fast:
needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets", "detect-tests"]
script:
- !reference [.base-script, script]
- - rspec_fail_fast tmp/matching_tests.txt "--tag ~quarantine"
+ - rspec_fail_fast "${RSPEC_MATCHING_TESTS_PATH}" "--tag ~quarantine"
artifacts:
expire_in: 7d
paths:
@@ -976,10 +978,10 @@ rspec foss-impact:
extends:
- .rspec-base-pg12-as-if-foss
- .rails:rules:rspec-foss-impact
- needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests as-if-foss"]
+ needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests"]
script:
- !reference [.base-script, script]
- - rspec_matched_foss_tests tmp/matching_foss_tests.txt "--tag ~quarantine"
+ - rspec_matched_foss_tests "${RSPEC_MATCHING_TESTS_PATH}" "--tag ~quarantine"
artifacts:
expire_in: 7d
paths:
diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml
index 17113b1245c..2631bae0c9a 100644
--- a/.gitlab/ci/setup.gitlab-ci.yml
+++ b/.gitlab/ci/setup.gitlab-ci.yml
@@ -110,10 +110,13 @@ generate-frontend-fixtures-mapping:
paths:
- ${FRONTEND_FIXTURES_MAPPING_PATH}
-.detect-test-base:
+detect-tests:
+ extends: .rails:rules:detect-tests
image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION}
needs: []
stage: prepare
+ variables:
+ RSPEC_TESTS_MAPPING_ENABLED: "true"
script:
- source ./scripts/utils.sh
- source ./scripts/rspec_helpers.sh
@@ -123,42 +126,23 @@ generate-frontend-fixtures-mapping:
- retrieve_frontend_fixtures_mapping
- |
if [ -n "$CI_MERGE_REQUEST_IID" ]; then
- tooling/bin/find_changes ${CHANGES_FILE};
- tooling/bin/find_tests ${CHANGES_FILE} ${MATCHED_TESTS_FILE};
- tooling/bin/find_changes ${CHANGES_FILE} ${MATCHED_TESTS_FILE} ${FRONTEND_FIXTURES_MAPPING_PATH};
- echo "Changed files: $(cat $CHANGES_FILE)";
- echo "Related rspec tests: $(cat $MATCHED_TESTS_FILE)";
+ mkdir -p $(dirname "$RSPEC_CHANGED_FILES_PATH")
+ tooling/bin/find_changes ${RSPEC_CHANGED_FILES_PATH};
+ tooling/bin/find_tests ${RSPEC_CHANGED_FILES_PATH} ${RSPEC_MATCHING_TESTS_PATH};
+ tooling/bin/find_changes ${RSPEC_CHANGED_FILES_PATH} ${RSPEC_MATCHING_TESTS_PATH} ${FRONTEND_FIXTURES_MAPPING_PATH};
+ echo "Changed files: $(cat $RSPEC_CHANGED_FILES_PATH)";
+ echo "Related rspec tests: $(cat $RSPEC_MATCHING_TESTS_PATH)";
fi
artifacts:
expire_in: 7d
paths:
- - ${CHANGES_FILE}
- - ${MATCHED_TESTS_FILE}
+ - ${RSPEC_CHANGED_FILES_PATH}
+ - ${RSPEC_MATCHING_TESTS_PATH}
- ${FRONTEND_FIXTURES_MAPPING_PATH}
-detect-tests:
- extends:
- - .detect-test-base
- - .rails:rules:detect-tests
- variables:
- RSPEC_TESTS_MAPPING_ENABLED: "true"
- CHANGES_FILE: tmp/changed_files.txt
- MATCHED_TESTS_FILE: tmp/matching_tests.txt
-
-detect-tests as-if-foss:
- extends:
- - .detect-test-base
- - .rails:rules:detect-tests
- - .as-if-foss
- variables:
- CHANGES_FILE: tmp/changed_foss_files.txt
- MATCHED_TESTS_FILE: tmp/matching_foss_tests.txt
- before_script:
- - '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb'
-
detect-previous-failed-tests:
extends:
- - .detect-test-base
+ - detect-tests
- .rails:rules:detect-previous-failed-tests
variables:
PREVIOUS_FAILED_TESTS_DIR: tmp/previous_failed_tests/
diff --git a/.gitlab/ci/static-analysis.gitlab-ci.yml b/.gitlab/ci/static-analysis.gitlab-ci.yml
index f3b4b31ad84..b2cbfedbfbd 100644
--- a/.gitlab/ci/static-analysis.gitlab-ci.yml
+++ b/.gitlab/ci/static-analysis.gitlab-ci.yml
@@ -123,7 +123,7 @@ rubocop:
if [ -z "${CI_MERGE_REQUEST_IID}" ] || [ "${RUN_ALL_RUBOCOP}" == "true" ]; then
run_timed_command "bundle exec rubocop --parallel"
else
- run_timed_command "bundle exec rubocop --parallel --force-exclusion $(cat tmp/changed_files.txt)"
+ run_timed_command "bundle exec rubocop --parallel --force-exclusion $(cat ${RSPEC_CHANGED_FILES_PATH})"
fi
qa:metadata-lint:
diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb
index 06a44b07f9f..c16b14ba152 100644
--- a/app/services/projects/destroy_service.rb
+++ b/app/services/projects/destroy_service.rb
@@ -67,9 +67,9 @@ module Projects
end
def remove_snippets
- # We're setting the hard_delete param because we dont need to perform the access checks within the service since
+ # We're setting the skip_authorization param because we dont need to perform the access checks within the service since
# the user has enough access rights to remove the project and its resources.
- response = ::Snippets::BulkDestroyService.new(current_user, project.snippets).execute(hard_delete: true)
+ response = ::Snippets::BulkDestroyService.new(current_user, project.snippets).execute(skip_authorization: true)
if response.error?
log_error("Snippet deletion failed on #{project.full_path} with the following message: #{response.message}")
diff --git a/app/services/snippets/bulk_destroy_service.rb b/app/services/snippets/bulk_destroy_service.rb
index 6eab9fb320e..9c6e1c14051 100644
--- a/app/services/snippets/bulk_destroy_service.rb
+++ b/app/services/snippets/bulk_destroy_service.rb
@@ -14,10 +14,10 @@ module Snippets
@snippets = snippets
end
- def execute(options = {})
+ def execute(skip_authorization: false)
return ServiceResponse.success(message: 'No snippets found.') if snippets.empty?
- user_can_delete_snippets! unless options[:hard_delete]
+ user_can_delete_snippets! unless skip_authorization
attempt_delete_repositories!
snippets.destroy_all # rubocop: disable Cop/DestroyAll
diff --git a/app/services/users/destroy_service.rb b/app/services/users/destroy_service.rb
index 5cd390141f0..173d030043c 100644
--- a/app/services/users/destroy_service.rb
+++ b/app/services/users/destroy_service.rb
@@ -58,7 +58,9 @@ module Users
MigrateToGhostUserService.new(user).execute(hard_delete: options[:hard_delete])
- response = Snippets::BulkDestroyService.new(current_user, user.snippets).execute(options)
+ skip_authorization = options.fetch(:hard_delete, false)
+ response = Snippets::BulkDestroyService.new(current_user, user.snippets)
+ .execute(skip_authorization: skip_authorization)
raise DestroyError, response.message if response.error?
# Rails attempts to load all related records into memory before
diff --git a/db/docs/spam_logs.yml b/db/docs/spam_logs.yml
index 35fa08ef46d..6e16b3600c8 100644
--- a/db/docs/spam_logs.yml
+++ b/db/docs/spam_logs.yml
@@ -3,7 +3,7 @@ table_name: spam_logs
classes:
- SpamLog
feature_categories:
-- authentication_and_authorization
-description: TODO
-introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/d20e75a8d80c2828336cd22897ea6868d666f8a5
+- instance_resiliency
+description: Logs users flagged by the Akismet anti-spam integration.
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/2266
milestone: '8.5'
diff --git a/doc/user/application_security/dast/index.md b/doc/user/application_security/dast/index.md
index a49dd8fd646..b8558fcedf2 100644
--- a/doc/user/application_security/dast/index.md
+++ b/doc/user/application_security/dast/index.md
@@ -1143,14 +1143,16 @@ To delete an on-demand scan:
1. In the saved scan's row select **More actions** (**{ellipsis_v}**), then select **Delete**.
1. Select **Delete** to confirm the deletion.
-### Site profile
+## Site profile
-A site profile describes the attributes of a web site to scan on demand with DAST. A site profile is
-required for an on-demand DAST scan.
+A site profile defines the attributes and configuration details of the deployed application,
+website, or API to be scanned by DAST. A site profile can be referenced in `.gitlab-ci.yml` and
+on-demand scans.
-A site profile contains the following:
+A site profile contains:
-- **Profile name**: A name you assign to the site to be scanned.
+- **Profile name**: A name you assign to the site to be scanned. While a site profile is referenced
+ in either `.gitlab-ci.yml` or an on-demand scan, it **cannot** be renamed.
- **Site type**: The type of target to be scanned, either website or API scan.
- **Target URL**: The URL that DAST runs against.
- **Excluded URLs**: A comma-separated list of URLs to exclude from the scan.
@@ -1168,7 +1170,7 @@ When an API site type is selected, a [host override](#host-override) is used to
When configured, request headers and password fields are encrypted using [`aes-256-gcm`](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard) before being stored in the database.
This data can only be read and decrypted with a valid secrets file.
-#### Site profile validation
+### Site profile validation
> - Site profile validation [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/233020) in GitLab 13.8.
> - Meta tag validation [introduced](https://gitlab.com/groups/gitlab-org/-/epics/6460) in GitLab 14.2.
@@ -1192,7 +1194,7 @@ All these methods are equivalent in functionality. Use whichever is feasible.
In [GitLab 14.2 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/324990), site profile
validation happens in a CI job using the [GitLab Runner](../../../ci/runners/index.md).
-#### Create a site profile
+### Create a site profile
To create a site profile:
@@ -1203,7 +1205,7 @@ To create a site profile:
The site profile is created.
-#### Edit a site profile
+### Edit a site profile
If a site profile is linked to a security policy, a user cannot edit the profile from this page. See
[Scan execution policies](../policies/scan-execution-policies.md)
@@ -1220,7 +1222,7 @@ To edit a site profile:
1. In the profile's row select the **More actions** (**{ellipsis_v}**) menu, then select **Edit**.
1. Edit the fields then select **Save profile**.
-#### Delete a site profile
+### Delete a site profile
If a site profile is linked to a security policy, a user cannot delete the profile from this page.
See [Scan execution policies](../policies/scan-execution-policies.md)
@@ -1234,7 +1236,7 @@ To delete a site profile:
1. In the profile's row, select the **More actions** (**{ellipsis_v}**) menu, then select **Delete**.
1. Select **Delete** to confirm the deletion.
-#### Validate a site profile
+### Validate a site profile
Validating a site is required to run an active scan.
@@ -1266,7 +1268,7 @@ To validate a site profile:
The site is validated and an active scan can run against it. A site profile's validation status is
revoked only when it's revoked manually, or its file, header, or meta tag is edited.
-#### Retry a failed validation
+### Retry a failed validation
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/322609) in GitLab 14.3.
> - [Deployed behind the `dast_failed_site_validations` flag](../../../administration/feature_flags.md), enabled by default.
@@ -1283,7 +1285,7 @@ To retry a site profile's failed validation:
1. Select the **Site Profiles** tab.
1. In the profile's row, select **Retry validation**.
-#### Revoke a site profile's validation status
+### Revoke a site profile's validation status
WARNING:
When a site profile's validation status is revoked, all site profiles that share the same URL also
@@ -1297,12 +1299,12 @@ To revoke a site profile's validation status:
The site profile's validation status is revoked.
-#### Validated site profile headers
+### Validated site profile headers
The following are code samples of how you can provide the required site profile header in your
application.
-##### Ruby on Rails example for on-demand scan
+#### Ruby on Rails example for on-demand scan
Here's how you can add a custom header in a Ruby on Rails application:
@@ -1315,7 +1317,7 @@ class DastWebsiteTargetController < ActionController::Base
end
```
-##### Django example for on-demand scan
+#### Django example for on-demand scan
Here's how you can add a
[custom header in Django](https://docs.djangoproject.com/en/2.2/ref/request-response/#setting-header-fields):
@@ -1329,7 +1331,7 @@ class DastWebsiteTargetView(View):
return response
```
-##### Node (with Express) example for on-demand scan
+#### Node (with Express) example for on-demand scan
Here's how you can add a
[custom header in Node (with Express)](https://expressjs.com/en/5x/api.html#res.append):
@@ -1341,22 +1343,26 @@ app.get('/dast-website-target', function(req, res) {
})
```
-### Scanner profile
+## Scanner profile
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/222767) in GitLab 13.4.
> - [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/225804) in GitLab 13.5: scan mode, AJAX spider, debug messages.
-A scanner profile defines the scanner settings used to run an on-demand scan:
+A scanner profile defines the configuration details of a security scanner. A scanner profile can be
+referenced in `.gitlab-ci.yml` and on-demand scans.
-- **Profile name:** A name you give the scanner profile. For example, "Spider_15".
+A scanner profile contains:
+
+- **Profile name:** A name you give the scanner profile. For example, "Spider_15". While a scanner
+ profile is referenced in either `.gitlab-ci.yml` or an on-demand scan, it **cannot** be renamed.
- **Scan mode:** A passive scan monitors all HTTP messages (requests and responses) sent to the target. An active scan attacks the target to find potential vulnerabilities.
- **Spider timeout:** The maximum number of minutes allowed for the spider to traverse the site.
- **Target timeout:** The maximum number of seconds DAST waits for the site to be available before
starting the scan.
-- **AJAX spider:** Run the AJAX spider, in addition to the traditional spider, to crawl the target site.
+- **AJAX spider:** Run the AJAX spider, in addition to the traditional spider, to crawl the target site.
- **Debug messages:** Include debug messages in the DAST console output.
-#### Create a scanner profile
+### Create a scanner profile
To create a scanner profile:
@@ -1366,7 +1372,7 @@ To create a scanner profile:
1. Complete the form. For details of each field, see [Scanner profile](#scanner-profile).
1. Select **Save profile**.
-#### Edit a scanner profile
+### Edit a scanner profile
If a scanner profile is linked to a security policy, a user cannot edit the profile from this page.
See [Scan execution policies](../policies/scan-execution-policies.md)
@@ -1381,7 +1387,7 @@ To edit a scanner profile:
1. Edit the form.
1. Select **Save profile**.
-#### Delete a scanner profile
+### Delete a scanner profile
If a scanner profile is linked to a security policy, a user cannot delete the profile from this
page. See [Scan execution policies](../policies/scan-execution-policies.md)
diff --git a/doc/user/application_security/security_dashboard/index.md b/doc/user/application_security/security_dashboard/index.md
index f3c834e06c7..5c5482c9f10 100644
--- a/doc/user/application_security/security_dashboard/index.md
+++ b/doc/user/application_security/security_dashboard/index.md
@@ -49,8 +49,11 @@ To reduce false negatives in [dependency scans](../../../user/application_securi
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/285477) in GitLab 13.11, date range slider to visualize data between given dates.
The project Security Dashboard shows the total number of vulnerabilities
-over time, with up to 365 days of historical data. Data refreshes daily at 01:15 UTC.
-It shows statistics for all vulnerabilities.
+over time, with up to 365 days of historical data. Data refresh begins daily at 01:15 UTC via a scheduled job.
+Each refresh captures a snapshot of open vulnerabilities. Data is not backported to prior days
+so vulnerabilities opened after the job has already run for the day will not be reflected in the
+counts until the following day's refresh job.
+Project Security Dashboards show statistics for all vulnerabilities with a current status of `Needs triage` or `Confirmed` .
To view total number of vulnerabilities over time:
diff --git a/doc/user/application_security/vulnerability_report/pipeline.md b/doc/user/application_security/vulnerability_report/pipeline.md
index 8d49c5adc51..6a5cfa76da0 100644
--- a/doc/user/application_security/vulnerability_report/pipeline.md
+++ b/doc/user/application_security/vulnerability_report/pipeline.md
@@ -83,11 +83,11 @@ incorporated once the pipeline finishes.
When a pipeline contains jobs that produce multiple security reports of the same type, it is possible that the same
vulnerability finding is present in multiple reports. This duplication is common when different scanners are used to
-increase coverage. The deduplication process allows you to maximize the vulnerability scanning coverage while reducing
+increase coverage, but can also exist within a single report. The deduplication process allows you to maximize the vulnerability scanning coverage while reducing
the number of findings you need to manage.
A finding is considered a duplicate of another finding when their [scan type](../terminology/index.md#scan-type-report-type),
-[location](../terminology/index.md#location-fingerprint) and
+[location](../terminology/index.md#location-fingerprint), and one or more of its
[identifiers](../../../development/integrations/secure.md#identifiers) are the same.
The scan type must match because each can have its own definition for the location of a vulnerability. For example,
@@ -95,8 +95,9 @@ static analyzers are able to locate a file path and line number, whereas a conta
name instead.
When comparing identifiers, GitLab does not compare `CWE` and `WASC` during deduplication because they are
-"type identifiers" and are used to classify groups of vulnerabilities. Including these identifiers results in
-many findings being incorrectly considered duplicates.
+"type identifiers" and are used to classify groups of vulnerabilities. Including these identifiers would result in
+many findings being incorrectly considered duplicates. Two findings are considered unique if none of their
+identifiers match.
In a set of duplicated findings, the first occurrence of a finding is kept and the remaining are skipped. Security
reports are processed in alphabetical file path order, and findings are processed sequentially in the order they
@@ -124,16 +125,17 @@ appear in a report.
- Location fingerprint: `adc83b19e793491b1c6ea0fd8b46cd9f32e592fc`
- Identifiers: CWE-798
- Deduplication result: duplicates because `CWE` identifiers are ignored.
-- Example 3: matching scan type, location and identifiers.
+- Example 3: matching scan type, location and an identifier.
- Finding
- Scan type: `container_scanning`
- Location fingerprint: `adc83b19e793491b1c6ea0fd8b46cd9f32e592fc`
- - Identifiers: CVE-2022-25510, CWE-259
+ - Identifiers: CVE-2019-12345, CVE-2022-25510, CWE-259
- Other Finding
- Scan type: `container_scanning`
- Location fingerprint: `adc83b19e793491b1c6ea0fd8b46cd9f32e592fc`
- Identifiers: CVE-2022-25510, CWE-798
- Deduplication result: duplicates because all criteria match, and type identifiers are ignored.
+ Only one identifier needs to match, in this case CVE-2022-25510.
The examples above don't include the raw location values. Each scan type defines its own
`fingerprint_data`, which is used to generate a `SHA1` hash that is used as the `location_fingerprint`.
diff --git a/lib/api/helpers/groups_helpers.rb b/lib/api/helpers/groups_helpers.rb
index 2b10eebb009..e9af50b80be 100644
--- a/lib/api/helpers/groups_helpers.rb
+++ b/lib/api/helpers/groups_helpers.rb
@@ -11,8 +11,7 @@ module API
optional :visibility, type: String,
values: Gitlab::VisibilityLevel.string_values,
desc: 'The visibility of the group'
- # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960
- optional :avatar, type: File, desc: 'Avatar image for the group' # rubocop:disable Scalability/FileUploads
+ optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for the group'
optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group'
optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users in this group to setup Two-factor authentication'
optional :two_factor_grace_period, type: Integer, desc: 'Time before Two-factor authentication is enforced'
diff --git a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb
index 16ab52349ed..43352b1bf91 100644
--- a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb
+++ b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb
@@ -32,7 +32,6 @@ module Gitlab
)
end
- relation = apply_additional_filters(relation, job_arguments: job_arguments, job_class: job_class)
next_batch_bounds = nil
relation.each_batch(of: batch_size, column: column_name) do |batch| # rubocop:disable Lint/UnreachableLoop
@@ -44,15 +43,6 @@ module Gitlab
next_batch_bounds
end
- # Deprecated
- #
- # Use `scope_to` to define additional filters on the migration job class.
- #
- # see https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#adding-additional-filters.
- def apply_additional_filters(relation, job_arguments: [], job_class: nil)
- relation
- end
-
private
def filter_batch(relation, table_name:, column_name:, job_class:, job_arguments: [])
diff --git a/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb
index ff90d2dbcac..49525479637 100644
--- a/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb
+++ b/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb
@@ -3,23 +3,9 @@
module Gitlab
module BackgroundMigration
module BatchingStrategies
- # Batching class to use for removing backfilled job artifact expire_at.
- # Batches will be scoped to records where either:
- # - expire_at is set to midnight on the 22nd of the month of the local timezone,
- # - record that has file_type = 3 (trace)
- #
- # If no more batches exist in the table, returns nil.
+ # Used to apply additional filters to the batching table, migrated to
+ # use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/96478
class RemoveBackfilledJobArtifactsExpireAtBatchingStrategy < PrimaryKeyBatchingStrategy
- EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE = <<~SQL
- EXTRACT(day FROM timezone('UTC', expire_at)) IN (21, 22, 23)
- AND EXTRACT(minute FROM timezone('UTC', expire_at)) IN (0, 30, 45)
- AND EXTRACT(second FROM timezone('UTC', expire_at)) = 0
- SQL
-
- def apply_additional_filters(relation, job_arguments: [], job_class: nil)
- relation.where(EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE)
- .or(relation.where(file_type: 3))
- end
end
end
end
diff --git a/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb
index 2fdd6f9e68c..d30263976e8 100644
--- a/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb
+++ b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb
@@ -26,13 +26,16 @@ module Gitlab
AND EXTRACT(second FROM timezone('UTC', expire_at)) = 0
SQL
+ scope_to ->(relation) {
+ relation.where(EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE)
+ .or(relation.where(file_type: 3))
+ }
+
def perform
each_sub_batch(
operation_name: :update_all
) do |sub_batch|
- sub_batch.where(EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE)
- .or(sub_batch.where(file_type: 3))
- .update_all(expire_at: nil)
+ sub_batch.update_all(expire_at: nil)
end
end
end
diff --git a/package.json b/package.json
index b3b8f66e0d0..051efa44b4a 100644
--- a/package.json
+++ b/package.json
@@ -14,7 +14,7 @@
"jest": "jest --config jest.config.js",
"jest-debug": "node --inspect-brk node_modules/.bin/jest --runInBand",
"jest:ci": "jest --config jest.config.js --ci --coverage --testSequencer ./scripts/frontend/parallel_ci_sequencer.js",
- "jest:ci:minimal": "jest --config jest.config.js --ci --coverage --findRelatedTests $(cat tmp/changed_files.txt) --passWithNoTests --testSequencer ./scripts/frontend/parallel_ci_sequencer.js",
+ "jest:ci:minimal": "jest --config jest.config.js --ci --coverage --findRelatedTests $(cat $RSPEC_MATCHING_TESTS_PATH) --passWithNoTests --testSequencer ./scripts/frontend/parallel_ci_sequencer.js",
"jest:integration": "jest --config jest.config.integration.js",
"lint:eslint": "node scripts/frontend/eslint.js",
"lint:eslint:fix": "node scripts/frontend/eslint.js --fix",
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh
index b31e3663eaa..5d7bd844c2c 100644
--- a/scripts/rspec_helpers.sh
+++ b/scripts/rspec_helpers.sh
@@ -269,7 +269,7 @@ function rspec_paralellized_job() {
debug_rspec_variables
if [[ -n $RSPEC_TESTS_MAPPING_ENABLED ]]; then
- tooling/bin/parallel_rspec --rspec_args "$(rspec_args "${rspec_opts}")" --filter "tmp/matching_tests.txt" || rspec_run_status=$?
+ tooling/bin/parallel_rspec --rspec_args "$(rspec_args "${rspec_opts}")" --filter "${RSPEC_MATCHING_TESTS_PATH}" || rspec_run_status=$?
else
tooling/bin/parallel_rspec --rspec_args "$(rspec_args "${rspec_opts}")" || rspec_run_status=$?
fi
@@ -360,9 +360,22 @@ function rspec_fail_fast() {
function rspec_matched_foss_tests() {
local test_file_count_threshold=20
local matching_tests_file=${1}
+ local foss_matching_tests_file="${matching_tests_file}-foss"
+
+ # Keep only files that exists (i.e. exclude EE speficic files)
+ cat ${matching_tests_file} | ruby -e 'puts $stdin.read.split(" ").select { |f| File.exist?(f) && f.include?("spec/") }.join(" ")' > "${foss_matching_tests_file}"
+
+ echo "Matching tests file:"
+ cat ${matching_tests_file}
+ echo -e "\n\n"
+
+ echo "FOSS matching tests file:"
+ cat ${foss_matching_tests_file}
+ echo -e "\n\n"
+
local rspec_opts=${2}
- local test_files="$(cat "${matching_tests_file}")"
- local test_file_count=$(wc -w "${matching_tests_file}" | awk {'print $1'})
+ local test_files="$(cat ${foss_matching_tests_file})"
+ local test_file_count=$(wc -w "${foss_matching_tests_file}" | awk {'print $1'})
if [[ "${test_file_count}" -gt "${test_file_count_threshold}" ]]; then
echo "This job is intentionally failed because there are more than ${test_file_count_threshold} FOSS test files matched,"
diff --git a/spec/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy_spec.rb b/spec/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy_spec.rb
index 4d8ec353a62..37fdd209622 100644
--- a/spec/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy_spec.rb
+++ b/spec/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy_spec.rb
@@ -77,25 +77,4 @@ RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchi
end
end
end
-
- context 'additional filters' do
- let(:strategy_with_filters) do
- Class.new(described_class) do
- def apply_additional_filters(relation, job_arguments:, job_class: nil)
- min_id = job_arguments.first
-
- relation.where.not(type: 'Project').where('id >= ?', min_id)
- end
- end
- end
-
- let(:batching_strategy) { strategy_with_filters.new(connection: ActiveRecord::Base.connection) }
- let!(:namespace5) { namespaces.create!(name: 'batchtest5', path: 'batch-test5', type: 'Project') }
-
- it 'applies additional filters' do
- batch_bounds = batching_strategy.next_batch(:namespaces, :id, batch_min_value: namespace4.id, batch_size: 3, job_arguments: [1])
-
- expect(batch_bounds).to eq([namespace4.id, namespace4.id])
- end
- end
end
diff --git a/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb b/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb
index f6c877c03ed..e296a46ea2f 100644
--- a/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb
+++ b/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb
@@ -2,100 +2,6 @@
require 'spec_helper'
-RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::RemoveBackfilledJobArtifactsExpireAtBatchingStrategy, '#next_batch' do # rubocop:disable Layout/LineLength
- let_it_be(:namespace) { table(:namespaces).create!(id: 1, name: 'user', path: 'user') }
- let_it_be(:project) do
- table(:projects).create!(
- id: 1,
- name: 'gitlab1',
- path: 'gitlab1',
- project_namespace_id: 1,
- namespace_id: namespace.id
- )
- end
-
- let(:batching_strategy) { described_class.new(connection: Ci::ApplicationRecord.connection) }
- let(:job_artifact) { table(:ci_job_artifacts, database: :ci) }
-
- # job artifacts expiring at midnight in various timezones
- let!(:ci_job_artifact_1) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) }
- let!(:ci_job_artifact_2) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-21 01:30:00.000')) }
- let!(:ci_job_artifact_3) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:00:00.000')) }
- let!(:ci_job_artifact_4) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:30:00.000')) }
- let!(:ci_job_artifact_5) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:00:00.000')) }
- let!(:ci_job_artifact_6) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:30:00.000')) }
- let!(:ci_job_artifact_7) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-23 06:45:00.000')) }
- # out ot scope job artifacts
- let!(:ci_job_artifact_8) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.001')) }
- let!(:ci_job_artifact_9) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-19 12:00:00.000')) }
- # job artifacts of trace type (file_type: 3)
- let!(:ci_job_artifact_10) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) }
- let!(:ci_job_artifact_11) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) }
- # out ot scope job artifacts
- let!(:ci_job_artifact_12) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-24 23:30:00.000')) }
- let!(:ci_job_artifact_13) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:30:00.000')) }
- # job artifacts of trace type (file_type: 3)
- let!(:ci_job_artifact_14) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) }
- let!(:ci_job_artifact_15) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) }
-
+RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::RemoveBackfilledJobArtifactsExpireAtBatchingStrategy do # rubocop:disable Layout/LineLength
it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchingStrategy }
-
- context 'when starting on the first batch' do
- it 'returns the bounds of the next batch' do
- batch_bounds = batching_strategy.next_batch(
- :ci_job_artifacts,
- :id,
- batch_min_value: ci_job_artifact_1.id,
- batch_size: 5,
- job_arguments: []
- )
- expect(batch_bounds).to eq([ci_job_artifact_1.id, ci_job_artifact_5.id])
- end
- end
-
- context 'when the range includes out of scope records' do
- it 'returns the bounds of the next batch, skipping records outside the scope' do
- batch_bounds = batching_strategy.next_batch(
- :ci_job_artifacts,
- :id,
- batch_min_value: ci_job_artifact_1.id,
- batch_size: 10,
- job_arguments: []
- )
- expect(batch_bounds).to eq([ci_job_artifact_1.id, ci_job_artifact_14.id])
- end
- end
-
- context 'when the range begins on out of scope records' do
- it 'returns the bounds of the next batch, skipping records outside the scope' do
- batch_bounds = batching_strategy.next_batch(
- :ci_job_artifacts,
- :id,
- batch_min_value: ci_job_artifact_8.id,
- batch_size: 3,
- job_arguments: []
- )
- expect(batch_bounds).to eq([ci_job_artifact_10.id, ci_job_artifact_14.id])
- end
- end
-
- context 'when no additional batch remain' do
- it 'returns nil' do
- batch_bounds = batching_strategy.next_batch(
- :ci_job_artifacts,
- :id,
- batch_min_value: ci_job_artifact_15.id + 1,
- batch_size: 10,
- job_arguments: []
- )
- expect(batch_bounds).to be_nil
- end
- end
-
- private
-
- def create_job_artifact(file_type:, expire_at:)
- job = table(:ci_builds, database: :ci).create!
- job_artifact.create!(job_id: job.id, expire_at: expire_at, project_id: project.id, file_type: file_type)
- end
end
diff --git a/spec/requests/api/groups_spec.rb b/spec/requests/api/groups_spec.rb
index 74be97fca2a..031e1a6c918 100644
--- a/spec/requests/api/groups_spec.rb
+++ b/spec/requests/api/groups_spec.rb
@@ -5,6 +5,7 @@ require 'spec_helper'
RSpec.describe API::Groups do
include GroupAPIHelpers
include UploadHelpers
+ include WorkhorseHelpers
let_it_be(:user1) { create(:user, can_create_group: false) }
let_it_be(:user2) { create(:user) }
@@ -872,21 +873,31 @@ RSpec.describe API::Groups do
group_param = {
avatar: fixture_file_upload(file_path)
}
- put api("/groups/#{group1.id}", user1), params: group_param
+ workhorse_form_with_file(
+ api("/groups/#{group1.id}", user1),
+ method: :put,
+ file_key: :avatar,
+ params: group_param
+ )
end
end
context 'when authenticated as the group owner' do
it 'updates the group' do
- put api("/groups/#{group1.id}", user1), params: {
- name: new_group_name,
- request_access_enabled: true,
- project_creation_level: "noone",
- subgroup_creation_level: "maintainer",
- default_branch_protection: ::Gitlab::Access::MAINTAINER_PROJECT_ACCESS,
- prevent_sharing_groups_outside_hierarchy: true,
- avatar: fixture_file_upload(file_path)
- }
+ workhorse_form_with_file(
+ api("/groups/#{group1.id}", user1),
+ method: :put,
+ file_key: :avatar,
+ params: {
+ name: new_group_name,
+ request_access_enabled: true,
+ project_creation_level: "noone",
+ subgroup_creation_level: "maintainer",
+ default_branch_protection: ::Gitlab::Access::MAINTAINER_PROJECT_ACCESS,
+ prevent_sharing_groups_outside_hierarchy: true,
+ avatar: fixture_file_upload(file_path)
+ }
+ )
expect(response).to have_gitlab_http_status(:ok)
expect(json_response['name']).to eq(new_group_name)
@@ -1787,7 +1798,12 @@ RSpec.describe API::Groups do
attrs[:avatar] = fixture_file_upload(file_path)
end
- post api("/groups", user3), params: params
+ workhorse_form_with_file(
+ api('/groups', user3),
+ method: :post,
+ file_key: :avatar,
+ params: params
+ )
end
end
diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb
index 955384e518c..e4ccd83e3ef 100644
--- a/spec/services/projects/destroy_service_spec.rb
+++ b/spec/services/projects/destroy_service_spec.rb
@@ -423,11 +423,11 @@ RSpec.describe Projects::DestroyService, :aggregate_failures, :event_store_publi
destroy_project(project, user)
end
- it 'calls the bulk snippet destroy service with the hard_delete param set to true' do
+ it 'calls the bulk snippet destroy service with the skip_authorization param set to true' do
expect(project.snippets.count).to eq 2
expect_next_instance_of(Snippets::BulkDestroyService, user, project.snippets) do |instance|
- expect(instance).to receive(:execute).with(hard_delete: true).and_call_original
+ expect(instance).to receive(:execute).with(skip_authorization: true).and_call_original
end
expect do
@@ -485,9 +485,11 @@ RSpec.describe Projects::DestroyService, :aggregate_failures, :event_store_publi
let!(:project_bot) { create(:user, :project_bot).tap { |user| project.add_maintainer(user) } }
it 'deletes bot user as well' do
- expect do
- destroy_project(project, user)
- end.to change { User.find_by(id: project_bot.id) }.to(nil)
+ expect_next_instance_of(Users::DestroyService, user) do |instance|
+ expect(instance).to receive(:execute).with(project_bot, skip_authorization: true).and_call_original
+ end
+
+ destroy_project(project, user)
end
end
diff --git a/spec/services/snippets/bulk_destroy_service_spec.rb b/spec/services/snippets/bulk_destroy_service_spec.rb
index 2d2bdd116d1..4142aa349e4 100644
--- a/spec/services/snippets/bulk_destroy_service_spec.rb
+++ b/spec/services/snippets/bulk_destroy_service_spec.rb
@@ -71,8 +71,8 @@ RSpec.describe Snippets::BulkDestroyService do
let(:error_message) { "You don't have access to delete these snippets." }
end
- context 'when hard_delete option is passed' do
- subject { described_class.new(service_user, snippets).execute(hard_delete: true) }
+ context 'when skip_authorization option is passed' do
+ subject { described_class.new(service_user, snippets).execute(skip_authorization: true) }
it 'returns a ServiceResponse success response' do
expect(subject).to be_success
diff --git a/spec/services/users/destroy_service_spec.rb b/spec/services/users/destroy_service_spec.rb
index 26e25cf6438..127ddeab3eb 100644
--- a/spec/services/users/destroy_service_spec.rb
+++ b/spec/services/users/destroy_service_spec.rb
@@ -73,7 +73,7 @@ RSpec.describe Users::DestroyService do
allow(user).to receive(:personal_projects).and_return([])
expect_next_instance_of(Snippets::BulkDestroyService) do |bulk_destroy_service|
- expect(bulk_destroy_service).to receive(:execute).with({ hard_delete: true }).and_call_original
+ expect(bulk_destroy_service).to receive(:execute).with({ skip_authorization: true }).and_call_original
end
service.execute(user, { hard_delete: true })
diff --git a/workhorse/internal/upstream/routes.go b/workhorse/internal/upstream/routes.go
index 288871d558d..057d3e7470c 100644
--- a/workhorse/internal/upstream/routes.go
+++ b/workhorse/internal/upstream/routes.go
@@ -329,6 +329,10 @@ func configureRoutes(u *upstream) {
u.route("POST", apiPattern+`v4/projects\z`, tempfileMultipartProxy),
u.route("PUT", apiProjectPattern+`\z`, tempfileMultipartProxy),
+ // Group Avatar
+ u.route("POST", apiPattern+`v4/groups\z`, tempfileMultipartProxy),
+ u.route("PUT", apiPattern+`v4/groups/[^/]+\z`, tempfileMultipartProxy),
+
// Explicitly proxy API requests
u.route("", apiPattern, proxy),
u.route("", ciAPIPattern, proxy),
diff --git a/workhorse/upload_test.go b/workhorse/upload_test.go
index fd4844220b9..81df444d158 100644
--- a/workhorse/upload_test.go
+++ b/workhorse/upload_test.go
@@ -135,6 +135,9 @@ func TestAcceleratedUpload(t *testing.T) {
{"POST", `/api/graphql`, false},
{"POST", `/api/v4/topics`, false},
{"PUT", `/api/v4/topics`, false},
+ {"POST", `/api/v4/groups`, false},
+ {"PUT", `/api/v4/groups/5`, false},
+ {"PUT", `/api/v4/groups/group%2Fsubgroup`, false},
{"PUT", "/api/v4/projects/9001/packages/nuget/v1/files", true},
{"PUT", "/api/v4/projects/group%2Fproject/packages/nuget/v1/files", true},
{"PUT", "/api/v4/projects/group%2Fsubgroup%2Fproject/packages/nuget/v1/files", true},