diff options
50 files changed, 1205 insertions, 627 deletions
diff --git a/.gitlab/issue_templates/Feature Flag Roll Out.md b/.gitlab/issue_templates/Feature Flag Roll Out.md index db99665e481..1b3d82cf522 100644 --- a/.gitlab/issue_templates/Feature Flag Roll Out.md +++ b/.gitlab/issue_templates/Feature Flag Roll Out.md @@ -62,8 +62,7 @@ _Consider adding links to check for Sentry errors, Production logs for 5xx, 302s - Ensure that the feature MRs have been deployed to non-production environments. - [ ] `/chatops run auto_deploy status <merge-commit-of-your-feature>` - [ ] Enable the feature globally on non-production environments. - - [ ] `/chatops run feature set <feature-flag-name> true --dev` - - [ ] `/chatops run feature set <feature-flag-name> true --staging` + - [ ] `/chatops run feature set <feature-flag-name> true --dev --staging` - [ ] Verify that the feature works as expected. Posting the QA result in this issue is preferable. The best environment to validate the feature in is [staging-canary](https://about.gitlab.com/handbook/engineering/infrastructure/environments/#staging-canary) as this is the first environment deployed to. Note you will need to make sure you are configured to use canary as outlined [here](https://about.gitlab.com/handbook/engineering/infrastructure/environments/canary-stage/) @@ -121,9 +120,7 @@ To do so, follow these steps: the feature can be officially announced in a release blog post. - [ ] `/chatops run release check <merge-request-url> <milestone>` - [ ] Consider cleaning up the feature flag from all environments by running these chatops command in `#production` channel. Otherwise these settings may override the default enabled. - - [ ] `/chatops run feature delete <feature-flag-name> --dev` - - [ ] `/chatops run feature delete <feature-flag-name> --staging` - - [ ] `/chatops run feature delete <feature-flag-name>` + - [ ] `/chatops run feature delete <feature-flag-name> --dev --staging --production` - [ ] Close [the feature issue](ISSUE LINK) to indicate the feature will be released in the current milestone. - [ ] Set the next milestone to this rollout issue for scheduling [the flag removal](#release-the-feature). - [ ] (Optional) You can [create a separate issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Feature%20Flag%20Cleanup) for scheduling the steps below to [Release the feature](#release-the-feature). @@ -159,9 +156,7 @@ You can either [create a follow-up issue for Feature Flag Cleanup](https://gitla - [ ] `/chatops run release check <merge-request-url> <milestone>` - [ ] Close [the feature issue](ISSUE LINK) to indicate the feature will be released in the current milestone. - [ ] If not already done, clean up the feature flag from all environments by running these chatops command in `#production` channel: - - [ ] `/chatops run feature delete <feature-flag-name> --dev` - - [ ] `/chatops run feature delete <feature-flag-name> --staging` - - [ ] `/chatops run feature delete <feature-flag-name>` + - [ ] `/chatops run feature delete <feature-flag-name> --dev --staging --production` - [ ] Close this rollout issue. ## Rollback Steps diff --git a/app/finders/packages/pypi/packages_finder.rb b/app/finders/packages/pypi/packages_finder.rb index 47cfb59944b..17138134eb3 100644 --- a/app/finders/packages/pypi/packages_finder.rb +++ b/app/finders/packages/pypi/packages_finder.rb @@ -4,6 +4,8 @@ module Packages module Pypi class PackagesFinder < ::Packages::GroupOrProjectPackageFinder def execute + return packages unless @params[:package_name] + packages.with_normalized_pypi_name(@params[:package_name]) end diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb index 7744e578df5..90a1bb4bc69 100644 --- a/app/models/packages/package.rb +++ b/app/models/packages/package.rb @@ -103,7 +103,15 @@ class Packages::Package < ApplicationRecord scope :for_projects, ->(project_ids) { where(project_id: project_ids) } scope :with_name, ->(name) { where(name: name) } scope :with_name_like, ->(name) { where(arel_table[:name].matches(name)) } - scope :with_normalized_pypi_name, ->(name) { where("LOWER(regexp_replace(name, '[-_.]+', '-', 'g')) = ?", name.downcase) } + + scope :with_normalized_pypi_name, ->(name) do + where( + "LOWER(regexp_replace(name, ?, '-', 'g')) = ?", + Gitlab::Regex::Packages::PYPI_NORMALIZED_NAME_REGEX_STRING, + name.downcase + ) + end + scope :search_by_name, ->(query) { fuzzy_search(query, [:name], use_minimum_char_limit: false) } scope :with_version, ->(version) { where(version: version) } scope :without_version_like, -> (version) { where.not(arel_table[:version].matches(version)) } @@ -315,6 +323,13 @@ class Packages::Package < ApplicationRecord ::Packages::MarkPackageFilesForDestructionWorker.perform_async(id) end + # As defined in PEP 503 https://peps.python.org/pep-0503/#normalized-names + def normalized_pypi_name + return name unless pypi? + + name.gsub(/#{Gitlab::Regex::Packages::PYPI_NORMALIZED_NAME_REGEX_STRING}/, '-').downcase + end + private def composer_tag_version? diff --git a/app/presenters/packages/pypi/package_presenter.rb b/app/presenters/packages/pypi/package_presenter.rb deleted file mode 100644 index a779ce41cf9..00000000000 --- a/app/presenters/packages/pypi/package_presenter.rb +++ /dev/null @@ -1,96 +0,0 @@ -# frozen_string_literal: true - -# Display package version data acording to PyPI -# Simple API: https://warehouse.pypa.io/api-reference/legacy/#simple-project-api -module Packages - module Pypi - class PackagePresenter - include API::Helpers::RelatedResourcesHelpers - - def initialize(packages, project_or_group) - @packages = packages - @project_or_group = project_or_group - end - - # Returns the HTML body for PyPI simple API. - # Basically a list of package download links for a specific - # package - def body - <<-HTML - <!DOCTYPE html> - <html> - <head> - <title>Links for #{escape(name)}</title> - </head> - <body> - <h1>Links for #{escape(name)}</h1> - #{links} - </body> - </html> - HTML - end - - private - - def links - refs = [] - - @packages.map do |package| - package_files = package.installable_package_files - - package_files.each do |file| - url = build_pypi_package_path(file) - - refs << package_link(url, package.pypi_metadatum.required_python, file.file_name) - end - end - - refs.join - end - - def package_link(url, required_python, filename) - "<a href=\"#{url}\" data-requires-python=\"#{escape(required_python)}\">#{filename}</a><br>" - end - - def build_pypi_package_path(file) - params = { - id: @project_or_group.id, - sha256: file.file_sha256, - file_identifier: file.file_name - } - - if project? - expose_url( - api_v4_projects_packages_pypi_files_file_identifier_path( - params, true - ) - ) + "#sha256=#{file.file_sha256}" - elsif group? - expose_url( - api_v4_groups___packages_pypi_files_file_identifier_path( - params, true - ) - ) + "#sha256=#{file.file_sha256}" - else - '' - end - end - - def name - @packages.first.name - end - - def escape(str) - ERB::Util.html_escape(str) - end - - def project? - @project_or_group.is_a?(::Project) - end - - def group? - @project_or_group.is_a?(::Group) - end - end - end -end diff --git a/app/presenters/packages/pypi/simple_index_presenter.rb b/app/presenters/packages/pypi/simple_index_presenter.rb new file mode 100644 index 00000000000..ffe4eeb9585 --- /dev/null +++ b/app/presenters/packages/pypi/simple_index_presenter.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# Display package repository index acording to PyPI +# Simple API: https://peps.python.org/pep-0503/ +module Packages + module Pypi + class SimpleIndexPresenter < SimplePresenterBase + private + + def links + refs = [] + + available_packages.each_batch do |batch| + batch.each do |package| + url = build_pypi_package_path(package) + + refs << package_link(url, package.pypi_metadatum.required_python, package.name) + end + end + + refs.join + end + + def build_pypi_package_path(package) + params = { + id: @project_or_group.id, + package_name: package.normalized_pypi_name + } + + if project? + expose_url( + api_v4_projects_packages_pypi_simple_package_name_path( + params, true + ) + ) + elsif group? + expose_url( + api_v4_groups___packages_pypi_simple_package_name_path( + params, true + ) + ) + end + end + + def body_name + @project_or_group.name + end + end + end +end diff --git a/app/presenters/packages/pypi/simple_package_versions_presenter.rb b/app/presenters/packages/pypi/simple_package_versions_presenter.rb new file mode 100644 index 00000000000..0baa0714463 --- /dev/null +++ b/app/presenters/packages/pypi/simple_package_versions_presenter.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +# Display package version data acording to PyPI +# Simple API: https://warehouse.pypa.io/api-reference/legacy/#simple-project-api +# Generates the HTML body for PyPI simple API. +# Basically a list of package download links for a specific +# package +module Packages + module Pypi + class SimplePackageVersionsPresenter < SimplePresenterBase + private + + def links + refs = [] + + available_packages.each_batch do |batch| + batch.each do |package| + package_files = package.installable_package_files + + package_files.each do |file| + url = build_pypi_package_file_path(file) + + refs << package_link(url, package.pypi_metadatum.required_python, file.file_name) + end + end + end + + refs.join + end + + def build_pypi_package_file_path(file) + params = { + id: @project_or_group.id, + sha256: file.file_sha256, + file_identifier: file.file_name + } + + if project? + expose_url( + api_v4_projects_packages_pypi_files_file_identifier_path( + params, true + ) + ) + "#sha256=#{file.file_sha256}" + elsif group? + expose_url( + api_v4_groups___packages_pypi_files_file_identifier_path( + params, true + ) + ) + "#sha256=#{file.file_sha256}" + end + end + + def body_name + @packages.first.name + end + end + end +end diff --git a/app/presenters/packages/pypi/simple_presenter_base.rb b/app/presenters/packages/pypi/simple_presenter_base.rb new file mode 100644 index 00000000000..a459319539c --- /dev/null +++ b/app/presenters/packages/pypi/simple_presenter_base.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +# Display package version data acording to PyPI +# Simple API: https://warehouse.pypa.io/api-reference/legacy/#simple-project-api +module Packages + module Pypi + class SimplePresenterBase + include API::Helpers::RelatedResourcesHelpers + + def initialize(packages, project_or_group) + @packages = packages + @project_or_group = project_or_group + end + + def body + <<-HTML + <!DOCTYPE html> + <html> + <head> + <title>Links for #{escape(body_name)}</title> + </head> + <body> + <h1>Links for #{escape(body_name)}</h1> + #{links} + </body> + </html> + HTML + end + + private + + def package_link(url, required_python, name) + "<a href=\"#{url}\" data-requires-python=\"#{escape(required_python)}\">#{name}</a>" + end + + def escape(str) + ERB::Util.html_escape(str) + end + + def project? + @project_or_group.is_a?(::Project) + end + + def group? + @project_or_group.is_a?(::Group) + end + + def available_packages + @packages.not_pending_destruction + end + end + end +end diff --git a/app/services/terraform/states/destroy_service.rb b/app/services/terraform/states/destroy_service.rb index 76fd0f9433d..728533a60ed 100644 --- a/app/services/terraform/states/destroy_service.rb +++ b/app/services/terraform/states/destroy_service.rb @@ -11,9 +11,7 @@ module Terraform return unless state.deleted_at? state.versions.each_batch(column: :version) do |batch| - batch.each do |version| - version.file.remove! - end + process_batch(batch) end state.destroy! @@ -22,6 +20,15 @@ module Terraform private attr_reader :state + + # Overridden in EE + def process_batch(batch) + batch.each do |version| + version.file.remove! + end + end end end end + +Terraform::States::DestroyService.prepend_mod diff --git a/app/views/admin/application_settings/_visibility_and_access.html.haml b/app/views/admin/application_settings/_visibility_and_access.html.haml index cc62ca354d1..96dcd7e1111 100644 --- a/app/views/admin/application_settings/_visibility_and_access.html.haml +++ b/app/views/admin/application_settings/_visibility_and_access.html.haml @@ -4,8 +4,7 @@ %fieldset = render 'shared/project_creation_levels', f: f, method: :default_project_creation, legend: s_('ProjectCreationLevel|Default project creation protection') = render_if_exists 'admin/application_settings/default_project_deletion_protection_setting', form: f - = render_if_exists 'admin/application_settings/default_delayed_project_deletion_setting', form: f - = render_if_exists 'admin/application_settings/default_project_deletion_adjourned_period_setting', form: f + = render_if_exists 'admin/application_settings/deletion_protection_settings', form: f .form-group.visibility-level-setting = f.label :default_project_visibility, class: 'label-bold' = render('shared/visibility_radios', model_method: :default_project_visibility, form: f, selected_level: @application_setting.default_project_visibility, form_model: Project.new) diff --git a/app/views/projects/_errors.html.haml b/app/views/projects/_errors.html.haml index 2dba22d3be6..5982aaf3622 100644 --- a/app/views/projects/_errors.html.haml +++ b/app/views/projects/_errors.html.haml @@ -1 +1 @@ -= form_errors(@project) += form_errors(@project, pajamas_alert: true) diff --git a/db/migrate/20220520122755_unlock_delayed_project_removal.rb b/db/migrate/20220520122755_unlock_delayed_project_removal.rb new file mode 100644 index 00000000000..dc76a95118b --- /dev/null +++ b/db/migrate/20220520122755_unlock_delayed_project_removal.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class UnlockDelayedProjectRemoval < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + class ApplicationSetting < MigrationRecord + self.table_name = 'application_settings' + end + + # As part of https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86568 the + # lock_delayed_project_removal setting is updated for the first time. No up + # migration is needed because the column existsted. However a down migration + # is needed to disable the settting because users would have no way to edit it + # and would have the cascading setting permanently locked on groups. + + def up + # no-op + end + + def down + ApplicationSetting.reset_column_information + + ApplicationSetting.update_all(lock_delayed_project_removal: false) + end +end diff --git a/db/post_migrate/20220610223040_add_index_on_available_pypi_packages.rb b/db/post_migrate/20220610223040_add_index_on_available_pypi_packages.rb new file mode 100644 index 00000000000..891272c8a51 --- /dev/null +++ b/db/post_migrate/20220610223040_add_index_on_available_pypi_packages.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddIndexOnAvailablePypiPackages < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + INDEX_NAME = 'index_packages_on_available_pypi_packages' + + def up + add_concurrent_index :packages_packages, + [:project_id, :id], + where: "status IN (0,1) AND package_type = 5 AND version IS NOT NULL", + name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :packages_packages, INDEX_NAME + end +end diff --git a/db/schema_migrations/20220520122755 b/db/schema_migrations/20220520122755 new file mode 100644 index 00000000000..5f627846e88 --- /dev/null +++ b/db/schema_migrations/20220520122755 @@ -0,0 +1 @@ +aac34066922d445a3dfe0c527b500000e0296cae84e38e9e31f721cd4b213d17
\ No newline at end of file diff --git a/db/schema_migrations/20220610223040 b/db/schema_migrations/20220610223040 new file mode 100644 index 00000000000..8aee145e29d --- /dev/null +++ b/db/schema_migrations/20220610223040 @@ -0,0 +1 @@ +1b670d02f021f81e3541a789915244238c78e234e6eb0e9a6476328f92686667
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 6973eed89d4..93a0598a961 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -28780,6 +28780,8 @@ CREATE INDEX index_packages_maven_metadata_on_path ON packages_maven_metadata US CREATE INDEX index_packages_nuget_dl_metadata_on_dependency_link_id ON packages_nuget_dependency_link_metadata USING btree (dependency_link_id); +CREATE INDEX index_packages_on_available_pypi_packages ON packages_packages USING btree (project_id, id) WHERE ((status = ANY (ARRAY[0, 1])) AND (package_type = 5) AND (version IS NOT NULL)); + CREATE INDEX index_packages_package_file_build_infos_on_package_file_id ON packages_package_file_build_infos USING btree (package_file_id); CREATE INDEX index_packages_package_file_build_infos_on_pipeline_id ON packages_package_file_build_infos USING btree (pipeline_id); diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index 43734b66232..08b27d100dc 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -61,6 +61,7 @@ allow the following for Gitaly Cluster to function properly: | GitLab | Praefect load balancer | `2305` | `3305` | | Praefect load balancer | Praefect | `2305` | `3305` | | Praefect | Gitaly | `8075` | `9999` | +| Praefect | GitLab (internal API) | `80` | `443` | | Gitaly | GitLab (internal API) | `80` | `443` | | Gitaly | Praefect load balancer | `2305` | `3305` | | Gitaly | Praefect | `2305` | `3305` | diff --git a/doc/administration/troubleshooting/elasticsearch.md b/doc/administration/troubleshooting/elasticsearch.md index 52de2f696dd..bfea8d2587a 100644 --- a/doc/administration/troubleshooting/elasticsearch.md +++ b/doc/administration/troubleshooting/elasticsearch.md @@ -219,7 +219,7 @@ The output from the last command is the key here. If it shows: If all the settings look correct and it is still not using Elasticsearch for the search function, it is best to escalate to GitLab support. This could be a bug/issue. -Moving past that, it is best to attempt the same [search via the Rails console](../../integration/elasticsearch.md#i-indexed-all-the-repositories-but-i-cant-get-any-hits-for-my-search-term-in-the-ui) +Moving past that, it is best to attempt the same [search via the Rails console](../../integration/advanced_search/elasticsearch_troubleshooting.md#i-indexed-all-the-repositories-but-i-cant-get-any-hits-for-my-search-term-in-the-ui) or the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html), and compare the results from what you see in GitLab. @@ -257,7 +257,7 @@ during the indexing of projects. If errors do occur, they stem from either the i - On the GitLab side. You need to rectify those. If they are not something you are familiar with, contact GitLab support for guidance. -- Within the Elasticsearch instance itself. See if the error is [documented and has a fix](../../integration/elasticsearch.md#troubleshooting). If not, speak with your Elasticsearch administrator. +- Within the Elasticsearch instance itself. See if the error is [documented and has a fix](../../integration/advanced_search/elasticsearch_troubleshooting.md). If not, speak with your Elasticsearch administrator. If the indexing process does not present errors, check the status of the indexed projects. You can do this via the following Rake tasks: @@ -375,7 +375,7 @@ If you still encounter issues after retrying the migration, reach out to GitLab ## Common issues -All common issues [should be documented](../../integration/elasticsearch.md#troubleshooting). If not, +All common issues [should be documented](../../integration/advanced_search/elasticsearch_troubleshooting.md). If not, feel free to update that page with issues you encounter and solutions. ## Replication diff --git a/doc/api/groups.md b/doc/api/groups.md index 7606ee5f2f6..8372d6deddd 100644 --- a/doc/api/groups.md +++ b/doc/api/groups.md @@ -1060,7 +1060,7 @@ Only available to group owners and administrators. This endpoint either: - Removes group, and queues a background job to delete all projects in the group as well. -- Since [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/33257), on [Premium](https://about.gitlab.com/pricing/) or higher tiers, marks a group for deletion. The deletion happens 7 days later by default, but this can be changed in the [instance settings](../user/admin_area/settings/visibility_and_access_controls.md#default-deletion-delay). +- Since [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/33257), on [Premium](https://about.gitlab.com/pricing/) or higher tiers, marks a group for deletion. The deletion happens 7 days later by default, but this can be changed in the [instance settings](../user/admin_area/settings/visibility_and_access_controls.md#deletion-protection). ```plaintext DELETE /groups/:id diff --git a/doc/api/packages/pypi.md b/doc/api/packages/pypi.md index 5ed162a3d05..3e23ded61f4 100644 --- a/doc/api/packages/pypi.md +++ b/doc/api/packages/pypi.md @@ -50,6 +50,47 @@ curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v This writes the downloaded file to `my.pypi.package-0.0.1.tar.gz` in the current directory. +## Group-level simple API index + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327595) in GitLab 15.1. + +Returns a list of packages in the group as an HTML file: + +```plaintext +GET groups/:id/-/packages/pypi/simple +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | string | yes | The ID or full path of the group. | + +```shell +curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/-/packages/pypi/simple" +``` + +Example response: + +```html +<!DOCTYPE html> +<html> + <head> + <title>Links for Group</title> + </head> + <body> + <h1>Links for Group</h1> + <a href="https://gitlab.example.com/api/v4/groups/1/-/packages/pypi/simple/my-pypi-package" data-requires-python="">my.pypi.package</a><br><a href="https://gitlab.example.com/api/v4/groups/1/-/packages/pypi/simple/package-2" data-requires-python="3.8">package_2</a><br> + </body> +</html> +``` + +To write the output to a file: + +```shell +curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/-/packages/pypi/simple" >> simple_index.html +``` + +This writes the downloaded file to `simple_index.html` in the current directory. + ## Group level simple API entry point > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225545) in GitLab 13.12. @@ -57,7 +98,7 @@ directory. Returns the package descriptor as an HTML file: ```plaintext -GET groups/:id/packages/pypi/simple/:package_name +GET groups/:id/-/packages/pypi/simple/:package_name ``` | Attribute | Type | Required | Description | @@ -66,7 +107,7 @@ GET groups/:id/packages/pypi/simple/:package_name | `package_name` | string | yes | The name of the package. | ```shell -curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/packages/pypi/simple/my.pypi.package" +curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/-/packages/pypi/simple/my.pypi.package" ``` Example response: @@ -79,7 +120,7 @@ Example response: </head> <body> <h1>Links for my.pypi.package</h1> - <a href="https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1-py3-none-any.whl#sha256=5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff" data-requires-python=">=3.6">my.pypi.package-0.0.1-py3-none-any.whl</a><br><a href="https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/9s9w01b0bcd52b709ec052084e33a5517ffca96f7728ddd9f8866a30cdf76f2/my.pypi.package-0.0.1.tar.gz#sha256=9s9w011b0bcd52b709ec052084e33a5517ffca96f7728ddd9f8866a30cdf76f2" data-requires-python=">=3.6">my.pypi.package-0.0.1.tar.gz</a><br> + <a href="https://gitlab.example.com/api/v4/groups/1/-/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1-py3-none-any.whl#sha256=5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff" data-requires-python=">=3.6">my.pypi.package-0.0.1-py3-none-any.whl</a><br><a href="https://gitlab.example.com/api/v4/groups/1/-/packages/pypi/files/9s9w01b0bcd52b709ec052084e33a5517ffca96f7728ddd9f8866a30cdf76f2/my.pypi.package-0.0.1.tar.gz#sha256=9s9w011b0bcd52b709ec052084e33a5517ffca96f7728ddd9f8866a30cdf76f2" data-requires-python=">=3.6">my.pypi.package-0.0.1.tar.gz</a><br> </body> </html> ``` @@ -87,7 +128,7 @@ Example response: To write the output to a file: ```shell -curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/packages/pypi/simple/my.pypi.package" >> simple.html +curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/-/packages/pypi/simple/my.pypi.package" >> simple.html ``` This writes the downloaded file to `simple.html` in the current directory. @@ -122,6 +163,47 @@ curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v This writes the downloaded file to `my.pypi.package-0.0.1.tar.gz` in the current directory. +## Project-level simple API index + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327595) in GitLab 15.1. + +Returns a list of packages in the project as an HTML file: + +```plaintext +GET projects/:id/packages/pypi/simple +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | string | yes | The ID or full path of the project. | + +```shell +curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/projects/1/packages/pypi/simple" +``` + +Example response: + +```html +<!DOCTYPE html> +<html> + <head> + <title>Links for Project</title> + </head> + <body> + <h1>Links for Project</h1> + <a href="https://gitlab.example.com/api/v4/projects/1/packages/pypi/simple/my-pypi-package" data-requires-python="">my.pypi.package</a><br><a href="https://gitlab.example.com/api/v4/projects/1/packages/pypi/simple/package-2" data-requires-python="3.8">package_2</a><br> + </body> +</html> +``` + +To write the output to a file: + +```shell +curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/projects/1/packages/pypi/simple" >> simple_index.html +``` + +This writes the downloaded file to `simple_index.html` in the current directory. + ## Project-level simple API entry point > Introduced in GitLab 12.10. diff --git a/doc/api/projects.md b/doc/api/projects.md index 64a2e0d801b..3cb75c39980 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -2164,7 +2164,7 @@ This endpoint: administrators can [configure](../user/group/index.md#enable-delayed-project-deletion) projects within a group to be deleted after a delayed period. When enabled, actual deletion happens after the number of days specified in the - [default deletion delay](../user/admin_area/settings/visibility_and_access_controls.md#default-deletion-delay). + [default deletion delay](../user/admin_area/settings/visibility_and_access_controls.md#deletion-protection). WARNING: The default behavior of [Delayed Project deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) diff --git a/doc/api/repositories.md b/doc/api/repositories.md index a0f13bcce9e..1a1eee175f7 100644 --- a/doc/api/repositories.md +++ b/doc/api/repositories.md @@ -17,7 +17,7 @@ This command provides essentially the same functionality as the `git ls-tree` co WARNING: This endpoint is changing to keyset-based pagination. Iterating pages of results with a number (`?page=2`) [is deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67509). -Support for iterating with a number will become unsupported in GitLab 15.0. Use +Support for iterating with a number became supported in GitLab 15.0. Use the new [keyset pagination system](index.md#keyset-based-pagination) instead. ```plaintext diff --git a/doc/api/search.md b/doc/api/search.md index 8c681904e98..688dc8195cf 100644 --- a/doc/api/search.md +++ b/doc/api/search.md @@ -27,13 +27,13 @@ GET /search | `order_by` | string | no | Allowed values are `created_at` only. If this is not set, the results are either sorted by `created_at` in descending order for basic search, or by the most relevant documents when using advanced search.| | `sort` | string | no | Allowed values are `asc` or `desc` only. If this is not set, the results are either sorted by `created_at` in descending order for basic search, or by the most relevant documents when using advanced search.| -Search the expression in the specified scope. These scopes are supported: projects, issues, merge_requests, milestones, snippet_titles, users. +Search the expression in the specified scope. These scopes are supported: `projects`, `issues`, `merge_requests`, `milestones`, `snippet_titles`, `users`. -If Elasticsearch is enabled additional scopes available are blobs, wiki_blobs, notes, and commits. Find more about [the feature](../integration/elasticsearch.md). **(PREMIUM)** +If Elasticsearch is enabled additional scopes available are `blobs`, `wiki_blobs`, `notes`, and `commits`. Find more about [the feature](../integration/elasticsearch.md). **(PREMIUM)** The response depends on the requested scope. -### Scope: projects +### Scope: `projects` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=projects&search=flight" @@ -65,7 +65,7 @@ Example response: ] ``` -### Scope: issues +### Scope: `issues` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=issues&search=file" @@ -131,7 +131,7 @@ Example response: NOTE: `assignee` column is deprecated, now we show it as a single-sized array `assignees` to conform to the GitLab EE API. -### Scope: merge_requests +### Scope: `merge_requests` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=merge_requests&search=file" @@ -210,7 +210,7 @@ Example response: ] ``` -### Scope: milestones +### Scope: `milestones` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=milestones&search=release" @@ -235,7 +235,7 @@ Example response: ] ``` -### Scope: snippet_titles +### Scope: `snippet_titles` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=snippet_titles&search=sample" @@ -266,7 +266,7 @@ Example response: ] ``` -### Scope: wiki_blobs **(PREMIUM)** +### Scope: `wiki_blobs` **(PREMIUM)** > Moved to GitLab Premium in 13.9. @@ -450,13 +450,13 @@ GET /groups/:id/search | `order_by` | string | no | Allowed values are `created_at` only. If this is not set, the results are either sorted by `created_at` in descending order for basic search, or by the most relevant documents when using advanced search.| | `sort` | string | no | Allowed values are `asc` or `desc` only. If this is not set, the results are either sorted by `created_at` in descending order for basic search, or by the most relevant documents when using advanced search.| -Search the expression in the specified scope. These scopes are supported: projects, issues, merge_requests, milestones, users. +Search the expression in the specified scope. These scopes are supported: `projects`, `issues`, `merge_requests`, `milestones`, `users`. -If Elasticsearch is enabled additional scopes available are blobs, wiki_blobs, notes, and commits. Find more about [the feature](../integration/elasticsearch.md). **(PREMIUM)** +If Elasticsearch is enabled additional scopes available are `blobs`, `wiki_blobs`, `notes`, and `commits`. Find more about [the feature](../integration/elasticsearch.md). **(PREMIUM)** The response depends on the requested scope. -### Scope: projects +### Scope: `projects` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/3/search?scope=projects&search=flight" @@ -488,7 +488,7 @@ Example response: ] ``` -### Scope: issues +### Scope: `issues` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/3/search?scope=issues&search=file" @@ -554,7 +554,7 @@ Example response: NOTE: `assignee` column is deprecated, now we show it as a single-sized array `assignees` to conform to the GitLab EE API. -### Scope: merge_requests +### Scope: `merge_requests` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/3/search?scope=merge_requests&search=file" @@ -633,7 +633,7 @@ Example response: ] ``` -### Scope: milestones +### Scope: `milestones` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/3/search?scope=milestones&search=release" @@ -658,7 +658,7 @@ Example response: ] ``` -### Scope: wiki_blobs **(PREMIUM)** +### Scope: `wiki_blobs` **(PREMIUM)** > Moved to GitLab Premium in 13.9. @@ -689,7 +689,7 @@ Example response: NOTE: `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` is intended to be only the filename and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). -### Scope: commits **(PREMIUM)** +### Scope: `commits` **(PREMIUM)** > Moved to GitLab Premium in 13.9. @@ -724,7 +724,7 @@ Example response: ] ``` -### Scope: blobs **(PREMIUM)** +### Scope: `blobs` **(PREMIUM)** > Moved to GitLab Premium in 13.9. @@ -765,7 +765,7 @@ Example response: NOTE: `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` is intended to be only the filename and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). -### Scope: notes **(PREMIUM)** +### Scope: `notes` **(PREMIUM)** > Moved to GitLab Premium in 13.9. @@ -801,7 +801,7 @@ Example response: ] ``` -### Scope: users +### Scope: `users` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/3/search?scope=users&search=doe" @@ -837,17 +837,17 @@ GET /projects/:id/search | `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | | `scope` | string | yes | The scope to search in | | `search` | string | yes | The search query | -| `ref` | string | no | The name of a repository branch or tag to search on. The project's default branch is used by default. This is only applicable for scopes: commits, blobs, and wiki_blobs. | +| `ref` | string | no | The name of a repository branch or tag to search on. The project's default branch is used by default. This is only applicable for scopes: `commits`, `blobs`, and `wiki_blobs`. | | `state` | string | no | Filter by state. Issues and merge requests are supported; it is ignored for other scopes. | | `confidential` | boolean | no | Filter by confidentiality. Issues scope is supported; it is ignored for other scopes. | | `order_by` | string | no | Allowed values are `created_at` only. If this is not set, the results are either sorted by `created_at` in descending order for basic search, or by the most relevant documents when using advanced search.| | `sort` | string | no | Allowed values are `asc` or `desc` only. If this is not set, the results are either sorted by `created_at` in descending order for basic search, or by the most relevant documents when using advanced search.| -Search the expression in the specified scope. These scopes are supported: issues, merge_requests, milestones, notes, wiki_blobs, commits, blobs, users. +Search the expression in the specified scope. These scopes are supported: `issues`, `merge_requests`, `milestones`, `notes`, `wiki_blobs`, `commits`, `blobs`, `users`. The response depends on the requested scope. -### Scope: issues +### Scope: `issues` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/12/search?scope=issues&search=file" @@ -913,7 +913,7 @@ Example response: NOTE: `assignee` column is deprecated, now we show it as a single-sized array `assignees` to conform to the GitLab EE API. -### Scope: merge_requests +### Scope: `merge_requests` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=merge_requests&search=file" @@ -992,7 +992,7 @@ Example response: ] ``` -### Scope: milestones +### Scope: `milestones` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/12/search?scope=milestones&search=release" @@ -1017,7 +1017,7 @@ Example response: ] ``` -### Scope: notes **(PREMIUM)** +### Scope: `notes` **(PREMIUM)** > Moved to GitLab Premium in 13.9. @@ -1053,7 +1053,7 @@ Example response: ] ``` -### Scope: wiki_blobs **(PREMIUM)** +### Scope: `wiki_blobs` **(PREMIUM)** > Moved to GitLab Premium in 13.9. @@ -1101,7 +1101,7 @@ Example response: NOTE: `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` are intended to be only the filename and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). -### Scope: commits **(PREMIUM)** +### Scope: `commits` **(PREMIUM)** > Moved to GitLab Premium in 13.9. @@ -1136,7 +1136,7 @@ Example response: ] ``` -### Scope: blobs **(PREMIUM)** +### Scope: `blobs` **(PREMIUM)** > Moved to GitLab Premium in 13.9. @@ -1183,7 +1183,7 @@ Example response: NOTE: `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` is intended to be only the filename and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). -### Scope: users +### Scope: `users` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=users&search=doe" diff --git a/doc/api/settings.md b/doc/api/settings.md index 53c3b0a0f23..805797792de 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -281,10 +281,10 @@ listed in the descriptions of the relevant settings. | `default_project_visibility` | string | no | What visibility level new projects receive. Can take `private`, `internal` and `public` as a parameter. Default is `private`. | | `default_projects_limit` | integer | no | Project limit per user. Default is `100000`. | | `default_snippet_visibility` | string | no | What visibility level new snippets receive. Can take `private`, `internal` and `public` as a parameter. Default is `private`. | -| `delayed_project_deletion` **(PREMIUM SELF)** | boolean | no | Enable delayed project deletion by default in new groups. Default is `false`. | -| `delayed_group_deletion` **(PREMIUM SELF)** | boolean | no | Enable delayed group deletion by default in new groups. Requires both `delayed_group_deletion` to be true and `deletion_adjourned_period` to be greater than 0. Default is `true`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352959) in GitLab 15.0. | +| `delayed_project_deletion` **(PREMIUM SELF)** | boolean | no | Enable delayed project deletion by default in new groups. Default is `false`. [From GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/issues/352960), can only be enabled when `delayed_group_deletion` is true. | +| `delayed_group_deletion` **(PREMIUM SELF)** | boolean | no | Enable delayed group deletion. Default is `true`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352959) in GitLab 15.0. [From GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/issues/352960), disables and locks the group-level setting for delayed protect deletion when set to `false`. | | `delete_inactive_projects` | boolean | no | Enable inactive project deletion feature. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84519) in GitLab 14.10. [Became operational](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85689) in GitLab 15.0 (with feature flag `inactive_projects_deletion`, disabled by default). | -| `deletion_adjourned_period` **(PREMIUM SELF)** | integer | no | The number of days to wait before deleting a project or group that is marked for deletion. Value must be between 0 and 90. On every update, a hook on `deletion_adjourned_period` sets the value of `delayed_group_deletion` to `true` if `deletion_adjourned_period` is greater than 0 and `false` if `deletion_adjourned_period` is 0. +| `deletion_adjourned_period` **(PREMIUM SELF)** | integer | no | The number of days to wait before deleting a project or group that is marked for deletion. Value must be between `1` and `90`. Defaults to `7`. [From GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/issues/352960), a hook on `deletion_adjourned_period` sets the period to `1` on every update, and sets both `delayed_project_deletion` and `delayed_group_deletion` to `false` if the period is `0`. | | `diff_max_patch_bytes` | integer | no | Maximum [diff patch size](../user/admin_area/diff_limits.md), in bytes. | | `diff_max_files` | integer | no | Maximum [files in a diff](../user/admin_area/diff_limits.md). | | `diff_max_lines` | integer | no | Maximum [lines in a diff](../user/admin_area/diff_limits.md). | diff --git a/doc/development/code_review.md b/doc/development/code_review.md index 3425dc90284..a6976271ddf 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -15,35 +15,33 @@ code is effective, understandable, maintainable, and secure. ## Getting your merge request reviewed, approved, and merged -You are strongly encouraged to get your code **reviewed** by a -[reviewer](https://about.gitlab.com/handbook/engineering/workflow/code-review/#reviewer) as soon as -there is any code to review, to get a second opinion on the chosen solution and -implementation, and an extra pair of eyes looking for bugs, logic problems, or -uncovered edge cases. - -The default approach is to choose a reviewer from your group or team for the first review. -This is only a recommendation and the reviewer may be from a different team. -However, it is recommended to pick someone who is a [domain expert](#domain-experts). -If your merge request touches more than one domain (for example, Dynamic Analysis and GraphQL), ask for reviews from an expert from each domain. +Before you begin: -You can read more about the importance of involving reviewers in the section on the responsibility of the author below. +- Familiarize yourself with the [contribution acceptance criteria](contributing/merge_request_workflow.md#contribution-acceptance-criteria). +- If you need some guidance (for example, if it's your first merge request), feel free to ask + one of the [Merge request coaches](https://about.gitlab.com/company/team/?department=merge-request-coach). -If you need some guidance (for example, it's your first merge request), feel free to ask -one of the [Merge request coaches](https://about.gitlab.com/company/team/). +As soon as you have code to review, have the code **reviewed** by a [reviewer](https://about.gitlab.com/handbook/engineering/workflow/code-review/#reviewer). +This reviewer can be from your group or team, or a [domain expert](#domain-experts). +The reviewer can: -If you need assistance with security scans or comments, feel free to include the -Application Security Team (`@gitlab-com/gl-security/appsec`) in the review. +- Give you a second opinion on the chosen solution and implementation. +- Help look for bugs, logic problems, or uncovered edge cases. -Depending on the areas your merge request touches, it must be **approved** by one -or more [maintainers](https://about.gitlab.com/handbook/engineering/workflow/code-review/#maintainer). +For assistance with security scans or comments, include the Application Security Team (`@gitlab-com/gl-security/appsec`). -For approvals, we use the approval functionality found in the merge request -widget. For reviewers, we use the [reviewer functionality](../user/project/merge_requests/getting_started.md#reviewer) in the sidebar. +The reviewers use the [reviewer functionality](../user/project/merge_requests/getting_started.md#reviewer) in the sidebar. Reviewers can add their approval by [approving additionally](../user/project/merge_requests/approvals/index.md#approve-a-merge-request). +Depending on the areas your merge request touches, it must be **approved** by one +or more [maintainers](https://about.gitlab.com/handbook/engineering/workflow/code-review/#maintainer). +The **Approved** button is in the merge request widget. + Getting your merge request **merged** also requires a maintainer. If it requires more than one approval, the last maintainer to review and approve merges it. +Read more about [author responsibilities](#the-responsibility-of-the-merge-request-author) below. + ### Domain experts Domain experts are team members who have substantial experience with a specific technology, @@ -238,6 +236,8 @@ up confusion or verify that the end result matches what they had in mind, to database specialists to get input on the data model or specific queries, or to any other developer to get an in-depth review of the solution. +If your merge request touches more than one domain (for example, Dynamic Analysis and GraphQL), ask for reviews from an expert from each domain. + If an author is unsure if a merge request needs a [domain expert's](#domain-experts) opinion, then that indicates it does. Without it, it's unlikely they have the required level of confidence in their solution. @@ -270,7 +270,15 @@ This saves reviewers time and helps authors catch mistakes earlier. ### The responsibility of the reviewer -[Review the merge request](#reviewing-a-merge-request) thoroughly. When you are confident +[Review the merge request](#reviewing-a-merge-request) thoroughly. + +Verify that the merge request meets all [contribution acceptance criteria](contributing/merge_request_workflow.md#contribution-acceptance-criteria). + +If a merge request is too large, fixes more than one issue, or implements more +than one feature, you should guide the author towards spltting the merge request +into smaller merge requests. + +When you are confident that it meets all requirements, you should: - Select **Approve**. @@ -292,6 +300,12 @@ Because a maintainer's job only depends on their knowledge of the overall GitLab codebase, and not that of any specific domain, they can review, approve, and merge merge requests from any team and in any product area. +If a merge request is too large, fixes more than one issue, or implements more +than one feature, the maintainer can ask the author to make the merge request +smaller. Request the previous reviewer, or a merge request coach to help guide +the author on how to split the merge request, and to review the resulting +changes. + Maintainers do their best to also review the specifics of the chosen solution before merging, but as they are not necessarily [domain experts](#domain-experts), they may be poorly placed to do so without an unreasonable investment of time. In those cases, they diff --git a/doc/development/elasticsearch.md b/doc/development/elasticsearch.md index 3745a9e50ad..1b655cca4f8 100644 --- a/doc/development/elasticsearch.md +++ b/doc/development/elasticsearch.md @@ -69,7 +69,7 @@ The `whitespace` tokenizer was selected in order to have more control over how t Please see the `code` filter for an explanation on how tokens are split. NOTE: -The [Elasticsearch code_analyzer doesn't account for all code cases](../integration/elasticsearch.md#elasticsearch-code_analyzer-doesnt-account-for-all-code-cases). +The [Elasticsearch code_analyzer doesn't account for all code cases](../integration/advanced_search/elasticsearch_troubleshooting.md#elasticsearch-code_analyzer-doesnt-account-for-all-code-cases). #### `code_search_analyzer` diff --git a/doc/development/feature_flags/index.md b/doc/development/feature_flags/index.md index 54158de6893..b1caaef0942 100644 --- a/doc/development/feature_flags/index.md +++ b/doc/development/feature_flags/index.md @@ -141,7 +141,7 @@ push_frontend_feature_flag(:my_ops_flag, project, type: :ops) An `experiment` feature flag should conform to the same standards as a `development` feature flag, although the interface has some differences. An experiment feature flag should have a rollout issue, -created using the [Experiment Tracking template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/issue_templates/experiment_tracking_template.md). More information can be found in the [experiment guide](../experiment_guide/index.md). +created using the [Experiment Tracking template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/issue_templates/Experiment%20Rollout.md). More information can be found in the [experiment guide](../experiment_guide/index.md). ## Feature flag definition and validation diff --git a/doc/development/internal_api/index.md b/doc/development/internal_api/index.md index fad651709e3..3605172ceb5 100644 --- a/doc/development/internal_api/index.md +++ b/doc/development/internal_api/index.md @@ -254,7 +254,7 @@ recovery codes based on their SSH key. | Attribute | Type | Required | Description | |:----------|:-------|:---------|:------------| | `key_id` | integer | no | The ID of the SSH key used as found in the authorized-keys file or through the `/authorized_keys` check | -| `user_id` | integer | no | **Deprecated** User_id for which to generate new recovery codes | +| `user_id` | integer | no | **Deprecated** User ID for which to generate new recovery codes | ```plaintext GET /internal/two_factor_recovery_codes @@ -333,7 +333,7 @@ Example response: ## Authenticate Error Tracking requests -This endpoint will be called by the error tracking GO REST API application to authenticate a project. +This endpoint is called by the error tracking Go REST API application to authenticate a project. | Attribute | Type | Required | Description | |:-------------|:--------|:---------|:-------------------------------------------------------------------| diff --git a/doc/gitlab-basics/start-using-git.md b/doc/gitlab-basics/start-using-git.md index 57629ea942b..65680227ad0 100644 --- a/doc/gitlab-basics/start-using-git.md +++ b/doc/gitlab-basics/start-using-git.md @@ -149,7 +149,7 @@ between your computer and GitLab. 1. GitLab requests your username and password: - If you have 2FA enabled for your account, you must use a [Personal Access Token](../user/profile/personal_access_tokens.md) - with **read_repository** or **write_repository** permissions instead of your account's password. + with `read_repository` or `write_repository` permissions instead of your account's password. - If you don't have 2FA enabled, use your account's password. 1. To view the files, go to the new directory: diff --git a/doc/integration/advanced_search/elasticsearch_troubleshooting.md b/doc/integration/advanced_search/elasticsearch_troubleshooting.md new file mode 100644 index 00000000000..cd99b857399 --- /dev/null +++ b/doc/integration/advanced_search/elasticsearch_troubleshooting.md @@ -0,0 +1,274 @@ +--- +type: reference +stage: Data Stores +group: Global Search +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Elasticsearch troubleshooting **(PREMIUM SELF)** + +Use the following information to troubleshoot Elasticsearch issues. + +## View logs + +One of the most valuable tools for identifying issues with the Elasticsearch +integration are logs. The most relevant logs for this integration are: + +1. [`sidekiq.log`](../../administration/logs.md#sidekiqlog) - All of the + indexing happens in Sidekiq, so much of the relevant logs for the + Elasticsearch integration can be found in this file. +1. [`elasticsearch.log`](../../administration/logs.md#elasticsearchlog) - There + are additional logs specific to Elasticsearch that are sent to this file + that may contain useful diagnostic information about searching, + indexing or migrations. + +Here are some common pitfalls and how to overcome them. + +## How can I verify that my GitLab instance is using Elasticsearch? + +There are a couple of ways to achieve that: + +- Whenever you perform a search there is a link on the search results page + in the top right hand corner saying "Advanced search functionality is enabled". + This is always correctly identifying whether the current project/namespace + being searched is using Elasticsearch. + +- From the Admin Area under **Settings > Advanced Search** check that the + Advanced Search settings are checked. + + Those same settings there can be obtained from the Rails console if necessary: + + ```ruby + ::Gitlab::CurrentSettings.elasticsearch_search? # Whether or not searches will use Elasticsearch + ::Gitlab::CurrentSettings.elasticsearch_indexing? # Whether or not content will be indexed in Elasticsearch + ::Gitlab::CurrentSettings.elasticsearch_limit_indexing? # Whether or not Elasticsearch is limited only to certain projects/namespaces + ``` + +- If Elasticsearch is limited to specific namespaces and you need to know if + Elasticsearch is being used for a specific project or namespace, you can use + the Rails console: + + ```ruby + ::Gitlab::CurrentSettings.search_using_elasticsearch?(scope: Namespace.find_by_full_path("/my-namespace")) + ::Gitlab::CurrentSettings.search_using_elasticsearch?(scope: Project.find_by_full_path("/my-namespace/my-project")) + ``` + +## I updated GitLab and now I can't find anything + +We continuously make updates to our indexing strategies and aim to support +newer versions of Elasticsearch. When indexing changes are made, it may +be necessary for you to [reindex](../elasticsearch.md#zero-downtime-reindexing) after updating GitLab. + +## I indexed all the repositories but I can't get any hits for my search term in the UI + +Make sure you [indexed all the database data](../elasticsearch.md#enable-advanced-search). + +If there aren't any results (hits) in the UI search, check if you are seeing the same results via the rails console (`sudo gitlab-rails console`): + +```ruby +u = User.find_by_username('your-username') +s = SearchService.new(u, {:search => 'search_term', :scope => 'blobs'}) +pp s.search_objects.to_a +``` + +Beyond that, check via the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html) to see if the data shows up on the Elasticsearch side: + +```shell +curl --request GET <elasticsearch_server_ip>:9200/gitlab-production/_search?q=<search_term> +``` + +More [complex Elasticsearch API calls](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-filter-context.html) are also possible. + +It is important to understand at which level the problem is manifesting (UI, Rails code, Elasticsearch side) to be able to [troubleshoot further](../../administration/troubleshooting/elasticsearch.md#search-results-workflow). + +NOTE: +The above instructions are not to be used for scenarios that only index a [subset of namespaces](../elasticsearch.md#limit-the-number-of-namespaces-and-projects-that-can-be-indexed). + +See [Elasticsearch Index Scopes](../elasticsearch.md#advanced-search-index-scopes) for more information on searching for specific types of data. + +## I indexed all the repositories but then switched Elasticsearch servers and now I can't find anything + +You must re-run all the Rake tasks to reindex the database, repositories, and wikis. + +## The indexing process is taking a very long time + +The more data present in your GitLab instance, the longer the indexing process takes. + +## There are some projects that weren't indexed, but I don't know which ones + +You can run `sudo gitlab-rake gitlab:elastic:projects_not_indexed` to display projects that aren't indexed. + +## No new data is added to the Elasticsearch index when I push code + +NOTE: +This was [fixed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35936) in GitLab 13.2 and the Rake task is not available for versions greater than that. + +When performing the initial indexing of blobs, we lock all projects until the project finishes indexing. It could happen that an error during the process causes one or multiple projects to remain locked. To unlock them, run: + +```shell +sudo gitlab-rake gitlab:elastic:clear_locked_projects +``` + +## `Can't specify parent if no parent field has been configured` error + +If you enabled Elasticsearch before GitLab 8.12 and have not rebuilt indices, you get +exceptions in lots of different cases: + +```plaintext +Elasticsearch::Transport::Transport::Errors::BadRequest([400] { + "error": { + "root_cause": [{ + "type": "illegal_argument_exception", + "reason": "Can't specify parent if no parent field has been configured" + }], + "type": "illegal_argument_exception", + "reason": "Can't specify parent if no parent field has been configured" + }, + "status": 400 +}): +``` + +This is because we changed the index mapping in GitLab 8.12 and the old indices should be removed and built from scratch again, +see details in the [update guide](../../update/upgrading_from_source.md). + +## `Elasticsearch::Transport::Transport::Errors::BadRequest` + +If you have this exception (just like in the case above but the actual message is different) please check if you have the correct Elasticsearch version and you met the other [requirements](../elasticsearch.md#system-requirements). +There is also an easy way to check it automatically with `sudo gitlab-rake gitlab:check` command. + +## `Elasticsearch::Transport::Transport::Errors::RequestEntityTooLarge` + +```plaintext +[413] {"Message":"Request size exceeded 10485760 bytes"} +``` + +This exception is seen when your Elasticsearch cluster is configured to reject requests above a certain size (10MiB in this case). This corresponds to the `http.max_content_length` setting in `elasticsearch.yml`. Increase it to a larger size and restart your Elasticsearch cluster. + +AWS has [fixed limits](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/limits.html#network-limits) for this setting ("Maximum size of HTTP request payloads"), based on the size of the underlying instance. + +## My single node Elasticsearch cluster status never goes from `yellow` to `green` even though everything seems to be running properly + +**For a single node Elasticsearch cluster the functional cluster health status is yellow** (never green) because the primary shard is allocated but replicas cannot be as there is no other node to which Elasticsearch can assign a replica. This also applies if you are using the [Amazon OpenSearch](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/aes-handling-errors.html#aes-handling-errors-yellow-cluster-status) service. + +WARNING: +Setting the number of replicas to `0` is discouraged (this is not allowed in the GitLab Elasticsearch Integration menu). If you are planning to add more Elasticsearch nodes (for a total of more than 1 Elasticsearch) the number of replicas needs to be set to an integer value larger than `0`. Failure to do so results in lack of redundancy (losing one node corrupts the index). + +If you have a **hard requirement to have a green status for your single node Elasticsearch cluster**, please make sure you understand the risks outlined in the previous paragraph and then run the following query to set the number of replicas to `0`(the cluster no longer tries to create any shard replicas): + +```shell +curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ + --data '{ + "index" : { + "number_of_replicas" : 0 + } + }' +``` + +## `health check timeout: no Elasticsearch node available` error in Sidekiq + +If you're getting a `health check timeout: no Elasticsearch node available` error in Sidekiq during the indexing process: + +```plaintext +Gitlab::Elastic::Indexer::Error: time="2020-01-23T09:13:00Z" level=fatal msg="health check timeout: no Elasticsearch node available" +``` + +You probably have not used either `http://` or `https://` as part of your value in the **"URL"** field of the Elasticsearch Integration Menu. Please make sure you are using either `http://` or `https://` in this field as the [Elasticsearch client for Go](https://github.com/olivere/elastic) that we are using [needs the prefix for the URL to be accepted as valid](https://github.com/olivere/elastic/commit/a80af35aa41856dc2c986204e2b64eab81ccac3a). +After you have corrected the formatting of the URL, delete the index (via the [dedicated Rake task](../elasticsearch.md#gitlab-advanced-search-rake-tasks)) and [reindex the content of your instance](../elasticsearch.md#enable-advanced-search). + +## My Elasticsearch cluster has a plugin and the integration is not working + +Certain 3rd party plugins may introduce bugs in your cluster or for whatever +reason may be incompatible with our integration. You should try disabling +plugins so you can rule out the possibility that the plugin is causing the +problem. + +## Low-level troubleshooting + +There is a [more structured, lower-level troubleshooting document](../../administration/troubleshooting/elasticsearch.md) for when you experience other issues, including poor performance. + +## Elasticsearch `code_analyzer` doesn't account for all code cases + +The `code_analyzer` pattern and filter configuration is being evaluated for improvement. We have fixed [most edge cases](https://gitlab.com/groups/gitlab-org/-/epics/3621#note_363429094) that were not returning expected search results due to our pattern and filter configuration. + +Improvements to the `code_analyzer` pattern and filters are being discussed in [epic 3621](https://gitlab.com/groups/gitlab-org/-/epics/3621). + +## Some binary files may not be searchable by name + +In GitLab 13.9, a change was made where [binary file names are being indexed](https://gitlab.com/gitlab-org/gitlab/-/issues/301083). However, without indexing all projects' data from scratch, only binary files that are added or updated after the GitLab 13.9 release are searchable. + +## Last resort to recreate an index + +There may be cases where somehow data never got indexed and it's not in the +queue, or the index is somehow in a state where migrations just cannot +proceed. It is always best to try to troubleshoot the root cause of the problem +by [viewing the logs](#view-logs). + +If there are no other options, then you always have the option of recreating the +entire index from scratch. If you have a small GitLab installation, this can +sometimes be a quick way to resolve a problem, but if you have a large GitLab +installation, then this might take a very long time to complete. Until the +index is fully recreated, your index does not serve correct search results, +so you may want to disable **Search with Elasticsearch** while it is running. + +If you are sure you've read the above caveats and want to proceed, then you +should run the following Rake task to recreate the entire index from scratch: + +**For Omnibus installations** + +```shell +# WARNING: DO NOT RUN THIS UNTIL YOU READ THE DESCRIPTION ABOVE +sudo gitlab-rake gitlab:elastic:index +``` + +**For installations from source** + +```shell +# WARNING: DO NOT RUN THIS UNTIL YOU READ THE DESCRIPTION ABOVE +cd /home/git/gitlab +sudo -u git -H bundle exec rake gitlab:elastic:index +``` + +## How does Advanced Search handle private projects? + +Advanced Search stores all the projects in the same Elasticsearch indices, +however, searches only surface results that can be viewed by the user. +Advanced Search honors all permission checks in the application by +filtering out projects that a user does not have access to at search time. + +## Indexing fails with `error: elastic: Error 429 (Too Many Requests)` + +If `ElasticCommitIndexerWorker` Sidekiq workers are failing with this error during indexing, it usually means that Elasticsearch is unable to keep up with the concurrency of indexing request. To address change the following settings: + +- To decrease the indexing throughput you can decrease `Bulk request concurrency` (see [Advanced Search settings](../elasticsearch.md#advanced-search-configuration)). This is set to `10` by default, but you change it to as low as 1 to reduce the number of concurrent indexing operations. +- If changing `Bulk request concurrency` didn't help, you can use the [queue selector](../../administration/operations/extra_sidekiq_processes.md#queue-selector) option to [limit indexing jobs only to specific Sidekiq nodes](../elasticsearch.md#index-large-instances-with-dedicated-sidekiq-nodes-or-processes), which should reduce the number of indexing requests. + +## Indexing is very slow or fails with `rejected execution of coordinating operation` messages + +Bulk requests getting rejected by the Elasticsearch nodes are likely due to load and lack of available memory. +Ensure that your Elasticsearch cluster meets the [system requirements](../elasticsearch.md#system-requirements) and has enough resources +to perform bulk operations. See also the error ["429 (Too Many Requests)"](#indexing-fails-with-error-elastic-error-429-too-many-requests). + +## Access requirements for the self-managed AWS OpenSearch Service + +To use the self-managed AWS OpenSearch Service with GitLab, configure your instance's domain access policies +to contain the actions below. +See [Identity and Access Management in Amazon OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html) for details. + +```plaintext +es:ESHttpDelete +es:ESHttpGet +es:ESHttpHead +es:ESHttpPost +es:ESHttpPut +es:ESHttpPatch +``` + +## Role-mapping when using AWS Elasticsearch or AWS OpenSearch fine-grained access control + +When using fine-grained access control with an IAM role, you might encounter the following error: + +```plaintext +{"error":{"root_cause":[{"type":"security_exception","reason":"no permissions for [indices:data/write/bulk] and User [name=arn:aws:iam::xxx:role/INSERT_ROLE_NAME_HERE, backend_roles=[arn:aws:iam::xxx:role/INSERT_ROLE_NAME_HERE], requestedTenant=null]"}],"type":"security_exception","reason":"no permissions for [indices:data/write/bulk] and User [name=arn:aws:iam::xxx:role/INSERT_ROLE_NAME_HERE, backend_roles=[arn:aws:iam::xxx:role/INSERT_ROLE_NAME_HERE], requestedTenant=null]"},"status":403} +``` + +To fix this, you need to [map the roles to users](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/fgac.html#fgac-mapping) in Kibana. diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index 6574a13be9b..a014a73d9ac 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -448,8 +448,9 @@ fixed the cause of the failure, select "Retry migration", and the migration is s in the background. If you cannot get the migration to succeed, you may -consider the [last resort to recreate the index from -scratch](#last-resort-to-recreate-an-index). This may allow you to skip over +consider the +[last resort to recreate the index from scratch](advanced_search/elasticsearch_troubleshooting.md#last-resort-to-recreate-an-index). +This may allow you to skip over the problem because a newly created index skips all migrations as the index is recreated with the correct up-to-date schema. @@ -854,267 +855,3 @@ The use of Elasticsearch in GitLab is only ever as a secondary data store. This means that all of the data stored in Elasticsearch can always be derived again from other data sources, specifically PostgreSQL and Gitaly. Therefore, if the Elasticsearch data store is ever corrupted for whatever reason, you can reindex everything from scratch. - -## Troubleshooting - -One of the most valuable tools for identifying issues with the Elasticsearch -integration are logs. The most relevant logs for this integration are: - -1. [`sidekiq.log`](../administration/logs.md#sidekiqlog) - All of the - indexing happens in Sidekiq, so much of the relevant logs for the - Elasticsearch integration can be found in this file. -1. [`elasticsearch.log`](../administration/logs.md#elasticsearchlog) - There - are additional logs specific to Elasticsearch that are sent to this file - that may contain useful diagnostic information about searching, - indexing or migrations. - -Here are some common pitfalls and how to overcome them. - -### How can I verify that my GitLab instance is using Elasticsearch? - -There are a couple of ways to achieve that: - -- Whenever you perform a search there is a link on the search results page - in the top right hand corner saying "Advanced search functionality is enabled". - This is always correctly identifying whether the current project/namespace - being searched is using Elasticsearch. - -- From the Admin Area under **Settings > Advanced Search** check that the - Advanced Search settings are checked. - - Those same settings there can be obtained from the Rails console if necessary: - - ```ruby - ::Gitlab::CurrentSettings.elasticsearch_search? # Whether or not searches will use Elasticsearch - ::Gitlab::CurrentSettings.elasticsearch_indexing? # Whether or not content will be indexed in Elasticsearch - ::Gitlab::CurrentSettings.elasticsearch_limit_indexing? # Whether or not Elasticsearch is limited only to certain projects/namespaces - ``` - -- If Elasticsearch is limited to specific namespaces and you need to know if - Elasticsearch is being used for a specific project or namespace, you can use - the Rails console: - - ```ruby - ::Gitlab::CurrentSettings.search_using_elasticsearch?(scope: Namespace.find_by_full_path("/my-namespace")) - ::Gitlab::CurrentSettings.search_using_elasticsearch?(scope: Project.find_by_full_path("/my-namespace/my-project")) - ``` - -### I updated GitLab and now I can't find anything - -We continuously make updates to our indexing strategies and aim to support -newer versions of Elasticsearch. When indexing changes are made, it may -be necessary for you to [reindex](#zero-downtime-reindexing) after updating GitLab. - -### I indexed all the repositories but I can't get any hits for my search term in the UI - -Make sure you indexed all the database data [as stated above](#enable-advanced-search). - -If there aren't any results (hits) in the UI search, check if you are seeing the same results via the rails console (`sudo gitlab-rails console`): - -```ruby -u = User.find_by_username('your-username') -s = SearchService.new(u, {:search => 'search_term', :scope => 'blobs'}) -pp s.search_objects.to_a -``` - -Beyond that, check via the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html) to see if the data shows up on the Elasticsearch side: - -```shell -curl --request GET <elasticsearch_server_ip>:9200/gitlab-production/_search?q=<search_term> -``` - -More [complex Elasticsearch API calls](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-filter-context.html) are also possible. - -It is important to understand at which level the problem is manifesting (UI, Rails code, Elasticsearch side) to be able to [troubleshoot further](../administration/troubleshooting/elasticsearch.md#search-results-workflow). - -NOTE: -The above instructions are not to be used for scenarios that only index a [subset of namespaces](#limit-the-number-of-namespaces-and-projects-that-can-be-indexed). - -See [Elasticsearch Index Scopes](#advanced-search-index-scopes) for more information on searching for specific types of data. - -### I indexed all the repositories but then switched Elasticsearch servers and now I can't find anything - -You must re-run all the Rake tasks to reindex the database, repositories, and wikis. - -### The indexing process is taking a very long time - -The more data present in your GitLab instance, the longer the indexing process takes. - -### There are some projects that weren't indexed, but I don't know which ones - -You can run `sudo gitlab-rake gitlab:elastic:projects_not_indexed` to display projects that aren't indexed. - -### No new data is added to the Elasticsearch index when I push code - -NOTE: -This was [fixed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35936) in GitLab 13.2 and the Rake task is not available for versions greater than that. - -When performing the initial indexing of blobs, we lock all projects until the project finishes indexing. It could happen that an error during the process causes one or multiple projects to remain locked. To unlock them, run: - -```shell -sudo gitlab-rake gitlab:elastic:clear_locked_projects -``` - -### `Can't specify parent if no parent field has been configured` error - -If you enabled Elasticsearch before GitLab 8.12 and have not rebuilt indices, you get -exceptions in lots of different cases: - -```plaintext -Elasticsearch::Transport::Transport::Errors::BadRequest([400] { - "error": { - "root_cause": [{ - "type": "illegal_argument_exception", - "reason": "Can't specify parent if no parent field has been configured" - }], - "type": "illegal_argument_exception", - "reason": "Can't specify parent if no parent field has been configured" - }, - "status": 400 -}): -``` - -This is because we changed the index mapping in GitLab 8.12 and the old indices should be removed and built from scratch again, -see details in the [update guide](../update/upgrading_from_source.md). - -### `Elasticsearch::Transport::Transport::Errors::BadRequest` - -If you have this exception (just like in the case above but the actual message is different) please check if you have the correct Elasticsearch version and you met the other [requirements](#system-requirements). -There is also an easy way to check it automatically with `sudo gitlab-rake gitlab:check` command. - -### `Elasticsearch::Transport::Transport::Errors::RequestEntityTooLarge` - -```plaintext -[413] {"Message":"Request size exceeded 10485760 bytes"} -``` - -This exception is seen when your Elasticsearch cluster is configured to reject requests above a certain size (10MiB in this case). This corresponds to the `http.max_content_length` setting in `elasticsearch.yml`. Increase it to a larger size and restart your Elasticsearch cluster. - -AWS has [fixed limits](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/limits.html#network-limits) for this setting ("Maximum size of HTTP request payloads"), based on the size of the underlying instance. - -### My single node Elasticsearch cluster status never goes from `yellow` to `green` even though everything seems to be running properly - -**For a single node Elasticsearch cluster the functional cluster health status is yellow** (never green) because the primary shard is allocated but replicas cannot be as there is no other node to which Elasticsearch can assign a replica. This also applies if you are using the [Amazon OpenSearch](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/aes-handling-errors.html#aes-handling-errors-yellow-cluster-status) service. - -WARNING: -Setting the number of replicas to `0` is discouraged (this is not allowed in the GitLab Elasticsearch Integration menu). If you are planning to add more Elasticsearch nodes (for a total of more than 1 Elasticsearch) the number of replicas needs to be set to an integer value larger than `0`. Failure to do so results in lack of redundancy (losing one node corrupts the index). - -If you have a **hard requirement to have a green status for your single node Elasticsearch cluster**, please make sure you understand the risks outlined in the previous paragraph and then run the following query to set the number of replicas to `0`(the cluster no longer tries to create any shard replicas): - -```shell -curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ - --data '{ - "index" : { - "number_of_replicas" : 0 - } - }' -``` - -### `health check timeout: no Elasticsearch node available` error in Sidekiq - -If you're getting a `health check timeout: no Elasticsearch node available` error in Sidekiq during the indexing process: - -```plaintext -Gitlab::Elastic::Indexer::Error: time="2020-01-23T09:13:00Z" level=fatal msg="health check timeout: no Elasticsearch node available" -``` - -You probably have not used either `http://` or `https://` as part of your value in the **"URL"** field of the Elasticsearch Integration Menu. Please make sure you are using either `http://` or `https://` in this field as the [Elasticsearch client for Go](https://github.com/olivere/elastic) that we are using [needs the prefix for the URL to be accepted as valid](https://github.com/olivere/elastic/commit/a80af35aa41856dc2c986204e2b64eab81ccac3a). -After you have corrected the formatting of the URL, delete the index (via the [dedicated Rake task](#gitlab-advanced-search-rake-tasks)) and [reindex the content of your instance](#enable-advanced-search). - -### My Elasticsearch cluster has a plugin and the integration is not working - -Certain 3rd party plugins may introduce bugs in your cluster or for whatever -reason may be incompatible with our integration. You should try disabling -plugins so you can rule out the possibility that the plugin is causing the -problem. - -### Low-level troubleshooting - -There is a [more structured, lower-level troubleshooting document](../administration/troubleshooting/elasticsearch.md) for when you experience other issues, including poor performance. - -### Elasticsearch `code_analyzer` doesn't account for all code cases - -The `code_analyzer` pattern and filter configuration is being evaluated for improvement. We have fixed [most edge cases](https://gitlab.com/groups/gitlab-org/-/epics/3621#note_363429094) that were not returning expected search results due to our pattern and filter configuration. - -Improvements to the `code_analyzer` pattern and filters are being discussed in [epic 3621](https://gitlab.com/groups/gitlab-org/-/epics/3621). - -### Some binary files may not be searchable by name - -In GitLab 13.9, a change was made where [binary file names are being indexed](https://gitlab.com/gitlab-org/gitlab/-/issues/301083). However, without indexing all projects' data from scratch, only binary files that are added or updated after the GitLab 13.9 release are searchable. - -### Last resort to recreate an index - -There may be cases where somehow data never got indexed and it's not in the -queue, or the index is somehow in a state where migrations just cannot -proceed. It is always best to try to troubleshoot the root cause of the problem -using the above [troubleshooting](#troubleshooting) steps. - -If there are no other options, then you always have the option of recreating the -entire index from scratch. If you have a small GitLab installation, this can -sometimes be a quick way to resolve a problem, but if you have a large GitLab -installation, then this might take a very long time to complete. Until the -index is fully recreated, your index does not serve correct search results, -so you may want to disable **Search with Elasticsearch** while it is running. - -If you are sure you've read the above caveats and want to proceed, then you -should run the following Rake task to recreate the entire index from scratch: - -**For Omnibus installations** - -```shell -# WARNING: DO NOT RUN THIS UNTIL YOU READ THE DESCRIPTION ABOVE -sudo gitlab-rake gitlab:elastic:index -``` - -**For installations from source** - -```shell -# WARNING: DO NOT RUN THIS UNTIL YOU READ THE DESCRIPTION ABOVE -cd /home/git/gitlab -sudo -u git -H bundle exec rake gitlab:elastic:index -``` - -### How does Advanced Search handle private projects? - -Advanced Search stores all the projects in the same Elasticsearch indices, -however, searches only surface results that can be viewed by the user. -Advanced Search honors all permission checks in the application by -filtering out projects that a user does not have access to at search time. - -### Indexing fails with `error: elastic: Error 429 (Too Many Requests)` - -If `ElasticCommitIndexerWorker` Sidekiq workers are failing with this error during indexing, it usually means that Elasticsearch is unable to keep up with the concurrency of indexing request. To address change the following settings: - -- To decrease the indexing throughput you can decrease `Bulk request concurrency` (see [Advanced Search settings](#advanced-search-configuration)). This is set to `10` by default, but you change it to as low as 1 to reduce the number of concurrent indexing operations. -- If changing `Bulk request concurrency` didn't help, you can use the [queue selector](../administration/operations/extra_sidekiq_processes.md#queue-selector) option to [limit indexing jobs only to specific Sidekiq nodes](#index-large-instances-with-dedicated-sidekiq-nodes-or-processes), which should reduce the number of indexing requests. - -### Indexing is very slow or fails with `rejected execution of coordinating operation` messages - -Bulk requests getting rejected by the Elasticsearch nodes are likely due to load and lack of available memory. -Ensure that your Elasticsearch cluster meets the [system requirements](#system-requirements) and has enough resources -to perform bulk operations. See also the error ["429 (Too Many Requests)"](#indexing-fails-with-error-elastic-error-429-too-many-requests). - -### Access requirements for the self-managed AWS OpenSearch Service - -To use the self-managed AWS OpenSearch Service with GitLab, configure your instance's domain access policies -to contain the actions below. -See [Identity and Access Management in Amazon OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html) for details. - -```plaintext -es:ESHttpDelete -es:ESHttpGet -es:ESHttpHead -es:ESHttpPost -es:ESHttpPut -es:ESHttpPatch -``` - -### Role-mapping when using AWS Elasticsearch or AWS OpenSearch fine-grained access control - -When using fine-grained access control with an IAM role, you might encounter the following error: - -```plaintext -{"error":{"root_cause":[{"type":"security_exception","reason":"no permissions for [indices:data/write/bulk] and User [name=arn:aws:iam::xxx:role/INSERT_ROLE_NAME_HERE, backend_roles=[arn:aws:iam::xxx:role/INSERT_ROLE_NAME_HERE], requestedTenant=null]"}],"type":"security_exception","reason":"no permissions for [indices:data/write/bulk] and User [name=arn:aws:iam::xxx:role/INSERT_ROLE_NAME_HERE, backend_roles=[arn:aws:iam::xxx:role/INSERT_ROLE_NAME_HERE], requestedTenant=null]"},"status":403} -``` - -To fix this, you need to [map the roles to users](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/fgac.html#fgac-mapping) in Kibana. diff --git a/doc/topics/git/feature_branching.md b/doc/topics/git/feature_branching.md index d3b2510f4e8..4e9b43d66f8 100644 --- a/doc/topics/git/feature_branching.md +++ b/doc/topics/git/feature_branching.md @@ -16,7 +16,7 @@ comments: false ## Feature branching sample workflow -1. Create a new feature branch called 'squash_some_bugs' +1. Create a new feature branch called `squash_some_bugs` 1. Edit '`bugs.rb`' and remove all the bugs. 1. Commit 1. Push diff --git a/doc/user/admin_area/settings/account_and_limit_settings.md b/doc/user/admin_area/settings/account_and_limit_settings.md index 9fc50af9b27..bedd648b3e7 100644 --- a/doc/user/admin_area/settings/account_and_limit_settings.md +++ b/doc/user/admin_area/settings/account_and_limit_settings.md @@ -192,7 +192,7 @@ To set a limit on how long these sessions are valid: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1007) in GitLab 14.6 [with a flag](../../../administration/feature_flags.md) named `ff_limit_ssh_key_lifetime`. Disabled by default. > - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/346753) in GitLab 14.6. -> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/1007) in GitLab 14.7. [Feature flag ff_limit_ssh_key_lifetime](https://gitlab.com/gitlab-org/gitlab/-/issues/347408) removed. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/1007) in GitLab 14.7. [Feature flag `ff_limit_ssh_key_lifetime`](https://gitlab.com/gitlab-org/gitlab/-/issues/347408) removed. Users can optionally specify a lifetime for [SSH keys](../../ssh.md). diff --git a/doc/user/admin_area/settings/visibility_and_access_controls.md b/doc/user/admin_area/settings/visibility_and_access_controls.md index c07f4ea51f1..a0acc70829d 100644 --- a/doc/user/admin_area/settings/visibility_and_access_controls.md +++ b/doc/user/admin_area/settings/visibility_and_access_controls.md @@ -33,50 +33,70 @@ on the instance. To alter which roles have permission to create projects: - Developers and Maintainers. 1. Select **Save changes**. -## Restrict project deletion to Administrators **(PREMIUM SELF)** +## Restrict project deletion to administrators **(PREMIUM SELF)** -Anyone with the **Owner** role, either at the project or group level, can -delete a project. To allow only users with administrator access to delete projects: +> User interface [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/352960) in GitLab 15.1. -1. Sign in to GitLab as a user with Administrator access level. +By default both administrators and anyone with the **Owner** role can delete a project. To restrict project deletion to only administrators: + +1. Sign in to GitLab as a user with administrator access. 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Settings > General**. 1. Expand the **Visibility and access controls** section. -1. Scroll to **Default project deletion protection**, and select **Only admins can delete project**. +1. Scroll to: + - (GitLab 15.1 and later) **Allowed to delete projects**, and select **Administrators**. + - (GitLab 15.0 and earlier) **Default project deletion projection** and select **Only admins can delete project**. 1. Select **Save changes**. -## Default delayed project deletion **(PREMIUM SELF)** +## Deletion protection **(PREMIUM SELF)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/255449) in GitLab 14.2 for groups created after August 12, 2021. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/255449) in GitLab 14.2 for groups created after August 12, 2021. +> - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/352960) from default delayed project deletion in GitLab 15.1. -[Delayed project deletion](../../project/settings/index.md#delayed-project-deletion) allows projects in a group (not a personal namespace) -to be deleted after a period of delay. +Instance-level protection against accidental deletion of groups and projects. -To enable delayed project deletion by default in new groups: +### Retention period -1. Check the **Default delayed project deletion** checkbox. -1. Select **Save changes**. +> [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/352960) in GitLab 15.1. -## Default deletion delay **(PREMIUM SELF)** +Groups and projects will remain restorable within a defined retention period. By default this is 7 days but it can be changed. +Setting the retention period to `0` means that groups and project are removed immediately and cannot be restored. -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) in GitLab 12.6. +In GitLab 15.1 and later, the retention period must be between `1` and `90`. If the retention period was `0` before the 15.1 update, +then it will get automatically changed to `1` while also disabling deletion protection the next time any application setting is changed. -By default, a project marked for deletion is permanently removed with immediate effect. -See [delayed project deletion](../../project/settings/index.md#delayed-project-deletion) to learn more. -By default, a group marked for deletion is permanently removed after seven days. +### Delayed project deletion -WARNING: -The default behavior of [Delayed Project deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) in GitLab 12.6 was changed to -[Immediate deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) in GitLab 13.2. +> User interface [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/352960) in GitLab 15.1. -The default period is seven days, and can be changed. Setting this period to `0` enables immediate removal -of projects or groups. +Administrators can enable [delayed project deletion](../../project/settings/index.md#delayed-project-deletion) by default for +newly-created groups. Group owners can choose to disable this and existing groups will retain their existing setting. When enabled +deleted groups will remain restorable within a retention period. -To change this period: +To configure delayed project deletion: -1. Select the desired option. +1. Sign in to GitLab as a user with administrator access. +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > General**. +1. Expand the **Visibility and access controls** section. +1. Scroll to: + - (GitLab 15.1 and later) **Deletion projection** and select keep deleted groups and projects, and select a retention period. + - (GitLab 15.0 and earlier) **Default delayed project projection** and select **Enable delayed project deletion by + default for newly-created groups.** Then set a retention period in **Default deletion delay**. 1. Select **Save changes**. +Deletion protection is not available for projects only (without being also being enabled for groups). + +In GitLab 15.1, and later this setting is enforced on groups when disabled and it cannot be overridden. + +### Delayed group deletion + +> User interface [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352960) in GitLab 15.1. + +Groups will remain restorable if the retention period is `1` or more days. + +In GitLab 15.1 and later, delayed group deletion can be enabled by setting **Deletion projection** to **Keep deleted**. + ### Override defaults and delete immediately Alternatively, projects that are marked for removal can be deleted immediately. To do so: diff --git a/doc/user/application_security/dast/dast_troubleshooting.md b/doc/user/application_security/dast/dast_troubleshooting.md index 9969526c906..50570b89920 100644 --- a/doc/user/application_security/dast/dast_troubleshooting.md +++ b/doc/user/application_security/dast/dast_troubleshooting.md @@ -89,6 +89,16 @@ include: - template: DAST.latest.gitlab-ci.yml ``` +## Getting error `shell not found` when using DAST CI/CD template + +When including the DAST CI/CD template as described in the documentation, the job may fail, with an error like the following recorded in the job logs: + +```shell +shell not found +``` + +To avoid this error, make sure you are using the latest stable version of Docker. More information is available in [issue 358847](https://gitlab.com/gitlab-org/gitlab/-/issues/358847). + ## Lack of IPv6 support Due to the underlying [ZAProxy engine not supporting IPv6](https://github.com/zaproxy/zaproxy/issues/3705), DAST is unable to scan or crawl IPv6-based applications. diff --git a/doc/user/application_security/secret_detection/index.md b/doc/user/application_security/secret_detection/index.md index 8dfb077fd06..9805fb3b67c 100644 --- a/doc/user/application_security/secret_detection/index.md +++ b/doc/user/application_security/secret_detection/index.md @@ -221,7 +221,7 @@ simultaneously: - [Disabling predefined rules](index.md#disable-predefined-analyzer-rules). - [Overriding predefined rules](index.md#override-predefined-analyzer-rules). -- Modifying the default behavior of the Secret Detection analyzer by [synthesizing and passing a custom configuration](index.md#synthesize-a-custom-configuration). Available for only `nodejs-scan`, `gosec`, and `semgrep`. +- Modifying the default behavior of the Secret Detection analyzer by [synthesizing and passing a custom configuration](index.md#synthesize-a-custom-configuration). Customization allows replacing the default secret detection rules with rules that you define. diff --git a/doc/user/clusters/agent/install/index.md b/doc/user/clusters/agent/install/index.md index fe408e30320..5d017ce6dfb 100644 --- a/doc/user/clusters/agent/install/index.md +++ b/doc/user/clusters/agent/install/index.md @@ -20,29 +20,51 @@ Before you can install the agent in your cluster, you need: - [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine/docs/quickstart) - [Amazon Elastic Kubernetes Service (EKS)](https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html) - [Digital Ocean](https://docs.digitalocean.com/products/kubernetes/quickstart/) -- On self-managed GitLab instances, a GitLab administrator must set up the [agent server](../../../../administration/clusters/kas.md). Then it will be available by default at `wss://gitlab.example.com/-/kubernetes-agent/`. +- On self-managed GitLab instances, a GitLab administrator must set up the + [agent server](../../../../administration/clusters/kas.md). + Then it will be available by default at `wss://gitlab.example.com/-/kubernetes-agent/`. On GitLab.com, the agent server is available at `wss://kas.gitlab.com`. ## Installation steps To install the agent in your cluster: -1. [Choose a name for the agent](#agent-naming-convention). +1. Optional. [Create an agent configuration file](#create-an-agent-configuration-file). 1. [Register the agent with GitLab](#register-the-agent-with-gitlab). 1. [Install the agent in your cluster](#install-the-agent-in-the-cluster). <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch a GitLab 14.2 [walk-through of this process](https://www.youtube.com/watch?v=XuBpKtsgGkE). -### Agent naming convention +### Create an agent configuration file + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/259669) in GitLab 13.7, the agent configuration file can be added to multiple directories (or subdirectories) of the repository. +> - Group authorization was [introduced](https://gitlab.com/groups/gitlab-org/-/epics/5784) in GitLab 14.3. + +The agent uses a YAML file for configuration settings. You must create this file if: + +- You use [a GitOps workflow](../gitops.md#gitops-workflow-steps). +- You use [a GitLab CI/CD workflow](../ci_cd_workflow.md#gitlab-cicd-workflow-steps) and want to authorize a different project to use the agent. + +To create an agent configuration file: + +1. Choose a name for your agent. The agent name follows the + [DNS label standard from RFC 1123](https://tools.ietf.org/html/rfc1123). The name must: + + - Be unique in the project. + - Contain at most 63 characters. + - Contain only lowercase alphanumeric characters or `-`. + - Start with an alphanumeric character. + - End with an alphanumeric character. + +1. In the repository, create a directory in this location: -The agent name must follow the [DNS label standard from RFC 1123](https://tools.ietf.org/html/rfc1123). -The name must: + ```plaintext + .gitlab/agents/<agent-name> + ``` -- Be unique in the project. -- Contain at most 63 characters. -- Contain only lowercase alphanumeric characters or `-`. -- Start with an alphanumeric character. -- End with an alphanumeric character. +1. In the directory, create a `config.yaml` file. Ensure the filename ends in `.yaml`, not `.yml`. + +You can leave the file blank for now, and [configure it](#configure-your-agent) later. ### Register the agent with GitLab @@ -64,34 +86,13 @@ You must register an agent before you can install the agent in your cluster. To it must be in this project. Your cluster manifest files should also be in this project. 1. From the left sidebar, select **Infrastructure > Kubernetes clusters**. 1. Select **Connect a cluster (agent)**. - - If you want to create a configuration with CI/CD defaults, type a name that meets [the naming convention](#agent-naming-convention). + - If you want to create a configuration with CI/CD defaults, type a name. - If you already have an [agent configuration file](#create-an-agent-configuration-file), select it from the list. 1. Select **Register an agent**. -1. GitLab generates an access token for the agent. Securely store this token. You need it to install the agent in your cluster and to [update the agent](#update-the-agent-version) to another version. -1. Copy the command under **Recommended installation method**. You need it when you use the one-liner installation method to install the agent in your cluster. - -### Create an agent configuration file - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/259669) in GitLab 13.7, the agent configuration file can be added to multiple directories (or subdirectories) of the repository. -> - Group authorization was [introduced](https://gitlab.com/groups/gitlab-org/-/epics/5784) in GitLab 14.3. - -The agent uses a YAML file for configuration settings. You need a configuration file if: - -- You want to use [a GitOps workflow](../gitops.md#gitops-configuration-reference). -- You want to authorize a different project to use the agent for a [GitLab CI/CD workflow](../ci_cd_workflow.md#authorize-the-agent). - -To create an agent configuration file: - -1. In the repository, create a directory in this location. The `<agent-name>` must meet [the naming convention](#agent-naming-convention). - - ```plaintext - .gitlab/agents/<agent-name> - ``` - -1. In the directory, create a `config.yaml` file. Ensure the filename ends in `.yaml`, not `.yml`. -1. Add content to the `config.yaml` file: - - For a GitOps workflow, view [the configuration reference](../gitops.md#gitops-configuration-reference) for details. - - For a GitLab CI/CD workflow, view [the configuration reference](../ci_cd_workflow.md) for details. +1. GitLab generates an access token for the agent. Securely store this token. You need it to install the agent + in your cluster and to [update the agent](#update-the-agent-version) to another version. +1. Copy the command under **Recommended installation method**. You need it when you use + the one-liner installation method to install the agent in your cluster. ### Install the agent in the cluster @@ -128,10 +129,34 @@ By default, the Helm installation command generated by GitLab: To see the full list of customizations available, see the Helm chart's [default values file](https://gitlab.com/gitlab-org/charts/gitlab-agent/-/blob/main/values.yaml). +###### Use the agent behind an HTTP proxy + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/351867) in GitLab 15.0, the GitLab agent Helm chart supports setting environment variables. + +To configure an HTTP proxy when using the Helm chart, you can use the environment variables `HTTP_PROXY`, `HTTPS_PROXY`, +and `NO_PROXY`. Upper and lowercase are both acceptable. + +You can set these variables by using the `extraEnv` value, as a list of objects with keys `name` and `value`. +For example, to set only the environment variable `HTTPS_PROXY` to the value `https://example.com/proxy`, you can run: + +```shell +helm upgrade --install gitlab-agent gitlab/gitlab-agent \ + --set extraEnv[0].name=HTTPS_PROXY \ + --set extraEnv[0].value=https://example.com/proxy \ + ... +``` + #### Advanced installation method GitLab also provides a [KPT package for the agent](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/tree/master/build/deployment/gitlab-agent). This method provides greater flexibility, but is only recommended for advanced users. +### Configure your agent + +To configure your agent, add content to the `config.yaml` file: + +- [View the configuration reference](../gitops.md#gitops-configuration-reference) for a GitOps workflow. +- [View the configuration reference](../ci_cd_workflow.md) for a GitLab CI/CD workflow. + ## Install multiple agents in your cluster To install a second agent in your cluster, you can follow the [previous steps](#register-the-agent-with-gitlab) a second time. To avoid resource name collisions within the cluster, you must either: diff --git a/doc/user/group/index.md b/doc/user/group/index.md index 46bb3ad647c..e8f5d4ac4d9 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -418,7 +418,7 @@ This action removes the group. It also adds a background job to delete all proje Specifically: -- In [GitLab 12.8 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/33257), on [GitLab Premium](https://about.gitlab.com/pricing/premium/) or higher tiers, this action adds a background job to mark a group for deletion. By default, the job schedules the deletion 7 days in the future. You can modify this waiting period through the [instance settings](../admin_area/settings/visibility_and_access_controls.md#default-deletion-delay). +- In [GitLab 12.8 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/33257), on [GitLab Premium](https://about.gitlab.com/pricing/premium/) or higher tiers, this action adds a background job to mark a group for deletion. By default, the job schedules the deletion 7 days in the future. You can modify this waiting period through the [instance settings](../admin_area/settings/visibility_and_access_controls.md#deletion-protection). - In [GitLab 13.6 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/39504), if the user who sets up the deletion is removed from the group before the deletion happens, the job is cancelled, and the group is no longer scheduled for deletion. @@ -736,14 +736,16 @@ To disable group mentions: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) in GitLab 13.2. > - [Inheritance and enforcement added](https://gitlab.com/gitlab-org/gitlab/-/issues/321724) in GitLab 13.11. > - [Instance setting to enable by default added](https://gitlab.com/gitlab-org/gitlab/-/issues/255449) in GitLab 14.2. +> - [Instance setting is inherited and enforced when disabled](https://gitlab.com/gitlab-org/gitlab/-/issues/352960) in GitLab 15.1. -[Delayed project deletion](../project/settings/index.md#delayed-project-deletion) can be enabled for groups. When enabled, projects in -the group are deleted after a period of delay. During this period, projects are in a read-only state and can be restored. The default -period is seven days but [is configurable at the instance level](../admin_area/settings/visibility_and_access_controls.md#default-deletion-delay). +[Delayed project deletion](../project/settings/index.md#delayed-project-deletion) is locked and disabled unless the instance-level settings for +[deletion protection](../admin_area/settings/visibility_and_access_controls.md#deletion-protection) is enabled for either groups only or groups and projects. +When enabled on groups, projects in the group are deleted after a period of delay. During this period, projects are in a read-only state and can be restored. +The default period is seven days but [is configurable at the instance level](../admin_area/settings/visibility_and_access_controls.md#deletion-protection). On self-managed GitLab, projects are deleted immediately by default. In GitLab 14.2 and later, an administrator can -[change the default setting](../admin_area/settings/visibility_and_access_controls.md#default-delayed-project-deletion) +[change the default setting](../admin_area/settings/visibility_and_access_controls.md#deletion-protection) for projects in newly-created groups. On GitLab.com, see the [GitLab.com settings page](../gitlab_com/index.md#delayed-project-deletion) for diff --git a/doc/user/markdown.md b/doc/user/markdown.md index 300d1a705c3..b1e6fcb2f4e 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -790,12 +790,16 @@ do_this_and_do_that_and_another_thing but_emphasis is_desired _here_ ``` +<!-- vale gitlab.Spelling = NO --> + perform_complicated_task do_this_and_do_that_and_another_thing but_emphasis is_desired _here_ +<!-- vale gitlab.Spelling = YES --> + --- If you wish to emphasize only a part of a word, it can still be done with asterisks: diff --git a/doc/user/project/merge_requests/methods/index.md b/doc/user/project/merge_requests/methods/index.md index d8b4644b1b8..d3221162cfd 100644 --- a/doc/user/project/merge_requests/methods/index.md +++ b/doc/user/project/merge_requests/methods/index.md @@ -113,7 +113,7 @@ On GitLab.com and self-managed GitLab, by default this feature is not available. ask an administrator to [enable the feature flag](../../../../administration/feature_flags.md) named `rebase_without_ci_ui`. The feature is not ready for production use. -To rebase a merge request's branch without triggering a CI/CD pipeline, select +To rebase a merge request's branch without triggering a CI/CD pipeline, select **Rebase without pipeline** from the merge request reports section. This option is available when fast-forward merge is not possible but a conflict-free rebase is possible. diff --git a/doc/user/project/settings/index.md b/doc/user/project/settings/index.md index b4fa1e8e8c1..b1514203f4b 100644 --- a/doc/user/project/settings/index.md +++ b/doc/user/project/settings/index.md @@ -458,7 +458,7 @@ in GitLab 12.6, and then to [immediate deletion](https://gitlab.com/gitlab-org/g Projects in a group (not a personal namespace) can be deleted after a delay period. Multiple settings can affect whether delayed project deletion is enabled for a particular project: -- Self-managed instance [settings](../../admin_area/settings/visibility_and_access_controls.md#default-delayed-project-deletion). +- Self-managed instance [settings](../../admin_area/settings/visibility_and_access_controls.md#deletion-protection). You can enable delayed project deletion as the default setting for new groups, and configure the number of days for the delay. For GitLab.com, see the [GitLab.com settings](../../gitlab_com/index.md#delayed-project-deletion). - Group [settings](../../group/index.md#enable-delayed-project-deletion) to enabled delayed project deletion for all diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb index f11270457c9..5bf3c3b8aac 100644 --- a/lib/api/pypi_packages.rb +++ b/lib/api/pypi_packages.rb @@ -39,6 +39,51 @@ module API params :package_name do requires :package_name, type: String, file_path: true, desc: 'The PyPi package name' end + + def present_simple_index(group_or_project) + authorize_read_package!(group_or_project) + + packages = Packages::Pypi::PackagesFinder.new(current_user, group_or_project).execute + presenter = ::Packages::Pypi::SimpleIndexPresenter.new(packages, group_or_project) + + present_html(presenter.body) + end + + def present_simple_package(group_or_project) + authorize_read_package!(group_or_project) + track_simple_event(group_or_project, 'list_package') + + packages = Packages::Pypi::PackagesFinder.new(current_user, group_or_project, { package_name: params[:package_name] }).execute + empty_packages = packages.empty? + + redirect_registry_request(empty_packages, :pypi, package_name: params[:package_name]) do + not_found!('Package') if empty_packages + presenter = ::Packages::Pypi::SimplePackageVersionsPresenter.new(packages, group_or_project) + + present_html(presenter.body) + end + end + + def track_simple_event(group_or_project, event_name) + if group_or_project.is_a?(Project) + project = group_or_project + namespace = group_or_project.namespace + else + project = nil + namespace = group_or_project + end + + track_package_event(event_name, :pypi, project: project, namespace: namespace) + end + + def present_html(content) + # Adjusts grape output format + # to be HTML + content_type "text/html; charset=utf-8" + env['api.format'] = :binary + + body content + end end params do @@ -67,7 +112,18 @@ module API present_carrierwave_file!(package_file.file, supports_direct_download: true) end - desc 'The PyPi Simple Endpoint' do + desc 'The PyPi Simple Group Index Endpoint' do + detail 'This feature was introduced in GitLab 15.1' + end + + # An API entry point but returns an HTML file instead of JSON. + # PyPi simple API returns a list of packages as a simple HTML file. + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get 'simple', format: :txt do + present_simple_index(find_authorized_group!) + end + + desc 'The PyPi Simple Group Package Endpoint' do detail 'This feature was introduced in GitLab 12.10' end @@ -75,29 +131,11 @@ module API use :package_name end - # An Api entry point but returns an HTML file instead of JSON. + # An API entry point but returns an HTML file instead of JSON. # PyPi simple API returns the package descriptor as a simple HTML file. route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth get 'simple/*package_name', format: :txt do - group = find_authorized_group! - authorize_read_package!(group) - - track_package_event('list_package', :pypi) - - packages = Packages::Pypi::PackagesFinder.new(current_user, group, { package_name: params[:package_name] }).execute - empty_packages = packages.empty? - - redirect_registry_request(empty_packages, :pypi, package_name: params[:package_name]) do - not_found!('Package') if empty_packages - presenter = ::Packages::Pypi::PackagePresenter.new(packages, group) - - # Adjusts grape output format - # to be HTML - content_type "text/html; charset=utf-8" - env['api.format'] = :binary - - body presenter.body - end + present_simple_package(find_authorized_group!) end end end @@ -133,7 +171,18 @@ module API present_carrierwave_file!(package_file.file, supports_direct_download: true) end - desc 'The PyPi Simple Endpoint' do + desc 'The PyPi Simple Project Index Endpoint' do + detail 'This feature was introduced in GitLab 15.1' + end + + # An API entry point but returns an HTML file instead of JSON. + # PyPi simple API returns a list of packages as a simple HTML file. + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get 'simple', format: :txt do + present_simple_index(authorized_user_project) + end + + desc 'The PyPi Simple Project Package Endpoint' do detail 'This feature was introduced in GitLab 12.10' end @@ -141,28 +190,11 @@ module API use :package_name end - # An Api entry point but returns an HTML file instead of JSON. + # An API entry point but returns an HTML file instead of JSON. # PyPi simple API returns the package descriptor as a simple HTML file. route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth get 'simple/*package_name', format: :txt do - authorize_read_package!(authorized_user_project) - - track_package_event('list_package', :pypi, project: authorized_user_project, namespace: authorized_user_project.namespace) - - packages = Packages::Pypi::PackagesFinder.new(current_user, authorized_user_project, { package_name: params[:package_name] }).execute - empty_packages = packages.empty? - - redirect_registry_request(empty_packages, :pypi, package_name: params[:package_name]) do - not_found!('Package') if empty_packages - presenter = ::Packages::Pypi::PackagePresenter.new(packages, authorized_user_project) - - # Adjusts grape output format - # to be HTML - content_type "text/html; charset=utf-8" - env['api.format'] = :binary - - body presenter.body - end + present_simple_package(authorized_user_project) end desc 'The PyPi Package upload endpoint' do diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index c37cb9305ad..b0f4194b7a0 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -5,7 +5,7 @@ module Gitlab module Packages CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt conan_sources.tgz conan_export.tgz].freeze CONAN_PACKAGE_FILES = %w[conaninfo.txt conanmanifest.txt conan_package.tgz].freeze - + PYPI_NORMALIZED_NAME_REGEX_STRING = '[-_.]+' API_PATH_REGEX = %r{^/api/v\d+/(projects/[^/]+/|groups?/[^/]+/-/)?packages/[A-Za-z]+}.freeze def conan_package_reference_regex diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 2f767473457..c91b8f943d0 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -2622,6 +2622,9 @@ msgstr "" msgid "AdminSettings|Delete project after" msgstr "" +msgid "AdminSettings|Deletion protection" +msgstr "" + msgid "AdminSettings|Disable Elasticsearch until indexing completes." msgstr "" @@ -2691,6 +2694,9 @@ msgstr "" msgid "AdminSettings|Inactive project deletion" msgstr "" +msgid "AdminSettings|Keep deleted" +msgstr "" + msgid "AdminSettings|Keep the latest artifacts for all jobs in the latest successful pipelines" msgstr "" @@ -2736,6 +2742,9 @@ msgstr "" msgid "AdminSettings|No required pipeline" msgstr "" +msgid "AdminSettings|None, delete immediately" +msgstr "" + msgid "AdminSettings|Only enable search after installing the plugin, enabling indexing, and recreating the index." msgstr "" @@ -2766,6 +2775,9 @@ msgstr "" msgid "AdminSettings|Restrict group access by IP address. %{link_start}Learn more%{link_end}." msgstr "" +msgid "AdminSettings|Retention period that deleted groups and projects will remain restorable. Personal projects are always deleted immediately. Some groups can opt-out their projects." +msgstr "" + msgid "AdminSettings|Save %{name} limits" msgstr "" @@ -3249,6 +3261,9 @@ msgstr "" msgid "Administration" msgstr "" +msgid "Administrators" +msgstr "" + msgid "Admin|Additional users must be reviewed and approved by a system administrator. Learn more about %{help_link_start}usage caps%{help_link_end}." msgstr "" @@ -3819,6 +3834,9 @@ msgstr "" msgid "Allowed to create:" msgstr "" +msgid "Allowed to delete projects" +msgstr "" + msgid "Allowed to fail" msgstr "" @@ -11906,12 +11924,6 @@ msgstr "" msgid "Default branch and protected branches" msgstr "" -msgid "Default delayed project deletion" -msgstr "" - -msgid "Default deletion delay" -msgstr "" - msgid "Default description template for issues" msgstr "" @@ -11924,9 +11936,6 @@ msgstr "" msgid "Default first day of the week in calendars and date pickers." msgstr "" -msgid "Default project deletion protection" -msgstr "" - msgid "Default projects limit" msgstr "" @@ -13412,9 +13421,6 @@ msgstr "" msgid "Documents reindexed: %{processed_documents} (%{percentage}%%)" msgstr "" -msgid "Does not apply to projects in personal namespaces, which are deleted immediately on request." -msgstr "" - msgid "Does not delete the source branch." msgstr "" @@ -14042,9 +14048,6 @@ msgstr "" msgid "Enable container expiration caching." msgstr "" -msgid "Enable delayed project deletion by default for newly-created groups." -msgstr "" - msgid "Enable email notification" msgstr "" @@ -16800,9 +16803,6 @@ msgstr "" msgid "Geo|Primary site" msgstr "" -msgid "Geo|Project" -msgstr "" - msgid "Geo|Project (ID: %{project_id}) no longer exists on the primary. It is safe to remove this entry, as this will not remove any data on disk." msgstr "" @@ -16950,9 +16950,6 @@ msgstr "" msgid "Geo|Synced" msgstr "" -msgid "Geo|Synced at" -msgstr "" - msgid "Geo|Synchronization" msgstr "" @@ -26667,9 +26664,6 @@ msgstr "" msgid "Only active projects show up in the search and on the dashboard." msgstr "" -msgid "Only admins can delete project" -msgstr "" - msgid "Only allow anyone to register for accounts on GitLab instances that you intend to be used by anyone. Allowing anyone to register makes GitLab instances more vulnerable." msgstr "" @@ -26910,6 +26904,9 @@ msgstr "" msgid "Owner" msgstr "" +msgid "Owners and administrators" +msgstr "" + msgid "Owners can modify this selection." msgstr "" @@ -38226,9 +38223,6 @@ msgstr "" msgid "The number of changes to fetch from GitLab when cloning a repository. Lower values can speed up pipeline execution. Set to %{code_open}0%{code_close} or blank to fetch all branches and tags for each job" msgstr "" -msgid "The number of days that must pass between marking an entity for deletion and actually removing it. The delay can be between 0 and 90 days." -msgstr "" - msgid "The number of merge requests merged by month." msgstr "" @@ -44506,6 +44500,9 @@ msgstr "" msgid "can only have one escalation policy" msgstr "" +msgid "can't be enabled when delayed group deletion is disabled" +msgstr "" + msgid "can't be nil" msgstr "" @@ -45103,6 +45100,9 @@ msgstr "" msgid "following" msgstr "" +msgid "for" +msgstr "" + msgid "for %{link_to_merge_request} with %{link_to_merge_request_source_branch}" msgstr "" @@ -45150,6 +45150,12 @@ msgstr "" msgid "groups" msgstr "" +msgid "groups and projects" +msgstr "" + +msgid "groups only" +msgstr "" + msgid "has already been linked to another vulnerability" msgstr "" diff --git a/spec/finders/packages/pypi/packages_finder_spec.rb b/spec/finders/packages/pypi/packages_finder_spec.rb index 1a44fb99009..3957eb188da 100644 --- a/spec/finders/packages/pypi/packages_finder_spec.rb +++ b/spec/finders/packages/pypi/packages_finder_spec.rb @@ -12,59 +12,91 @@ RSpec.describe Packages::Pypi::PackagesFinder do let_it_be(:package3) { create(:pypi_package, name: package2.name, project: project) } let_it_be(:package4) { create(:pypi_package, name: package2.name, project: project2) } - let(:package_name) { package2.name } + shared_examples 'when no package is found' do + context 'non-existing package' do + let(:package_name) { 'none' } - describe 'execute' do - subject { described_class.new(user, scope, package_name: package_name).execute } + it { expect(subject).to be_empty } + end + end - shared_examples 'when no package is found' do - context 'non-existing package' do - let(:package_name) { 'none' } + shared_examples 'when package_name param is a non-normalized name' do + context 'non-existing package' do + let(:package_name) { package2.name.upcase.tr('-', '.') } - it { expect(subject).to be_empty } - end + it { expect(subject).to be_empty } end + end - shared_examples 'when package_name param is a non-normalized name' do - context 'non-existing package' do - let(:package_name) { package2.name.upcase.tr('-', '.') } + describe '#execute' do + subject { described_class.new(user, scope, package_name: package_name).execute } - it { expect(subject).to be_empty } + context 'with package name param' do + let(:package_name) { package2.name } + + context 'within a project' do + let(:scope) { project } + + it { is_expected.to contain_exactly(package2, package3) } + + it_behaves_like 'when no package is found' + it_behaves_like 'when package_name param is a non-normalized name' end - end - context 'within a project' do - let(:scope) { project } + context 'within a group' do + let(:scope) { group } - it { is_expected.to contain_exactly(package2, package3) } + it { expect(subject).to be_empty } - it_behaves_like 'when no package is found' - it_behaves_like 'when package_name param is a non-normalized name' - end + context 'user with access to only one project' do + before do + project2.add_developer(user) + end - context 'within a group' do - let(:scope) { group } + it { is_expected.to contain_exactly(package4) } - it { expect(subject).to be_empty } + it_behaves_like 'when no package is found' + it_behaves_like 'when package_name param is a non-normalized name' - context 'user with access to only one project' do - before do - project2.add_developer(user) + context 'user with access to multiple projects' do + before do + project.add_developer(user) + end + + it { is_expected.to contain_exactly(package2, package3, package4) } + end end + end + end - it { is_expected.to contain_exactly(package4) } + context 'without package_name param' do + let(:package_name) { nil } - it_behaves_like 'when no package is found' - it_behaves_like 'when package_name param is a non-normalized name' + context 'within a group' do + let(:scope) { group } - context ' user with access to multiple projects' do + context 'user with access to only one project' do before do - project.add_developer(user) + project2.add_developer(user) end - it { is_expected.to contain_exactly(package2, package3, package4) } + it { is_expected.to contain_exactly(package4) } + + context 'user with access to multiple projects' do + before do + project.add_developer(user) + end + + it { is_expected.to contain_exactly(package1, package2, package3, package4) } + end end end + + context 'within a project' do + let(:scope) { project } + + it { is_expected.to contain_exactly(package1, package2, package3) } + end end end end diff --git a/spec/models/packages/package_spec.rb b/spec/models/packages/package_spec.rb index a9ed811e77d..06f02f021cf 100644 --- a/spec/models/packages/package_spec.rb +++ b/spec/models/packages/package_spec.rb @@ -1336,4 +1336,24 @@ RSpec.describe Packages::Package, type: :model do end end end + + describe '#normalized_pypi_name' do + let_it_be(:package) { create(:pypi_package) } + + subject { package.normalized_pypi_name } + + where(:package_name, :normalized_name) do + 'ASDF' | 'asdf' + 'a.B_c-d' | 'a-b-c-d' + 'a-------b....c___d' | 'a-b-c-d' + end + + with_them do + before do + package.update_column(:name, package_name) + end + + it { is_expected.to eq(normalized_name) } + end + end end diff --git a/spec/presenters/packages/pypi/simple_index_presenter_spec.rb b/spec/presenters/packages/pypi/simple_index_presenter_spec.rb new file mode 100644 index 00000000000..d915706577f --- /dev/null +++ b/spec/presenters/packages/pypi/simple_index_presenter_spec.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ::Packages::Pypi::SimpleIndexPresenter, :aggregate_failures do + using RSpec::Parameterized::TableSyntax + + let_it_be(:group) { create(:group) } + let_it_be(:project) { create(:project, group: group) } + let_it_be(:package_name) { 'sample-project' } + let_it_be(:package1) { create(:pypi_package, project: project, name: package_name, version: '1.0.0') } + let_it_be(:package2) { create(:pypi_package, project: project, name: package_name, version: '2.0.0') } + + let(:packages) { project.packages } + + describe '#body' do + subject(:presenter) { described_class.new(packages, project_or_group).body } + + shared_examples_for "pypi package presenter" do + where(:version, :expected_version) do + '>=2.7' | '>=2.7' + '"><script>alert(1)</script>' | '"><script>alert(1)</script>' + '>=2.7, !=3.0' | '>=2.7, !=3.0' + end + + with_them do + let(:python_version) { version } + let(:expected_python_version) { expected_version } + + before do + package1.pypi_metadatum.update_column(:required_python, python_version) + package2.pypi_metadatum.update_column(:required_python, '') + end + + it 'contains links for all packages' do + expect(presenter).to include(expected_link1) + expect(presenter).to include(expected_link2) + end + end + end + + context 'for project' do + let(:project_or_group) { project } + let(:expected_link1) { "<a href=\"http://localhost/api/v4/projects/#{project.id}/packages/pypi/simple/#{package1.normalized_pypi_name}\" data-requires-python=\"#{expected_python_version}\">#{package1.name}</a>" } # rubocop:disable Layout/LineLength + let(:expected_link2) { "<a href=\"http://localhost/api/v4/projects/#{project.id}/packages/pypi/simple/#{package2.normalized_pypi_name}\" data-requires-python=\"\">#{package2.name}</a>" } # rubocop:disable Layout/LineLength + + it_behaves_like 'pypi package presenter' + end + + context 'for group' do + let(:project_or_group) { group } + let(:expected_link1) { "<a href=\"http://localhost/api/v4/groups/#{group.id}/-/packages/pypi/simple/#{package1.normalized_pypi_name}\" data-requires-python=\"#{expected_python_version}\">#{package1.name}</a>" } # rubocop:disable Layout/LineLength + let(:expected_link2) { "<a href=\"http://localhost/api/v4/groups/#{group.id}/-/packages/pypi/simple/#{package2.normalized_pypi_name}\" data-requires-python=\"\">#{package2.name}</a>" } # rubocop:disable Layout/LineLength + + it_behaves_like 'pypi package presenter' + end + + context 'with package files pending destruction' do + let_it_be(:package_pending_destruction) do + create(:package, :pending_destruction, project: project, name: "package_pending_destruction") + end + + let(:project_or_group) { project } + + it { is_expected.not_to include(package_pending_destruction.name) } + end + end +end diff --git a/spec/presenters/packages/pypi/package_presenter_spec.rb b/spec/presenters/packages/pypi/simple_package_versions_presenter_spec.rb index b19abdbc17a..be454e5168c 100644 --- a/spec/presenters/packages/pypi/package_presenter_spec.rb +++ b/spec/presenters/packages/pypi/simple_package_versions_presenter_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe ::Packages::Pypi::PackagePresenter do +RSpec.describe ::Packages::Pypi::SimplePackageVersionsPresenter, :aggregate_failures do using RSpec::Parameterized::TableSyntax let_it_be(:group) { create(:group) } @@ -11,14 +11,13 @@ RSpec.describe ::Packages::Pypi::PackagePresenter do let_it_be(:package1) { create(:pypi_package, project: project, name: package_name, version: '1.0.0') } let_it_be(:package2) { create(:pypi_package, project: project, name: package_name, version: '2.0.0') } - let(:packages) { [package1, package2] } - let(:file) { package.package_files.first } let(:filename) { file.file_name } - - subject(:presenter) { described_class.new(packages, project_or_group).body} + let(:packages) { project.packages } describe '#body' do + subject(:presenter) { described_class.new(packages, project_or_group).body } + shared_examples_for "pypi package presenter" do where(:version, :expected_version, :with_package1) do '>=2.7' | '>=2.7' | true @@ -32,29 +31,31 @@ RSpec.describe ::Packages::Pypi::PackagePresenter do let(:package) { with_package1 ? package1 : package2 } before do - package.pypi_metadatum.required_python = python_version + package.pypi_metadatum.update_column(:required_python, python_version) end - it { is_expected.to include expected_file } + it { is_expected.to include expected_link } end end context 'for project' do let(:project_or_group) { project } - let(:expected_file) { "<a href=\"http://localhost/api/v4/projects/#{project.id}/packages/pypi/files/#{file.file_sha256}/#{filename}#sha256=#{file.file_sha256}\" data-requires-python=\"#{expected_python_version}\">#{filename}</a><br>" } + let(:expected_link) { "<a href=\"http://localhost/api/v4/projects/#{project.id}/packages/pypi/files/#{file.file_sha256}/#{filename}#sha256=#{file.file_sha256}\" data-requires-python=\"#{expected_python_version}\">#{filename}</a>" } # rubocop:disable Layout/LineLength it_behaves_like 'pypi package presenter' end context 'for group' do let(:project_or_group) { group } - let(:expected_file) { "<a href=\"http://localhost/api/v4/groups/#{group.id}/-/packages/pypi/files/#{file.file_sha256}/#{filename}#sha256=#{file.file_sha256}\" data-requires-python=\"#{expected_python_version}\">#{filename}</a><br>" } + let(:expected_link) { "<a href=\"http://localhost/api/v4/groups/#{group.id}/-/packages/pypi/files/#{file.file_sha256}/#{filename}#sha256=#{file.file_sha256}\" data-requires-python=\"#{expected_python_version}\">#{filename}</a>" } # rubocop:disable Layout/LineLength it_behaves_like 'pypi package presenter' end context 'with package files pending destruction' do - let_it_be(:package_file_pending_destruction) { create(:package_file, :pending_destruction, package: package1, file_name: "package_file_pending_destruction") } + let_it_be(:package_file_pending_destruction) do + create(:package_file, :pending_destruction, package: package1, file_name: "package_file_pending_destruction") + end let(:project_or_group) { project } diff --git a/spec/requests/api/pypi_packages_spec.rb b/spec/requests/api/pypi_packages_spec.rb index 8fa5f409298..a24b852cdac 100644 --- a/spec/requests/api/pypi_packages_spec.rb +++ b/spec/requests/api/pypi_packages_spec.rb @@ -17,7 +17,68 @@ RSpec.describe API::PypiPackages do let(:headers) { {} } - context 'simple API endpoint' do + context 'simple index API endpoint' do + let_it_be(:package) { create(:pypi_package, project: project) } + let_it_be(:package2) { create(:pypi_package, project: project) } + + subject { get api(url), headers: headers } + + describe 'GET /api/v4/groups/:id/-/packages/pypi/simple' do + let(:url) { "/groups/#{group.id}/-/packages/pypi/simple" } + let(:snowplow_gitlab_standard_context) { { project: project, namespace: project.namespace } } + + it_behaves_like 'pypi simple index API endpoint' + it_behaves_like 'rejects PyPI access with unknown group id' + + context 'deploy tokens' do + let_it_be(:group_deploy_token) { create(:group_deploy_token, deploy_token: deploy_token, group: group) } + + before do + project.update_column(:visibility_level, Gitlab::VisibilityLevel::PRIVATE) + group.update_column(:visibility_level, Gitlab::VisibilityLevel::PRIVATE) + end + + it_behaves_like 'deploy token for package GET requests' + + context 'with group path as id' do + let(:url) { "/groups/#{CGI.escape(group.full_path)}/-/packages/pypi/simple"} + + it_behaves_like 'deploy token for package GET requests' + end + end + + context 'job token' do + before do + project.update_column(:visibility_level, Gitlab::VisibilityLevel::PRIVATE) + group.update_column(:visibility_level, Gitlab::VisibilityLevel::PRIVATE) + group.add_developer(user) + end + + it_behaves_like 'job token for package GET requests' + end + + it_behaves_like 'a pypi user namespace endpoint' + end + + describe 'GET /api/v4/projects/:id/packages/pypi/simple' do + let(:package_name) { package.name } + let(:url) { "/projects/#{project.id}/packages/pypi/simple" } + let(:snowplow_gitlab_standard_context) { { project: nil, namespace: group } } + + it_behaves_like 'pypi simple index API endpoint' + it_behaves_like 'rejects PyPI access with unknown project id' + it_behaves_like 'deploy token for package GET requests' + it_behaves_like 'job token for package GET requests' + + context 'with project path as id' do + let(:url) { "/projects/#{CGI.escape(project.full_path)}/packages/pypi/simple" } + + it_behaves_like 'deploy token for package GET requests' + end + end + end + + context 'simple package API endpoint' do let_it_be(:package) { create(:pypi_package, project: project) } subject { get api(url), headers: headers } @@ -25,7 +86,7 @@ RSpec.describe API::PypiPackages do describe 'GET /api/v4/groups/:id/-/packages/pypi/simple/:package_name' do let(:package_name) { package.name } let(:url) { "/groups/#{group.id}/-/packages/pypi/simple/#{package_name}" } - let(:snowplow_gitlab_standard_context) { {} } + let(:snowplow_gitlab_standard_context) { { project: nil, namespace: group } } it_behaves_like 'pypi simple API endpoint' it_behaves_like 'rejects PyPI access with unknown group id' diff --git a/spec/services/terraform/states/destroy_service_spec.rb b/spec/services/terraform/states/destroy_service_spec.rb index 16fcc8f93db..5acf32cd73c 100644 --- a/spec/services/terraform/states/destroy_service_spec.rb +++ b/spec/services/terraform/states/destroy_service_spec.rb @@ -4,7 +4,8 @@ require 'spec_helper' RSpec.describe Terraform::States::DestroyService do let_it_be(:state) { create(:terraform_state, :with_version, :deletion_in_progress) } - let_it_be(:file) { double } + + let(:file) { instance_double(Terraform::StateUploader, relative_path: 'path') } before do allow_next_found_instance_of(Terraform::StateVersion) do |version| diff --git a/spec/support/shared_examples/requests/api/pypi_packages_shared_examples.rb b/spec/support/shared_examples/requests/api/pypi_packages_shared_examples.rb index aff086d1ba3..795545e4ad1 100644 --- a/spec/support/shared_examples/requests/api/pypi_packages_shared_examples.rb +++ b/spec/support/shared_examples/requests/api/pypi_packages_shared_examples.rb @@ -124,6 +124,23 @@ RSpec.shared_examples 'PyPI package versions' do |user_type, status, add_member end end +RSpec.shared_examples 'PyPI package index' do |user_type, status, add_member = true| + context "for user type #{user_type}" do + before do + project.send("add_#{user_type}", user) if add_member && user_type != :anonymous + group.send("add_#{user_type}", user) if add_member && user_type != :anonymous + end + + it 'returns the package index' do + subject + + expect(response.body).to match(package.name) + end + + it_behaves_like 'returning response status', status + end +end + RSpec.shared_examples 'PyPI package download' do |user_type, status, add_member = true| context "for user type #{user_type}" do before do @@ -259,6 +276,45 @@ RSpec.shared_examples 'pypi simple API endpoint' do end end +RSpec.shared_examples 'pypi simple index API endpoint' do + using RSpec::Parameterized::TableSyntax + + context 'with valid project' do + where(:visibility_level, :user_role, :member, :user_token, :shared_examples_name, :expected_status) do + :public | :developer | true | true | 'PyPI package index' | :success + :public | :guest | true | true | 'PyPI package index' | :success + :public | :developer | true | false | 'PyPI package index' | :success + :public | :guest | true | false | 'PyPI package index' | :success + :public | :developer | false | true | 'PyPI package index' | :success + :public | :guest | false | true | 'PyPI package index' | :success + :public | :developer | false | false | 'PyPI package index' | :success + :public | :guest | false | false | 'PyPI package index' | :success + :public | :anonymous | false | true | 'PyPI package index' | :success + :private | :developer | true | true | 'PyPI package index' | :success + :private | :guest | true | true | 'process PyPI api request' | :forbidden + :private | :developer | true | false | 'process PyPI api request' | :unauthorized + :private | :guest | true | false | 'process PyPI api request' | :unauthorized + :private | :developer | false | true | 'process PyPI api request' | :not_found + :private | :guest | false | true | 'process PyPI api request' | :not_found + :private | :developer | false | false | 'process PyPI api request' | :unauthorized + :private | :guest | false | false | 'process PyPI api request' | :unauthorized + :private | :anonymous | false | true | 'process PyPI api request' | :unauthorized + end + + with_them do + let(:token) { user_token ? personal_access_token.token : 'wrong' } + let(:headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, token) } + + before do + project.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s)) + group.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s)) + end + + it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status], params[:member] + end + end +end + RSpec.shared_examples 'pypi file download endpoint' do using RSpec::Parameterized::TableSyntax |