diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-11-21 06:11:27 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-11-21 06:11:27 +0300 |
| commit | 7619a0da6b4cc22b1eb8ecf3f943f6a28eac9937 (patch) | |
| tree | 0e1bfce99d2bb1aa35b73abc853964ef852252aa | |
| parent | b605abacd983a19c30dc364e8014fc758f496e98 (diff) | |
Add latest changes from gitlab-org/gitlab@master
45 files changed, 292 insertions, 57 deletions
diff --git a/.rubocop_todo/style/symbol_proc.yml b/.rubocop_todo/style/symbol_proc.yml index f13bf0c2e2e..ac3dec87a5c 100644 --- a/.rubocop_todo/style/symbol_proc.yml +++ b/.rubocop_todo/style/symbol_proc.yml @@ -201,9 +201,7 @@ Style/SymbolProc: - 'spec/lib/gitlab/usage/metrics/instrumentations/database_metric_spec.rb' - 'spec/lib/omni_auth/strategies/jwt_spec.rb' - 'spec/models/merge_request_diff_spec.rb' - - 'spec/models/namespace/root_storage_statistics_spec.rb' - 'spec/models/operations/feature_flags/strategy_spec.rb' - - 'spec/models/project_statistics_spec.rb' - 'spec/models/repository_spec.rb' - 'spec/models/user_spec.rb' - 'spec/presenters/projects/import_export/project_export_presenter_spec.rb' @@ -199,9 +199,9 @@ gem 'seed-fu', '~> 2.3.7' # rubocop:todo Gemfile/MissingFeatureCategory gem 'elasticsearch-model', '~> 7.2' # rubocop:todo Gemfile/MissingFeatureCategory gem 'elasticsearch-rails', '~> 7.2', require: 'elasticsearch/rails/instrumentation' # rubocop:todo Gemfile/MissingFeatureCategory gem 'elasticsearch-api', '7.13.3' # rubocop:todo Gemfile/MissingFeatureCategory -gem 'aws-sdk-core', '~> 3.186.0' # rubocop:todo Gemfile/MissingFeatureCategory +gem 'aws-sdk-core', '~> 3.187.0' # rubocop:todo Gemfile/MissingFeatureCategory gem 'aws-sdk-cloudformation', '~> 1' # rubocop:todo Gemfile/MissingFeatureCategory -gem 'aws-sdk-s3', '~> 1.136.0' # rubocop:todo Gemfile/MissingFeatureCategory +gem 'aws-sdk-s3', '~> 1.137.0' # rubocop:todo Gemfile/MissingFeatureCategory gem 'faraday_middleware-aws-sigv4', '~>0.3.0' # rubocop:todo Gemfile/MissingFeatureCategory gem 'typhoeus', '~> 1.4.0' # Used with Elasticsearch to support http keep-alive connections # rubocop:todo Gemfile/MissingFeatureCategory diff --git a/Gemfile.checksum b/Gemfile.checksum index e94f6bdf91c..97a2f334c35 100644 --- a/Gemfile.checksum +++ b/Gemfile.checksum @@ -37,9 +37,9 @@ {"name":"aws-eventstream","version":"1.2.0","platform":"ruby","checksum":"ffa53482c92880b001ff2fb06919b9bb82fd847cbb0fa244985d2ebb6dd0d1df"}, {"name":"aws-partitions","version":"1.761.0","platform":"ruby","checksum":"291e444e1edfc92c5521a6dbdd1236ccc3f122b3520163b2be6ec5b6ef350ef2"}, {"name":"aws-sdk-cloudformation","version":"1.41.0","platform":"ruby","checksum":"31e47539719734413671edf9b1a31f8673fbf9688549f50c41affabbcb1c6b26"}, -{"name":"aws-sdk-core","version":"3.186.0","platform":"ruby","checksum":"5ed564f83f334010c532d55f215068cc833aad40be41fe3dc851b08f1321f4a7"}, +{"name":"aws-sdk-core","version":"3.187.0","platform":"ruby","checksum":"b0b776d8be7fcc2b008fae511c02a7a381f31c5836319ccf86d4d97439d42e87"}, {"name":"aws-sdk-kms","version":"1.64.0","platform":"ruby","checksum":"40de596c95047bfc6e1aacea24f3df6241aa716b6f7ce08ac4c5f7e3120395ad"}, -{"name":"aws-sdk-s3","version":"1.136.0","platform":"ruby","checksum":"3547302a85d51de6cc75b48fb37d328f65f6526e7fc73a27a5b1b871f99a8d63"}, +{"name":"aws-sdk-s3","version":"1.137.0","platform":"ruby","checksum":"580011295202bf134d62473916933ef00ad9b9076264d8de8bd00e3c4e5bebab"}, {"name":"aws-sigv4","version":"1.6.0","platform":"ruby","checksum":"ca9e6a15cd424f1f32b524b9760995331459bc22e67d3daad4fcf0c0084b087d"}, {"name":"axe-core-api","version":"4.6.0","platform":"ruby","checksum":"1b0ddec3353f108dc10363baf2282f43a5ff7f13d4e25f99071294e78f8a6c62"}, {"name":"axe-core-rspec","version":"4.6.0","platform":"ruby","checksum":"11c25bc9dd388c137ba4e5e63d64d20092bf22c884d8ffc829a22acfbacd747f"}, diff --git a/Gemfile.lock b/Gemfile.lock index 78421e05b9c..a9e524b52ff 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -290,7 +290,7 @@ GEM aws-sdk-cloudformation (1.41.0) aws-sdk-core (~> 3, >= 3.99.0) aws-sigv4 (~> 1.1) - aws-sdk-core (3.186.0) + aws-sdk-core (3.187.0) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.5) @@ -298,7 +298,7 @@ GEM aws-sdk-kms (1.64.0) aws-sdk-core (~> 3, >= 3.165.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.136.0) + aws-sdk-s3 (1.137.0) aws-sdk-core (~> 3, >= 3.181.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.6) @@ -1794,8 +1794,8 @@ DEPENDENCIES autoprefixer-rails (= 10.2.5.1) awesome_print aws-sdk-cloudformation (~> 1) - aws-sdk-core (~> 3.186.0) - aws-sdk-s3 (~> 1.136.0) + aws-sdk-core (~> 3.187.0) + aws-sdk-s3 (~> 1.137.0) axe-core-rspec babosa (~> 2.0) base32 (~> 0.3.0) diff --git a/app/models/vulnerability.rb b/app/models/vulnerability.rb index 7c6c6e8c7e4..89a234b86af 100644 --- a/app/models/vulnerability.rb +++ b/app/models/vulnerability.rb @@ -5,9 +5,9 @@ class Vulnerability < ApplicationRecord include EachBatch include IgnorableColumns - ignore_column %i[epic_id milestone_id last_edited_at start_date start_date_sourcing_milestone_id], + ignore_column %i[epic_id milestone_id last_edited_at start_date start_date_sourcing_milestone_id updated_by_id], remove_with: '16.9', - remove_after: '2024-01-13' + remove_after: '2024-01-19' alias_attribute :vulnerability_id, :id diff --git a/db/docs/activity_pub_releases_subscriptions.yml b/db/docs/activity_pub_releases_subscriptions.yml index d759aada5a9..6075847c780 100644 --- a/db/docs/activity_pub_releases_subscriptions.yml +++ b/db/docs/activity_pub_releases_subscriptions.yml @@ -8,4 +8,6 @@ description: Stores subscriptions from external users through ActivityPub for pr releases introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/132889 milestone: '16.6' -gitlab_schema: gitlab_main +gitlab_schema: gitlab_main_cell +sharding_key: + project_id: projects diff --git a/db/docs/audit_events_streaming_http_group_namespace_filters.yml b/db/docs/audit_events_streaming_http_group_namespace_filters.yml index df08e8b57d2..204ebcd3feb 100644 --- a/db/docs/audit_events_streaming_http_group_namespace_filters.yml +++ b/db/docs/audit_events_streaming_http_group_namespace_filters.yml @@ -7,4 +7,6 @@ feature_categories: description: Represents a subgroup or project filter for audit event streaming on groups introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/135080 milestone: '16.6' -gitlab_schema: gitlab_main +gitlab_schema: gitlab_main_cell +sharding_key: + namespace_id: namespaces diff --git a/db/docs/labels.yml b/db/docs/labels.yml index 83956783891..7cccf039993 100644 --- a/db/docs/labels.yml +++ b/db/docs/labels.yml @@ -12,3 +12,6 @@ description: Information related to labels, which can be associated with groups introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/03654a6abf47c88b8b980a6707874ff78080d2fe milestone: '7.2' gitlab_schema: gitlab_main_cell +sharding_key: + project_id: projects + group_id: namespaces diff --git a/db/docs/merge_requests.yml b/db/docs/merge_requests.yml index 7e888184b6a..c24f6fd53b1 100644 --- a/db/docs/merge_requests.yml +++ b/db/docs/merge_requests.yml @@ -8,3 +8,5 @@ description: This is the main table that stores information about project merge introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/6d460aa2d6b3959593c168eed181516036525393 milestone: "<6.0" gitlab_schema: gitlab_main_cell +sharding_key: + target_project_id: projects diff --git a/db/gitlab_schemas/gitlab_ci.yaml b/db/gitlab_schemas/gitlab_ci.yaml index a756fa2fdd3..14911929d87 100644 --- a/db/gitlab_schemas/gitlab_ci.yaml +++ b/db/gitlab_schemas/gitlab_ci.yaml @@ -5,3 +5,4 @@ allow_cross_joins: allow_cross_transactions: - gitlab_internal - gitlab_shared +cell_local: true diff --git a/db/gitlab_schemas/gitlab_internal.yaml b/db/gitlab_schemas/gitlab_internal.yaml index 7093c980daa..bdeeee333c2 100644 --- a/db/gitlab_schemas/gitlab_internal.yaml +++ b/db/gitlab_schemas/gitlab_internal.yaml @@ -1,2 +1,3 @@ name: gitlab_internal description: Schema for all internal tables, like `pg_*` +cell_local: false diff --git a/db/gitlab_schemas/gitlab_main.yaml b/db/gitlab_schemas/gitlab_main.yaml index ed918e7c57d..878811e77c0 100644 --- a/db/gitlab_schemas/gitlab_main.yaml +++ b/db/gitlab_schemas/gitlab_main.yaml @@ -5,3 +5,4 @@ allow_cross_joins: allow_cross_transactions: - gitlab_internal - gitlab_shared +cell_local: false diff --git a/db/gitlab_schemas/gitlab_main_cell.yaml b/db/gitlab_schemas/gitlab_main_cell.yaml index 53f6f30dfbd..ac09f091d88 100644 --- a/db/gitlab_schemas/gitlab_main_cell.yaml +++ b/db/gitlab_schemas/gitlab_main_cell.yaml @@ -9,3 +9,4 @@ allow_cross_transactions: - gitlab_main allow_cross_foreign_keys: - gitlab_main +cell_local: true diff --git a/db/gitlab_schemas/gitlab_main_clusterwide.yaml b/db/gitlab_schemas/gitlab_main_clusterwide.yaml index 8f07f0caf4f..c026e4dd20a 100644 --- a/db/gitlab_schemas/gitlab_main_clusterwide.yaml +++ b/db/gitlab_schemas/gitlab_main_clusterwide.yaml @@ -19,3 +19,4 @@ allow_cross_foreign_keys: # temporarily allow FKs between clusterwide till all tables # are moved to either _clusterwide or _cell - gitlab_main +cell_local: false diff --git a/db/gitlab_schemas/gitlab_pm.yaml b/db/gitlab_schemas/gitlab_pm.yaml index 8ea8e49376b..240e9816492 100644 --- a/db/gitlab_schemas/gitlab_pm.yaml +++ b/db/gitlab_schemas/gitlab_pm.yaml @@ -5,3 +5,4 @@ allow_cross_joins: allow_cross_transactions: - gitlab_internal - gitlab_shared +cell_local: true diff --git a/db/gitlab_schemas/gitlab_shared.yaml b/db/gitlab_schemas/gitlab_shared.yaml index 549797b606b..739fe052903 100644 --- a/db/gitlab_schemas/gitlab_shared.yaml +++ b/db/gitlab_schemas/gitlab_shared.yaml @@ -6,3 +6,4 @@ allow_cross_joins: - gitlab_internal allow_cross_transactions: - gitlab_internal +cell_local: false diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 68196cbec93..e183518a111 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -27117,6 +27117,7 @@ Represents a vulnerability. | <a id="vulnerabilitylocation"></a>`location` | [`VulnerabilityLocation`](#vulnerabilitylocation) | Location metadata for the vulnerability. Its fields depend on the type of security scan that found the vulnerability. | | <a id="vulnerabilitymergerequest"></a>`mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request that fixes the vulnerability. | | <a id="vulnerabilitymessage"></a>`message` **{warning-solid}** | [`String`](#string) | **Deprecated** in 16.1. message field has been removed from security reports schema. | +| <a id="vulnerabilitypresentondefaultbranch"></a>`presentOnDefaultBranch` | [`Boolean!`](#boolean) | Indicates whether the vulnerability is present on the default branch or not. | | <a id="vulnerabilityprimaryidentifier"></a>`primaryIdentifier` | [`VulnerabilityIdentifier`](#vulnerabilityidentifier) | Primary identifier of the vulnerability. | | <a id="vulnerabilityproject"></a>`project` | [`Project`](#project) | Project on which the vulnerability was found. | | <a id="vulnerabilityreporttype"></a>`reportType` | [`VulnerabilityReportType`](#vulnerabilityreporttype) | Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST, SECRET_DETECTION, COVERAGE_FUZZING, API_FUZZING, CLUSTER_IMAGE_SCANNING, GENERIC). `Scan Type` in the UI. | diff --git a/doc/api/project_vulnerabilities.md b/doc/api/project_vulnerabilities.md index c15a7452219..e1b0dedc0b9 100644 --- a/doc/api/project_vulnerabilities.md +++ b/doc/api/project_vulnerabilities.md @@ -10,6 +10,7 @@ type: reference, api > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10242) in GitLab 12.6. > - `last_edited_at` [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/268154) in GitLab 16.7. > - `start_date` [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/268154) in GitLab 16.7. +> - `updated_by_id` [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/268154) in GitLab 16.7. WARNING: This API is in the process of being deprecated and considered unstable. @@ -99,8 +100,7 @@ Example response: "severity": "low", "state": "detected", "title": "Regular Expression Denial of Service in debug", - "updated_at": "2020-04-07T14:01:04.655Z", - "updated_by_id": null + "updated_at": "2020-04-07T14:01:04.655Z" } ] ``` @@ -185,8 +185,7 @@ Example response: "severity": "low", "state": "detected", "title": "Regular Expression Denial of Service in debug", - "updated_at": "2020-04-07T14:01:04.655Z", - "updated_by_id": null + "updated_at": "2020-04-07T14:01:04.655Z" } ``` diff --git a/doc/api/vulnerabilities.md b/doc/api/vulnerabilities.md index bddedb05c2c..ed5638915e0 100644 --- a/doc/api/vulnerabilities.md +++ b/doc/api/vulnerabilities.md @@ -9,6 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10242) in GitLab 12.6. > - `last_edited_at` [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/268154) in GitLab 16.7. > - `start_date` [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/268154) in GitLab 16.7. +> - `updated_by_id` [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/268154) in GitLab 16.7. NOTE: The former Vulnerabilities API was renamed to Vulnerability Findings API @@ -62,7 +63,6 @@ Example response: "full_name": "gitlab-examples / security / security-reports" }, "author_id": 1, - "updated_by_id": null, "last_edited_by_id": null, "closed_by_id": null, "due_date": null, @@ -110,7 +110,6 @@ Example response: "full_name": "gitlab-examples / security / security-reports" }, "author_id": 1, - "updated_by_id": null, "last_edited_by_id": null, "closed_by_id": null, "due_date": null, @@ -158,7 +157,6 @@ Example response: "full_name": "gitlab-examples / security / security-reports" }, "author_id": 1, - "updated_by_id": null, "last_edited_by_id": null, "closed_by_id": null, "due_date": null, @@ -206,7 +204,6 @@ Example response: "full_name": "gitlab-examples / security / security-reports" }, "author_id": 1, - "updated_by_id": null, "last_edited_by_id": null, "closed_by_id": null, "due_date": null, @@ -254,7 +251,6 @@ Example response: "full_name": "gitlab-examples / security / security-reports" }, "author_id": 1, - "updated_by_id": null, "last_edited_by_id": null, "closed_by_id": null, "due_date": null, diff --git a/doc/architecture/blueprints/organization/isolation.md b/doc/architecture/blueprints/organization/isolation.md index 238269c4329..467bd1932bd 100644 --- a/doc/architecture/blueprints/organization/isolation.md +++ b/doc/architecture/blueprints/organization/isolation.md @@ -65,7 +65,7 @@ These are: The major constraint these POCs were trying to overcome was that there is no standard way in the GitLab application or database to even determine what Organization (or Project or namespace) a piece of data belongs to. This means that the first step is to implement a standard way to efficiently find the parent Organization for any model or row in the database. -The proposed solution is ensuring that every single table that exists in the `gitlab_main_cell` and `gitlab_ci_cell` (Cell-local) databases must include a valid sharding key that is either `project_id` or `namespace_id`. +The proposed solution is ensuring that every single table that exists in the `gitlab_main_cell`, `gitlab_ci` and `gitlab_pm` (Cell-local) databases must include a valid sharding key that is a reference to `projects`, `namespaces` or `organizations`. At first we considered enforcing everything to have an `organization_id`, but we determined that this would be too expensive to update for customers that need to migrate large Groups out of the default Organization. The added benefit is that more than half of our tables already have one of these columns. Additionally, if we can't consistently attribute data to a top-level Group, then we won't be able to validate if a top-level Group is safe to be moved to a new Organization. @@ -79,7 +79,7 @@ We can also use these sharding keys to help us decide whether: ## Detailed steps -1. Implement developer facing documentation explaining the requirement to add these sharding keys and how they should choose between `project_id` and `namespace_id`. +1. Implement developer facing documentation explaining the requirement to add these sharding keys and how they should choose. 1. Add a way to declare a sharding key in `db/docs` and automatically populate it for all tables that already have a sharding key 1. Implement automation in our CI pipelines and/or DB migrations that makes it impossible to create new tables without a sharding key. 1. Implement a way for people to declare a desired sharding key in `db/docs` as @@ -107,7 +107,7 @@ We can also use these sharding keys to help us decide whether: automated MRs for the sharding keys that can be automatically inferred and automate creating issues for all the sharding keys that can't be automatically inferred -1. Validate that all existing `project_id` and `namespace_id` columns on all Cell-local tables can reliably be assumed to be the sharding key. This requires assigning issues to teams to confirm that these columns aren't used for some other purpose that would actually not be suitable. If there is an issue with a table we need to migrate and rename these columns, and then add a new `project_id` or `namespace_id` column with the correct sharding key. +1. Validate that all existing sharding key columns on all Cell-local tables can reliably be assumed to be the sharding key. This requires assigning issues to teams to confirm that these columns aren't used for some other purpose that would actually not be suitable. 1. We allow customers to create new Organizations without the option to migrate namespaces into them. All namespaces need to be newly created in their new Organization. 1. Implement new functionality in GitLab similar to the [POC](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/131968), which allows a namespace owner to see if their namespace is fully isolated. 1. Implement functionality that allows namespace owners to migrate an existing namespace from one Organization to another. Most likely this will be existing customers that want to migrate their namespace out of the default Organization into a newly created Organization. Only isolated namespaces as implemented in the previous step will be allowed to move. diff --git a/doc/ci/pipelines/settings.md b/doc/ci/pipelines/settings.md index 321eae183eb..4768c04c748 100644 --- a/doc/ci/pipelines/settings.md +++ b/doc/ci/pipelines/settings.md @@ -103,6 +103,24 @@ To avoid this scenario: For more information, see [Deployment safety](../environments/deployment_safety.md#prevent-outdated-deployment-jobs). +## Restrict roles that can cancel pipelines or jobs **(PREMIUM ALL)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/410634) in GitLab 16.7 [with a flag](../../administration/feature_flags.md) named `restrict_pipeline_cancellation_by_role`. Disabled by default. + +You can customize which roles have permission to cancel pipelines or jobs. + +By default, users with at least the Developer role can cancel pipelines or jobs. +You can restrict cancellation permission to only users with at least the Maintainer role, +or completely prevent cancellation of any pipelines or jobs. + +To change the permissions to cancel pipelines or jobs: + +1. On the left sidebar, select **Search or go to** and find your project. +1. Select **Settings > CI/CD**. +1. Expand **General pipelines**. +1. Select an option from **Minimum role required to cancel a pipeline or job**. +1. Select **Save changes**. + ## Specify a custom CI/CD configuration file > Support for external `.gitlab-ci.yml` locations [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14376) in GitLab 12.6. diff --git a/doc/development/database/multiple_databases.md b/doc/development/database/multiple_databases.md index 22dc637b515..84af28c81e2 100644 --- a/doc/development/database/multiple_databases.md +++ b/doc/development/database/multiple_databases.md @@ -64,6 +64,86 @@ After a schema has been assigned, the merge request pipeline might fail due to o - [Cross-database transactions](#fixing-cross-database-transactions) - [Cross-database foreign keys](#foreign-keys-that-cross-databases) +### Defining a sharding key for all cell-local tables + +All tables with the following `gitlab_schema` are considered "cell-local": + +- `gitlab_main_cell` +- `gitlab_ci` + +All newly created cell-local tables are required to have a `sharding_key` +defined in the corresponding `db/docs/` file for that table. + +The purpose of the sharding key is documented in the +[Organization isolation blueprint](../../architecture/blueprints/organization/isolation.md), +but in short this column is used to provide a standard way of determining which +Organization owns a particular row in the database. The column will be used in +the future to enforce constraints on data not cross Organization boundaries. It +will also be used in the future to provide a uniform way to migrate data +between Cells. + +The actual name of the foreign key can be anything but it must reference a row +in `projects` or `groups`. The following are examples of valid sharding keys: + +- The table entries belong to a project only: + + ```yaml + sharding_key: + project_id: projects + ``` + +- The table entries belong to a project and the foreign key is `target_project_id`: + + ```yaml + sharding_key: + target_project_id: projects + ``` + +- The table entries belong to a namespace/group only: + + ```yaml + sharding_key: + namespace_id: namespaces + ``` + +- The table entries belong to a namespace/group only and the foreign key is `group_id`: + + ```yaml + sharding_key: + group_id: namespaces + ``` + +- The table entries belong to a namespace or a project: + + ```yaml + sharding_key: + project_id: projects + namespace_id: namespaces + ``` + +#### The sharding key must be immutable + +The choice of a `sharding_key` should always be immutable. Therefore, if your feature +requires a user experience which allows data to be moved between projects or +groups/namespaces, then you may need to redesign the move feature to create new rows. An +example of this can be seen in the +[move an issue feature](../../user/project/issues/managing_issues.md#move-an-issue). +This feature does not actually change the `project_id` column for an existing +`issues` row but instead creates a new `issues` row and creates a link in the +database from the original `issues` row. If there is a particularly challenging +existing feature that needs to allow moving data you will need to reach out to +the Tenant Scale team early on to discuss options for how to manage the +sharding key. + +#### Using the same sharding key for projects and namespaces + +Developers may also choose to use `namespace_id` only for tables that can +belong to a project where the feature used by the table is being developed +following the +[Consolidating Groups and Projects blueprint](../../architecture/blueprints/consolidating_groups_and_projects/index.md). +In that case the `namespace_id` would need to be the ID of the +`ProjectNamespace` and not the group that the namespace belongs to. + ### The impact of `gitlab_schema` The usage of `gitlab_schema` has a significant impact on the application. diff --git a/doc/user/analytics/img/enhanced_issue_analytics_v16_7.png b/doc/user/analytics/img/enhanced_issue_analytics_v16_7.png Binary files differnew file mode 100644 index 00000000000..519e56acaa5 --- /dev/null +++ b/doc/user/analytics/img/enhanced_issue_analytics_v16_7.png diff --git a/doc/user/analytics/img/issues_closed_analytics_v16_4.png b/doc/user/analytics/img/issues_closed_analytics_v16_4.png Binary files differdeleted file mode 100644 index 5e1fe4eaa8c..00000000000 --- a/doc/user/analytics/img/issues_closed_analytics_v16_4.png +++ /dev/null diff --git a/doc/user/analytics/issue_analytics.md b/doc/user/analytics/issue_analytics.md index 8f29c008d75..35771a26366 100644 --- a/doc/user/analytics/issue_analytics.md +++ b/doc/user/analytics/issue_analytics.md @@ -50,7 +50,7 @@ available. This feature is not ready for production use. Enhanced issue analytics display the additional metric "Issues closed", which represents the total number of resolved issues in your project over a selected period. You can use this metric to improve the overall turn-around time and value delivered to your customers. - + ## Drill into the information diff --git a/doc/user/group/issues_analytics/img/enhanced_issue_analytics_v16_7.png b/doc/user/group/issues_analytics/img/enhanced_issue_analytics_v16_7.png Binary files differnew file mode 100644 index 00000000000..519e56acaa5 --- /dev/null +++ b/doc/user/group/issues_analytics/img/enhanced_issue_analytics_v16_7.png diff --git a/doc/user/group/issues_analytics/img/issues_closed_analytics_v16_4.png b/doc/user/group/issues_analytics/img/issues_closed_analytics_v16_4.png Binary files differdeleted file mode 100644 index 5e1fe4eaa8c..00000000000 --- a/doc/user/group/issues_analytics/img/issues_closed_analytics_v16_4.png +++ /dev/null diff --git a/doc/user/group/issues_analytics/index.md b/doc/user/group/issues_analytics/index.md index 4f1c7b4be7a..3dc3248ab6a 100644 --- a/doc/user/group/issues_analytics/index.md +++ b/doc/user/group/issues_analytics/index.md @@ -50,7 +50,7 @@ available. This feature is not ready for production use. Enhanced issue analytics display the additional metric "Issues closed", which represents the total number of resolved issues in your group over a selected period. You can use this metric to improve the overall turn-around time and value delivered to your customers. - + ## Drill into the information diff --git a/doc/user/permissions.md b/doc/user/permissions.md index 38854d3daef..ce3e6722a79 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -268,26 +268,27 @@ More details about the permissions for some project-level features follow. | Action | Non-member | Guest | Reporter | Developer | Maintainer | Owner | |---------------------------------------------------------------------------------------------------------------------------|------------|---------|----------|-----------|------------|-------| -| See that artifacts exist | ✓ (3) | ✓ (3) | ✓ | ✓ | ✓ | ✓ | -| View a list of jobs | ✓ (1) | ✓ (2) | ✓ | ✓ | ✓ | ✓ | -| View and download artifacts | ✓ (1) | ✓ (2) | ✓ | ✓ | ✓ | ✓ | -| View [environments](../ci/environments/index.md) | ✓ (3) | ✓ (3) | ✓ | ✓ | ✓ | ✓ | -| View job logs and job details page | ✓ (1) | ✓ (2) | ✓ | ✓ | ✓ | ✓ | -| View pipelines and pipeline details pages | ✓ (1) | ✓ (2) | ✓ | ✓ | ✓ | ✓ | -| View pipelines tab in MR | ✓ (3) | ✓ (3) | ✓ | ✓ | ✓ | ✓ | -| [View vulnerabilities in a pipeline](application_security/vulnerability_report/pipeline.md#view-vulnerabilities-in-a-pipeline) | | ✓ (2) | ✓ | ✓ | ✓ | ✓ | +| See that artifacts exist | ✓ (3) | ✓ (3) | ✓ | ✓ | ✓ | ✓ | +| View a list of jobs | ✓ (1) | ✓ (2) | ✓ | ✓ | ✓ | ✓ | +| View and download artifacts | ✓ (1) | ✓ (2) | ✓ | ✓ | ✓ | ✓ | +| View [environments](../ci/environments/index.md) | ✓ (3) | ✓ (3) | ✓ | ✓ | ✓ | ✓ | +| View job logs and job details page | ✓ (1) | ✓ (2) | ✓ | ✓ | ✓ | ✓ | +| View pipelines and pipeline details pages | ✓ (1) | ✓ (2) | ✓ | ✓ | ✓ | ✓ | +| View pipelines tab in MR | ✓ (3) | ✓ (3) | ✓ | ✓ | ✓ | ✓ | +| [View vulnerabilities in a pipeline](application_security/vulnerability_report/pipeline.md#view-vulnerabilities-in-a-pipeline) | | ✓ (2) | ✓ | ✓ | ✓ | ✓ | | View and download project-level [Secure Files](../api/secure_files.md) | | | | ✓ | ✓ | ✓ | -| Cancel and retry jobs | | | | ✓ | ✓ | ✓ | +| Retry jobs | | | | ✓ | ✓ | ✓ | +| Cancel jobs | | | | ✓ (7) | ✓ (7) | ✓ (7) | | Create new [environments](../ci/environments/index.md) | | | | ✓ | ✓ | ✓ | -| Delete job logs or job artifacts | | | | ✓ (4) | ✓ | ✓ | +| Delete job logs or job artifacts | | | | ✓ (4) | ✓ | ✓ | | Run CI/CD pipeline | | | | ✓ | ✓ | ✓ | -| Run CI/CD pipeline for a protected branch | | | | ✓ (5) | ✓ (5) | ✓ | +| Run CI/CD pipeline for a protected branch | | | | ✓ (5) | ✓ (5) | ✓ | | Stop [environments](../ci/environments/index.md) | | | | ✓ | ✓ | ✓ | -| Run deployment job for a protected environment | | | ✓ (5) | ✓ (6) | ✓ (6) | ✓ | +| Run deployment job for a protected environment | | | ✓ (5) | ✓ (6) | ✓ (6) | ✓ | | View a job with [debug logging](../ci/variables/index.md#enable-debug-logging) | | | | ✓ | ✓ | ✓ | | Use pipeline editor | | | | ✓ | ✓ | ✓ | | Run [interactive web terminals](../ci/interactive_web_terminal/index.md) | | | | ✓ | ✓ | ✓ | -| Add project runners to project | | | | | ✓ | ✓ | +| Add project runners to project | | | | | ✓ | ✓ | | Clear runner caches manually | | | | | ✓ | ✓ | | Enable shared runners in project | | | | | ✓ | ✓ | | Manage CI/CD settings | | | | | ✓ | ✓ | @@ -308,6 +309,7 @@ More details about the permissions for some project-level features follow. run for a non-protected branch. 5. If the user is [allowed to merge or push to the protected branch](../ci/pipelines/index.md#pipeline-security-on-protected-branches). 6. If the user if [part of a group with at least the Reporter role](../ci/environments/protected_environments.md#deployment-only-access-to-protected-environments) +7. Cancellation permissions can be [customized via the pipeline settings](../ci/pipelines/settings.md#restrict-roles-that-can-cancel-pipelines-or-jobs) <!-- markdownlint-enable MD029 --> diff --git a/doc/user/project/merge_requests/status_checks.md b/doc/user/project/merge_requests/status_checks.md index c330af0fc9b..5f50faeaeb0 100644 --- a/doc/user/project/merge_requests/status_checks.md +++ b/doc/user/project/merge_requests/status_checks.md @@ -172,8 +172,7 @@ To retry a failed status check: An organization might have a policy that does not allow merging merge requests if external status checks do not pass. However, the details in the widget are for informational -purposes only. GitLab does not prevent merging of merge requests that fail status checks. -Support to allow merges to be blocked when external status checks fail is proposed in epic [&8516](https://gitlab.com/groups/gitlab-org/-/epics/8516). +purposes only. NOTE: GitLab cannot guarantee that the external status checks are properly processed by diff --git a/lib/gitlab/database/dictionary.rb b/lib/gitlab/database/dictionary.rb index c766010a80f..e77647681e8 100644 --- a/lib/gitlab/database/dictionary.rb +++ b/lib/gitlab/database/dictionary.rb @@ -57,6 +57,10 @@ module Gitlab data['gitlab_schema'] end + def sharding_key + data['sharding_key'] + end + def schema?(schema_name) gitlab_schema == schema_name.to_s end diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 7c48aea929f..e6f7dbec69c 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -88,6 +88,10 @@ module Gitlab # rubocop:enable Gitlab/DocUrl end + def self.cell_local?(schema) + Gitlab::Database.all_gitlab_schemas[schema.to_s].cell_local + end + def self.cross_joins_allowed?(table_schemas, all_tables) return true unless table_schemas.many? diff --git a/lib/gitlab/database/gitlab_schema_info.rb b/lib/gitlab/database/gitlab_schema_info.rb index 20d2b31a65c..b7ec3dfc893 100644 --- a/lib/gitlab/database/gitlab_schema_info.rb +++ b/lib/gitlab/database/gitlab_schema_info.rb @@ -14,6 +14,7 @@ module Gitlab :allow_cross_transactions, :allow_cross_foreign_keys, :file_path, + :cell_local, keyword_init: true ) do def initialize(*) diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 9d48cec55f4..c79432f36cc 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -226,7 +226,8 @@ module Gitlab params 'email1@example.com email2@example.com (up to 6 emails)' types Issue condition do - Feature.enabled?(:issue_email_participants, parent) && + quick_action_target.persisted? && + Feature.enabled?(:issue_email_participants, parent) && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) end command :invite_email do |emails = ""| diff --git a/locale/gitlab.pot b/locale/gitlab.pot index c5e4593a9cb..61eded19625 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -26565,6 +26565,11 @@ msgstr "" msgid "IssuesAnalytics|Last 12 months (%{chartDateRange})" msgstr "" +msgid "IssuesAnalytics|Last month (%{dateRange})" +msgid_plural "IssuesAnalytics|Last %{monthsCount} months (%{dateRange})" +msgstr[0] "" +msgstr[1] "" + msgid "IssuesAnalytics|Opened" msgstr "" @@ -26574,10 +26579,8 @@ msgstr "" msgid "IssuesAnalytics|Sorry, your filter produced no results" msgstr "" -msgid "IssuesAnalytics|This month (%{dateRange})" -msgid_plural "IssuesAnalytics|Last %{monthsCount} months (%{dateRange})" -msgstr[0] "" -msgstr[1] "" +msgid "IssuesAnalytics|This month (%{currentMonthYear})" +msgstr "" msgid "IssuesAnalytics|To widen your search, change or remove filters in the filter bar above." msgstr "" diff --git a/spec/db/docs_spec.rb b/spec/db/docs_spec.rb index 19edf3da0d5..5c19db0238c 100644 --- a/spec/db/docs_spec.rb +++ b/spec/db/docs_spec.rb @@ -15,6 +15,7 @@ RSpec.shared_examples 'validate dictionary' do |objects, directory_path, require milestone gitlab_schema schema_inconsistencies + sharding_key ] end diff --git a/spec/lib/gitlab/ci/config/external/file/local_spec.rb b/spec/lib/gitlab/ci/config/external/file/local_spec.rb index 0643bf0c046..b961ee0d190 100644 --- a/spec/lib/gitlab/ci/config/external/file/local_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/local_spec.rb @@ -269,8 +269,8 @@ RSpec.describe Gitlab::Ci::Config::External::File::Local, feature_category: :pip context_sha: sha, type: :local, location: 'lib/gitlab/ci/templates/existent-file.yml', - blob: "http://localhost/#{project.full_path}/-/blob/#{sha}/lib/gitlab/ci/templates/existent-file.yml", - raw: "http://localhost/#{project.full_path}/-/raw/#{sha}/lib/gitlab/ci/templates/existent-file.yml", + blob: "http://#{Gitlab.config.gitlab.host}/#{project.full_path}/-/blob/#{sha}/lib/gitlab/ci/templates/existent-file.yml", + raw: "http://#{Gitlab.config.gitlab.host}/#{project.full_path}/-/raw/#{sha}/lib/gitlab/ci/templates/existent-file.yml", extra: {} ) } diff --git a/spec/lib/gitlab/database/gitlab_schema_spec.rb b/spec/lib/gitlab/database/gitlab_schema_spec.rb index 3573ce7ee14..7fca47c707c 100644 --- a/spec/lib/gitlab/database/gitlab_schema_spec.rb +++ b/spec/lib/gitlab/database/gitlab_schema_spec.rb @@ -273,4 +273,16 @@ RSpec.describe Gitlab::Database::GitlabSchema, feature_category: :database do end end end + + describe '.cell_local?' do + it 'is true for cell local tables and false otherwise' do + expect(described_class.cell_local?('gitlab_ci')).to eq(true) + expect(described_class.cell_local?('gitlab_pm')).to eq(true) + expect(described_class.cell_local?('gitlab_main_cell')).to eq(true) + expect(described_class.cell_local?('gitlab_main')).to eq(false) + expect(described_class.cell_local?('gitlab_main_clusterwide')).to eq(false) + expect(described_class.cell_local?('gitlab_shared')).to eq(false) + expect(described_class.cell_local?('gitlab_internal')).to eq(false) + end + end end diff --git a/spec/lib/gitlab/database/sharding_key_spec.rb b/spec/lib/gitlab/database/sharding_key_spec.rb new file mode 100644 index 00000000000..5bee095ac8d --- /dev/null +++ b/spec/lib/gitlab/database/sharding_key_spec.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'new tables missing sharding_key', feature_category: :cell do + # Specific tables can be temporarily exempt from this requirement. You must add an issue link in a comment next to + # the table name to remove this once a decision has been made. + let(:allowed_to_be_missing_sharding_key) do + [ + 'abuse_report_assignees' # https://gitlab.com/gitlab-org/gitlab/-/issues/432365 + ] + end + + let(:starting_from_milestone) { 16.6 } + + let(:allowed_sharding_key_referenced_tables) { %w[projects namespaces organizations] } + + it 'requires a sharding_key for all cell-local tables, after milestone 16.6', :aggregate_failures do + tables_missing_sharding_key(starting_from_milestone: starting_from_milestone).each do |table_name| + expect(allowed_to_be_missing_sharding_key).to include(table_name), error_message(table_name) + end + end + + it 'ensures all sharding_key columns exist and reference projects, namespaces or organizations', + :aggregate_failures do + all_tables_to_sharding_key.each do |table_name, sharding_key| + sharding_key.each do |column_name, referenced_table_name| + expect(column_exists?(table_name, column_name)).to eq(true), + "Could not find sharding key column #{table_name}.#{column_name}" + expect(referenced_table_name).to be_in(allowed_sharding_key_referenced_tables) + end + end + end + + it 'only allows `allowed_to_be_missing_sharding_key` to include tables that are missing a sharding_key', + :aggregate_failures do + allowed_to_be_missing_sharding_key.each do |exempted_table| + expect(tables_missing_sharding_key(starting_from_milestone: starting_from_milestone)).to include(exempted_table), + "`#{exempted_table}` is not missing a `sharding_key`. " \ + "You must remove this table from the `allowed_to_be_missing_sharding_key` list." + end + end + + private + + def error_message(table_name) + <<~HEREDOC + The table `#{table_name}` is missing a `sharding_key` in the `db/docs` YML file. + Starting from GitLab #{starting_from_milestone}, we expect all new tables to define a `sharding_key`. + + To choose an appropriate sharding_key for this table please refer + to our guidelines at https://docs.gitlab.com/ee/development/database/multiple_databases.html#defining-a-sharding-key-for-all-cell-local-tables, or consult with the Tenant Scale group. + HEREDOC + end + + def tables_missing_sharding_key(starting_from_milestone:) + ::Gitlab::Database::Dictionary.entries.select do |entry| + entry.sharding_key.blank? && + entry.milestone.to_f >= starting_from_milestone && + ::Gitlab::Database::GitlabSchema.cell_local?(entry.gitlab_schema) + end.map(&:table_name) + end + + def all_tables_to_sharding_key + entries_with_sharding_key = ::Gitlab::Database::Dictionary.entries.select do |entry| + entry.sharding_key.present? && + ::Gitlab::Database::GitlabSchema.cell_local?(entry.gitlab_schema) + end + + entries_with_sharding_key.to_h do |entry| + [entry.table_name, entry.sharding_key] + end + end + + def column_exists?(table_name, column_name) + sql = <<~SQL + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'public' AND + table_name = '#{table_name}' AND + column_name = '#{column_name}'; + SQL + + result = ApplicationRecord.connection.execute(sql) + + result.count > 0 + end +end diff --git a/spec/models/namespace/root_storage_statistics_spec.rb b/spec/models/namespace/root_storage_statistics_spec.rb index 4b66b7532a7..9a66b4745c0 100644 --- a/spec/models/namespace/root_storage_statistics_spec.rb +++ b/spec/models/namespace/root_storage_statistics_spec.rb @@ -12,7 +12,7 @@ RSpec.describe Namespace::RootStorageStatistics, type: :model, feature_category: describe '.for_namespace_ids' do it 'returns only requested namespaces' do stats = create_list(:namespace_root_storage_statistics, 3) - namespace_ids = stats[0..1].map { |s| s.namespace_id } + namespace_ids = stats[0..1].map(&:namespace_id) requested_stats = described_class.for_namespace_ids(namespace_ids).pluck(:namespace_id) diff --git a/spec/models/project_statistics_spec.rb b/spec/models/project_statistics_spec.rb index dd7989244d4..211ac257c53 100644 --- a/spec/models/project_statistics_spec.rb +++ b/spec/models/project_statistics_spec.rb @@ -15,8 +15,8 @@ RSpec.describe ProjectStatistics do describe '.for_project_ids' do it 'returns only requested projects' do stats = create_list(:project_statistics, 3) - project_ids = stats[0..1].map { |s| s.project_id } - expected_ids = stats[0..1].map { |s| s.id } + project_ids = stats[0..1].map(&:project_id) + expected_ids = stats[0..1].map(&:id) requested_stats = described_class.for_project_ids(project_ids).pluck(:id) diff --git a/spec/requests/api/graphql/project/alert_management/integrations_spec.rb b/spec/requests/api/graphql/project/alert_management/integrations_spec.rb index e48db541e1f..c4d3a217027 100644 --- a/spec/requests/api/graphql/project/alert_management/integrations_spec.rb +++ b/spec/requests/api/graphql/project/alert_management/integrations_spec.rb @@ -67,7 +67,7 @@ RSpec.describe 'getting Alert Management Integrations', feature_category: :incid 'name' => 'Prometheus', 'active' => prometheus_integration.manual_configuration?, 'token' => project_alerting_setting.token, - 'url' => "http://localhost/#{project.full_path}/prometheus/alerts/notify.json", + 'url' => "http://#{Gitlab.config.gitlab.host}/#{project.full_path}/prometheus/alerts/notify.json", 'apiUrl' => prometheus_integration.api_url ) ] diff --git a/spec/requests/api/snippets_spec.rb b/spec/requests/api/snippets_spec.rb index 0b97bb5c443..b43f98e5323 100644 --- a/spec/requests/api/snippets_spec.rb +++ b/spec/requests/api/snippets_spec.rb @@ -115,8 +115,8 @@ RSpec.describe API::Snippets, :aggregate_failures, factory_default: :keep, featu public_snippet.id, public_snippet_other.id) expect(json_response.map { |snippet| snippet['web_url'] }).to contain_exactly( - "http://localhost/-/snippets/#{public_snippet.id}", - "http://localhost/-/snippets/#{public_snippet_other.id}") + "http://#{Gitlab.config.gitlab.host}/-/snippets/#{public_snippet.id}", + "http://#{Gitlab.config.gitlab.host}/-/snippets/#{public_snippet_other.id}") expect(json_response[0]['files'].first).to eq snippet_blob_file(public_snippet_other.blobs.first) expect(json_response[1]['files'].first).to eq snippet_blob_file(public_snippet.blobs.first) end diff --git a/spec/services/quick_actions/interpret_service_spec.rb b/spec/services/quick_actions/interpret_service_spec.rb index 1c9c6323e96..0b09bb353d5 100644 --- a/spec/services/quick_actions/interpret_service_spec.rb +++ b/spec/services/quick_actions/interpret_service_spec.rb @@ -2384,6 +2384,18 @@ RSpec.describe QuickActions::InterpretService, feature_category: :team_planning end end end + + it 'is part of the available commands' do + expect(service.available_commands(issuable)).to include(a_hash_including(name: :invite_email)) + end + + context 'with non-persisted issue' do + let(:issuable) { build(:issue) } + + it 'is not part of the available commands' do + expect(service.available_commands(issuable)).not_to include(a_hash_including(name: :invite_email)) + end + end end context 'severity command' do diff --git a/spec/support/helpers/snippet_helpers.rb b/spec/support/helpers/snippet_helpers.rb index 1ec50bce070..242661a4e23 100644 --- a/spec/support/helpers/snippet_helpers.rb +++ b/spec/support/helpers/snippet_helpers.rb @@ -8,7 +8,7 @@ module SnippetHelpers def snippet_blob_file(blob) { "path" => blob.path, - "raw_url" => gitlab_raw_snippet_blob_url(blob.container, blob.path, host: 'localhost') + "raw_url" => gitlab_raw_snippet_blob_url(blob.container, blob.path, host: Gitlab.config.gitlab.host) } end end |
