diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-06-30 00:08:46 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-06-30 00:08:46 +0300 |
commit | 4f1b5d2757835a204a7b882e2bea32dd8c9dfa6b (patch) | |
tree | 0617ad58ec0f7e8fcbc02bae09976299f9d0a24e | |
parent | 1bbd0179d7ed8fb17c0574aa74ef491e53c833a7 (diff) |
Add latest changes from gitlab-org/gitlab@master
39 files changed, 528 insertions, 202 deletions
diff --git a/.rubocop_todo/layout/empty_line_after_magic_comment.yml b/.rubocop_todo/layout/empty_line_after_magic_comment.yml index 71a05b0146d..698247e419c 100644 --- a/.rubocop_todo/layout/empty_line_after_magic_comment.yml +++ b/.rubocop_todo/layout/empty_line_after_magic_comment.yml @@ -289,7 +289,6 @@ Layout/EmptyLineAfterMagicComment: - 'ee/spec/features/projects/kerberos_clone_instructions_spec.rb' - 'ee/spec/features/projects/settings/merge_request_approvals_settings_spec.rb' - 'ee/spec/features/projects/settings/merge_requests_settings_spec.rb' - - 'ee/spec/features/trial_registrations/company_information_spec.rb' - 'ee/spec/finders/auth/group_saml_identity_finder_spec.rb' - 'ee/spec/finders/geo/design_registry_finder_spec.rb' - 'ee/spec/finders/geo/project_registry_status_finder_spec.rb' diff --git a/.rubocop_todo/layout/line_length.yml b/.rubocop_todo/layout/line_length.yml index daf734eb8ed..3c6a0e8adda 100644 --- a/.rubocop_todo/layout/line_length.yml +++ b/.rubocop_todo/layout/line_length.yml @@ -1494,7 +1494,6 @@ Layout/LineLength: - 'ee/spec/features/search/elastic/project_search_spec.rb' - 'ee/spec/features/search/elastic/snippet_search_spec.rb' - 'ee/spec/features/subscriptions_spec.rb' - - 'ee/spec/features/trial_registrations/company_information_spec.rb' - 'ee/spec/features/users/login_spec.rb' - 'ee/spec/finders/analytics/devops_adoption/enabled_namespaces_finder_spec.rb' - 'ee/spec/finders/analytics/devops_adoption/snapshots_finder_spec.rb' diff --git a/.rubocop_todo/rspec/context_wording.yml b/.rubocop_todo/rspec/context_wording.yml index 64b2c69e6de..08b9c1d50c6 100644 --- a/.rubocop_todo/rspec/context_wording.yml +++ b/.rubocop_todo/rspec/context_wording.yml @@ -168,7 +168,6 @@ RSpec/ContextWording: - 'ee/spec/features/promotion_spec.rb' - 'ee/spec/features/protected_branches_spec.rb' - 'ee/spec/features/signup_spec.rb' - - 'ee/spec/features/trial_registrations/company_information_spec.rb' - 'ee/spec/features/users/login_spec.rb' - 'ee/spec/features/users/signup_spec.rb' - 'ee/spec/finders/approval_rules/group_finder_spec.rb' diff --git a/.rubocop_todo/rspec/expect_in_hook.yml b/.rubocop_todo/rspec/expect_in_hook.yml index dd217e07249..cfd7e0e8e06 100644 --- a/.rubocop_todo/rspec/expect_in_hook.yml +++ b/.rubocop_todo/rspec/expect_in_hook.yml @@ -13,7 +13,6 @@ RSpec/ExpectInHook: - 'ee/spec/features/projects/feature_flags/user_deletes_feature_flag_spec.rb' - 'ee/spec/features/projects/settings/ee/service_desk_setting_spec.rb' - 'ee/spec/features/signup_spec.rb' - - 'ee/spec/features/trial_registrations/company_information_spec.rb' - 'ee/spec/finders/license_template_finder_spec.rb' - 'ee/spec/finders/projects/integrations/jira/issues_finder_spec.rb' - 'ee/spec/finders/template_finder_spec.rb' diff --git a/app/assets/stylesheets/fonts.scss b/app/assets/stylesheets/fonts.scss index bc49d17fcbb..e4497c60ff8 100644 --- a/app/assets/stylesheets/fonts.scss +++ b/app/assets/stylesheets/fonts.scss @@ -9,10 +9,21 @@ Usage: font-weight: 100 900; font-display: optional; font-style: normal; - font-named-instance: 'Regular'; /* stylelint-disable property-no-unknown */ + /* stylelint-disable-next-line property-no-unknown */ + font-named-instance: 'Regular'; src: font-url('gitlab-sans/GitLabSans.woff2') format('woff2'); } +@font-face { + font-family: 'GitLab Sans'; + font-weight: 100 900; + font-display: optional; + font-style: italic; + /* stylelint-disable-next-line property-no-unknown */ + font-named-instance: 'Regular'; + src: font-url('gitlab-sans/GitLabSans-Italic.woff2') format('woff2'); +} + /* ------------------------------------------------------- Monospaced font: GitLab Mono. diff --git a/config/feature_flags/development/p_ci_builds_metadata_foreign_key.yml b/config/feature_flags/development/p_ci_builds_metadata_foreign_key.yml new file mode 100644 index 00000000000..9d6c6e9612e --- /dev/null +++ b/config/feature_flags/development/p_ci_builds_metadata_foreign_key.yml @@ -0,0 +1,8 @@ +--- +name: p_ci_builds_metadata_foreign_key +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/124865 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/414396 +milestone: '16.2' +type: development +group: group::pipeline execution +default_enabled: false diff --git a/doc/architecture/blueprints/cells/cells-feature-contributions-forks.md b/doc/architecture/blueprints/cells/cells-feature-contributions-forks.md index 3e498c24144..8a67383c5e4 100644 --- a/doc/architecture/blueprints/cells/cells-feature-contributions-forks.md +++ b/doc/architecture/blueprints/cells/cells-feature-contributions-forks.md @@ -30,13 +30,13 @@ with various usage patterns: Forks allow users not having write access to parent project to make changes. The forking workflow is especially important for the Open Source community which is able to contribute back to public projects. However, it is equally important in some companies which prefer the strong split -of responsibilites and tighter access control. The access to project is restricted +of responsibilities and tighter access control. The access to project is restricted to designated list of developers. Forks enable: -- tigther control of who can modify the upstream project -- split of the responsibilites: parent project might use CI configuration connecting to production systems +- tighter control of who can modify the upstream project +- split of the responsibilities: parent project might use CI configuration connecting to production systems - run CI pipelines in context of fork in much more restrictive environment - consider all forks to be unveted which reduces risks of leaking secrets, or any other information tied with the project diff --git a/doc/development/database/batched_background_migrations.md b/doc/development/database/batched_background_migrations.md index 6006d3d465e..7398308e37d 100644 --- a/doc/development/database/batched_background_migrations.md +++ b/doc/development/database/batched_background_migrations.md @@ -538,6 +538,136 @@ Bumping the [import/export version](../../user/project/settings/import_export.md be required, if importing a project from a prior version of GitLab requires the data to be in the new format. +## Managing + +NOTE: +BBM management takes place through `chatops` integration, which is limited to GitLab team members only. + +### List batched background migrations + +To list the batched background migrations in the system, run this command: + +`/chatops run batched_background_migrations list` + +This command supports the following options: + +- Database selection: + - `--database DATABASE_NAME`: Connects to the given database: + - `main`: Uses the main database (default). + - `ci`: Uses the CI database. +- Environment selection: + - `--dev`: Uses the `dev` environment. + - `--staging`: Uses the `staging` environment. + - `--staging_ref`: Uses the `staging_ref` environment. + - `--production` : Uses the `production` environment (default). + +Output example: + +![List command](img/list_v15_4.png) + +NOTE: +ChatOps returns 20 batched background migrations order by `created_at` (DESC). + +### Monitor the progress and status of a batched background migration + +To see the status and progress of a specific batched background migration, run this command: + +`/chatops run batched_background_migrations status MIGRATION_ID` + +This command supports the following options: + +- Database selection: + - `--database DATABASE_NAME`: Connects to the given database: + - `main`: Uses the main database (default) + - `ci`: Uses the CI database +- Environment selection: + - `--dev`: Uses the `dev` environment. + - `--staging`: Uses the `staging` environment. + - `--staging_ref`: Uses the `staging_ref` environment. + - `--production` : Uses the `production` environment (default). + +Output example: + +![Status command](img/status_v15_4.png) + +`Progress` represents the percentage of the background migration that has been completed. + +Definitions of the batched background migration states: + +- **Active:** Either: + - Ready to be picked by the runner. + - Running batched jobs. +- **Finalizing:** Running batched jobs. +- **Failed:** Failed batched background migration. +- **Finished:** Completed batched background migration. +- **Paused:** Not visible to the runner. + +### Pause a batched background migration + +If you want to pause a batched background migration, you need to run the following command: + +`/chatops run batched_background_migrations pause MIGRATION_ID` + +This command supports the following options: + +- Database selection: + - `--database DATABASE_NAME`: Connects to the given database: + - `main`: Uses the main database (default). + - `ci`: Uses the CI database. +- Environment selection: + - `--dev`: Uses the `dev` environment. + - `--staging`: Uses the `staging` environment. + - `--staging_ref`: Uses the `staging_ref` environment. + - `--production` : Uses the `production` environment (default). + +Output example: + +![Pause command](img/pause_v15_4.png) + +NOTE: +You can pause only `active` batched background migrations. + +### Resume a batched background migration + +If you want to resume a batched background migration, you need to run the following command: + +`/chatops run batched_background_migrations resume MIGRATION_ID` + +This command supports the following options: + +- Database selection: + - `--database DATABASE_NAME`: Connects to the given database: + - `main`: Uses the main database (default). + - `ci`: Uses the CI database. +- Environment selection: + - `--dev`: Uses the `dev` environment. + - `--staging`: Uses the `staging` environment. + - `--staging_ref`: Uses the `staging_ref` environment. + - `--production` : Uses the `production` environment (default). + +Output example: + +![Resume command](img/resume_v15_4.png) + +NOTE: +You can resume only `active` batched background migrations + +### Enable or disable background migrations + +In extremely limited circumstances, a GitLab administrator can disable either or +both of these [feature flags](../../administration/feature_flags.md): + +- `execute_background_migrations` +- `execute_batched_migrations_on_schedule` + +These flags are enabled by default. Disable them only as a last resort +to limit database operations in special circumstances, like database host maintenance. + +WARNING: +Do not disable either of these flags unless you fully understand the ramifications. If you disable +the `execute_background_migrations` or `execute_batched_migrations_on_schedule` feature flag, +GitLab upgrades might fail and data loss might occur. + ## Batched background migrations for EE-only features All the background migration classes for EE-only features should be present in GitLab FOSS. @@ -808,115 +938,6 @@ for more details. The batched background migrations framework has ChatOps support. Using ChatOps, GitLab engineers can interact with the batched background migrations present in the system. -#### List batched background migrations - -To list the batched background migrations in the system, run this command: - -`/chatops run batched_background_migrations list` - -This command supports the following options: - -- Database selection: - - `--database DATABASE_NAME`: Connects to the given database: - - `main`: Uses the main database (default). - - `ci`: Uses the CI database. -- Environment selection: - - `--dev`: Uses the `dev` environment. - - `--staging`: Uses the `staging` environment. - - `--staging_ref`: Uses the `staging_ref` environment. - - `--production` : Uses the `production` environment (default). - -Output example: - -![List command](img/list_v15_4.png) - -NOTE: -ChatOps returns 20 batched background migrations order by `created_at` (DESC). - -#### Monitor the progress and status of a batched background migration - -To see the status and progress of a specific batched background migration, run this command: - -`/chatops run batched_background_migrations status MIGRATION_ID` - -This command supports the following options: - -- Database selection: - - `--database DATABASE_NAME`: Connects to the given database: - - `main`: Uses the main database (default) - - `ci`: Uses the CI database -- Environment selection: - - `--dev`: Uses the `dev` environment. - - `--staging`: Uses the `staging` environment. - - `--staging_ref`: Uses the `staging_ref` environment. - - `--production` : Uses the `production` environment (default). - -Output example: - -![Status command](img/status_v15_4.png) - -`Progress` represents the percentage of the background migration that has been completed. - -Definitions of the batched background migration states: - -- **Active:** Either: - - Ready to be picked by the runner. - - Running batched jobs. -- **Finalizing:** Running batched jobs. -- **Failed:** Failed batched background migration. -- **Finished:** Completed batched background migration. -- **Paused:** Not visible to the runner. - -#### Pause a batched background migration - -If you want to pause a batched background migration, you need to run the following command: - -`/chatops run batched_background_migrations pause MIGRATION_ID` - -This command supports the following options: - -- Database selection: - - `--database DATABASE_NAME`: Connects to the given database: - - `main`: Uses the main database (default). - - `ci`: Uses the CI database. -- Environment selection: - - `--dev`: Uses the `dev` environment. - - `--staging`: Uses the `staging` environment. - - `--staging_ref`: Uses the `staging_ref` environment. - - `--production` : Uses the `production` environment (default). - -Output example: - -![Pause command](img/pause_v15_4.png) - -NOTE: -You can pause only `active` batched background migrations. - -#### Resume a batched background migration - -If you want to resume a batched background migration, you need to run the following command: - -`/chatops run batched_background_migrations resume MIGRATION_ID` - -This command supports the following options: - -- Database selection: - - `--database DATABASE_NAME`: Connects to the given database: - - `main`: Uses the main database (default). - - `ci`: Uses the CI database. -- Environment selection: - - `--dev`: Uses the `dev` environment. - - `--staging`: Uses the `staging` environment. - - `--staging_ref`: Uses the `staging_ref` environment. - - `--production` : Uses the `production` environment (default). - -Output example: - -![Resume command](img/resume_v15_4.png) - -NOTE: -You can resume only `active` batched background migrations - ### Viewing failure error logs You can view failures in two ways: diff --git a/doc/development/policies.md b/doc/development/policies.md index b50d654ed28..65f96ccea4e 100644 --- a/doc/development/policies.md +++ b/doc/development/policies.md @@ -18,7 +18,7 @@ Permissions are broken into two parts: `conditions` and `rules`. Conditions are ### Conditions -Conditions are defined by the `condition` method, and are given a name and a block. The block is executed in the context of the policy object - so it can access `@user` and `@subject`, as well as call any methods defined on the policy. Note that `@user` may be nil (in the anonymous case), but `@subject` is guaranteed to be a real instance of the subject class. +Conditions are defined by the `condition` method, and are given a name and a block. The block is executed in the context of the policy object - so it can access `@user` and `@subject`, as well as call any methods defined on the policy. `@user` may be nil (in the anonymous case), but `@subject` is guaranteed to be a real instance of the subject class. ```ruby class FooPolicy < BasePolicy @@ -42,7 +42,7 @@ Conditions are cached according to their scope. Scope and ordering is covered la ### Rules -A `rule` is a logical combination of conditions and other rules, that are configured to enable or prevent certain abilities. It is important to note that the rule configuration is static - a rule's logic cannot touch the database or know about `@user` or `@subject`. This allows us to cache only at the condition level. Rules are specified through the `rule` method, which takes a block of DSL configuration, and returns an object that responds to `#enable` or `#prevent`: +A `rule` is a logical combination of conditions and other rules, that are configured to enable or prevent certain abilities. The rule configuration is static - a rule's logic cannot touch the database or know about `@user` or `@subject`. This allows us to cache only at the condition level. Rules are specified through the `rule` method, which takes a block of DSL configuration, and returns an object that responds to `#enable` or `#prevent`: ```ruby class FooPolicy < BasePolicy @@ -67,7 +67,7 @@ Within the rule DSL, you can use: - A regular word mentions a condition by name - a rule that is in effect when that condition is truthy. - `~` indicates negation, also available as `negate`. - `&` and `|` are logical combinations, also available as `all?(...)` and `any?(...)`. -- `can?(:other_ability)` delegates to the rules that apply to `:other_ability`. Note that this is distinct from the instance method `can?`, which can check dynamically - this only configures a delegation to another ability. +- `can?(:other_ability)` delegates to the rules that apply to `:other_ability`. This is distinct from the instance method `can?`, which can check dynamically - this only configures a delegation to another ability. `~`, `&` and `|` operators are overridden methods in [`DeclarativePolicy::Rule::Base`](https://gitlab.com/gitlab-org/declarative-policy/-/blob/main/lib/declarative_policy/rule.rb). @@ -124,7 +124,7 @@ heuristic of how expensive they are to calculate. The sorting is dynamic and cache-aware, so that previously calculated conditions are considered first, before computing other conditions. -Note that the score is chosen by a developer via the `score:` parameter +The score is chosen by a developer via the `score:` parameter in a `condition` to denote how expensive evaluating this rule would be relative to other rules. @@ -173,7 +173,7 @@ class FooPolicy < BasePolicy end ``` -includes all rules from `ProjectPolicy`. The delegated conditions are evaluated with the correct delegated subject, and are sorted along with the regular rules in the policy. Note that only the relevant rules for a particular ability are actually considered. +includes all rules from `ProjectPolicy`. The delegated conditions are evaluated with the correct delegated subject, and are sorted along with the regular rules in the policy. Only the relevant rules for a particular ability are actually considered. ### Overrides diff --git a/doc/development/testing_guide/contract/provider_tests.md b/doc/development/testing_guide/contract/provider_tests.md index cb3aeae529d..71940941d51 100644 --- a/doc/development/testing_guide/contract/provider_tests.md +++ b/doc/development/testing_guide/contract/provider_tests.md @@ -10,7 +10,7 @@ This tutorial guides you through writing a provider test from scratch. It is a c ## Create the skeleton -Provider tests are quite simple. The goal is to set up the test data and then link that with the corresponding contract. Start by creating a file called `get_discussions_helper.rb` under `spec/contracts/provider/pact_helpers/project/merge_request`. Note that the files are called `helpers` to match how they are called by Pact in the Rake tasks, which are set up at the end of this tutorial. +Provider tests are quite simple. The goal is to set up the test data and then link that with the corresponding contract. Start by creating a file called `get_discussions_helper.rb` under `spec/contracts/provider/pact_helpers/project/merge_request`. The files are called `helpers` to match how they are called by Pact in the Rake tasks, which are set up at the end of this tutorial. For more information about how the contract test directory is structured, see [Test suite folder structure](index.md#test-suite-folder-structure). @@ -122,7 +122,7 @@ To create the test data, create `show_state.rb` under `spec/contracts/provider/s ### Default user in `spec/contracts/provider/spec_helper.rb` -Before you create the test data, note that a default user is created in the [`spec_helper`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/spec/contracts/provider/spec_helper.rb), which is the user being used for the test runs. This user is configured using `RSpec.configure`, as Pact actually is built on top of RSpec. This step allows us to configure the user before any of the test runs. +Before you create the test data, a default user is created in the [`spec_helper`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/spec/contracts/provider/spec_helper.rb), which is the user being used for the test runs. This user is configured using `RSpec.configure`, as Pact actually is built on top of RSpec. This step allows us to configure the user before any of the test runs. ```ruby RSpec.configure do |config| diff --git a/doc/development/testing_guide/end_to_end/style_guide.md b/doc/development/testing_guide/end_to_end/style_guide.md index 32d8bf339ed..f5a3fa2fc51 100644 --- a/doc/development/testing_guide/end_to_end/style_guide.md +++ b/doc/development/testing_guide/end_to_end/style_guide.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w This document describes the conventions used at GitLab for writing End-to-end (E2E) tests using the GitLab QA project. -Please note that this guide is an extension of the primary [testing standards and style guidelines](../index.md). If this guide defines a rule that contradicts the primary guide, this guide takes precedence. +This guide is an extension of the primary [testing standards and style guidelines](../index.md). If this guide defines a rule that contradicts the primary guide, this guide takes precedence. ## `click_` versus `go_to_` diff --git a/doc/development/testing_guide/review_apps.md b/doc/development/testing_guide/review_apps.md index 1e5ee9f3003..b4ae23336d5 100644 --- a/doc/development/testing_guide/review_apps.md +++ b/doc/development/testing_guide/review_apps.md @@ -204,10 +204,10 @@ subgraph "CNG-mirror pipeline" - If the `review-deploy` job keeps failing (and a manual retry didn't help), please post a message in the `#g_qe_engineering_productivity` channel and/or create a `~"Engineering Productivity"` `~"ep::review apps"` `~"type::bug"` - issue with a link to your merge request. Note that the deployment failure can + issue with a link to your merge request. The deployment failure can reveal an actual problem introduced in your merge request (that is, this isn't necessarily a transient failure)! -- If the `review-qa-smoke` or `review-qa-reliable` job keeps failing (note that we already retry them once), +- If the `review-qa-smoke` or `review-qa-reliable` job keeps failing (we already retry them once), please check the job's logs: you could discover an actual problem introduced in your merge request. You can also download the artifacts to see screenshots of the page at the time the failures occurred. If you don't find the cause of the diff --git a/doc/install/aws/gitlab_hybrid_on_aws.md b/doc/install/aws/gitlab_hybrid_on_aws.md index 75b39a387dc..84b5728dbd8 100644 --- a/doc/install/aws/gitlab_hybrid_on_aws.md +++ b/doc/install/aws/gitlab_hybrid_on_aws.md @@ -157,8 +157,8 @@ On Demand pricing is used in this table for comparisons, but should not be used | Sidekiq | 2 vCPU, 8 GB | | | | Supporting services such as NGINX, Prometheus, etc | 2 vCPU, 8 GB | | | | **GitLab Ref Arch Raw Total K8s Node Capacity** | 16 vCPU, 32 GB | | | -| One Node for Overhead and Miscellaneous (EKS Cluster AutoScaler, Grafana, Prometheus, etc) | + 8 vCPU, 16GB | | | -| **Grand Total w/ Overheads**<br />Minimum hosts = 3 | 24 vCPU, 48 GB | **c5.2xlarge** <br />(8vCPU/16GB) x 3 nodes<br />24 vCPU, 48 GB | $1.02/hr | +| One Node for Overhead and Miscellaneous (EKS Cluster AutoScaler, Grafana, Prometheus, etc) | + 8 vCPU, 16 GB | | | +| **Grand Total w/ Overheads**<br />Minimum hosts = 3 | 24 vCPU, 48 GB | **c5.2xlarge** <br />(8vCPU/16 GB) x 3 nodes<br />24 vCPU, 48 GB | $1.02/hr | | **Idle Configuration (Scaled-In)** | 16 vCPU, 32 GB | **c5.2xlarge** x 2 | $0.68/hr | NOTE: @@ -168,7 +168,7 @@ If EKS node autoscaling is employed, it is likely that your average loading will | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------- | ------------------------------- | ------------------------------- | | **Bastion Host (Quick Start)** | 1 HA instance in ASG | **t2.micro** for prod, **m4.2xlarge** for performance testing | | | | **PostgreSQL**<br />AWS Amazon RDS PostgreSQL Nodes Configuration (GPT tested) | 2vCPU, 7.5 GB<br />Tested with Graviton ARM | **db.r6g.large** x 3 nodes <br />(6vCPU, 48 GB) | 3 nodes x $0.26 = $0.78/hr | 3 nodes x $0.26 = $0.78/hr | -| **Redis** | 1vCPU, 3.75GB<br />(across 12 nodes for Redis Cache, Redis Queues/Shared State, Sentinel Cache, Sentinel Queues/Shared State) | **cache.m6g.large** x 3 nodes<br />(6vCPU, 19GB) | 3 nodes x $0.15 = $0.45/hr | 2 nodes x $0.15 = $0.30/hr | +| **Redis** | 1vCPU, 3.75GB<br />(across 12 nodes for Redis Cache, Redis Queues/Shared State, Sentinel Cache, Sentinel Queues/Shared State) | **cache.m6g.large** x 3 nodes<br />(6vCPU, 19 GB) | 3 nodes x $0.15 = $0.45/hr | 2 nodes x $0.15 = $0.30/hr | | **<u>Gitaly Cluster</u>** [Details](gitlab_sre_for_aws.md#gitaly-sre-considerations) | [Gitaly & Praefect Must Have an Uneven Node Count for HA](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | | | | | Gitaly Instances (in ASG) | 12 vCPU, 45GB<br />(across 3 nodes) | **m5.xlarge** x 3 nodes<br />(48 vCPU, 180 GB) | $0.192 x 3 = $0.58/hr | $0.192 x 3 = $0.58/hr | | | The GitLab Reference architecture for 2K is not Highly Available and therefore has a single Gitaly no Praefect. AWS Quick Starts MUST be HA, so it implements Praefect from the 3K Ref Architecture to meet that requirement | | | | @@ -211,7 +211,7 @@ On Demand pricing is used in this table for comparisons, but should not be used | Supporting services such as NGINX, Prometheus, etc | [2 allocations](../../administration/reference_architectures/3k_users.md#cluster-topology) x ([2 vCPU and 7.5 GB](../../administration/reference_architectures/3k_users.md#cluster-topology)) = <br />4 vCPU, 15 GB | | | | **GitLab Ref Arch Raw Total K8s Node Capacity** | 32 vCPU, 56 GB | | | | One Node for Overhead and Miscellaneous (EKS Cluster AutoScaler, Grafana, Prometheus, etc) | + 16 vCPU, 32GB | | | -| **Grand Total w/ Overheads Full Scale**<br />Minimum hosts = 3 | 48 vCPU, 88 GB | **c5.2xlarge** (8vCPU/16GB) x 5 nodes<br />40 vCPU, 80 GB<br />[Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/3K/3k-QuickStart-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_124216/3k-QuickStart-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_124216_results.txt) | $1.70/hr | +| **Grand Total w/ Overheads Full Scale**<br />Minimum hosts = 3 | 48 vCPU, 88 GB | **c5.2xlarge** (8vCPU/16 GB) x 5 nodes<br />40 vCPU, 80 GB<br />[Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/3K/3k-QuickStart-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_124216/3k-QuickStart-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_124216_results.txt) | $1.70/hr | | **Possible Idle Configuration (Scaled-In 75% - round up)**<br />Pod autoscaling must be also adjusted to enable lower idling configuration. | 24 vCPU, 48 GB | c5.2xlarge x 4 | $1.36/hr | Other combinations of node type and quantity can be used to meet the Grand Total. Due to the properties of pods, hosts that are overly small may have significant unused capacity. @@ -223,7 +223,7 @@ If EKS node autoscaling is employed, it is likely that your average loading will | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------- | ------------------------------- | ------------------------------------------------------------ | | **Bastion Host (Quick Start)** | 1 HA instance in ASG | **t2.micro** for prod, **m4.2xlarge** for performance testing | | | | **PostgreSQL**<br />Amazon RDS PostgreSQL Nodes Configuration (GPT tested) | 18vCPU, 36 GB <br />(across 9 nodes for PostgreSQL, PgBouncer, Consul)<br />Tested with Graviton ARM | **db.r6g.xlarge** x 3 nodes <br />(12vCPU, 96 GB) | 3 nodes x $0.52 = $1.56/hr | 3 nodes x $0.52 = $1.56/hr | -| **Redis** | 6vCPU, 18GB<br />(across 6 nodes for Redis Cache, Sentinel) | **cache.m6g.large** x 3 nodes<br />(6vCPU, 19GB) | 3 nodes x $0.15 = $0.45/hr | 2 nodes x $0.15 = $0.30/hr | +| **Redis** | 6vCPU, 18 GB<br />(across 6 nodes for Redis Cache, Sentinel) | **cache.m6g.large** x 3 nodes<br />(6vCPU, 19 GB) | 3 nodes x $0.15 = $0.45/hr | 2 nodes x $0.15 = $0.30/hr | | **<u>Gitaly Cluster</u>** [Details](gitlab_sre_for_aws.md#gitaly-sre-considerations) | | | | | | Gitaly Instances (in ASG) | 12 vCPU, 45GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | **m5.large** x 3 nodes<br />(12 vCPU, 48 GB) | $0.192 x 3 = $0.58/hr | [Gitaly & Praefect Must Have an Uneven Node Count for HA](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | | Praefect (Instances in ASG with load balancer) | 6 vCPU, 5.4 GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | **c5.large** x 3 nodes<br />(6 vCPU, 12 GB) | $0.09 x 3 = $0.21/hr | [Gitaly & Praefect Must Have an Uneven Node Count for HA](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | @@ -264,8 +264,8 @@ On Demand pricing is used in this table for comparisons, but should not be used | Sidekiq | [8 pods](https://gitlab.com/gitlab-org/charts/gitlab/-/blob/master/examples/ref/5k.yaml#L24) x ([1 vCPU & 2 GB](../../administration/reference_architectures/5k_users.md#sidekiq)) = <br />8 vCPU, 16 GB | | | | Supporting services such as NGINX, Prometheus, etc | [2 allocations](../../administration/reference_architectures/5k_users.md#cluster-topology) x ([2 vCPU and 7.5 GB](../../administration/reference_architectures/5k_users.md#cluster-topology)) = <br />4 vCPU, 15 GB | | | | **GitLab Ref Arch Raw Total K8s Node Capacity** | 62 vCPU, 96.5 GB | | | -| One Node for Quick Start Overhead and Miscellaneous (EKS Cluster AutoScaler, Grafana, Prometheus, etc) | + 8 vCPU, 16GB | | | -| **Grand Total w/ Overheads Full Scale**<br />Minimum hosts = 3 | 70 vCPU, 112.5 GB | **c5.2xlarge** (8vCPU/16GB) x 9 nodes<br />72 vCPU, 144 GB<br />[Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/5K/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128_results.txt) | $2.38/hr | +| One Node for Quick Start Overhead and Miscellaneous (EKS Cluster AutoScaler, Grafana, Prometheus, etc) | + 8 vCPU, 16 GB | | | +| **Grand Total w/ Overheads Full Scale**<br />Minimum hosts = 3 | 70 vCPU, 112.5 GB | **c5.2xlarge** (8vCPU/16 GB) x 9 nodes<br />72 vCPU, 144 GB<br />[Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/5K/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128_results.txt) | $2.38/hr | | **Possible Idle Configuration (Scaled-In 75% - round up)**<br />Pod autoscaling must be also adjusted to enable lower idling configuration. | 24 vCPU, 48 GB | c5.2xlarge x 7 | $1.85/hr | Other combinations of node type and quantity can be used to meet the Grand Total. Due to the CPU and memory requirements of pods, hosts that are overly small may have significant unused capacity. @@ -330,9 +330,9 @@ If EKS node autoscaling is employed, it is likely that your average loading will | ------------------------------------------------------------ | ------------------------------ | ------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | | **Bastion Host (Quick Start)** | 1 HA instance in ASG | **t2.micro** for prod, **m4.2xlarge** for performance testing | | | | **PostgreSQL**<br />Amazon RDS PostgreSQL Nodes Configuration (GPT tested) | 36vCPU, 102 GB <br />(across 9 nodes for PostgreSQL, PgBouncer, Consul) | **db.r6g.2xlarge** x 3 nodes <br />(24vCPU, 192 GB) | 3 nodes x $1.04 = $3.12/hr | 3 nodes x $1.04 = $3.12/hr | -| **Redis** | 30vCPU, 114GB<br />(across 12 nodes for Redis Cache, Redis Queues/Shared State, Sentinel Cache, Sentinel Queues/Shared State) | **cache.m5.2xlarge** x 3 nodes<br />(24vCPU, 78GB) | 3 nodes x $0.62 = $1.86/hr | 2 nodes x $0.62 = $1.24/hr | +| **Redis** | 30vCPU, 114 GB<br />(across 12 nodes for Redis Cache, Redis Queues/Shared State, Sentinel Cache, Sentinel Queues/Shared State) | **cache.m5.2xlarge** x 3 nodes<br />(24vCPU, 78GB) | 3 nodes x $0.62 = $1.86/hr | 2 nodes x $0.62 = $1.24/hr | | **<u>Gitaly Cluster</u>** [Details](gitlab_sre_for_aws.md#gitaly-sre-considerations) | | | | | -| Gitaly Instances (in ASG) | 48 vCPU, 180GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | **m5.4xlarge** x 3 nodes<br />(48 vCPU, 180 GB) | $0.77 x 3 = $2.31/hr | [Gitaly & Praefect Must Have an Uneven Node Count for HA](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | +| Gitaly Instances (in ASG) | 48 vCPU, 180 GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | **m5.4xlarge** x 3 nodes<br />(48 vCPU, 180 GB) | $0.77 x 3 = $2.31/hr | [Gitaly & Praefect Must Have an Uneven Node Count for HA](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | | Praefect (Instances in ASG with load balancer) | 6 vCPU, 5.4 GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | **c5.large** x 3 nodes<br />(6 vCPU, 12 GB) | $0.09 x 3 = $0.21/hr | [Gitaly & Praefect Must Have an Uneven Node Count for HA](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | | Praefect PostgreSQL(1) (Amazon RDS) | 6 vCPU, 5.4 GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | Not applicable; reuses GitLab PostgreSQL | $0 | | | Internal Load Balancing Node | 2 vCPU, 1.8 GB | AWS ELB | $0.10/hr | $0.10/hr | @@ -383,7 +383,7 @@ If EKS node autoscaling is employed, it is likely that your average loading will | ------------------------------------------------------------ | ------------------------------------------------------------ | --------------------------------------------------------- | ------------------------------- | ------------------------------------------------------------ | | **Bastion Host (Quick Start)** | 1 HA instance in ASG | **t2.micro** for prod, **m4.2xlarge** for performance testing | | | | **PostgreSQL**<br />Amazon RDS PostgreSQL Nodes Configuration (GPT tested) | 96vCPU, 360 GB <br />(across 3 nodes) | **db.r6g.8xlarge** x 3 nodes <br />(96vCPU, 768 GB total) | 3 nodes x $4.15 = $12.45/hr | 3 nodes x $4.15 = $12.45/hr | -| **Redis** | 30vCPU, 114GB<br />(across 12 nodes for Redis Cache, Redis Queues/Shared State, Sentinel Cache, Sentinel Queues/Shared State) | **cache.m6g.2xlarge** x 3 nodes<br />(24vCPU, 78GB total) | 3 nodes x $0.60 = $1.80/hr | 2 nodes x $0.60 = $1.20/hr | +| **Redis** | 30vCPU, 114 GB<br />(across 12 nodes for Redis Cache, Redis Queues/Shared State, Sentinel Cache, Sentinel Queues/Shared State) | **cache.m6g.2xlarge** x 3 nodes<br />(24vCPU, 78GB total) | 3 nodes x $0.60 = $1.80/hr | 2 nodes x $0.60 = $1.20/hr | | **<u>Gitaly Cluster</u>** [Details](gitlab_sre_for_aws.md#gitaly-sre-considerations) | | | | | | Gitaly Instances (in ASG) | 64 vCPU, 240GB x [3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | **m5.16xlarge** x 3 nodes<br />(64 vCPU, 256 GB each) | $3.07 x 3 = $9.21/hr | [Gitaly & Praefect Must Have an Uneven Node Count for HA](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | | Praefect (Instances in ASG with load balancer) | 4 vCPU, 3.6 GB x [3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | **c5.xlarge** x 3 nodes<br />(4 vCPU, 8 GB each) | $0.17 x 3 = $0.51/hr | [Gitaly & Praefect Must Have an Uneven Node Count for HA](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | diff --git a/doc/update/background_migrations.md b/doc/update/background_migrations.md index 3f8b7be2891..b56067fad79 100644 --- a/doc/update/background_migrations.md +++ b/doc/update/background_migrations.md @@ -78,7 +78,7 @@ sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::Database::Ba > - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/329511) in GitLab 13.12. > - Enabled on GitLab.com. > - Recommended for production use. -> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-background-migrations). +> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](../development/database/batched_background_migrations.md#enable-or-disable-background-migrations). There can be [risks when disabling released features](../administration/feature_flags.md#risks-when-disabling-released-features). Refer to this feature's version history for more details. @@ -126,7 +126,7 @@ If you get this error, [check the batched background migration options](#databas ### Pause batched background migrations in GitLab 14.x To pause an ongoing batched background migration, -[disable the batched background migrations feature](#enable-or-disable-background-migrations). +[disable the batched background migrations feature](../development/database/batched_background_migrations.md#enable-or-disable-background-migrations). Disabling the feature completes the current batch of migrations, then waits to start the next batch until after the feature is enabled again. @@ -225,22 +225,6 @@ ApplicationSetting.update_all(database_max_running_batched_background_migrations ## Troubleshooting -### Enable or disable background migrations - -In extremely limited circumstances, a GitLab administrator can disable either or -both of these [feature flags](../administration/feature_flags.md): - -- `execute_background_migrations` -- `execute_batched_migrations_on_schedule` - -These flags are enabled by default. Disable them only as a last resort -to limit database operations in special circumstances, like database host maintenance. - -WARNING: -Do not disable either of these flags unless you fully understand the ramifications. If you disable -the `execute_background_migrations` or `execute_batched_migrations_on_schedule` feature flag, -GitLab upgrades might fail and data loss might occur. - ### Database migrations failing because of batched background migration not finished When updating to GitLab 14.2 or later there might be a database migration failing with a message like: diff --git a/doc/user/project/repository/code_suggestions.md b/doc/user/project/repository/code_suggestions.md index c15619279a8..0caa71afdd7 100644 --- a/doc/user/project/repository/code_suggestions.md +++ b/doc/user/project/repository/code_suggestions.md @@ -47,11 +47,19 @@ Code Suggestions may produce [low-quality or incomplete suggestions](#model-accu The best results from Code Suggestions are expected [for languages the Google Vertex AI Codey APIs](https://cloud.google.com/vertex-ai/docs/generative-ai/code/code-models-overview#supported_coding_languages) directly support: +- C++ +- C# - Go - Google SQL - Java - JavaScript +- Kotlin +- PHP - Python +- Ruby +- Rust +- Scala +- Swift - TypeScript Supported [code infrastructure interfaces](https://cloud.google.com/vertex-ai/docs/generative-ai/code/code-models-overview#supported_code_infrastructure_interfaces) include: @@ -185,22 +193,17 @@ mitigate abuse or misuse. GitLab may disable this feature for any or all custome Code Suggestions is a generative artificial intelligence (AI) model. Your personal access token enables a secure API connection to GitLab.com. -This API connection securely transmits a context window from your IDE/editor to the Code Suggestions GitLab hosted service, +This API connection securely transmits a context window from your IDE/editor to the Code Suggestions GitLab hosted service which calls Google Vertex AI Codey APIs, and the generated suggestion is transmitted back to your IDE/editor. GitLab currently leverages [Google Cloud's Vertex AI Codey API models](https://cloud.google.com/vertex-ai/docs/generative-ai/code/code-models-overview). ### Data privacy -Code Suggestions operate completely in the GitLab infrastructure, providing the same level of -[security](https://about.gitlab.com/security/) as any other features of GitLab, and processing any personal -data in accordance with our [Privacy Statement](https://about.gitlab.com/privacy/). +No new additional data is collected to enable this feature. Private non-public GitLab customer data is +not used as training data. -No new additional data is collected to enable this feature. The content of your GitLab hosted source code is -not used as training data. Source code inference against the Code Suggestions model is not used to re-train the model. -Your data also never leaves GitLab. All training and inference is done in GitLab.com infrastructure. - -[Read more about the security of GitLab.com](https://about.gitlab.com/security/faq/). +Learn more about Google Vertex AI Codey APIs [Data Governance](https://cloud.google.com/vertex-ai/docs/generative-ai/data-governance) ### Self-managed instance data privacy @@ -221,6 +224,10 @@ Code suggestions are routed through Google Vertex AI Codey APIs. Learn more abou Google Vertex AI Codey APIs are not trained on private non-public GitLab customer or user data. +Google has [shared the following](https://ai.google/discover/foundation-models/) about the data Codey models are trained on: + +> Codey is our family of foundational coding models built on PaLM 2. Codey was fine-tuned on a large dataset of high quality, permissively licensed code from external sources + ## Progressive enhancement This feature is designed as a progressive enhancement to developer's IDEs. diff --git a/lib/api/concerns/packages/npm_endpoints.rb b/lib/api/concerns/packages/npm_endpoints.rb index 460b694c707..8f92889dedc 100644 --- a/lib/api/concerns/packages/npm_endpoints.rb +++ b/lib/api/concerns/packages/npm_endpoints.rb @@ -130,6 +130,8 @@ module API track_package_event(:create_tag, :npm, project: project, namespace: project.namespace) + enqueue_sync_metadata_cache_worker(project, package_name) + ::Packages::Npm::CreateTagService.new(package, tag).execute no_content! @@ -164,6 +166,8 @@ module API track_package_event(:delete_tag, :npm, project: project, namespace: project.namespace) + enqueue_sync_metadata_cache_worker(project, package_name) + ::Packages::RemoveTagService.new(package_tag).execute no_content! @@ -210,14 +214,19 @@ module API not_found!('Packages') if packages.empty? - if endpoint_scope == :project && Feature.enabled?(:npm_metadata_cache, project) && - metadata_cache&.file&.exists? - metadata_cache.touch_last_downloaded_at - present_carrierwave_file!(metadata_cache.file) - else - present ::Packages::Npm::PackagePresenter.new(generate_metadata_service(packages).execute), - with: ::API::Entities::NpmPackage + if endpoint_scope == :project && Feature.enabled?(:npm_metadata_cache, project) + if metadata_cache&.file&.exists? + metadata_cache.touch_last_downloaded_at + present_carrierwave_file!(metadata_cache.file) + + break + end + + enqueue_sync_metadata_cache_worker(project, package_name) end + + present ::Packages::Npm::PackagePresenter.new(generate_metadata_service(packages).execute), + with: ::API::Entities::NpmPackage end end diff --git a/lib/api/helpers/packages/npm.rb b/lib/api/helpers/packages/npm.rb index c1c6b1478d8..d253be0b8cf 100644 --- a/lib/api/helpers/packages/npm.rb +++ b/lib/api/helpers/packages/npm.rb @@ -84,6 +84,12 @@ module API end strong_memoize_attr :project_id_or_nil + def enqueue_sync_metadata_cache_worker(project, package_name) + return unless Feature.enabled?(:npm_metadata_cache, project) + + ::Packages::Npm::CreateMetadataCacheWorker.perform_async(project.id, package_name) + end + private def top_namespace_from(package_name) diff --git a/lib/api/npm_project_packages.rb b/lib/api/npm_project_packages.rb index 61409909b06..a7463f4a5cd 100644 --- a/lib/api/npm_project_packages.rb +++ b/lib/api/npm_project_packages.rb @@ -80,6 +80,7 @@ module API if created_package[:status] == :error render_api_error!(created_package[:message], created_package[:http_status]) else + enqueue_sync_metadata_cache_worker(project, created_package.name) track_package_event('push_package', :npm, category: 'API::NpmPackages', project: project, namespace: project.namespace) created_package end diff --git a/lib/api/package_files.rb b/lib/api/package_files.rb index 7ff49f326d9..6a02769519f 100644 --- a/lib/api/package_files.rb +++ b/lib/api/package_files.rb @@ -14,6 +14,7 @@ module API urgency :low helpers ::API::Helpers::PackagesHelpers + helpers ::API::Helpers::Packages::Npm params do requires :id, types: [String, Integer], desc: 'ID or URL-encoded path of the project' @@ -70,6 +71,8 @@ module API destroy_conditionally!(package_file) do |package_file| package_file.pending_destruction! + + enqueue_sync_metadata_cache_worker(user_project, package.name) if package.npm? end end end diff --git a/lib/api/project_packages.rb b/lib/api/project_packages.rb index 43bd15931ef..2aa6858e41d 100644 --- a/lib/api/project_packages.rb +++ b/lib/api/project_packages.rb @@ -15,6 +15,7 @@ module API urgency :low helpers ::API::Helpers::PackagesHelpers + helpers ::API::Helpers::Packages::Npm helpers do def package strong_memoize(:package) do # rubocop:disable Gitlab/StrongMemoizeAttr @@ -133,6 +134,8 @@ module API destroy_conditionally!(package) do |package| ::Packages::MarkPackageForDestructionService.new(container: package, current_user: current_user).execute + + enqueue_sync_metadata_cache_worker(user_project, package.name) if package.npm? end end end diff --git a/lib/banzai/filter/references/external_issue_reference_filter.rb b/lib/banzai/filter/references/external_issue_reference_filter.rb index 1061a9917dd..ead816abab3 100644 --- a/lib/banzai/filter/references/external_issue_reference_filter.rb +++ b/lib/banzai/filter/references/external_issue_reference_filter.rb @@ -24,8 +24,15 @@ module Banzai # # Returns a String replaced with the return of the block. def references_in(text, pattern = object_reference_pattern) - text.gsub(pattern) do |match| - yield match, $~[:issue] + case pattern + when Regexp + text.gsub(pattern) do |match| + yield match, $~[:issue] + end + when Gitlab::UntrustedRegexp + pattern.replace_gsub(text) do |match| + yield match, match[:issue] + end end end diff --git a/lib/banzai/filter/references/reference_filter.rb b/lib/banzai/filter/references/reference_filter.rb index a687ae2882e..5353d3f4e49 100644 --- a/lib/banzai/filter/references/reference_filter.rb +++ b/lib/banzai/filter/references/reference_filter.rb @@ -206,7 +206,8 @@ module Banzai end def replace_text_when_pattern_matches(node, index, pattern) - return unless node.text =~ pattern + return if pattern.is_a?(Gitlab::UntrustedRegexp) && !pattern.match?(node.text) + return if pattern.is_a?(Regexp) && !(pattern =~ node.text) content = node.to_html html = yield content diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 49d3c270bac..0b775781007 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.34.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.35.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 49d3c270bac..0b775781007 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.34.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.35.0' build: stage: build diff --git a/lib/gitlab/database/ci_builds_partitioning.rb b/lib/gitlab/database/ci_builds_partitioning.rb new file mode 100644 index 00000000000..a8a935b8c39 --- /dev/null +++ b/lib/gitlab/database/ci_builds_partitioning.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class CiBuildsPartitioning + include AsyncDdlExclusiveLeaseGuard + + ATTEMPTS = 5 + LOCK_TIMEOUT = 10.seconds + LEASE_TIMEOUT = 20.minutes + + def initialize(logger: Gitlab::AppLogger) + @connection = ::Ci::ApplicationRecord.connection + @timing_configuration = Array.new(ATTEMPTS) { [LOCK_TIMEOUT, 3.minutes] } + @logger = logger + end + + def execute + return unless can_execute? + + try_obtain_lease do + lock_retries.run(raise_on_exhaustion: true) do + connection.execute(create_foreign_key_sql) + end + + log_info('Foreign key successfully created') + end + + rescue StandardError => e + log_info("Failed to create foreign key: #{e.message}") + end + + private + + attr_reader :connection, :timing_configuration, :logger + + def can_execute? + return false if process_disabled? + return false unless Gitlab.com? + + if foreign_key_exists? + log_info('Foreign key exists, nothing to do') + + return false + end + + if vacuum_running? + log_info('Autovacuum detected') + + return false + end + + true + end + + def process_disabled? + ::Feature.disabled?(:p_ci_builds_metadata_foreign_key) + end + + def foreign_key_exists? + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::PostgresForeignKey + .by_constrained_table_name_or_identifier(:p_ci_builds_metadata) + .by_referenced_table_name(:p_ci_builds) + .by_name(:temp_fk_e20479742e_p) + .exists? + end + end + + def vacuum_running? + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::PostgresAutovacuumActivity + .wraparound_prevention + .for_tables(%i[ci_builds ci_builds_metadata]) + .any? + end + end + + def lock_retries + Gitlab::Database::WithLockRetries.new( + timing_configuration: timing_configuration, + connection: connection, + logger: logger, + klass: self.class + ) + end + + def create_foreign_key_sql + <<~SQL.squish + SET LOCAL statement_timeout TO '11s'; + + LOCK TABLE ci_builds, p_ci_builds, p_ci_builds_metadata IN ACCESS EXCLUSIVE MODE; + + ALTER TABLE p_ci_builds_metadata + ADD CONSTRAINT temp_fk_e20479742e_p + FOREIGN KEY (partition_id, build_id) + REFERENCES p_ci_builds (partition_id, id) + ON UPDATE CASCADE ON DELETE CASCADE; + SQL + end + + def log_info(message) + logger.info(message: message, class: self.class.to_s) + end + + def connection_db_config + ::Ci::ApplicationRecord.connection_db_config + end + + def lease_timeout + LEASE_TIMEOUT + end + end + end +end diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 4a1b0be848e..9c860ebc6aa 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -59,6 +59,7 @@ module Gitlab # most bloated indexes for reindexing. def self.perform_with_heuristic(candidate_indexes = Gitlab::Database::PostgresIndex.reindexing_support, maximum_records: DEFAULT_INDEXES_PER_INVOCATION) IndexSelection.new(candidate_indexes).take(maximum_records).each do |index| + Gitlab::Database::CiBuildsPartitioning.new.execute Coordinator.new(index).perform end end diff --git a/package.json b/package.json index 0e264cdcea5..0749ab639a4 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "@gitlab/at.js": "1.5.7", "@gitlab/cluster-client": "^1.2.0", "@gitlab/favicon-overlay": "2.0.0", - "@gitlab/fonts": "^1.2.0", + "@gitlab/fonts": "^1.3.0", "@gitlab/svgs": "3.53.0", "@gitlab/ui": "64.13.4", "@gitlab/visual-review-tools": "1.7.3", diff --git a/spec/frontend/vue_shared/components/runner_instructions/instructions/runner_docker_instructions_spec.js b/spec/frontend/vue_shared/components/runner_instructions/instructions/runner_docker_instructions_spec.js index 94823bb640b..b94d8c1de21 100644 --- a/spec/frontend/vue_shared/components/runner_instructions/instructions/runner_docker_instructions_spec.js +++ b/spec/frontend/vue_shared/components/runner_instructions/instructions/runner_docker_instructions_spec.js @@ -2,6 +2,7 @@ import { shallowMount } from '@vue/test-utils'; import { GlButton } from '@gitlab/ui'; import RunnerDockerInstructions from '~/vue_shared/components/runner_instructions/instructions/runner_docker_instructions.vue'; +import { DOCS_URL } from 'jh_else_ce/lib/utils/url_utility'; describe('RunnerDockerInstructions', () => { let wrapper; @@ -25,8 +26,6 @@ describe('RunnerDockerInstructions', () => { }); it('renders link', () => { - expect(findButton().attributes('href')).toBe( - 'https://docs.gitlab.com/runner/install/docker.html', - ); + expect(findButton().attributes('href')).toBe(`${DOCS_URL}/runner/install/docker.html`); }); }); diff --git a/spec/frontend/vue_shared/components/runner_instructions/instructions/runner_kubernetes_instructions_spec.js b/spec/frontend/vue_shared/components/runner_instructions/instructions/runner_kubernetes_instructions_spec.js index 9d6658e002c..f0b033a2ca2 100644 --- a/spec/frontend/vue_shared/components/runner_instructions/instructions/runner_kubernetes_instructions_spec.js +++ b/spec/frontend/vue_shared/components/runner_instructions/instructions/runner_kubernetes_instructions_spec.js @@ -2,6 +2,7 @@ import { shallowMount } from '@vue/test-utils'; import { GlButton } from '@gitlab/ui'; import RunnerKubernetesInstructions from '~/vue_shared/components/runner_instructions/instructions/runner_kubernetes_instructions.vue'; +import { DOCS_URL } from 'jh_else_ce/lib/utils/url_utility'; describe('RunnerKubernetesInstructions', () => { let wrapper; @@ -25,8 +26,6 @@ describe('RunnerKubernetesInstructions', () => { }); it('renders link', () => { - expect(findButton().attributes('href')).toBe( - 'https://docs.gitlab.com/runner/install/kubernetes.html', - ); + expect(findButton().attributes('href')).toBe(`${DOCS_URL}/runner/install/kubernetes.html`); }); }); diff --git a/spec/helpers/tree_helper_spec.rb b/spec/helpers/tree_helper_spec.rb index e13b83feefd..1ca5b8eb954 100644 --- a/spec/helpers/tree_helper_spec.rb +++ b/spec/helpers/tree_helper_spec.rb @@ -274,6 +274,7 @@ RSpec.describe TreeHelper do describe '.fork_modal_options' do let_it_be(:blob) { project.repository.blob_at('refs/heads/master', @path) } + let(:fork_path) { "/#{project.path_with_namespace}/-/forks/new" } before do allow(helper).to receive(:current_user).and_return(user) @@ -282,7 +283,7 @@ RSpec.describe TreeHelper do subject { helper.fork_modal_options(project, blob) } it 'returns correct fork path' do - expect(subject).to match a_hash_including(fork_path: '/namespace1/project-1/-/forks/new', fork_modal_id: nil) + expect(subject).to match a_hash_including(fork_path: fork_path, fork_modal_id: nil) end context 'when show_edit_button true' do @@ -292,7 +293,7 @@ RSpec.describe TreeHelper do it 'returns correct fork path and modal id' do expect(subject).to match a_hash_including( - fork_path: '/namespace1/project-1/-/forks/new', + fork_path: fork_path, fork_modal_id: 'modal-confirm-fork-edit') end end @@ -304,7 +305,7 @@ RSpec.describe TreeHelper do it 'returns correct fork path and modal id' do expect(subject).to match a_hash_including( - fork_path: '/namespace1/project-1/-/forks/new', + fork_path: fork_path, fork_modal_id: 'modal-confirm-fork-webide') end end diff --git a/spec/lib/api/helpers/packages/npm_spec.rb b/spec/lib/api/helpers/packages/npm_spec.rb index cfb68d2c53e..bd28fdadf02 100644 --- a/spec/lib/api/helpers/packages/npm_spec.rb +++ b/spec/lib/api/helpers/packages/npm_spec.rb @@ -3,6 +3,15 @@ require 'spec_helper' RSpec.describe ::API::Helpers::Packages::Npm, feature_category: :package_registry do # rubocop: disable RSpec/FilePath + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + let_it_be(:namespace) { group } + let_it_be(:project) { create(:project, :public, namespace: namespace) } + let_it_be(:package) { create(:npm_package, project: project) } + + let(:package_name) { package.name } + let(:params) { { id: project.id } } + let(:endpoint_scope) { :project } let(:object) { klass.new(params) } let(:klass) do Struct.new(:params) do @@ -11,12 +20,6 @@ RSpec.describe ::API::Helpers::Packages::Npm, feature_category: :package_registr end end - let_it_be(:user) { create(:user) } - let_it_be(:group) { create(:group) } - let_it_be(:namespace) { group } - let_it_be(:project) { create(:project, :public, namespace: namespace) } - let_it_be(:package) { create(:npm_package, project: project) } - before do allow(object).to receive(:endpoint_scope).and_return(endpoint_scope) allow(object).to receive(:current_user).and_return(user) @@ -25,12 +28,7 @@ RSpec.describe ::API::Helpers::Packages::Npm, feature_category: :package_registr describe '#finder_for_endpoint_scope' do subject { object.finder_for_endpoint_scope(package_name) } - let(:package_name) { package.name } - context 'when called with project scope' do - let(:params) { { id: project.id } } - let(:endpoint_scope) { :project } - it 'returns a PackageFinder for project scope' do expect(::Packages::Npm::PackageFinder).to receive(:new).with(package_name, project: project) @@ -142,4 +140,10 @@ RSpec.describe ::API::Helpers::Packages::Npm, feature_category: :package_registr end end end + + describe '#enqueue_sync_metadata_cache_worker' do + it_behaves_like 'enqueue a worker to sync a metadata cache' do + subject { object.enqueue_sync_metadata_cache_worker(project, package_name) } + end + end end diff --git a/spec/lib/banzai/filter/references/external_issue_reference_filter_spec.rb b/spec/lib/banzai/filter/references/external_issue_reference_filter_spec.rb index 79500f43394..86fb7d3964d 100644 --- a/spec/lib/banzai/filter/references/external_issue_reference_filter_spec.rb +++ b/spec/lib/banzai/filter/references/external_issue_reference_filter_spec.rb @@ -223,7 +223,7 @@ RSpec.describe Banzai::Filter::References::ExternalIssueReferenceFilter, feature end context "jira project" do - let_it_be(:service) { create(:jira_integration, project: project) } + let_it_be_with_reload(:service) { create(:jira_integration, project: project) } let(:reference) { issue.to_reference } @@ -250,6 +250,36 @@ RSpec.describe Banzai::Filter::References::ExternalIssueReferenceFilter, feature expect(filter(act).to_html).to eq exp end end + + context 'with a custom regex' do + before do + service.jira_tracker_data.update!(jira_issue_regex: '[JIRA]{2,}-\\d+') + end + + context "with right markdown" do + let(:issue) { ExternalIssue.new("JIRA-123", project) } + + it_behaves_like "external issue tracker" + end + + context "with a single-letter prefix" do + let(:issue) { ExternalIssue.new("J-123", project) } + + it "ignores reference" do + exp = act = "Issue #{reference}" + expect(filter(act).to_html).to eq exp + end + end + + context "with wrong markdown" do + let(:issue) { ExternalIssue.new("#123", project) } + + it "ignores reference" do + exp = act = "Issue #{reference}" + expect(filter(act).to_html).to eq exp + end + end + end end context "ewm project" do diff --git a/spec/requests/api/npm_group_packages_spec.rb b/spec/requests/api/npm_group_packages_spec.rb index efb606b23d7..fe0bf1d8b46 100644 --- a/spec/requests/api/npm_group_packages_spec.rb +++ b/spec/requests/api/npm_group_packages_spec.rb @@ -172,12 +172,31 @@ RSpec.describe API::NpmGroupPackages, feature_category: :package_registry do it_behaves_like 'handling create dist tag requests', scope: :group do let(:url) { api("/groups/#{group.id}/-/packages/npm/-/package/#{package_name}/dist-tags/#{tag_name}") } end + + it_behaves_like 'enqueue a worker to sync a metadata cache' do + let(:tag_name) { 'test' } + let(:url) { api("/groups/#{group.id}/-/packages/npm/-/package/#{package_name}/dist-tags/#{tag_name}") } + let(:env) { { 'api.request.body': package.version } } + let(:headers) { build_token_auth_header(personal_access_token.token) } + + subject { put(url, env: env, headers: headers) } + end end describe 'DELETE /api/v4/packages/npm/-/package/*package_name/dist-tags/:tag' do it_behaves_like 'handling delete dist tag requests', scope: :group do let(:url) { api("/groups/#{group.id}/-/packages/npm/-/package/#{package_name}/dist-tags/#{tag_name}") } end + + it_behaves_like 'enqueue a worker to sync a metadata cache' do + let_it_be(:package_tag) { create(:packages_tag, package: package) } + + let(:tag_name) { package_tag.name } + let(:url) { api("/groups/#{group.id}/-/packages/npm/-/package/#{package_name}/dist-tags/#{tag_name}") } + let(:headers) { build_token_auth_header(personal_access_token.token) } + + subject { delete(url, headers: headers) } + end end describe 'POST /api/v4/groups/:id/-/packages/npm/-/npm/v1/security/advisories/bulk' do diff --git a/spec/requests/api/npm_instance_packages_spec.rb b/spec/requests/api/npm_instance_packages_spec.rb index 63a50e7f511..4f965d86d66 100644 --- a/spec/requests/api/npm_instance_packages_spec.rb +++ b/spec/requests/api/npm_instance_packages_spec.rb @@ -65,12 +65,31 @@ RSpec.describe API::NpmInstancePackages, feature_category: :package_registry do it_behaves_like 'handling create dist tag requests', scope: :instance do let(:url) { api("/packages/npm/-/package/#{package_name}/dist-tags/#{tag_name}") } end + + it_behaves_like 'enqueue a worker to sync a metadata cache' do + let(:tag_name) { 'test' } + let(:url) { api("/packages/npm/-/package/#{package_name}/dist-tags/#{tag_name}") } + let(:env) { { 'api.request.body': package.version } } + let(:headers) { build_token_auth_header(personal_access_token.token) } + + subject { put(url, env: env, headers: headers) } + end end describe 'DELETE /api/v4/packages/npm/-/package/*package_name/dist-tags/:tag' do it_behaves_like 'handling delete dist tag requests', scope: :instance do let(:url) { api("/packages/npm/-/package/#{package_name}/dist-tags/#{tag_name}") } end + + it_behaves_like 'enqueue a worker to sync a metadata cache' do + let_it_be(:package_tag) { create(:packages_tag, package: package) } + + let(:tag_name) { package_tag.name } + let(:url) { api("/packages/npm/-/package/#{package_name}/dist-tags/#{tag_name}") } + let(:headers) { build_token_auth_header(personal_access_token.token) } + + subject { delete(url, headers: headers) } + end end describe 'POST /api/v4/packages/npm/-/npm/v1/security/advisories/bulk' do diff --git a/spec/requests/api/npm_project_packages_spec.rb b/spec/requests/api/npm_project_packages_spec.rb index 33418e85601..955013845da 100644 --- a/spec/requests/api/npm_project_packages_spec.rb +++ b/spec/requests/api/npm_project_packages_spec.rb @@ -52,6 +52,8 @@ RSpec.describe API::NpmProjectPackages, feature_category: :package_registry do .to change { npm_metadata_cache.reload.last_downloaded_at }.from(nil).to(instance_of(ActiveSupport::TimeWithZone)) end + it_behaves_like 'does not enqueue a worker to sync a metadata cache' + context 'when npm_metadata_cache disabled' do before do stub_feature_flags(npm_metadata_cache: false) @@ -66,6 +68,7 @@ RSpec.describe API::NpmProjectPackages, feature_category: :package_registry do end it_behaves_like 'generates metadata response "on-the-fly"' + it_behaves_like 'enqueue a worker to sync a metadata cache' end end end @@ -81,12 +84,31 @@ RSpec.describe API::NpmProjectPackages, feature_category: :package_registry do it_behaves_like 'handling create dist tag requests', scope: :project do let(:url) { api("/projects/#{project.id}/packages/npm/-/package/#{package_name}/dist-tags/#{tag_name}") } end + + it_behaves_like 'enqueue a worker to sync a metadata cache' do + let(:tag_name) { 'test' } + let(:url) { api("/projects/#{project.id}/packages/npm/-/package/#{package_name}/dist-tags/#{tag_name}") } + let(:env) { { 'api.request.body': package.version } } + let(:headers) { build_token_auth_header(personal_access_token.token) } + + subject { put(url, env: env, headers: headers) } + end end describe 'DELETE /api/v4/projects/:id/packages/npm/-/package/*package_name/dist-tags/:tag' do it_behaves_like 'handling delete dist tag requests', scope: :project do let(:url) { api("/projects/#{project.id}/packages/npm/-/package/#{package_name}/dist-tags/#{tag_name}") } end + + it_behaves_like 'enqueue a worker to sync a metadata cache' do + let_it_be(:package_tag) { create(:packages_tag, package: package) } + + let(:tag_name) { package_tag.name } + let(:url) { api("/projects/#{project.id}/packages/npm/-/package/#{package_name}/dist-tags/#{tag_name}") } + let(:headers) { build_token_auth_header(personal_access_token.token) } + + subject { delete(url, headers: headers) } + end end describe 'POST /api/v4/projects/:id/packages/npm/-/npm/v1/security/advisories/bulk' do @@ -339,6 +361,10 @@ RSpec.describe API::NpmProjectPackages, feature_category: :package_registry do it_behaves_like 'handling upload with different authentications' end + it_behaves_like 'enqueue a worker to sync a metadata cache' do + let(:package_name) { "@#{group.path}/my_package_name" } + end + context 'with an existing package' do let_it_be(:second_project) { create(:project, namespace: namespace) } @@ -383,6 +409,10 @@ RSpec.describe API::NpmProjectPackages, feature_category: :package_registry do expect(response).to have_gitlab_http_status(:forbidden) end + + it_behaves_like 'does not enqueue a worker to sync a metadata cache' do + subject { upload_package_with_token } + end end context 'with dependencies' do diff --git a/spec/requests/api/project_packages_spec.rb b/spec/requests/api/project_packages_spec.rb index b84b7e9c52d..09991be998a 100644 --- a/spec/requests/api/project_packages_spec.rb +++ b/spec/requests/api/project_packages_spec.rb @@ -660,6 +660,12 @@ RSpec.describe API::ProjectPackages, feature_category: :package_registry do expect(response).to have_gitlab_http_status(:no_content) end + it_behaves_like 'enqueue a worker to sync a metadata cache' do + let(:package_name) { package1.name } + + subject { delete api(package_url, user) } + end + context 'with JOB-TOKEN auth' do let(:job) { create(:ci_build, :running, user: user, project: project) } @@ -692,6 +698,14 @@ RSpec.describe API::ProjectPackages, feature_category: :package_registry do delete api(package_url, user) end + + it_behaves_like 'does not enqueue a worker to sync a metadata cache' do + before do + project.add_maintainer(user) + end + + subject { delete api(package_url, user) } + end end end end diff --git a/spec/support/matchers/have_native_text_validation_message.rb b/spec/support/matchers/have_native_text_validation_message.rb new file mode 100644 index 00000000000..3923086dc70 --- /dev/null +++ b/spec/support/matchers/have_native_text_validation_message.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +RSpec::Matchers.define :have_native_text_validation_message do |field| + match do |page| + message = page.find_field(field).native.attribute('validationMessage') + expect(message).to match(/Please fill [a-z]+ this field./) + end +end diff --git a/spec/support/shared_examples/npm_sync_metadata_cache_worker_shared_examples.rb b/spec/support/shared_examples/npm_sync_metadata_cache_worker_shared_examples.rb new file mode 100644 index 00000000000..de2dc4c3725 --- /dev/null +++ b/spec/support/shared_examples/npm_sync_metadata_cache_worker_shared_examples.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +RSpec.shared_examples 'does not enqueue a worker to sync a metadata cache' do + it 'does not enqueue a worker to sync a metadata cache' do + expect(Packages::Npm::CreateMetadataCacheWorker).not_to receive(:perform_async) + + subject + end +end + +RSpec.shared_examples 'enqueue a worker to sync a metadata cache' do + before do + project.add_maintainer(user) + end + + it 'enqueues a worker to create a metadata cache' do + expect(Packages::Npm::CreateMetadataCacheWorker) + .to receive(:perform_async).with(project.id, package_name) + + subject + end + + context 'with npm_metadata_cache disabled' do + before do + stub_feature_flags(npm_metadata_cache: false) + end + + it_behaves_like 'does not enqueue a worker to sync a metadata cache' + end +end diff --git a/yarn.lock b/yarn.lock index f2c061419a3..bdbfd253701 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1108,10 +1108,10 @@ resolved "https://registry.yarnpkg.com/@gitlab/favicon-overlay/-/favicon-overlay-2.0.0.tgz#2f32d0b6a4d5b8ac44e2927083d9ab478a78c984" integrity sha512-GNcORxXJ98LVGzOT9dDYKfbheqH6lNgPDD72lyXRnQIH7CjgGyos8i17aSBPq1f4s3zF3PyedFiAR4YEZbva2Q== -"@gitlab/fonts@^1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@gitlab/fonts/-/fonts-1.2.0.tgz#140bea5316e71d9b20d084fb10030db4e4ae73bd" - integrity sha512-9wn+xnMgzvs9EAwMU0kbkcX5IKLXDOVUf6oBOi0zrFpwtM0wY4pfXkswSMl1kfMyNn6nW2n35g7sbv/Uy/QezQ== +"@gitlab/fonts@^1.3.0": + version "1.3.0" + resolved "https://registry.yarnpkg.com/@gitlab/fonts/-/fonts-1.3.0.tgz#df89c1bb6714e4a8a5d3272568aa4de7fb337267" + integrity sha512-DoMUIN3DqjEn7wvcxBg/b7Ite5fTdF5EmuOZoBRo2j0UBGweDXmNBi+9HrTZs4cBU660dOxcf1hATFcG3npbPg== "@gitlab/stylelint-config@4.1.0": version "4.1.0" |