diff options
34 files changed, 356 insertions, 68 deletions
diff --git a/app/assets/javascripts/import_entities/import_groups/components/import_actions_cell.vue b/app/assets/javascripts/import_entities/import_groups/components/import_actions_cell.vue index cb147330264..678efc536f2 100644 --- a/app/assets/javascripts/import_entities/import_groups/components/import_actions_cell.vue +++ b/app/assets/javascripts/import_entities/import_groups/components/import_actions_cell.vue @@ -47,6 +47,7 @@ export default { <gl-button variant="confirm" category="secondary" + data-testid="import-group-button" @click="importGroup({ migrateProjects: true })" >{{ isFinished ? __('Re-import with projects') : __('Import with projects') }}</gl-button > @@ -58,7 +59,6 @@ export default { no-caret variant="confirm" category="secondary" - data-qa-selector="import_group_button" > <gl-disclosure-dropdown-item @action="importGroup({ migrateProjects: false })"> <template #list-item> diff --git a/app/services/metrics/dashboard/base_service.rb b/app/services/metrics/dashboard/base_service.rb index 5975fa28b0b..7863af5affc 100644 --- a/app/services/metrics/dashboard/base_service.rb +++ b/app/services/metrics/dashboard/base_service.rb @@ -10,7 +10,6 @@ module Metrics STAGES = ::Gitlab::Metrics::Dashboard::Stages SEQUENCE = [ STAGES::CommonMetricsInserter, - STAGES::MetricEndpointInserter, STAGES::VariableEndpointInserter, STAGES::PanelIdsInserter, STAGES::TrackPanelType, diff --git a/app/services/metrics/dashboard/gitlab_alert_embed_service.rb b/app/services/metrics/dashboard/gitlab_alert_embed_service.rb index 33c93b25c71..6bf46fbc80e 100644 --- a/app/services/metrics/dashboard/gitlab_alert_embed_service.rb +++ b/app/services/metrics/dashboard/gitlab_alert_embed_service.rb @@ -12,7 +12,6 @@ module Metrics include Gitlab::Utils::StrongMemoize SEQUENCE = [ - STAGES::MetricEndpointInserter, STAGES::PanelIdsInserter ].freeze diff --git a/app/services/metrics/dashboard/pod_dashboard_service.rb b/app/services/metrics/dashboard/pod_dashboard_service.rb index c83f8618460..e2d7a081a3e 100644 --- a/app/services/metrics/dashboard/pod_dashboard_service.rb +++ b/app/services/metrics/dashboard/pod_dashboard_service.rb @@ -10,7 +10,6 @@ module Metrics DASHBOARD_VERSION = '3a91b32f91b2dd3d90275333c0ea3630b3f3f37c4296ede5b5eef59bf523d66b' SEQUENCE = [ - STAGES::MetricEndpointInserter, STAGES::VariableEndpointInserter, STAGES::PanelIdsInserter ].freeze diff --git a/app/services/metrics/dashboard/predefined_dashboard_service.rb b/app/services/metrics/dashboard/predefined_dashboard_service.rb index abdef66c2e0..1777a94bdd0 100644 --- a/app/services/metrics/dashboard/predefined_dashboard_service.rb +++ b/app/services/metrics/dashboard/predefined_dashboard_service.rb @@ -10,7 +10,6 @@ module Metrics DASHBOARD_NAME = nil SEQUENCE = [ - STAGES::MetricEndpointInserter, STAGES::VariableEndpointInserter, STAGES::PanelIdsInserter ].freeze diff --git a/app/services/metrics/dashboard/system_dashboard_service.rb b/app/services/metrics/dashboard/system_dashboard_service.rb index 1bd31b2ba21..b84cf187d2c 100644 --- a/app/services/metrics/dashboard/system_dashboard_service.rb +++ b/app/services/metrics/dashboard/system_dashboard_service.rb @@ -15,7 +15,6 @@ module Metrics STAGES::CommonMetricsInserter, STAGES::CustomMetricsInserter, STAGES::CustomMetricsDetailsInserter, - STAGES::MetricEndpointInserter, STAGES::VariableEndpointInserter, STAGES::PanelIdsInserter ].freeze diff --git a/app/services/metrics/dashboard/transient_embed_service.rb b/app/services/metrics/dashboard/transient_embed_service.rb index 29ea9909a36..8b0ba992342 100644 --- a/app/services/metrics/dashboard/transient_embed_service.rb +++ b/app/services/metrics/dashboard/transient_embed_service.rb @@ -28,11 +28,6 @@ module Metrics invalid_embed_json!(e.message) end - override :sequence - def sequence - [STAGES::MetricEndpointInserter] - end - override :identifiers def identifiers Digest::SHA256.hexdigest(params[:embed_json]) diff --git a/config/feature_flags/development/hamilton_seat_management.yml b/config/feature_flags/development/hamilton_seat_management.yml new file mode 100644 index 00000000000..15ac8e07f68 --- /dev/null +++ b/config/feature_flags/development/hamilton_seat_management.yml @@ -0,0 +1,8 @@ +--- +name: hamilton_seat_management +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/126964 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/419175 +milestone: '16.3' +type: development +group: group::purchase +default_enabled: false diff --git a/danger/bulk_database_actions/Dangerfile b/danger/bulk_database_actions/Dangerfile new file mode 100644 index 00000000000..a8cc7bea000 --- /dev/null +++ b/danger/bulk_database_actions/Dangerfile @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +bulk_database_actions.add_comment_for_bulk_database_action_method_usage diff --git a/danger/plugins/bulk_database_actions.rb b/danger/plugins/bulk_database_actions.rb new file mode 100644 index 00000000000..cc1b21a6673 --- /dev/null +++ b/danger/plugins/bulk_database_actions.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require_relative '../../tooling/danger/bulk_database_actions' + +module Danger + class BulkDatabaseActions < ::Danger::Plugin + include Tooling::Danger::BulkDatabaseActions + end +end diff --git a/db/migrate/20230718094246_add_namespace_id_to_notes.rb b/db/migrate/20230718094246_add_namespace_id_to_notes.rb new file mode 100644 index 00000000000..f2f3ef401a4 --- /dev/null +++ b/db/migrate/20230718094246_add_namespace_id_to_notes.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class AddNamespaceIdToNotes < Gitlab::Database::Migration[2.1] + enable_lock_retries! + + def up + add_column :notes, :namespace_id, :bigint + end + + def down + remove_column :notes, :namespace_id + end +end diff --git a/db/post_migrate/20230718094501_prepare_notes_namespace_id_index.rb b/db/post_migrate/20230718094501_prepare_notes_namespace_id_index.rb new file mode 100644 index 00000000000..41f57814c41 --- /dev/null +++ b/db/post_migrate/20230718094501_prepare_notes_namespace_id_index.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class PrepareNotesNamespaceIdIndex < Gitlab::Database::Migration[2.1] + INDEX_NAME = 'index_notes_on_namespace_id' + + # TODO: Index to be created synchronously as part of https://gitlab.com/gitlab-org/gitlab/-/issues/416127 + def up + prepare_async_index :notes, :namespace_id, name: INDEX_NAME + end + + def down + unprepare_async_index :notes, :namespace_id, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20230724150939_index_projects_on_namespace_id_and_repository_size_limit.rb b/db/post_migrate/20230724150939_index_projects_on_namespace_id_and_repository_size_limit.rb new file mode 100644 index 00000000000..b066cb248fb --- /dev/null +++ b/db/post_migrate/20230724150939_index_projects_on_namespace_id_and_repository_size_limit.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class IndexProjectsOnNamespaceIdAndRepositorySizeLimit < Gitlab::Database::Migration[2.1] + INDEX_NAME = "index_projects_on_namespace_id_and_repository_size_limit" + + disable_ddl_transaction! + + def up + add_concurrent_index :projects, [:namespace_id, :repository_size_limit], name: INDEX_NAME + end + + def down + remove_concurrent_index :projects, [:namespace_id, :repository_size_limit], name: INDEX_NAME + end +end diff --git a/db/schema_migrations/20230718094246 b/db/schema_migrations/20230718094246 new file mode 100644 index 00000000000..4d42af51f7d --- /dev/null +++ b/db/schema_migrations/20230718094246 @@ -0,0 +1 @@ +8cb2cd90109dcc4f29ab34fa4caca617fe7bf42493e3a6712b66978480f06940
\ No newline at end of file diff --git a/db/schema_migrations/20230718094501 b/db/schema_migrations/20230718094501 new file mode 100644 index 00000000000..3db259aa8ba --- /dev/null +++ b/db/schema_migrations/20230718094501 @@ -0,0 +1 @@ +636bd7305c03f94cd161dc21deda05c289946b31a3c04805c08273f9a7067729
\ No newline at end of file diff --git a/db/schema_migrations/20230724150939 b/db/schema_migrations/20230724150939 new file mode 100644 index 00000000000..c3190c22f27 --- /dev/null +++ b/db/schema_migrations/20230724150939 @@ -0,0 +1 @@ +4047c6746335957fc3d274036e20a176080fb7aa695a98667ed9170370960212
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index fc0b145ad38..d1d26f4abd3 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -19080,7 +19080,8 @@ CREATE TABLE notes ( confidential boolean, last_edited_at timestamp with time zone, internal boolean DEFAULT false NOT NULL, - id bigint NOT NULL + id bigint NOT NULL, + namespace_id bigint ); CREATE SEQUENCE notes_id_seq @@ -32728,6 +32729,8 @@ CREATE INDEX index_projects_on_name_trigram ON projects USING gin (name gin_trgm CREATE INDEX index_projects_on_namespace_id_and_id ON projects USING btree (namespace_id, id); +CREATE INDEX index_projects_on_namespace_id_and_repository_size_limit ON projects USING btree (namespace_id, repository_size_limit); + CREATE INDEX index_projects_on_path_and_id ON projects USING btree (path, id); CREATE INDEX index_projects_on_path_trigram ON projects USING gin (path gin_trgm_ops); diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index fd86b249a38..288f04277b6 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -6865,6 +6865,29 @@ Input type: `UserAchievementsDeleteInput` | <a id="mutationuserachievementsdeleteerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | <a id="mutationuserachievementsdeleteuserachievement"></a>`userAchievement` | [`UserAchievement`](#userachievement) | Deleted user achievement. | +### `Mutation.userAddOnAssignmentCreate` + +WARNING: +**Introduced** in 16.3. +This feature is an Experiment. It can be changed or removed at any time. + +Input type: `UserAddOnAssignmentCreateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationuseraddonassignmentcreateaddonpurchaseid"></a>`addOnPurchaseId` | [`GitlabSubscriptionsAddOnPurchaseID!`](#gitlabsubscriptionsaddonpurchaseid) | Global ID of AddOnPurchase to be assinged to. | +| <a id="mutationuseraddonassignmentcreateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationuseraddonassignmentcreateuserid"></a>`userId` | [`UserID!`](#userid) | Global ID of user to be assigned. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationuseraddonassignmentcreateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationuseraddonassignmentcreateerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + ### `Mutation.userCalloutCreate` Input type: `UserCalloutCreateInput` diff --git a/doc/architecture/blueprints/rate_limiting/index.md b/doc/architecture/blueprints/rate_limiting/index.md index f42a70aa97a..7af50097e97 100644 --- a/doc/architecture/blueprints/rate_limiting/index.md +++ b/doc/architecture/blueprints/rate_limiting/index.md @@ -186,6 +186,24 @@ Things we want to build and support by default: 1. Logging that will expose limits applied in Kibana. 1. An automatically generated documentation page describing all the limits. +### Support rate limits based on resources used + +One of the problems of our rate limiting system is that values are static +(e.g. 100 requests per minutes) and irrespective of the complexity or resources +used by the operation. For example: + +- Firing 100 requests per minute to fetch a simple resource can have very different + implications than creating a CI pipeline. +- Each pipeline creation action can perform very differently depending on the + pipeline being created (small MR pipeline VS large scheduled pipeline). +- Paginating resources after an offset of 1000 starts to become expensive on the database. + +We should allow some rate limits to be defiened as `computing score / period` where for +computing score we calculate the milliseconds accumulated (for all requests executed +and inflight) within a given period (for example: 1 minute). + +This way if a user is sending expensive requests they are likely to hit the rate limit earlier. + ### API to expose limits and policies Once we have an established a consistent way to define application limits we diff --git a/doc/ci/runners/saas/macos_saas_runner.md b/doc/ci/runners/saas/macos_saas_runner.md index a559fc7d53e..8550aeb6477 100644 --- a/doc/ci/runners/saas/macos_saas_runner.md +++ b/doc/ci/runners/saas/macos_saas_runner.md @@ -117,6 +117,54 @@ variables: HOMEBREW_NO_AUTO_UPDATE: 1 ``` +## Optimizing Cocoapods + +If you use Cocoapods in a project, you should consider the following optimizations to improve CI performance. + +### Cocoapods CDN + +You can use CDN access to download packages from the CDN instead of having to clone an entire +project repository. CDN access is available in Cocoapods 1.8 or later. + +To enable CDN access, ensure your Podfile starts with: + +```ruby +source 'https://cdn.cocoapods.org/' +``` + +NOTE: +All GitLab SaaS runners include CDN support in Cocoapods. + +### Use GitLab caching + +Use caching in Cocoapods packages in GitLab to only run `pod install` +when pods change, which can improve build performance. + +To [configure caching](../../../ci/caching/index.md) for your project: + +1. Add the `cache` configuration to your `.gitlab-ci.yml` file: + + ```yaml + cache: + key: + files: + - Podfile.lock + paths: + - Pods + ``` + +1. Add the [`cocoapods-check`](https://guides.cocoapods.org/plugins/optimising-ci-times.html) plugin to your project. +1. Update the job script to check for installed dependencies before it calls `pod install`: + + ```shell + bundle exec pod check || bundle exec pod install + ``` + +### Include pods in source control + +You can also [include the pods directory in source control](https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control). This eliminates the need to install pods as part of the CI job, +but it does increase the overall size of your project's repository. + ## Known issues and usage constraints - If the VM image does not include the specific software version you need for your job, the required software must be fetched and installed. This causes an increase in job execution time. diff --git a/doc/development/database_review.md b/doc/development/database_review.md index 42021a5ae95..d742fe8a54b 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -29,6 +29,8 @@ A database review is required for: These metrics could have complex queries over large tables. See the [Analytics Instrumentation Guide](https://about.gitlab.com/handbook/product/analytics-instrumentation-guide/) for implementation details. +- Changes that use [`update`, `delete`, `update_all`, `delete_all` or `destroy_all`](#preparation-when-using-update-delete-update_all-delete_all-or-destroy_all) + methods on an ActiveRecord object. A database reviewer is expected to look out for overly complex queries in the change and review those closer. If the author does not @@ -216,6 +218,15 @@ Include in the MR description: - If you're adding a composite index, another index might become redundant, so remove that in the same migration. For example adding `index(column_A, column_B, column_C)` makes the indexes `index(column_A, column_B)` and `index(column_A)` redundant. +#### Preparation when using `update`, `delete`, `update_all`, `delete_all` or `destroy_all` + +Using these ActiveRecord methods requires extra care because they modify data and can perform poorly, or they +can destroy data if improperly scoped. These methods are also incompatible with Common Table Expression (CTE) +statements. Danger will comment on a Merge Request Diff when these methods are used. + +Follow documentation for [preparation when adding or modifying queries](#preparation-when-adding-or-modifying-queries) +to add the raw SQL query and query plan to the Merge Request description, and request a database review. + ### How to review for database - Check migrations diff --git a/doc/user/analytics/analytics_dashboards.md b/doc/user/analytics/analytics_dashboards.md index 2bba2d6df17..cdc0b164e99 100644 --- a/doc/user/analytics/analytics_dashboards.md +++ b/doc/user/analytics/analytics_dashboards.md @@ -40,7 +40,7 @@ Instead, you can create a custom dashboard with a similar style. ### Value Stream Management -- **Value Streams Dashboard** displays metrics related to [DevOps performance, security exposure, and workstream optimization](../analytics/value_streams_dashboard.md#devops-metrics-comparison-panel). +- **Value Streams Dashboard** displays metrics related to [DevOps performance, security exposure, and workstream optimization](../analytics/value_streams_dashboard.md#devsecops-metrics-comparison-panel). ## Custom dashboards diff --git a/doc/user/analytics/value_streams_dashboard.md b/doc/user/analytics/value_streams_dashboard.md index 39975476778..450ddd52415 100644 --- a/doc/user/analytics/value_streams_dashboard.md +++ b/doc/user/analytics/value_streams_dashboard.md @@ -10,13 +10,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - Released in GitLab 15.11 as an Open [Beta](../../policy/experiment-beta-support.md#beta) feature [with a flag](../../administration/feature_flags.md) named `group_analytics_dashboards_page`. Enabled by default. > - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/392734) in GitLab 16.0. Feature flag `group_analytics_dashboards_page` removed. -You can leave feedback on dashboard bugs or functionality in [issue 381787](https://gitlab.com/gitlab-org/gitlab/-/issues/381787). +You can leave feedback on dashboard bugs or functionality in [issue 419488](https://gitlab.com/gitlab-org/gitlab/-/issues/419488). For more information, see also the [Value Stream Management category direction page](https://about.gitlab.com/direction/plan/value_stream_management/). The Value Streams Dashboard is a customizable dashboard you can use to identify trends, patterns, and opportunities for digital transformation improvements. - -With the Value Streams Dashboard, you can compare software delivery metrics. -This comparison can help you understand whether projects and groups are improving. +The centralized UI in Value Streams Dashboard acts as the single source of truth (SSOT), where all stakeholders can access and view the same set of metrics that are relevant to the organization. The Value Streams Dashboard includes the following metrics: @@ -24,14 +22,16 @@ The Value Streams Dashboard includes the following metrics: - [Value Stream Analytics (VSA) - flow metrics](../group/value_stream_analytics/index.md) - [Vulnerabilities](https://gitlab.com/gitlab-org/gitlab/-/security/vulnerability_report) metrics. -The Value Streams Dashboard allows you to: +With the Value Streams Dashboard, you can: -- Aggregate data records from different APIs. -- Track software performance (DORA) and flow of value (VSA) across the organization. +- Track and compare the above metrics over a period of time. +- Identify downward trends early on. +- Understand security exposure. +- Drill down into individual projects or metrics to take actions for improvement. -## DevOps metrics comparison panel +## DevSecOps metrics comparison panel -The DevOps metrics comparison displays DORA4 and flow metrics for a group or project in the +The DevSecOps metrics comparison displays DORA4, vulnerability, and flow metrics for a group or project in the month-to-date, last month, the month before, and the past 180 days. This visualization helps you get a high-level custom view over multiple DevOps metrics and @@ -60,15 +60,21 @@ For example, if a project has a high score for Deployment Frequency (Velocity), Prerequisite: -- To view the value streams dashboard for a group, you must have at least the Reporter role for the group. +- You must have at least the Reporter role for the group. To view the value streams dashboard: -1. On the left sidebar, at the top, select **Search GitLab** (**{search}**) to find your project or group. -1. Select **Analyze > Value stream analytics**. -1. Below the **Filter results** text box, in the **Lifecycle metrics** row, select **Value Streams Dashboard / DORA**. -1. Optional. To open the new page, append this path `/analytics/dashboards/value_streams_dashboard` to the group URL -(for example, `https://gitlab.com/groups/gitlab-org/-/analytics/dashboards/value_streams_dashboard`). +- From Analytics Dashboards: + + 1. On the group left sidebar, at the top, select **Search GitLab** (**{search}**) to find your group. + 1. Select **Analyze > Analytics Dashboards**. + +- From Value Stream Analytics: + + 1. On the left sidebar, at the top, select **Search GitLab** (**{search}**) to find your project or group. + 1. Select **Analyze > Value stream analytics**. + 1. Below the **Filter results** text box, in the **Lifecycle metrics** row, select **Value Streams Dashboard / DORA**. + 1. Optional. To open the new page, append this path `/analytics/dashboards/value_streams_dashboard` to the group URL (for example, `https://gitlab.com/groups/gitlab-org/-/analytics/dashboards/value_streams_dashboard`). ## Customize the dashboard panels diff --git a/doc/user/workspace/index.md b/doc/user/workspace/index.md index a344946ddbc..d862e1098a6 100644 --- a/doc/user/workspace/index.md +++ b/doc/user/workspace/index.md @@ -19,8 +19,6 @@ A workspace is a virtual sandbox environment for your code in GitLab. You can us Each workspace includes its own set of dependencies, libraries, and tools, which you can customize to meet the specific needs of each project. Workspaces use the AMD64 architecture. -For a demo of this feature, see [GitLab Workspaces Demo](https://go.gitlab.com/qtu66q). - ## Set up a workspace ### Prerequisites @@ -197,6 +195,12 @@ GitLab uses the Linux root group ID permission to create, update, or delete file If you have a container image that does not support arbitrary user IDs, you cannot create, update, or delete files in a workspace. To create a container image that supports arbitrary user IDs, see the [OpenShift documentation](https://docs.openshift.com/container-platform/4.12/openshift_images/create-images.html#use-uid_create-images). +## Related topics + +- [Quickstart guide for GitLab remote development workspaces](https://go.gitlab.com/AVKFvy) +- [Set up your infrastructure for on-demand, cloud-based development environments in GitLab](https://go.gitlab.com/dp75xo) +- [GitLab workspaces demo](https://go.gitlab.com/qtu66q) + ## Troubleshooting When working with workspaces, you might encounter the following issues. diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 5986c5de441..850c89c1fb1 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -984,9 +984,11 @@ excluded_attributes: notes: - :noteable_id - :review_id + - :namespace_id commit_notes: - :noteable_id - :review_id + - :namespace_id label_links: - :label_id - :target_id diff --git a/locale/gitlab.pot b/locale/gitlab.pot index ff6a5725931..87718d3fa4f 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -14911,9 +14911,6 @@ msgstr "" msgid "Date range too large" msgstr "" -msgid "Date to enact enforcement on newly created namespaces" -msgstr "" - msgid "DateRange|%{start_date}–%{end_date}" msgstr "" diff --git a/package.json b/package.json index cfcf21c2613..afb282e0d34 100644 --- a/package.json +++ b/package.json @@ -146,7 +146,7 @@ "gettext-parser": "^6.0.0", "graphql": "^15.7.2", "graphql-tag": "^2.11.0", - "gridstack": "^8.3.0", + "gridstack": "^8.4.0", "highlight.js": "^11.8.0", "immer": "^9.0.15", "ipaddr.js": "^1.9.1", diff --git a/scripts/setup/as-if-jh.sh b/scripts/setup/as-if-jh.sh index 445a988b996..ffc3c6582db 100755 --- a/scripts/setup/as-if-jh.sh +++ b/scripts/setup/as-if-jh.sh @@ -19,7 +19,7 @@ download_jh_path() { echoinfo "Downloading ${path}" - curl --location --output "${output}" --header "Private-Token: ${ADD_JH_FILES_TOKEN}" --get --data-urlencode "sha=${JH_BRANCH}" --data-urlencode "path=${path}" "https://gitlab.com/api/v4/projects/${GITLAB_JH_MIRROR_PROJECT}/repository/archive" + curl -f --location --output "${output}" --header "Private-Token: ${ADD_JH_FILES_TOKEN}" --get --data-urlencode "sha=${JH_BRANCH}" --data-urlencode "path=${path}" "https://gitlab.com/api/v4/projects/${GITLAB_JH_MIRROR_PROJECT}/repository/archive" tar -zxf "${output}" --strip-component 1 rm "${output}" diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb index c22292cb82c..bb50c7cf69d 100644 --- a/spec/db/schema_spec.rb +++ b/spec/db/schema_spec.rb @@ -14,7 +14,8 @@ RSpec.describe 'Database schema', feature_category: :database do # but in Search::NamespaceIndexAssignment model, only `search_index_id` is used as foreign key and indexed search_namespace_index_assignments: [%w[search_index_id index_type]], slack_integrations_scopes: [%w[slack_api_scope_id]], - namespaces: %w[organization_id] # this index is added in an async manner, hence it needs to be ignored in the first phase. + namespaces: %w[organization_id], # this index is added in an async manner, hence it needs to be ignored in the first phase. + notes: %w[namespace_id] # this index is added in an async manner, hence it needs to be ignored in the first phase. }.with_indifferent_access.freeze TABLE_PARTITIONS = %w[ci_builds_metadata].freeze @@ -82,7 +83,7 @@ RSpec.describe 'Database schema', feature_category: :database do merge_requests_compliance_violations: %w[target_project_id], merge_request_diff_commits: %w[commit_author_id committer_id], namespaces: %w[owner_id parent_id], - notes: %w[author_id commit_id noteable_id updated_by_id resolved_by_id confirmed_by_id discussion_id], + notes: %w[author_id commit_id noteable_id updated_by_id resolved_by_id confirmed_by_id discussion_id namespace_id], notification_settings: %w[source_id], oauth_access_grants: %w[resource_owner_id application_id], oauth_access_tokens: %w[resource_owner_id application_id], diff --git a/spec/lib/gitlab/metrics/dashboard/finder_spec.rb b/spec/lib/gitlab/metrics/dashboard/finder_spec.rb index d3cb9760052..ce114e9413f 100644 --- a/spec/lib/gitlab/metrics/dashboard/finder_spec.rb +++ b/spec/lib/gitlab/metrics/dashboard/finder_spec.rb @@ -31,13 +31,6 @@ RSpec.describe Gitlab::Metrics::Dashboard::Finder, :use_clean_rails_memory_store it_behaves_like 'misconfigured dashboard service response', :unprocessable_entity end - context 'when the dashboard contains a metric without a query' do - let(:dashboard) { { 'panel_groups' => [{ 'panels' => [{ 'metrics' => [{ 'id' => 'mock' }] }] }] } } - let(:project) { project_with_dashboard(dashboard_path, dashboard.to_yaml) } - - it_behaves_like 'misconfigured dashboard service response', :unprocessable_entity - end - context 'when the system dashboard is specified' do let(:dashboard_path) { system_dashboard_path } diff --git a/spec/lib/gitlab/metrics/dashboard/processor_spec.rb b/spec/lib/gitlab/metrics/dashboard/processor_spec.rb index 52908a0b339..9bf4a7f761a 100644 --- a/spec/lib/gitlab/metrics/dashboard/processor_spec.rb +++ b/spec/lib/gitlab/metrics/dashboard/processor_spec.rb @@ -15,7 +15,6 @@ RSpec.describe Gitlab::Metrics::Dashboard::Processor do Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter, Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter, Gitlab::Metrics::Dashboard::Stages::CustomMetricsDetailsInserter, - Gitlab::Metrics::Dashboard::Stages::MetricEndpointInserter, Gitlab::Metrics::Dashboard::Stages::PanelIdsInserter, Gitlab::Metrics::Dashboard::Stages::UrlValidator ] @@ -91,8 +90,7 @@ RSpec.describe Gitlab::Metrics::Dashboard::Processor do context 'when the dashboard should not include project metrics' do let(:sequence) do [ - Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter, - Gitlab::Metrics::Dashboard::Stages::MetricEndpointInserter + Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter ] end @@ -105,16 +103,6 @@ RSpec.describe Gitlab::Metrics::Dashboard::Processor do expect(metrics).to eq %w(metric_b metric_a2 metric_a1) end end - - context 'when sample_metrics are requested' do - let(:process_params) { [project, dashboard_yml, sequence, { environment: environment, sample_metrics: true }] } - - it 'includes a sample metrics path for the prometheus endpoint with each metric' do - expect(all_metrics).to satisfy_all do |metric| - metric[:prometheus_endpoint_path] == sample_metrics_path(metric[:id]) - end - end - end end context 'when there are no alerts' do @@ -153,12 +141,6 @@ RSpec.describe Gitlab::Metrics::Dashboard::Processor do it_behaves_like 'errors with message', 'Each "panel" must define an array :metrics' end - - context 'when the dashboard contains a metric which is missing a query' do - let(:dashboard_yml) { { panel_groups: [{ panels: [{ metrics: [{}] }] }] } } - - it_behaves_like 'errors with message', 'Each "metric" must define one of :query or :query_range' - end end private @@ -177,7 +159,6 @@ RSpec.describe Gitlab::Metrics::Dashboard::Processor do unit: metric.unit, label: metric.legend, metric_id: metric.id, - prometheus_endpoint_path: prometheus_path(metric.query), edit_path: edit_metric_path(metric) } end diff --git a/spec/tooling/danger/bulk_database_actions_spec.rb b/spec/tooling/danger/bulk_database_actions_spec.rb new file mode 100644 index 00000000000..584730ba308 --- /dev/null +++ b/spec/tooling/danger/bulk_database_actions_spec.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +require 'gitlab-dangerfiles' +require 'danger' +require 'danger/plugins/internal/helper' +require 'gitlab/dangerfiles/spec_helper' +require 'rspec-parameterized' + +require_relative '../../../tooling/danger/bulk_database_actions' +require_relative '../../../tooling/danger/project_helper' + +RSpec.describe Tooling::Danger::BulkDatabaseActions, feature_category: :tooling do + include_context "with dangerfile" + + let(:fake_danger) { DangerSpecHelper.fake_danger.include(described_class) } + let(:fake_project_helper) { instance_double(Tooling::Danger::ProjectHelper) } + + let(:mr_url) { 'https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1' } + let(:doc_link) { described_class::DOCUMENTATION_LINK } + + let(:comment_text) { "\n#{described_class::COMMENT_TEXT}" } + + let(:file_lines) do + file_diff.map { |line| line.delete_prefix('+') } + end + + let(:file_diff) do + [ + "+ def execute", + "+ pat_family.active.#{method_call}", + "+", + "+ ServiceResponse.success", + "+ end" + ] + end + + before do + allow(bulk_database_actions).to receive(:project_helper).and_return(fake_project_helper) + allow(bulk_database_actions.project_helper).to receive(:file_lines).and_return(file_lines) + allow(bulk_database_actions.helper).to receive(:added_files).and_return([filename]) + allow(bulk_database_actions.helper).to receive(:changed_lines).with(filename).and_return(file_diff) + allow(bulk_database_actions.helper).to receive(:mr_web_url).and_return(mr_url) + end + + subject(:bulk_database_actions) { fake_danger.new(helper: fake_helper) } + + describe '#add_comment_for_bulk_database_action_method_usage' do + context 'when file is a non-spec Ruby file' do + let(:filename) { 'app/services/personal_access_tokens/revoke_token_family_service.rb' } + + using RSpec::Parameterized::TableSyntax + + where(:method_call, :expect_comment?) do + 'update_all(revoked: true)' | true + 'destroy_all' | true + 'delete_all' | true + 'update(revoked: true)' | true + 'delete' | true + 'update_two_factor' | false + 'delete_keys(key)' | false + 'destroy_hook(hook)' | false + 'destroy_all_merged' | false + 'update_all_mirrors' | false + end + + with_them do + it "correctly handles potential bulk database action" do + if expect_comment? + expect(bulk_database_actions).to receive(:markdown).with(comment_text, file: filename, line: 2) + else + expect(bulk_database_actions).not_to receive(:markdown) + end + + bulk_database_actions.add_comment_for_bulk_database_action_method_usage + end + end + end + + context 'for spec directories' do + let(:method_call) { 'update_all(revoked: true)' } + + shared_examples 'no Danger comment' do + it 'does not comment on the bulk update action usage' do + expect(bulk_database_actions).not_to receive(:markdown) + + bulk_database_actions.add_comment_for_bulk_database_action_method_usage + end + end + + context 'for FOSS spec file' do + let(:filename) { 'spec/services/personal_access_tokens/revoke_token_family_service_spec.rb' } + + it_behaves_like 'no Danger comment' + end + + context 'for EE spec file' do + let(:filename) { 'ee/spec/services/personal_access_tokens/revoke_token_family_service_spec.rb' } + + it_behaves_like 'no Danger comment' + end + + context 'for JiHu spec file' do + let(:filename) { 'jh/spec/services/personal_access_tokens/revoke_token_family_service_spec.rb' } + + it_behaves_like 'no Danger comment' + end + end + end +end diff --git a/tooling/danger/bulk_database_actions.rb b/tooling/danger/bulk_database_actions.rb new file mode 100644 index 00000000000..9a26243fb2d --- /dev/null +++ b/tooling/danger/bulk_database_actions.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require_relative 'suggestor' + +module Tooling + module Danger + module BulkDatabaseActions + include ::Tooling::Danger::Suggestor + + BULK_UPDATE_METHODS_REGEX = /\W(((update|delete|destroy)_all)|delete|update)(\(|\s+|$)/ + + DOCUMENTATION_LINK = 'https://docs.gitlab.com/ee/development/database_review.html#preparation-when-using-update-delete-update_all-and-destroy_all' + COMMENT_TEXT = + "When using `update`, `delete`, `update_all`, `delete_all` or `destroy_all` you must include the full " \ + "database query and query execution plan in the merge request description, and request a ~database review. " \ + "This comment can be ignored if the object is not an ActiveRecord class, since no database query " \ + "would be generated. For more information, see [Database Review documentation](#{DOCUMENTATION_LINK}).".freeze + + def add_comment_for_bulk_database_action_method_usage + changed_ruby_files.each do |filename| + add_suggestion( + filename: filename, + regex: BULK_UPDATE_METHODS_REGEX, + comment_text: COMMENT_TEXT + ) + end + end + + def changed_ruby_files + helper.added_files.select { |f| f.end_with?('.rb') && !f.start_with?('spec/', 'ee/spec/', 'jh/spec/') } + end + end + end +end diff --git a/yarn.lock b/yarn.lock index b50a0ba3861..0171b999690 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6746,10 +6746,10 @@ graphql@^15.7.2: resolved "https://registry.yarnpkg.com/graphql/-/graphql-15.7.2.tgz#85ab0eeb83722977151b3feb4d631b5f2ab287ef" integrity sha512-AnnKk7hFQFmU/2I9YSQf3xw44ctnSFCfp3zE0N6W174gqe9fWG/2rKaKxROK7CcI3XtERpjEKFqts8o319Kf7A== -gridstack@^8.3.0: - version "8.3.0" - resolved "https://registry.yarnpkg.com/gridstack/-/gridstack-8.3.0.tgz#4c79f8b8c4cffeb3664266108e38ed91b3d0f7b4" - integrity sha512-RcL2xskAYKOpakvpSwHdKheG7C7YgNY7777C5m+T1JMjSgcmEc3qPBM573l0NuyjMz4Errx1/3p+rMgUfF4+mw== +gridstack@^8.4.0: + version "8.4.0" + resolved "https://registry.yarnpkg.com/gridstack/-/gridstack-8.4.0.tgz#7af49159f9dc144c89a2c56246e1710406f75fcf" + integrity sha512-qLJuJrBy9bbG3hI+h2cEhiuZ51J3MyEMmv5AXg7MCFiBeG8A4HyIUytueqtD/oZcA3Pccq2Xoj7GrwpmKOS3ig== gzip-size@^6.0.0: version "6.0.0" |